diff --git a/cmake/cmake.version b/cmake/cmake.version index 70dd4aeaa47b90966962b31cb306806ea7917415..fe35fbe7bd7822b447828f34627c1a9421393a62 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.7.2.alpha") + SET(TD_VER_NUMBER "3.1.0.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/cmake/geos_CMakeLists.txt.in b/cmake/geos_CMakeLists.txt.in index 44b3ec027cec62336ed77dd22c02ab64a5b2cf17..f939ccead045fcab4c3aee14e86bb014ae39e8cb 100644 --- a/cmake/geos_CMakeLists.txt.in +++ b/cmake/geos_CMakeLists.txt.in @@ -2,11 +2,11 @@ # geos ExternalProject_Add(geos GIT_REPOSITORY https://github.com/libgeos/geos.git - GIT_TAG 3.11.2 + GIT_TAG 3.12.0 SOURCE_DIR "${TD_CONTRIB_DIR}/geos" BINARY_DIR "" CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - ) \ No newline at end of file + ) diff --git a/cmake/rocksdb_CMakeLists.txt.in b/cmake/rocksdb_CMakeLists.txt.in index f238ed20af79ba74d07927eb66b35456a00b279e..45599d82e3a8750f8b6e4823c0f8eb8dc8a79f24 100644 --- a/cmake/rocksdb_CMakeLists.txt.in +++ b/cmake/rocksdb_CMakeLists.txt.in @@ -5,8 +5,8 @@ if (${BUILD_CONTRIB}) URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.1.1.tar.gz URL_HASH MD5=3b4c97ee45df9c8a5517308d31ab008b DOWNLOAD_NO_PROGRESS 1 - DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" - SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" + DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" + SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" @@ -18,8 +18,8 @@ else() URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.1.1.tar.gz URL_HASH MD5=3b4c97ee45df9c8a5517308d31ab008b DOWNLOAD_NO_PROGRESS 1 - DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" - SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" + DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" + SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" diff --git a/docs/en/12-taos-sql/01-data-type.md b/docs/en/12-taos-sql/01-data-type.md index 13007d5bb1beef28a7307b648754ee2bced41a21..b9d51bcfcdbd2044d6fc8a66399832a732fda6f0 100644 --- a/docs/en/12-taos-sql/01-data-type.md +++ b/docs/en/12-taos-sql/01-data-type.md @@ -42,10 +42,20 @@ In TDengine, the data types below can be used when specifying a column or tag. | 14 | NCHAR | User Defined | Multi-byte string that can include multi byte characters like Chinese characters. Each character of NCHAR type consumes 4 bytes storage. The string value should be quoted with single quotes. Literal single quote inside the string must be preceded with backslash, like `\'`. The length must be specified when defining a column or tag of NCHAR type, for example nchar(10) means it can store at most 10 characters of nchar type and will consume fixed storage of 40 bytes. An error will be reported if the string value exceeds the length defined. | | 15 | JSON | | JSON type can only be used on tags. A tag of json type is excluded with any other tags of any other type. | | 16 | VARCHAR | User-defined | Alias of BINARY | +| 16 | GEOMETRY | User-defined | Geometry | :::note +- Each row of the table cannot be longer than 48KB (64KB since version 3.0.5.0) (note that each BINARY/NCHAR/GEOMETRY column takes up an additional 2 bytes of storage space). - Only ASCII visible characters are suggested to be used in a column or tag of BINARY type. Multi-byte characters must be stored in NCHAR type. - The length of BINARY can be up to 16,374(data column is 65,517 and tag column is 16,382 since version 3.0.5.0) bytes. The string value must be quoted with single quotes. You must specify a length in bytes for a BINARY value, for example binary(20) for up to twenty single-byte characters. If the data exceeds the specified length, an error will occur. The literal single quote inside the string must be preceded with back slash like `\'` +- The maximum length of the GEOMETRY data column is 65,517 bytes, and the maximum length of the tag column is 16,382 bytes. Supports POINT, LINESTRING, and POLYGON subtypes of 2D. The following table describes the length calculation method: + + | # | **Syntax** | **MinLen** | **MaxLen** | **Growth of each point** | + |---|--------------------------------------|------------|------------|--------------------------| + | 1 | POINT(1.0 1.0) | 21 | 21 | NA | + | 2 | LINESTRING(1.0 1.0, 2.0 2.0) | 9+2*16 | 9+4094*16 | +16 | + | 3 | POLYGON((1.0 1.0, 2.0 2.0, 1.0 1.0)) | 13+3*16 | 13+4094*16 | +16 | + - Numeric values in SQL statements will be determined as integer or float type according to whether there is decimal point or whether scientific notation is used, so attention must be paid to avoid overflow. For example, 9999999999999999999 will be considered as overflow because it exceeds the upper limit of long integer, but 9999999999999999999.0 will be considered as a legal float number. ::: diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index e783d61497a7fe153b1f9697cf143728bfdcc7c6..e7f3aa8d1bd1a2b4c675af22813e1775c6105b7c 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -36,8 +36,6 @@ database_option: { | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value | WAL_RETENTION_SIZE value - | WAL_ROLL_PERIOD value - | WAL_SEGMENT_SIZE value } ``` @@ -75,10 +73,8 @@ database_option: { - TABLE_PREFIX: The prefix in the table name that is ignored when distributing a table to a vgroup when it's a positive number, or only the prefix is used when distributing a table to a vgroup, the default value is 0; For example, if the table name v30001, then "0001" is used if TSDB_PREFIX is set to 2 but "v3" is used if TSDB_PREFIX is set to -2; It can help you to control the distribution of tables. - TABLE_SUFFIX: The suffix in the table name that is ignored when distributing a table to a vgroup when it's a positive number, or only the suffix is used when distributing a table to a vgroup, the default value is 0; For example, if the table name v30001, then "v300" is used if TSDB_SUFFIX is set to 2 but "01" is used if TSDB_SUFFIX is set to -2; It can help you to control the distribution of tables. - TSDB_PAGESIZE: The page size of the data storage engine in a vnode. The unit is KB. The default is 4 KB. The range is 1 to 16384, that is, 1 KB to 16 MB. -- WAL_RETENTION_PERIOD: specifies the maximum time of which WAL files are to be kept for consumption. This parameter is used for data subscription. Enter a time in seconds. The default value 0. A value of 0 indicates that WAL files are not required to keep for consumption. Alter it with a proper value at first to create topics. +- WAL_RETENTION_PERIOD: specifies the maximum time of which WAL files are to be kept for consumption. This parameter is used for data subscription. Enter a time in seconds. The default value is 3600, which means the data in latest 3600 seconds will be kept in WAL for data subscription. Please adjust this parameter to a more proper value for your data subscription. - WAL_RETENTION_SIZE: specifies the maximum total size of which WAL files are to be kept for consumption. This parameter is used for data subscription. Enter a size in KB. The default value is 0. A value of 0 indicates that the total size of WAL files to keep for consumption has no upper limit. -- WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after TSDB data in memory are flushed to disk. -- WAL_SEGMENT_SIZE: specifies the maximum size of a WAL file. After the current WAL file reaches this size, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after TSDB data in memory are flushed to disk. ### Example Statement ```sql diff --git a/docs/en/12-taos-sql/03-table.md b/docs/en/12-taos-sql/03-table.md index 7f39fb58673d1b79d184884087f09327568d67ca..10c44848c95632777c98b936929f615c550accb2 100644 --- a/docs/en/12-taos-sql/03-table.md +++ b/docs/en/12-taos-sql/03-table.md @@ -9,27 +9,27 @@ You create standard tables and subtables with the `CREATE TABLE` statement. ```sql CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definition] ...) [table_options] - + CREATE TABLE create_subtable_clause - + CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definition] ...) [TAGS (create_definition [, create_definition] ...)] [table_options] - + create_subtable_clause: { create_subtable_clause [create_subtable_clause] ... | [IF NOT EXISTS] [db_name.]tb_name USING [db_name.]stb_name [(tag_name [, tag_name] ...)] TAGS (tag_value [, tag_value] ...) } - + create_definition: col_name column_definition - + column_definition: type_name [comment 'string_value'] - + table_options: table_option ... - + table_option: { COMMENT 'string_value' | WATERMARK duration[,duration] @@ -45,9 +45,9 @@ table_option: { 1. The first column of a table MUST be of type TIMESTAMP. It is automatically set as the primary key. 2. The maximum length of the table name is 192 bytes. -3. The maximum length of each row is 48k(64k since version 3.0.5.0) bytes, please note that the extra 2 bytes used by each BINARY/NCHAR column are also counted. +3. The maximum length of each row is 48k(64k since version 3.0.5.0) bytes, please note that the extra 2 bytes used by each BINARY/NCHAR/GEOMETRY column are also counted. 4. The name of the subtable can only consist of characters from the English alphabet, digits and underscore. Table names can't start with a digit. Table names are case insensitive. -5. The maximum length in bytes must be specified when using BINARY or NCHAR types. +5. The maximum length in bytes must be specified when using BINARY/NCHAR/GEOMETRY types. 6. Escape character "\`" can be used to avoid the conflict between table names and reserved keywords, above rules will be bypassed when using escape character on table names, but the upper limit for the name length is still valid. The table names specified using escape character are case sensitive. For example \`aBc\` and \`abc\` are different table names but `abc` and `aBc` are same table names because they are both converted to `abc` internally. Only ASCII visible characters can be used with escape character. @@ -58,7 +58,7 @@ table_option: { 3. MAX_DELAY: specifies the maximum latency for pushing computation results. The default value is 15 minutes or the value of the INTERVAL parameter, whichever is smaller. Enter a value between 0 and 15 minutes in milliseconds, seconds, or minutes. You can enter multiple values separated by commas (,). Note: Retain the default value if possible. Configuring a small MAX_DELAY may cause results to be frequently pushed, affecting storage and query performance. This parameter applies only to supertables and takes effect only when the RETENTIONS parameter has been specified for the database. 4. ROLLUP: specifies aggregate functions to roll up. Rolling up a function provides downsampled results based on multiple axes. This parameter applies only to supertables and takes effect only when the RETENTIONS parameter has been specified for the database. You can specify only one function to roll up. The rollup takes effect on all columns except TS. Enter one of the following values: avg, sum, min, max, last, or first. 5. SMA: specifies functions on which to enable small materialized aggregates (SMA). SMA is user-defined precomputation of aggregates based on data blocks. Enter one of the following values: max, min, or sum This parameter can be used with supertables and standard tables. -6. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The default value is 0, i.e. never expire. +6. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The default value is 0, i.e. never expire. ## Create Subtables @@ -88,7 +88,7 @@ You can create multiple subtables in a single SQL statement provided that all su ```sql ALTER TABLE [db_name.]tb_name alter_table_clause - + alter_table_clause: { alter_table_options | ADD COLUMN col_name column_type @@ -96,10 +96,10 @@ alter_table_clause: { | MODIFY COLUMN col_name column_type | RENAME COLUMN old_col_name new_col_name } - + alter_table_options: alter_table_option ... - + alter_table_option: { TTL value | COMMENT 'string_value' @@ -142,15 +142,15 @@ ALTER TABLE tb_name RENAME COLUMN old_col_name new_col_name ```sql ALTER TABLE [db_name.]tb_name alter_table_clause - + alter_table_clause: { alter_table_options | SET TAG tag_name = new_tag_value } - + alter_table_options: alter_table_option ... - + alter_table_option: { TTL value | COMMENT 'string_value' diff --git a/docs/en/12-taos-sql/04-stable.md b/docs/en/12-taos-sql/04-stable.md index 3a4d6cc590279ad00a20a11ff95a53b4284a7b08..66d93aed580ae04e8293950b253f1b6429660b72 100644 --- a/docs/en/12-taos-sql/04-stable.md +++ b/docs/en/12-taos-sql/04-stable.md @@ -51,6 +51,11 @@ DESCRIBE [db_name.]stb_name; ### View tag information for all child tables in the supertable +``` +SHOW TABLE TAGS FROM table_name [FROM db_name]; +SHOW TABLE TAGS FROM [db_name.]table_name; +``` + ``` taos> SHOW TABLE TAGS FROM st1; tbname | id | loc | diff --git a/docs/en/12-taos-sql/16-operators.md b/docs/en/12-taos-sql/16-operators.md index 32ad4e7075b6a510cb537016effb6064e6c51794..9328d1688a0a2e74cad78da01fbf26c508c20159 100644 --- a/docs/en/12-taos-sql/16-operators.md +++ b/docs/en/12-taos-sql/16-operators.md @@ -39,7 +39,7 @@ TDengine supports the `UNION` and `UNION ALL` operations. UNION ALL collects all | 3 | \>, < | All types except BLOB, MEDIUMBLOB, and JSON | Greater than and less than | | 4 | \>=, <= | All types except BLOB, MEDIUMBLOB, and JSON | Greater than or equal to and less than or equal to | | 5 | IS [NOT] NULL | All types | Indicates whether the value is null | -| 6 | [NOT] BETWEEN AND | All types except BLOB, MEDIUMBLOB, and JSON | Closed interval comparison | +| 6 | [NOT] BETWEEN AND | All types except BLOB, MEDIUMBLOB, JSON and GEOMETRY | Closed interval comparison | | 7 | IN | All types except BLOB, MEDIUMBLOB, and JSON; the primary key (timestamp) is also not supported | Equal to any value in the list | | 8 | LIKE | BINARY, NCHAR, and VARCHAR | Wildcard match | | 9 | MATCH, NMATCH | BINARY, NCHAR, and VARCHAR | Regular expression match | diff --git a/docs/en/12-taos-sql/20-keywords.md b/docs/en/12-taos-sql/20-keywords.md index a2191c87ee17444d5341b8e97b33a954be84dc98..3c441ed8d40f1028ca2accfa4f4413e259e01152 100644 --- a/docs/en/12-taos-sql/20-keywords.md +++ b/docs/en/12-taos-sql/20-keywords.md @@ -334,8 +334,6 @@ The following list shows all reserved keywords: - WAL_LEVEL - WAL_RETENTION_PERIOD - WAL_RETENTION_SIZE -- WAL_ROLL_PERIOD -- WAL_SEGMENT_SIZE - WATERMARK - WHERE - WINDOW_CLOSE diff --git a/docs/en/12-taos-sql/22-meta.md b/docs/en/12-taos-sql/22-meta.md index 194c75b19c327b71bf82f318380f930e7a217dc6..2d7c517dcaadf5dbc3e033e3d16c012f36966a67 100644 --- a/docs/en/12-taos-sql/22-meta.md +++ b/docs/en/12-taos-sql/22-meta.md @@ -74,38 +74,36 @@ Provides information about the cluster. Provides information about user-created databases. Similar to SHOW DATABASES. -| # | **Column** | **Data Type** | **Description** | -| --- | :------------------: | ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | name | BINARY(32) | Database name | -| 2 | create_time | TIMESTAMP | Creation time | -| 3 | ntables | INT | Number of standard tables and subtables (not including supertables) | -| 4 | vgroups | INT | Number of vgroups. It should be noted that `vnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 6 | replica | INT | Number of replicas. It should be noted that `replica` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 7 | strict | BINARY(4) | Obsoleted | -| 8 | duration | INT | Duration for storage of single files. It should be noted that `duration` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 9 | keep | INT | Data retention period. It should be noted that `keep` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 10 | buffer | INT | Write cache size per vnode, in MB. It should be noted that `buffer` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 11 | pagesize | INT | Page size for vnode metadata storage engine, in KB. It should be noted that `pagesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 12 | pages | INT | Number of pages per vnode metadata storage engine. It should be noted that `pages` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 13 | minrows | INT | Maximum number of records per file block. It should be noted that `minrows` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 14 | maxrows | INT | Minimum number of records per file block. It should be noted that `maxrows` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 15 | comp | INT | Compression method. It should be noted that `comp` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 16 | precision | BINARY(2) | Time precision. It should be noted that `precision` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 17 | status | BINARY(10) | Current database status | -| 18 | retentions | BINARY (60) | Aggregation interval and retention period. It should be noted that `retentions` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 19 | single_stable | BOOL | Whether the database can contain multiple supertables. It should be noted that `single_stable` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 20 | cachemodel | BINARY(60) | Caching method for the newest data. It should be noted that `cachemodel` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 21 | cachesize | INT | Memory per vnode used for caching the newest data. It should be noted that `cachesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 22 | wal_level | INT | WAL level. It should be noted that `wal_level` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 23 | wal_fsync_period | INT | Interval at which WAL is written to disk. It should be noted that `wal_fsync_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 24 | wal_retention_period | INT | WAL retention period, in seconds. It should be noted that `wal_retention_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 25 | wal_retention_size | INT | Maximum WAL size. It should be noted that `wal_retention_size` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 26 | wal_roll_period | INT | WAL rotation period. It should be noted that `wal_roll_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 27 | wal_segment_size | BIGINT | WAL file size. It should be noted that `wal_segment_size` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 28 | stt_trigger | SMALLINT | The threshold for number of files to trigger file merging. It should be noted that `stt_trigger` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 29 | table_prefix | SMALLINT | The prefix length in the table name that is ignored when distributing table to vnode based on table name. It should be noted that `table_prefix` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 30 | table_suffix | SMALLINT | The suffix length in the table name that is ignored when distributing table to vnode based on table name. It should be noted that `table_suffix` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 31 | tsdb_pagesize | INT | The page size for internal storage engine, its unit is KB. It should be noted that `tsdb_pagesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| # | **Column** | **Data Type** | **Description** | +| --- | :------------------: | ---------------- | ------------------------------------------------ | +| 1| name| BINARY(32)| Database name | +| 2 | create_time | TIMESTAMP | Creation time | +| 3 | ntables | INT | Number of standard tables and subtables (not including supertables) | +| 4 | vgroups | INT | Number of vgroups. It should be noted that `vnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 6 | replica | INT | Number of replicas. It should be noted that `replica` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 7 | strict | BINARY(4) | Obsoleted | +| 8 | duration | INT | Duration for storage of single files. It should be noted that `duration` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 9 | keep | INT | Data retention period. It should be noted that `keep` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 10 | buffer | INT | Write cache size per vnode, in MB. It should be noted that `buffer` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 11 | pagesize | INT | Page size for vnode metadata storage engine, in KB. It should be noted that `pagesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 12 | pages | INT | Number of pages per vnode metadata storage engine. It should be noted that `pages` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 13 | minrows | INT | Maximum number of records per file block. It should be noted that `minrows` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 14 | maxrows | INT | Minimum number of records per file block. It should be noted that `maxrows` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 15 | comp | INT | Compression method. It should be noted that `comp` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 16 | precision | BINARY(2) | Time precision. It should be noted that `precision` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 17 | status | BINARY(10) | Current database status | +| 18 | retentions | BINARY (60) | Aggregation interval and retention period. It should be noted that `retentions` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 19 | single_stable | BOOL | Whether the database can contain multiple supertables. It should be noted that `single_stable` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 20 | cachemodel | BINARY(60) | Caching method for the newest data. It should be noted that `cachemodel` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 21 | cachesize | INT | Memory per vnode used for caching the newest data. It should be noted that `cachesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 22 | wal_level | INT | WAL level. It should be noted that `wal_level` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 23 | wal_fsync_period | INT | Interval at which WAL is written to disk. It should be noted that `wal_fsync_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 24 | wal_retention_period | INT | WAL retention period, in second. It should be noted that `wal_retention_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 25 | wal_retention_size | INT | Maximum WAL size. It should be noted that `wal_retention_size` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 26 | stt_trigger | SMALLINT | The threshold for number of files to trigger file merging. It should be noted that `stt_trigger` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 27 | table_prefix | SMALLINT | The prefix length in the table name that is ignored when distributing table to vnode based on table name. It should be noted that `table_prefix` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 28 | table_suffix | SMALLINT | The suffix length in the table name that is ignored when distributing table to vnode based on table name. It should be noted that `table_suffix` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 29 | tsdb_pagesize | INT | The page size for internal storage engine, its unit is KB. It should be noted that `tsdb_pagesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | ## INS_FUNCTIONS diff --git a/docs/en/12-taos-sql/24-show.md b/docs/en/12-taos-sql/24-show.md index bd4a60b20e87e21f8948aa64ed0f9bb86da6b6c6..b663fbd43521d03931deb7e3f9bfc65ac99d85ff 100644 --- a/docs/en/12-taos-sql/24-show.md +++ b/docs/en/12-taos-sql/24-show.md @@ -101,6 +101,7 @@ Note: TDengine Enterprise Edition only. ```sql SHOW INDEXES FROM tbl_name [FROM db_name]; +SHOW INDEXES FROM [db_name.]tbl_name; ``` Shows indices that have been created. @@ -326,6 +327,7 @@ Note that only the information about the data blocks in the data file will be di ```sql SHOW TAGS FROM child_table_name [FROM db_name]; +SHOW TAGS FROM [db_name.]child_table_name; ``` Shows all tag information in a subtable. diff --git a/docs/en/12-taos-sql/27-index.md b/docs/en/12-taos-sql/27-index.md index 33e511b8a090cafa25af4c349f68f277042526c9..a89c8929c1b632e5a2deb8d493459d58f57b0cc1 100644 --- a/docs/en/12-taos-sql/27-index.md +++ b/docs/en/12-taos-sql/27-index.md @@ -56,6 +56,7 @@ DROP INDEX index_name; ````sql SHOW INDEXES FROM tbl_name [FROM db_name]; +SHOW INDEXES FROM [db_name.]tbl_name ; ```` Shows indices that have been created for the specified database or table. diff --git a/docs/en/12-taos-sql/29-changes.md b/docs/en/12-taos-sql/29-changes.md index 086aee59fe0583e905c04320c9fdd3a4a93351c7..bbb52db4d9100c6643b6ec84460f273e9be3f2e6 100644 --- a/docs/en/12-taos-sql/29-changes.md +++ b/docs/en/12-taos-sql/29-changes.md @@ -18,6 +18,7 @@ description: This document describes how TDengine SQL has changed in version 3.0 | 8 | Mixed operations | Enhanced | Mixing scalar and vector operations in queries has been enhanced and is supported in all SELECT clauses. | 9 | Tag operations | Added | Tag columns can be used in queries and clauses like data columns. | 10 | Timeline clauses and time functions in supertables | Enhanced | When PARTITION BY is not used, data in supertables is merged into a single timeline. +| 11 | GEOMETRY | Added | Geometry ## SQL Syntax @@ -33,7 +34,7 @@ The following data types can be used in the schema for standard tables. | 6 | ALTER USER | Modified | Deprecated | 7 | COMPACT VNODES | Not supported | Compacted the data on a vnode. Not supported. | 8 | CREATE ACCOUNT | Deprecated| This Enterprise Edition-only statement has been removed. It returns the error "This statement is no longer supported." -| 9 | CREATE DATABASE | Modified | Deprecated +| 9 | CREATE DATABASE | Modified | Deprecated | 10 | CREATE DNODE | Modified | Now supports specifying hostname and port separately | 11 | CREATE INDEX | Added | Creates an SMA index. | 12 | CREATE MNODE | Added | Creates an mnode. diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index 56f5e20cb44be5ebaa9f112c70233f1b6bd5a6be..a98683d43c169c6a2f76dc154035c0af84464287 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -31,7 +31,8 @@ Websocket connections are supported on all platforms that can run Go. | connector-rust version | TDengine version | major features | | :----------------: | :--------------: | :--------------------------------------------------: | -| v0.8.12 | 3.0.5.0 or later | TMQ: Get consuming progress and seek offset to consume. | +| v0.9.2 | 3.0.7.0 or later | STMT: Get tag_fields and col_fields under ws. | +| v0.8.12 | 3.0.5.0 | TMQ: Get consuming progress and seek offset to consume. | | v0.8.0 | 3.0.4.0 | Support schemaless insert. | | v0.7.6 | 3.0.3.0 | Support req_id in query. | | v0.6.0 | 3.0.0.0 | Base features. | diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 2d213feceb109b6ed7b6bd2a90e36619ca53b2ca..a080ef75b45c3f0ed1fc6330d9c3de528f896948 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -166,7 +166,7 @@ Please note the `taoskeeper` needs to be installed and running to create the `lo | Attribute | Description | | ------------- | ---------------------------------------------------------------------------- | -| Applicable | Server Only | +| Applicable | Server and Client | | Meaning | Switch for allowing TDengine to collect and report service usage information | | Value Range | 0: Not allowed; 1: Allowed | | Default Value | 1 | @@ -174,7 +174,7 @@ Please note the `taoskeeper` needs to be installed and running to create the `lo | Attribute | Description | | ------------- | ---------------------------------------------------------------------------- | -| Applicable | Server Only | +| Applicable | Server and Client | | Meaning | Switch for allowing TDengine to collect and report crash related information | | Value Range | 0,1 0: Not allowed; 1: allowed | | Default Value | 1 | @@ -722,6 +722,16 @@ The charset that takes effect is UTF-8. | Value Range | 0: not change; 1: change by modification | | Default Value | 0 | +### keepTimeOffset + +| Attribute | Description | +| ------------- | ------------------------- | +| Applicable | Server Only | +| Meaning | Latency of data migration | +| Unit | hour | +| Value Range | 0-23 | +| Default Value | 0 | + ### tmqMaxTopicNum | Attribute | Description | @@ -788,3 +798,4 @@ The charset that takes effect is UTF-8. | 53 | udf | Yes | Yes | | | 54 | enableCoreFile | Yes | Yes | | | 55 | ttlChangeOnWrite | No | Yes | | +| 56 | keepTimeOffset | Yes | Yes | | diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index 79a6badfead70c27fc344b1e506aa8ea5afb624d..3e51aa72bb85841e219f89c1b91b4ff4e4f791cc 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -30,7 +30,8 @@ Websocket 连接支持所有能运行 Rust 的平台。 | Rust 连接器版本 | TDengine 版本 | 主要功能 | | :----------------: | :--------------: | :--------------------------------------------------: | -| v0.8.12 | 3.0.5.0 or later | 消息订阅：获取消费进度及按照指定进度开始消费。 | +| v0.9.2 | 3.0.7.0 or later | STMT：ws 下获取 tag_fields、col_fields。 | +| v0.8.12 | 3.0.5.0 | 消息订阅：获取消费进度及按照指定进度开始消费。 | | v0.8.0 | 3.0.4.0 | 支持无模式写入。 | | v0.7.6 | 3.0.3.0 | 支持在请求中使用 req_id。 | | v0.6.0 | 3.0.0.0 | 基础功能。 | diff --git a/docs/zh/08-connector/_verify_windows.mdx b/docs/zh/08-connector/_verify_windows.mdx index 850fb5735de4ab4a094c4beb05c6bb6eb2f3e9f3..bd9547f93703318cf654a069ab85916ba4b7ff89 100644 --- a/docs/zh/08-connector/_verify_windows.mdx +++ b/docs/zh/08-connector/_verify_windows.mdx @@ -2,10 +2,10 @@ ```text taos> show databases; - name | create_time | vgroups | ntables | replica | strict | duration | keep | buffer | pagesize | pages | minrows | maxrows | comp | precision | status | retention | single_stable | cachemodel | cachesize | wal_level | wal_fsync_period | wal_retention_period | wal_retention_size | wal_roll_period | wal_seg_size | -========================================================================================================================================================================================================================================================================================================================================================================================================================================================================= - information_schema | NULL | NULL | 14 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - performance_schema | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | + name | create_time | vgroups | ntables | replica | strict | duration | keep | buffer | pagesize | pages | minrows | maxrows | comp | precision | status | retention | single_stable | cachemodel | cachesize | wal_level | wal_fsync_period | wal_retention_period | wal_retention_size | +=============================================================================================================================================================================================================================================================================================================================================================================================================================== + information_schema | NULL | NULL | 14 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | + performance_schema | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | test | 2022-08-04 16:46:40.506 | 2 | 0 | 1 | off | 14400m | 5256000m,5256000m,5256000m | 96 | 4 | 256 | 100 | 4096 | 2 | ms | ready | NULL | false | none | 1 | 1 | 3000 | 0 | 0 | 0 | 0 | Query OK, 3 rows in database (0.123000s) diff --git a/docs/zh/12-taos-sql/01-data-type.md b/docs/zh/12-taos-sql/01-data-type.md index 4a4c1d6ec69b95e8cf3c38b5b5a8d2a4cb335d62..1df07e7e7f009d202c6fe71a1940dc6ab388d845 100644 --- a/docs/zh/12-taos-sql/01-data-type.md +++ b/docs/zh/12-taos-sql/01-data-type.md @@ -42,12 +42,21 @@ CREATE DATABASE db_name PRECISION 'ns'; | 14 | NCHAR | 自定义 | 记录包含多字节字符在内的字符串，如中文字符。每个 NCHAR 字符占用 4 字节的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 `\'`。NCHAR 使用时须指定字符串大小，类型为 NCHAR(10) 的列表示此列的字符串最多存储 10 个 NCHAR 字符。如果用户字符串长度超出声明长度，将会报错。 | | 15 | JSON | | JSON 数据类型， 只有 Tag 可以是 JSON 格式 | | 16 | VARCHAR | 自定义 | BINARY 类型的别名 | +| 17 | GEOMETRY | 自定义 | 几何类型 | :::note -- 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置）。 +- 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）（注意：每个 BINARY/NCHAR/GEOMETRY 类型的列还会额外占用 2 个字节的存储位置）。 - 虽然 BINARY 类型在底层存储上支持字节型的二进制字符，但不同编程语言对二进制数据的处理方式并不保证一致，因此建议在 BINARY 类型中只存储 ASCII 可见字符，而避免存储不可见字符。多字节的数据，例如中文字符，则需要使用 NCHAR 类型进行保存。如果强行使用 BINARY 类型保存中文字符，虽然有时也能正常读写，但并不带有字符集信息，很容易出现数据乱码甚至数据损坏等情况。 - BINARY 类型理论上最长可以有 16,374（从 3.0.5.0 版本开始，数据列为 65,517，标签列为 16,382） 字节。BINARY 仅支持字符串输入，字符串两端需使用单引号引用。使用时须指定大小，如 BINARY(20) 定义了最长为 20 个单字节字符的字符串，每个字符占 1 字节的存储空间，总共固定占用 20 字节的空间，此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\'`。 +- GEOMETRY 类型数据列为最大长度为 65,517 字节，标签列最大长度为 16,382 字节。支持 2D 的 POINT、LINESTRING 和 POLYGON 子类型数据。长度计算方式如下表所示： + + | # | **语法** | **最小长度** | **最大长度** | **每组坐标长度增长** | + |---|--------------------------------------|----------|------------|--------------| + | 1 | POINT(1.0 1.0) | 21 | 21 | 无 | + | 2 | LINESTRING(1.0 1.0, 2.0 2.0) | 9+2*16 | 9+4094*16 | +16 | + | 3 | POLYGON((1.0 1.0, 2.0 2.0, 1.0 1.0)) | 13+3*16 | 13+4094*16 | +16 | + - SQL 语句中的数值类型将依据是否存在小数点，或使用科学计数法表示，来判断数值类型是否为整型或者浮点型，因此在使用时要注意相应类型越界的情况。例如，9999999999999999999 会认为超过长整型的上边界而溢出，而 9999999999999999999.0 会被认为是有效的浮点数。 ::: diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index 75e01b9ab59ac810a2449ee176ad301857300948..ac435debea6bc2652f9ba0b30cadb90866ee1d22 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -36,7 +36,6 @@ database_option: { | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value | WAL_RETENTION_SIZE value - | WAL_SEGMENT_SIZE value } ``` @@ -74,10 +73,8 @@ database_option: { - TABLE_PREFIX：当其为正值时，在决定把一个表分配到哪个 vgroup 时要忽略表名中指定长度的前缀；当其为负值时，在决定把一个表分配到哪个 vgroup 时只使用表名中指定长度的前缀；例如，假定表名为 "v30001"，当 TSDB_PREFIX = 2 时 使用 "0001" 来决定分配到哪个 vgroup ，当 TSDB_PREFIX = -2 时使用 "v3" 来决定分配到哪个 vgroup - TABLE_SUFFIX：当其为正值时，在决定把一个表分配到哪个 vgroup 时要忽略表名中指定长度的后缀；当其为负值时，在决定把一个表分配到哪个 vgroup 时只使用表名中指定长度的后缀；例如，假定表名为 "v30001"，当 TSDB_SUFFIX = 2 时 使用 "v300" 来决定分配到哪个 vgroup ，当 TSDB_SUFFIX = -2 时使用 "01" 来决定分配到哪个 vgroup。 - TSDB_PAGESIZE：一个 VNODE 中时序数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB到 16 MB。 -- WAL_RETENTION_PERIOD: 为了数据订阅消费，需要WAL日志文件额外保留的最大时长策略。WAL日志清理，不受订阅客户端消费状态影响。单位为 s。默认为 0，表示无需为订阅保留。新建订阅，应先设置恰当的时长策略。 +- WAL_RETENTION_PERIOD: 为了数据订阅消费，需要WAL日志文件额外保留的最大时长策略。WAL日志清理，不受订阅客户端消费状态影响。单位为 s。默认为 3600，表示在 WAL 保留最近 3600 秒的数据，请根据数据订阅的需要修改这个参数为适当值。 - WAL_RETENTION_SIZE：为了数据订阅消费，需要WAL日志文件额外保留的最大累计大小策略。单位为 KB。默认为 0，表示累计大小无上限。 -- WAL_ROLL_PERIOD：wal 文件切换时长，单位为 s。当WAL文件创建并写入后，经过该时间，会自动创建一个新的WAL文件。默认为 0，即仅在TSDB落盘时创建新文件。 -- WAL_SEGMENT_SIZE：wal 单个文件大小，单位为 KB。当前写入文件大小超过上限后会自动创建一个新的WAL文件。默认为 0，即仅在TSDB落盘时创建新文件。 ### 创建数据库示例 ```sql diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index 8ef07c313a1992cd2e174b90ec8c488d56755e9f..9258258263282c6d3f9df5aa8dc650ec511a9680 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -43,10 +43,10 @@ table_option: { 1. 表的第一个字段必须是 TIMESTAMP，并且系统自动将其设为主键； 2. 表名最大长度为 192； -3. 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）;（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） -4. 表名，超级表名，以及子表名只能由字母、数字和下划线组成，且不能以数字开头，不区分大小写 -5. 使用数据类型 binary 或 nchar，需指定其最长的字节数，如 binary(20)，表示 20 字节； -6. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"。 如果不加转义符，表名会被默认转换成小组；加上转义符可以保留表名中的大小写属性。 +3. 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）;（注意：每个 BINARY/NCHAR/GEOMETRY 类型的列还会额外占用 2 个字节的存储位置） +4. 子表名只能由字母、数字和下划线组成，且不能以数字开头，不区分大小写 +5. 使用数据类型 BINARY/NCHAR/GEOMETRY，需指定其最长的字节数，如 BINARY(20)，表示 20 字节； +6. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"，可以让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 例如：\`aBc\` 和 \`abc\` 是不同的表名，但是 abc 和 aBc 是相同的表名。 **参数说明** diff --git a/docs/zh/12-taos-sql/04-stable.md b/docs/zh/12-taos-sql/04-stable.md index 93decf190dc5bb1f37d4c896d25c86c158c572e1..853d2bf981a665fbfa4a439e57cd27cbbf7ccd84 100644 --- a/docs/zh/12-taos-sql/04-stable.md +++ b/docs/zh/12-taos-sql/04-stable.md @@ -51,6 +51,11 @@ DESCRIBE [db_name.]stb_name; ### 获取超级表中所有子表的标签信息 +``` +SHOW TABLE TAGS FROM table_name [FROM db_name]; +SHOW TABLE TAGS FROM [db_name.]table_name; +``` + ``` taos> SHOW TABLE TAGS FROM st1; tbname | id | loc | diff --git a/docs/zh/12-taos-sql/16-operators.md b/docs/zh/12-taos-sql/16-operators.md index 48e9991799abf99ca868fc30e34f0435054afa0b..0636121edda2d2cb25e54861db4f18b45c2491ee 100644 --- a/docs/zh/12-taos-sql/16-operators.md +++ b/docs/zh/12-taos-sql/16-operators.md @@ -39,7 +39,7 @@ TDengine 支持 `UNION ALL` 和 `UNION` 操作符。UNION ALL 将查询返回的 | 3 | \>, < | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型 | 大于，小于 | | 4 | \>=, <= | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型 | 大于等于，小于等于 | | 5 | IS [NOT] NULL | 所有类型 | 是否为空值 | -| 6 | [NOT] BETWEEN AND | 除 BOOL、BLOB、MEDIUMBLOB 和 JSON 外的所有类型 | 闭区间比较 | +| 6 | [NOT] BETWEEN AND | 除 BOOL、BLOB、MEDIUMBLOB、JSON 和 GEOMETRY 外的所有类型 | 闭区间比较 | | 7 | IN | 除 BLOB、MEDIUMBLOB 和 JSON 外的所有类型，且不可以为表的时间戳主键列 | 与列表内的任意值相等 | | 8 | LIKE | BINARY、NCHAR 和 VARCHAR | 通配符匹配 | | 9 | MATCH, NMATCH | BINARY、NCHAR 和 VARCHAR | 正则表达式匹配 | diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index d416febd55521853812fbe6952d403f4d265ac4e..35dafc52efcce1412d1ddc6871d8e4eafc1f0d4e 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -334,8 +334,6 @@ description: TDengine 保留关键字的详细列表 - WAL_LEVEL - WAL_RETENTION_PERIOD - WAL_RETENTION_SIZE -- WAL_ROLL_PERIOD -- WAL_SEGMENT_SIZE - WATERMARK - WHERE - WINDOW_CLOSE diff --git a/docs/zh/12-taos-sql/22-meta.md b/docs/zh/12-taos-sql/22-meta.md index 62356b76d1e254fd90f7c9e13790bfccf041ce85..a5396d860762fbd68ca24c8bad4a71c808e63272 100644 --- a/docs/zh/12-taos-sql/22-meta.md +++ b/docs/zh/12-taos-sql/22-meta.md @@ -74,38 +74,36 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 提供用户创建的数据库对象的相关信息。也可以使用 SHOW DATABASES 来查询这些信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :------------------: | ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | name | BINARY(32) | 数据库名 | -| 2 | create_time | TIMESTAMP | 创建时间 | -| 3 | ntables | INT | 数据库中表的数量，包含子表和普通表但不包含超级表 | -| 4 | vgroups | INT | 数据库中有多少个 vgroup。需要注意，`vgroups` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 6 | replica | INT | 副本数。需要注意，`replica` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 7 | strict | BINARY(4) | 废弃参数 | -| 8 | duration | INT | 单文件存储数据的时间跨度。需要注意，`duration` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 9 | keep | INT | 数据保留时长。需要注意，`keep` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 10 | buffer | INT | 每个 vnode 写缓存的内存块大小，单位 MB。需要注意，`buffer` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 11 | pagesize | INT | 每个 VNODE 中元数据存储引擎的页大小，单位为 KB。需要注意，`pagesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 12 | pages | INT | 每个 vnode 元数据存储引擎的缓存页个数。需要注意，`pages` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 13 | minrows | INT | 文件块中记录的最大条数。需要注意，`minrows` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 14 | maxrows | INT | 文件块中记录的最小条数。需要注意，`maxrows` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 15 | comp | INT | 数据压缩方式。需要注意，`comp` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 16 | precision | BINARY(2) | 时间分辨率。需要注意，`precision` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 17 | status | BINARY(10) | 数据库状态 | -| 18 | retentions | BINARY (60) | 数据的聚合周期和保存时长。需要注意，`retentions` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 19 | single_stable | BOOL | 表示此数据库中是否只可以创建一个超级表。需要注意，`single_stable` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 20 | cachemodel | BINARY(60) | 表示是否在内存中缓存子表的最近数据。需要注意，`cachemodel` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 21 | cachesize | INT | 表示每个 vnode 中用于缓存子表最近数据的内存大小。需要注意，`cachesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 22 | wal_level | INT | WAL 级别。需要注意，`wal_level` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 23 | wal_fsync_period | INT | 数据落盘周期。需要注意，`wal_fsync_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 24 | wal_retention_period | INT | WAL 的保存时长，单位为秒。需要注意，`wal_retention_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 25 | wal_retention_size | INT | WAL 的保存上限。需要注意，`wal_retention_size` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 26 | wal_roll_period | INT | wal 文件切换时长。需要注意，`wal_roll_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 27 | wal_segment_size | BIGINT | wal 单个文件大小。需要注意，`wal_segment_size` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 28 | stt_trigger | SMALLINT | 触发文件合并的落盘文件的个数。需要注意，`stt_trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 29 | table_prefix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。需要注意，`table_prefix` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 30 | table_suffix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度。需要注意，`table_suffix` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 31 | tsdb_pagesize | INT | 时序数据存储引擎中的页大小。需要注意，`tsdb_pagesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| # | **列名** | **数据类型** | **说明** | +| --- | :------------------: | ---------------- | ------------------------------------------------ | +| 1 | name | BINARY(32) | 数据库名 | +| 2 | create_time | TIMESTAMP | 创建时间 | +| 3 | ntables | INT | 数据库中表的数量，包含子表和普通表但不包含超级表 | +| 4 | vgroups | INT | 数据库中有多少个 vgroup。需要注意，`vgroups` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 6 | replica | INT | 副本数。需要注意，`replica` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 7 | strict | BINARY(4) | 废弃参数 | +| 8 | duration | INT | 单文件存储数据的时间跨度。需要注意，`duration` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 9 | keep | INT | 数据保留时长。需要注意，`keep` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 10 | buffer | INT | 每个 vnode 写缓存的内存块大小，单位 MB。需要注意，`buffer` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 11 | pagesize | INT | 每个 VNODE 中元数据存储引擎的页大小，单位为 KB。需要注意，`pagesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 12 | pages | INT | 每个 vnode 元数据存储引擎的缓存页个数。需要注意，`pages` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 13 | minrows | INT | 文件块中记录的最大条数。需要注意，`minrows` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 14 | maxrows | INT | 文件块中记录的最小条数。需要注意，`maxrows` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 15 | comp | INT | 数据压缩方式。需要注意，`comp` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 16 | precision | BINARY(2) | 时间分辨率。需要注意，`precision` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 17 | status | BINARY(10) | 数据库状态 | +| 18 | retentions | BINARY (60) | 数据的聚合周期和保存时长。需要注意，`retentions` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 19 | single_stable | BOOL | 表示此数据库中是否只可以创建一个超级表。需要注意，`single_stable` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 20 | cachemodel | BINARY(60) | 表示是否在内存中缓存子表的最近数据。需要注意，`cachemodel` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 21 | cachesize | INT | 表示每个 vnode 中用于缓存子表最近数据的内存大小。需要注意，`cachesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 22 | wal_level | INT | WAL 级别。需要注意，`wal_level` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 23 | wal_fsync_period | INT | 数据落盘周期。需要注意，`wal_fsync_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 24 | wal_retention_period | INT | WAL 的保存时长，单位为秒。需要注意，`wal_retention_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 25 | wal_retention_size | INT | WAL 的保存上限。需要注意，`wal_retention_size` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 26 | stt_trigger | SMALLINT | 触发文件合并的落盘文件的个数。需要注意，`stt_trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 27 | table_prefix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。需要注意，`table_prefix` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 28 | table_suffix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度。需要注意，`table_suffix` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 29 | tsdb_pagesize | INT | 时序数据存储引擎中的页大小。需要注意，`tsdb_pagesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | ## INS_FUNCTIONS diff --git a/docs/zh/12-taos-sql/24-show.md b/docs/zh/12-taos-sql/24-show.md index f3397ae82dcc0465fb23d2f9770025e7b46f6a48..6e102e2356daa15bd766639028fbf8ebcedd96b2 100644 --- a/docs/zh/12-taos-sql/24-show.md +++ b/docs/zh/12-taos-sql/24-show.md @@ -101,6 +101,7 @@ SHOW GRANTS; ```sql SHOW INDEXES FROM tbl_name [FROM db_name]; +SHOW INDEXES FROM [db_name.]tbl_name; ``` 显示已创建的索引。 @@ -269,6 +270,7 @@ Query OK, 24 row(s) in set (0.002444s) ```sql SHOW TAGS FROM child_table_name [FROM db_name]; +SHOW TAGS FROM [db_name.]child_table_name; ``` 显示子表的标签信息。 diff --git a/docs/zh/12-taos-sql/27-index.md b/docs/zh/12-taos-sql/27-index.md index 51cf5f4e4e012839d093a791e84c8d49b62261c1..da8f38eb225933b89f5b403e374dea3a1604c364 100644 --- a/docs/zh/12-taos-sql/27-index.md +++ b/docs/zh/12-taos-sql/27-index.md @@ -57,6 +57,7 @@ DROP INDEX index_name; ````sql SHOW INDEXES FROM tbl_name [FROM db_name]; +SHOW INDEXES FROM [db_name.]tbl_name; ```` 显示在所指定的数据库或表上已创建的索引。 diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index 27dd3294b74943a289f25246e8a08a6ef416cd48..2a1e5f092cec8ea4fd0f8c77b44843d04071e259 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -18,6 +18,7 @@ description: "TDengine 3.0 版本的语法变更说明" | 8 | 混合运算 | 增强 | 查询中的混合运算（标量运算和矢量运算混合）全面增强，SELECT的各个子句均全面支持符合语法语义的混合运算。 | 9 | 标签运算 | 新增 |在查询中，标签列可以像普通列一样参与各种运算，用于各种子句。 | 10 | 时间线子句和时间函数用于超级表查询 | 增强 |没有PARTITION BY时，超级表的数据会被合并成一条时间线。 +| 11 | GEOMETRY | 新增 | 几何类型。 ## SQL 语句变更 @@ -33,7 +34,7 @@ description: "TDengine 3.0 版本的语法变更说明" | 6 | ALTER USER | 调整 | 废除 | 7 | COMPACT VNODES | 暂不支持 | 整理指定VNODE的数据。3.0.0版本暂不支持。 | 8 | CREATE ACCOUNT | 废除 | 2.x中为企业版功能，3.0不再支持。语法暂时保留了，执行报“This statement is no longer supported”错误。 -| 9 | CREATE DATABASE | 调整 |

废除

新增

调整

+| 9 | CREATE DATABASE | 调整 |

废除

新增

调整

| 10 | CREATE DNODE | 调整 | 新增主机名和端口号分开指定语法 | 11 | CREATE INDEX | 新增 | 创建SMA索引。 | 12 | CREATE MNODE | 新增 | 创建管理节点。 diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index a4d6c9910c6a7fd14ddf722a88475824f1cc9271..ba5eb6385920d398108598f0872ab7c2916ac609 100755 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -184,7 +184,7 @@ taos -C | 属性 | 说明 | | -------- | ------------------------ | -| 适用范围 | 仅服务端适用 | +| 适用范围 | 客户端和服务端都适用 | | 含义 | 是否上传 telemetry | | 取值范围 | 0,1 0: 不上传；1：上传 | | 缺省值 | 1 | @@ -193,7 +193,7 @@ taos -C | 属性 | 说明 | | -------- | ------------------------ | -| 适用范围 | 仅服务端适用 | +| 适用范围 | 客户端和服务端都适用 | | 含义 | 是否上传 crash 信息 | | 取值范围 | 0,1 0: 不上传；1：上传 | | 缺省值 | 1 | @@ -726,6 +726,16 @@ charset 的有效值是 UTF-8。 | 取值范围 | 0: 不改变；1：改变 | | 缺省值 | 0 | +### keepTimeOffset + +| 属性 | 说明 | +| -------- | ------------------ | +| 适用范围 | 仅服务端适用 | +| 含义 | 迁移操作的延时 | +| 单位 | 小时 | +| 取值范围 | 0-23 | +| 缺省值 | 0 | + ### tmqMaxTopicNum | 属性 | 说明 | @@ -803,6 +813,7 @@ charset 的有效值是 UTF-8。 | 53 | udf | 是 | 是 | | | 54 | enableCoreFile | 是 | 是 | | | 55 | ttlChangeOnWrite | 否 | 是 | | +| 56 | keepTimeOffset | 是 | 是 | | ## 2.x->3.0 的废弃参数 @@ -817,76 +828,74 @@ charset 的有效值是 UTF-8。 | 7 | offlineThreshold | 是 | 否 | 3.0 行为未知 | | 8 | role | 是 | 否 | 由 supportVnode 决定是否能够创建 | | 9 | dnodeNopLoop | 是 | 否 | 2.6 文档中未找到此参数 | -| 10 | keepTimeOffset | 是 | 否 | 2.6 文档中未找到此参数 | -| 11 | rpcTimer | 是 | 否 | 3.0 行为未知 | -| 12 | rpcMaxTime | 是 | 否 | 3.0 行为未知 | -| 13 | rpcForceTcp | 是 | 否 | 默认为 TCP | -| 14 | tcpConnTimeout | 是 | 否 | 3.0 行为未知 | -| 15 | syncCheckInterval | 是 | 否 | 3.0 行为未知 | -| 16 | maxTmrCtrl | 是 | 否 | 3.0 行为未知 | -| 17 | monitorReplica | 是 | 否 | 由 RAFT 协议管理多副本 | -| 18 | smlTagNullName | 是 | 否 | 3.0 行为未知 | -| 20 | ratioOfQueryCores | 是 | 否 | 由 线程池 相关配置参数决定 | -| 21 | maxStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 22 | maxFirstStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 23 | retryStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 24 | streamCompDelayRatio | 是 | 否 | 3.0 行为未知 | -| 25 | maxVgroupsPerDb | 是 | 否 | 由 create db 的参数 vgroups 指定实际 vgroups 数量 | -| 26 | maxTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 27 | minTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 28 | tableIncStepPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 29 | cache | 是 | 否 | 由 buffer 代替 cache\*blocks | -| 30 | blocks | 是 | 否 | 由 buffer 代替 cache\*blocks | -| 31 | days | 是 | 否 | 由 create db 的参数 duration 取代 | -| 32 | keep | 是 | 否 | 由 create db 的参数 keep 取代 | -| 33 | minRows | 是 | 否 | 由 create db 的参数 minRows 取代 | -| 34 | maxRows | 是 | 否 | 由 create db 的参数 maxRows 取代 | -| 35 | quorum | 是 | 否 | 由 RAFT 协议决定 | -| 36 | comp | 是 | 否 | 由 create db 的参数 comp 取代 | -| 37 | walLevel | 是 | 否 | 由 create db 的参数 wal_level 取代 | -| 38 | fsync | 是 | 否 | 由 create db 的参数 wal_fsync_period 取代 | -| 39 | replica | 是 | 否 | 由 create db 的参数 replica 取代 | -| 40 | partitions | 是 | 否 | 3.0 行为未知 | -| 41 | update | 是 | 否 | 允许更新部分列 | -| 42 | cachelast | 是 | 否 | 由 create db 的参数 cacheModel 取代 | -| 43 | maxSQLLength | 是 | 否 | SQL 上限为 1MB，无需参数控制 | -| 44 | maxWildCardsLength | 是 | 否 | 3.0 行为未知 | -| 45 | maxRegexStringLen | 是 | 否 | 3.0 行为未知 | -| 46 | maxNumOfOrderedRes | 是 | 否 | 3.0 行为未知 | -| 47 | maxConnections | 是 | 否 | 取决于系统配置和系统处理能力，详见后面的 Note | -| 48 | mnodeEqualVnodeNum | 是 | 否 | 3.0 行为未知 | -| 49 | http | 是 | 否 | http 服务由 taosAdapter 提供 | -| 50 | httpEnableRecordSql | 是 | 否 | taosd 不提供 http 服务 | -| 51 | httpMaxThreads | 是 | 否 | taosd 不提供 http 服务 | -| 52 | restfulRowLimit | 是 | 否 | taosd 不提供 http 服务 | -| 53 | httpDbNameMandatory | 是 | 否 | taosd 不提供 http 服务 | -| 54 | httpKeepAlive | 是 | 否 | taosd 不提供 http 服务 | -| 55 | enableRecordSql | 是 | 否 | 3.0 行为未知 | -| 56 | maxBinaryDisplayWidth | 是 | 否 | 3.0 行为未知 | -| 57 | stream | 是 | 否 | 默认启用连续查询 | -| 58 | retrieveBlockingModel | 是 | 否 | 3.0 行为未知 | -| 59 | tsdbMetaCompactRatio | 是 | 否 | 3.0 行为未知 | -| 60 | defaultJSONStrType | 是 | 否 | 3.0 行为未知 | -| 61 | walFlushSize | 是 | 否 | 3.0 行为未知 | -| 62 | keepTimeOffset | 是 | 否 | 3.0 行为未知 | -| 63 | flowctrl | 是 | 否 | 3.0 行为未知 | -| 64 | slaveQuery | 是 | 否 | 3.0 行为未知： slave vnode 是否能够处理查询？ | -| 65 | adjustMaster | 是 | 否 | 3.0 行为未知 | -| 66 | topicBinaryLen | 是 | 否 | 3.0 行为未知 | -| 67 | telegrafUseFieldNum | 是 | 否 | 3.0 行为未知 | -| 68 | deadLockKillQuery | 是 | 否 | 3.0 行为未知 | -| 69 | clientMerge | 是 | 否 | 3.0 行为未知 | -| 70 | sdbDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 71 | odbcDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 72 | httpDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 73 | monDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 74 | cqDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 75 | shortcutFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 76 | probeSeconds | 是 | 否 | 3.0 行为未知 | -| 77 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | -| 78 | probeInterval | 是 | 否 | 3.0 行为未知 | -| 79 | lossyColumns | 是 | 否 | 3.0 行为未知 | -| 80 | fPrecision | 是 | 否 | 3.0 行为未知 | -| 81 | dPrecision | 是 | 否 | 3.0 行为未知 | -| 82 | maxRange | 是 | 否 | 3.0 行为未知 | -| 83 | range | 是 | 否 | 3.0 行为未知 | +| 10 | rpcTimer | 是 | 否 | 3.0 行为未知 | +| 11 | rpcMaxTime | 是 | 否 | 3.0 行为未知 | +| 12 | rpcForceTcp | 是 | 否 | 默认为 TCP | +| 13 | tcpConnTimeout | 是 | 否 | 3.0 行为未知 | +| 14 | syncCheckInterval | 是 | 否 | 3.0 行为未知 | +| 15 | maxTmrCtrl | 是 | 否 | 3.0 行为未知 | +| 16 | monitorReplica | 是 | 否 | 由 RAFT 协议管理多副本 | +| 17 | smlTagNullName | 是 | 否 | 3.0 行为未知 | +| 18 | ratioOfQueryCores | 是 | 否 | 由 线程池 相关配置参数决定 | +| 19 | maxStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 20 | maxFirstStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 21 | retryStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 22 | streamCompDelayRatio | 是 | 否 | 3.0 行为未知 | +| 23 | maxVgroupsPerDb | 是 | 否 | 由 create db 的参数 vgroups 指定实际 vgroups 数量 | +| 24 | maxTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 25 | minTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 26 | tableIncStepPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 27 | cache | 是 | 否 | 由 buffer 代替 cache\*blocks | +| 28 | blocks | 是 | 否 | 由 buffer 代替 cache\*blocks | +| 29 | days | 是 | 否 | 由 create db 的参数 duration 取代 | +| 30 | keep | 是 | 否 | 由 create db 的参数 keep 取代 | +| 31 | minRows | 是 | 否 | 由 create db 的参数 minRows 取代 | +| 32 | maxRows | 是 | 否 | 由 create db 的参数 maxRows 取代 | +| 33 | quorum | 是 | 否 | 由 RAFT 协议决定 | +| 34 | comp | 是 | 否 | 由 create db 的参数 comp 取代 | +| 35 | walLevel | 是 | 否 | 由 create db 的参数 wal_level 取代 | +| 36 | fsync | 是 | 否 | 由 create db 的参数 wal_fsync_period 取代 | +| 37 | replica | 是 | 否 | 由 create db 的参数 replica 取代 | +| 38 | partitions | 是 | 否 | 3.0 行为未知 | +| 39 | update | 是 | 否 | 允许更新部分列 | +| 40 | cachelast | 是 | 否 | 由 create db 的参数 cacheModel 取代 | +| 41 | maxSQLLength | 是 | 否 | SQL 上限为 1MB，无需参数控制 | +| 42 | maxWildCardsLength | 是 | 否 | 3.0 行为未知 | +| 43 | maxRegexStringLen | 是 | 否 | 3.0 行为未知 | +| 44 | maxNumOfOrderedRes | 是 | 否 | 3.0 行为未知 | +| 45 | maxConnections | 是 | 否 | 取决于系统配置和系统处理能力，详见后面的 Note | +| 46 | mnodeEqualVnodeNum | 是 | 否 | 3.0 行为未知 | +| 47 | http | 是 | 否 | http 服务由 taosAdapter 提供 | +| 48 | httpEnableRecordSql | 是 | 否 | taosd 不提供 http 服务 | +| 49 | httpMaxThreads | 是 | 否 | taosd 不提供 http 服务 | +| 50 | restfulRowLimit | 是 | 否 | taosd 不提供 http 服务 | +| 51 | httpDbNameMandatory | 是 | 否 | taosd 不提供 http 服务 | +| 52 | httpKeepAlive | 是 | 否 | taosd 不提供 http 服务 | +| 53 | enableRecordSql | 是 | 否 | 3.0 行为未知 | +| 54 | maxBinaryDisplayWidth | 是 | 否 | 3.0 行为未知 | +| 55 | stream | 是 | 否 | 默认启用连续查询 | +| 56 | retrieveBlockingModel | 是 | 否 | 3.0 行为未知 | +| 57 | tsdbMetaCompactRatio | 是 | 否 | 3.0 行为未知 | +| 58 | defaultJSONStrType | 是 | 否 | 3.0 行为未知 | +| 59 | walFlushSize | 是 | 否 | 3.0 行为未知 | +| 60 | flowctrl | 是 | 否 | 3.0 行为未知 | +| 61 | slaveQuery | 是 | 否 | 3.0 行为未知： slave vnode 是否能够处理查询？ | +| 62 | adjustMaster | 是 | 否 | 3.0 行为未知 | +| 63 | topicBinaryLen | 是 | 否 | 3.0 行为未知 | +| 64 | telegrafUseFieldNum | 是 | 否 | 3.0 行为未知 | +| 65 | deadLockKillQuery | 是 | 否 | 3.0 行为未知 | +| 66 | clientMerge | 是 | 否 | 3.0 行为未知 | +| 67 | sdbDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 68 | odbcDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 69 | httpDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 70 | monDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 71 | cqDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 72 | shortcutFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 73 | probeSeconds | 是 | 否 | 3.0 行为未知 | +| 74 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | +| 75 | probeInterval | 是 | 否 | 3.0 行为未知 | +| 76 | lossyColumns | 是 | 否 | 3.0 行为未知 | +| 77 | fPrecision | 是 | 否 | 3.0 行为未知 | +| 78 | dPrecision | 是 | 否 | 3.0 行为未知 | +| 79 | maxRange | 是 | 否 | 3.0 行为未知 | +| 80 | range | 是 | 否 | 3.0 行为未知 | diff --git a/include/common/tcommon.h b/include/common/tcommon.h index ea17262abdb280618ae00814b3dcd23ca87a0865..bdfb1d32b4642e265b468e4f4e3a7bd476b07ff9 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -54,6 +54,11 @@ typedef struct SSessionKey { uint64_t groupId; } SSessionKey; +typedef struct SVersionRange { + uint64_t minVer; + uint64_t maxVer; +} SVersionRange; + static inline int winKeyCmprImpl(const void* pKey1, const void* pKey2) { SWinKey* pWin1 = (SWinKey*)pKey1; SWinKey* pWin2 = (SWinKey*)pKey2; @@ -131,10 +136,10 @@ static inline int STupleKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, enum { TMQ_MSG_TYPE__DUMMY = 0, - TMQ_MSG_TYPE__POLL_RSP, + TMQ_MSG_TYPE__POLL_DATA_RSP, TMQ_MSG_TYPE__POLL_META_RSP, TMQ_MSG_TYPE__EP_RSP, - TMQ_MSG_TYPE__TAOSX_RSP, + TMQ_MSG_TYPE__POLL_DATA_META_RSP, TMQ_MSG_TYPE__WALINFO_RSP, TMQ_MSG_TYPE__END_RSP, }; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index cd8e0642cf2dcae26ae288e421083f8accae54d4..c0412d2617500196d2d4d31f9ff49f7bede9a08d 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -177,7 +177,6 @@ static FORCE_INLINE void colDataSetDouble(SColumnInfoData* pColumnInfoData, uint int32_t getJsonValueLen(const char* data); int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); -int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData); int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, @@ -187,6 +186,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); + int32_t colDataGetRowLength(const SColumnInfoData* pColumnInfoData, int32_t rowIdx); void colDataTrim(SColumnInfoData* pColumnInfoData); @@ -208,7 +208,6 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock); size_t blockDataGetSerialMetaSize(uint32_t numOfCols); int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); -int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); @@ -237,11 +236,10 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColIn SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); +int32_t blockGetEncodeSize(const SSDataBlock* pBlock); int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols); const char* blockDecode(SSDataBlock* pBlock, const char* pData); -void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag); -void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); @@ -251,9 +249,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf); -static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { - return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock); -} +void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList); #ifdef __cplusplus } diff --git a/include/common/tglobal.h b/include/common/tglobal.h index bc4037c64234540f53b390cca8f82a7feb286a8e..4f2ed2b0659f8572974028594bb3a9feb1b910a3 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -34,6 +34,7 @@ extern char tsFirst[]; extern char tsSecond[]; extern char tsLocalFqdn[]; extern char tsLocalEp[]; +extern char tsVersionName[]; extern uint16_t tsServerPort; extern int32_t tsVersion; extern int32_t tsStatusInterval; @@ -48,6 +49,7 @@ extern int32_t tsMaxNumOfDistinctResults; extern int32_t tsCompatibleModel; extern bool tsPrintAuth; extern int64_t tsTickPerMin[3]; +extern int64_t tsTickPerHour[3]; extern int32_t tsCountAlwaysReturnValue; extern float tsSelectivityRatio; extern int32_t tsTagFilterResCacheSize; @@ -83,8 +85,14 @@ extern int64_t tsVndCommitMaxIntervalMs; extern int64_t tsMndSdbWriteDelta; extern int64_t tsMndLogRetention; extern int8_t tsGrant; +extern int32_t tsMndGrantMode; extern bool tsMndSkipGrant; +// dnode +extern int64_t tsDndStart; +extern int64_t tsDndStartOsUptime; +extern int64_t tsDndUpTime; + // monitor extern bool tsEnableMonitor; extern int32_t tsMonitorInterval; @@ -185,6 +193,7 @@ extern bool tsDisableStream; extern int64_t tsStreamBufferSize; extern int64_t tsCheckpointInterval; extern bool tsFilterScalarMode; +extern int32_t tsKeepTimeOffset; extern int32_t tsMaxStreamBackendCache; extern int32_t tsPQSortMemThreshold; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0c58b470c2301c35b5b38a0a90aab4541308b502..e772a47e3dc79aec6f474f1f9ad93b4398c67f6e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1144,6 +1144,7 @@ typedef struct { char timezone[TD_TIMEZONE_LEN]; // tsTimezone char locale[TD_LOCALE_LEN]; // tsLocale char charset[TD_LOCALE_LEN]; // tsCharset + int8_t ttlChangeOnWrite; } SClusterCfg; typedef struct { @@ -1180,6 +1181,8 @@ typedef struct { typedef struct { int8_t syncState; int8_t syncRestore; + int64_t syncTerm; + int64_t roleTimeMs; } SMnodeLoad; typedef struct { @@ -1495,6 +1498,7 @@ int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesR typedef struct { char name[TSDB_CONFIG_OPTION_LEN + 1]; char value[TSDB_CONFIG_VALUE_LEN + 1]; + char scope[TSDB_CONFIG_SCOPE_LEN + 1]; } SVariablesInfo; typedef struct { @@ -2894,6 +2898,12 @@ enum { TMQ_OFFSET__SNAPSHOT_META = 3, }; +enum { + WITH_DATA = 0, + WITH_META = 1, + ONLY_META = 2, +}; + typedef struct { int8_t type; union { @@ -3004,6 +3014,7 @@ typedef struct { char* sql; char* ast; int64_t deleteMark; + int64_t lastTs; } SMCreateSmaReq; int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 7d12c2a1d6186171e1bf7c43d342e013970ebd86..232551007d2c7ec9c038b9d31420606b95bd5cb9 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -252,7 +252,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "stream-recover-finish", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_SCAN_HISTORY, "stream-scan-history", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_SCAN_HISTORY_FINISH, "stream-scan-history-finish", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_TRANSFER_STATE, "stream-transfer-state", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECK, "stream-task-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECKPOINT, "stream-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_REPORT_CHECKPOINT, "stream-report-checkpoint", NULL, NULL) @@ -297,8 +299,7 @@ enum { TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, "vnode-stream-recover1", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, "vnode-stream-recover2", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_SCAN_HISTORY, "vnode-stream-scan-history", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_CHECK_POINT_SOURCE, "vnode-stream-checkpoint-source", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 5410e9af887043930fc039a4c5de3466e4198437..8a6b7b50209e0d35247cc3211f71ad43d991fa14 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -193,166 +193,166 @@ #define TK_INTERVAL 175 #define TK_COUNT 176 #define TK_LAST_ROW 177 -#define TK_TOPIC 178 -#define TK_META 179 -#define TK_CONSUMER 180 -#define TK_GROUP 181 -#define TK_DESC 182 -#define TK_DESCRIBE 183 -#define TK_RESET 184 -#define TK_QUERY 185 -#define TK_CACHE 186 -#define TK_EXPLAIN 187 -#define TK_ANALYZE 188 -#define TK_VERBOSE 189 -#define TK_NK_BOOL 190 -#define TK_RATIO 191 -#define TK_NK_FLOAT 192 -#define TK_OUTPUTTYPE 193 -#define TK_AGGREGATE 194 -#define TK_BUFSIZE 195 -#define TK_LANGUAGE 196 -#define TK_REPLACE 197 -#define TK_STREAM 198 -#define TK_INTO 199 -#define TK_PAUSE 200 -#define TK_RESUME 201 -#define TK_TRIGGER 202 -#define TK_AT_ONCE 203 -#define TK_WINDOW_CLOSE 204 -#define TK_IGNORE 205 -#define TK_EXPIRED 206 -#define TK_FILL_HISTORY 207 -#define TK_UPDATE 208 -#define TK_SUBTABLE 209 -#define TK_UNTREATED 210 -#define TK_KILL 211 -#define TK_CONNECTION 212 -#define TK_TRANSACTION 213 -#define TK_BALANCE 214 -#define TK_VGROUP 215 -#define TK_LEADER 216 -#define TK_MERGE 217 -#define TK_REDISTRIBUTE 218 -#define TK_SPLIT 219 -#define TK_DELETE 220 -#define TK_INSERT 221 -#define TK_NULL 222 -#define TK_NK_QUESTION 223 -#define TK_NK_ARROW 224 -#define TK_ROWTS 225 -#define TK_QSTART 226 -#define TK_QEND 227 -#define TK_QDURATION 228 -#define TK_WSTART 229 -#define TK_WEND 230 -#define TK_WDURATION 231 -#define TK_IROWTS 232 -#define TK_ISFILLED 233 -#define TK_CAST 234 -#define TK_NOW 235 -#define TK_TODAY 236 -#define TK_TIMEZONE 237 -#define TK_CLIENT_VERSION 238 -#define TK_SERVER_VERSION 239 -#define TK_SERVER_STATUS 240 -#define TK_CURRENT_USER 241 -#define TK_CASE 242 -#define TK_WHEN 243 -#define TK_THEN 244 -#define TK_ELSE 245 -#define TK_BETWEEN 246 -#define TK_IS 247 -#define TK_NK_LT 248 -#define TK_NK_GT 249 -#define TK_NK_LE 250 -#define TK_NK_GE 251 -#define TK_NK_NE 252 -#define TK_MATCH 253 -#define TK_NMATCH 254 -#define TK_CONTAINS 255 -#define TK_IN 256 -#define TK_JOIN 257 -#define TK_INNER 258 -#define TK_SELECT 259 -#define TK_DISTINCT 260 -#define TK_WHERE 261 -#define TK_PARTITION 262 -#define TK_BY 263 -#define TK_SESSION 264 -#define TK_STATE_WINDOW 265 -#define TK_EVENT_WINDOW 266 -#define TK_SLIDING 267 -#define TK_FILL 268 -#define TK_VALUE 269 -#define TK_VALUE_F 270 -#define TK_NONE 271 -#define TK_PREV 272 -#define TK_NULL_F 273 -#define TK_LINEAR 274 -#define TK_NEXT 275 -#define TK_HAVING 276 -#define TK_RANGE 277 -#define TK_EVERY 278 -#define TK_ORDER 279 -#define TK_SLIMIT 280 -#define TK_SOFFSET 281 -#define TK_LIMIT 282 -#define TK_OFFSET 283 -#define TK_ASC 284 -#define TK_NULLS 285 -#define TK_ABORT 286 -#define TK_AFTER 287 -#define TK_ATTACH 288 -#define TK_BEFORE 289 -#define TK_BEGIN 290 -#define TK_BITAND 291 -#define TK_BITNOT 292 -#define TK_BITOR 293 -#define TK_BLOCKS 294 -#define TK_CHANGE 295 -#define TK_COMMA 296 -#define TK_CONCAT 297 -#define TK_CONFLICT 298 -#define TK_COPY 299 -#define TK_DEFERRED 300 -#define TK_DELIMITERS 301 -#define TK_DETACH 302 -#define TK_DIVIDE 303 -#define TK_DOT 304 -#define TK_EACH 305 -#define TK_FAIL 306 -#define TK_FILE 307 -#define TK_FOR 308 -#define TK_GLOB 309 -#define TK_ID 310 -#define TK_IMMEDIATE 311 -#define TK_IMPORT 312 -#define TK_INITIALLY 313 -#define TK_INSTEAD 314 -#define TK_ISNULL 315 -#define TK_KEY 316 -#define TK_MODULES 317 -#define TK_NK_BITNOT 318 -#define TK_NK_SEMI 319 -#define TK_NOTNULL 320 -#define TK_OF 321 -#define TK_PLUS 322 -#define TK_PRIVILEGE 323 -#define TK_RAISE 324 -#define TK_RESTRICT 325 -#define TK_ROW 326 -#define TK_SEMI 327 -#define TK_STAR 328 -#define TK_STATEMENT 329 -#define TK_STRICT 330 -#define TK_STRING 331 -#define TK_TIMES 332 -#define TK_VALUES 333 -#define TK_VARIABLE 334 -#define TK_VIEW 335 -#define TK_WAL 336 - +#define TK_META 178 +#define TK_ONLY 179 +#define TK_TOPIC 180 +#define TK_CONSUMER 181 +#define TK_GROUP 182 +#define TK_DESC 183 +#define TK_DESCRIBE 184 +#define TK_RESET 185 +#define TK_QUERY 186 +#define TK_CACHE 187 +#define TK_EXPLAIN 188 +#define TK_ANALYZE 189 +#define TK_VERBOSE 190 +#define TK_NK_BOOL 191 +#define TK_RATIO 192 +#define TK_NK_FLOAT 193 +#define TK_OUTPUTTYPE 194 +#define TK_AGGREGATE 195 +#define TK_BUFSIZE 196 +#define TK_LANGUAGE 197 +#define TK_REPLACE 198 +#define TK_STREAM 199 +#define TK_INTO 200 +#define TK_PAUSE 201 +#define TK_RESUME 202 +#define TK_TRIGGER 203 +#define TK_AT_ONCE 204 +#define TK_WINDOW_CLOSE 205 +#define TK_IGNORE 206 +#define TK_EXPIRED 207 +#define TK_FILL_HISTORY 208 +#define TK_UPDATE 209 +#define TK_SUBTABLE 210 +#define TK_UNTREATED 211 +#define TK_KILL 212 +#define TK_CONNECTION 213 +#define TK_TRANSACTION 214 +#define TK_BALANCE 215 +#define TK_VGROUP 216 +#define TK_LEADER 217 +#define TK_MERGE 218 +#define TK_REDISTRIBUTE 219 +#define TK_SPLIT 220 +#define TK_DELETE 221 +#define TK_INSERT 222 +#define TK_NULL 223 +#define TK_NK_QUESTION 224 +#define TK_NK_ARROW 225 +#define TK_ROWTS 226 +#define TK_QSTART 227 +#define TK_QEND 228 +#define TK_QDURATION 229 +#define TK_WSTART 230 +#define TK_WEND 231 +#define TK_WDURATION 232 +#define TK_IROWTS 233 +#define TK_ISFILLED 234 +#define TK_CAST 235 +#define TK_NOW 236 +#define TK_TODAY 237 +#define TK_TIMEZONE 238 +#define TK_CLIENT_VERSION 239 +#define TK_SERVER_VERSION 240 +#define TK_SERVER_STATUS 241 +#define TK_CURRENT_USER 242 +#define TK_CASE 243 +#define TK_WHEN 244 +#define TK_THEN 245 +#define TK_ELSE 246 +#define TK_BETWEEN 247 +#define TK_IS 248 +#define TK_NK_LT 249 +#define TK_NK_GT 250 +#define TK_NK_LE 251 +#define TK_NK_GE 252 +#define TK_NK_NE 253 +#define TK_MATCH 254 +#define TK_NMATCH 255 +#define TK_CONTAINS 256 +#define TK_IN 257 +#define TK_JOIN 258 +#define TK_INNER 259 +#define TK_SELECT 260 +#define TK_DISTINCT 261 +#define TK_WHERE 262 +#define TK_PARTITION 263 +#define TK_BY 264 +#define TK_SESSION 265 +#define TK_STATE_WINDOW 266 +#define TK_EVENT_WINDOW 267 +#define TK_SLIDING 268 +#define TK_FILL 269 +#define TK_VALUE 270 +#define TK_VALUE_F 271 +#define TK_NONE 272 +#define TK_PREV 273 +#define TK_NULL_F 274 +#define TK_LINEAR 275 +#define TK_NEXT 276 +#define TK_HAVING 277 +#define TK_RANGE 278 +#define TK_EVERY 279 +#define TK_ORDER 280 +#define TK_SLIMIT 281 +#define TK_SOFFSET 282 +#define TK_LIMIT 283 +#define TK_OFFSET 284 +#define TK_ASC 285 +#define TK_NULLS 286 +#define TK_ABORT 287 +#define TK_AFTER 288 +#define TK_ATTACH 289 +#define TK_BEFORE 290 +#define TK_BEGIN 291 +#define TK_BITAND 292 +#define TK_BITNOT 293 +#define TK_BITOR 294 +#define TK_BLOCKS 295 +#define TK_CHANGE 296 +#define TK_COMMA 297 +#define TK_CONCAT 298 +#define TK_CONFLICT 299 +#define TK_COPY 300 +#define TK_DEFERRED 301 +#define TK_DELIMITERS 302 +#define TK_DETACH 303 +#define TK_DIVIDE 304 +#define TK_DOT 305 +#define TK_EACH 306 +#define TK_FAIL 307 +#define TK_FILE 308 +#define TK_FOR 309 +#define TK_GLOB 310 +#define TK_ID 311 +#define TK_IMMEDIATE 312 +#define TK_IMPORT 313 +#define TK_INITIALLY 314 +#define TK_INSTEAD 315 +#define TK_ISNULL 316 +#define TK_KEY 317 +#define TK_MODULES 318 +#define TK_NK_BITNOT 319 +#define TK_NK_SEMI 320 +#define TK_NOTNULL 321 +#define TK_OF 322 +#define TK_PLUS 323 +#define TK_PRIVILEGE 324 +#define TK_RAISE 325 +#define TK_RESTRICT 326 +#define TK_ROW 327 +#define TK_SEMI 328 +#define TK_STAR 329 +#define TK_STATEMENT 330 +#define TK_STRICT 331 +#define TK_STRING 332 +#define TK_TIMES 333 +#define TK_VALUES 334 +#define TK_VARIABLE 335 +#define TK_VIEW 336 +#define TK_WAL 337 #define TK_NK_SPACE 600 diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 3f53976c67834285593ec45e4cfb98001f14ea52..3bef15f3a7c49b7a89112344b67182b3da9f3696 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -55,6 +55,9 @@ typedef struct { void* pStateBackend; struct SStorageAPI api; + + int8_t fillHistory; + STimeWindow winRange; } SReadHandle; // in queue mode, data streams are seperated by msg @@ -193,14 +196,6 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key); -/** - * return the scan info, in the form of tuple of two items, including table uid and current timestamp - * @param tinfo - * @param uid - * @param ts - * @return - */ -int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts); SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo); @@ -220,15 +215,22 @@ void* qExtractReaderFromStreamScanner(void* scanner); int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner); -int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo); -int32_t qStreamSourceRecoverStep1(qTaskInfo_t tinfo, int64_t ver); -int32_t qStreamSourceRecoverStep2(qTaskInfo_t tinfo, int64_t ver); +int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo); +int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow); +int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow); int32_t qStreamRecoverFinish(qTaskInfo_t tinfo); -int32_t qStreamRestoreParam(qTaskInfo_t tinfo); +int32_t qRestoreStreamOperatorOption(qTaskInfo_t tinfo); bool qStreamRecoverScanFinished(qTaskInfo_t tinfo); -void qStreamCloseTsdbReader(void* task); +bool qStreamRecoverScanStep1Finished(qTaskInfo_t tinfo); +bool qStreamRecoverScanStep2Finished(qTaskInfo_t tinfo); +int32_t qStreamRecoverSetAllStepFinished(qTaskInfo_t tinfo); void resetTaskInfo(qTaskInfo_t tinfo); +void qResetStreamInfoTimeWindow(qTaskInfo_t tinfo); + +int32_t qStreamOperatorReleaseState(qTaskInfo_t tInfo); +int32_t qStreamOperatorReloadState(qTaskInfo_t tInfo); + #ifdef __cplusplus } #endif diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 41516adc73be1bde7aa1235c25554c5736028e0f..2fbd7851e89911e61328b30bd88506d73cc486ad 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -123,8 +123,8 @@ typedef struct SSnapContext { SHashObj* suidInfo; SArray* idList; int32_t index; - bool withMeta; - bool queryMeta; // true-get meta, false-get data + int8_t withMeta; + int8_t queryMeta; // true-get meta, false-get data } SSnapContext; typedef struct { @@ -234,29 +234,6 @@ typedef struct SStoreSnapshotFn { int32_t (*getTableInfoFromSnapshot)(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid); } SStoreSnapshotFn; -/** -void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); -void metaReaderReleaseLock(SMetaReader *pReader); -void metaReaderClear(SMetaReader *pReader); -int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); -int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); -int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); -const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); -int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); - -int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); -int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); -bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); -int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, - bool *acquired); -int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, - int32_t payloadLen, double selectivityRatio); -tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); -int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); -int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t -payloadLen); - */ - typedef struct SStoreMeta { SMTbCursor* (*openTableMetaCursor)(void* pVnode); // metaOpenTbCursor void (*closeTableMetaCursor)(SMTbCursor* pTbCur); // metaCloseTbCursor @@ -403,7 +380,7 @@ typedef struct SStateStore { SStreamStateCur* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key); struct SStreamFileState* (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, - uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark); + uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark, const char*id); void (*streamFileStateDestroy)(struct SStreamFileState* pFileState); void (*streamFileStateClear)(struct SStreamFileState* pFileState); @@ -415,6 +392,7 @@ typedef struct SStateStore { int32_t (*streamStateCommit)(SStreamState* pState); void (*streamStateDestroy)(SStreamState* pState, bool remove); int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); + void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts); } SStateStore; typedef struct SStorageAPI { diff --git a/include/libs/function/function.h b/include/libs/function/function.h index c92ce254a8bfb157f8f844fbdcc195014ab0c0cb..2e3cd670d73651494aafc148a474dc8d2f48ad3c 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -129,30 +129,38 @@ typedef struct SSerializeDataHandle { } SSerializeDataHandle; // incremental state storage + +typedef struct SBackendCfWrapper { + void *rocksdb; + void **pHandle; + void *writeOpts; + void *readOpts; + void **cfOpts; + void *dbOpt; + void *param; + void *env; + SListNode *pComparNode; + void *pBackend; + void *compactFactory; + TdThreadRwlock rwLock; + bool remove; + int64_t backendId; + char idstr[64]; +} SBackendCfWrapper; typedef struct STdbState { - void *rocksdb; - void **pHandle; - void *writeOpts; - void *readOpts; - void **cfOpts; - void *dbOpt; + SBackendCfWrapper *pBackendCfWrapper; + int64_t backendCfWrapperId; + char idstr[64]; + struct SStreamTask *pOwner; - void *param; - void *env; - SListNode *pComparNode; - void *pBackend; - char idstr[64]; - void *compactFactory; - TdThreadRwlock rwLock; - - void *db; - void *pStateDb; - void *pFuncStateDb; - void *pFillStateDb; // todo refactor - void *pSessionStateDb; - void *pParNameDb; - void *pParTagDb; - void *txn; + void *db; + void *pStateDb; + void *pFuncStateDb; + void *pFillStateDb; // todo refactor + void *pSessionStateDb; + void *pParNameDb; + void *pParTagDb; + void *txn; } STdbState; typedef struct { diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 40a71050420c89d8c7cc44f1ae27124a51fa3e89..f0a715e6511af931b7e580dae9a83cf7d447e371 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -36,9 +36,10 @@ extern "C" { #define SHOW_CREATE_TB_RESULT_FIELD1_LEN (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) #define SHOW_CREATE_TB_RESULT_FIELD2_LEN (TSDB_MAX_ALLOWED_SQL_LEN * 3) -#define SHOW_LOCAL_VARIABLES_RESULT_COLS 2 +#define SHOW_LOCAL_VARIABLES_RESULT_COLS 3 #define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) +#define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) #define SHOW_ALIVE_RESULT_COLS 1 @@ -319,19 +320,22 @@ typedef struct SIndexOptions { SNode* pInterval; SNode* pOffset; SNode* pSliding; + int8_t tsPrecision; SNode* pStreamOptions; } SIndexOptions; typedef struct SCreateIndexStmt { - ENodeType type; - EIndexType indexType; - bool ignoreExists; - char indexDbName[TSDB_DB_NAME_LEN]; - char indexName[TSDB_INDEX_NAME_LEN]; - char dbName[TSDB_DB_NAME_LEN]; - char tableName[TSDB_TABLE_NAME_LEN]; - SNodeList* pCols; - SIndexOptions* pOptions; + ENodeType type; + EIndexType indexType; + bool ignoreExists; + char indexDbName[TSDB_DB_NAME_LEN]; + char indexName[TSDB_INDEX_NAME_LEN]; + char dbName[TSDB_DB_NAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; + SNodeList* pCols; + SIndexOptions* pOptions; + SNode* pPrevQuery; + SMCreateSmaReq* pReq; } SCreateIndexStmt; typedef struct SDropIndexStmt { @@ -362,7 +366,7 @@ typedef struct SCreateTopicStmt { char subDbName[TSDB_DB_NAME_LEN]; char subSTbName[TSDB_TABLE_NAME_LEN]; bool ignoreExists; - bool withMeta; + int8_t withMeta; SNode* pQuery; SNode* pWhere; } SCreateTopicStmt; diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 68bc27737819cc405096d388f5fd55ff188dec3e..063318332a7cdc0a66c983fdfa464ca8f1bf9a8e 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -55,6 +55,7 @@ typedef struct SLogicNode { EGroupAction groupAction; EOrder inputTsOrder; EOrder outputTsOrder; + bool forceCreateNonBlockingOptr; // true if the operator can use non-blocking(pipeline) mode } SLogicNode; typedef enum EScanType { @@ -105,6 +106,7 @@ typedef struct SScanLogicNode { bool hasNormalCols; // neither tag column nor primary key tag column bool sortPrimaryKey; bool igLastNull; + bool groupOrderScan; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -318,6 +320,7 @@ typedef struct SPhysiNode { struct SPhysiNode* pParent; SNode* pLimit; SNode* pSlimit; + bool forceCreateNonBlockingOptr; } SPhysiNode; typedef struct SScanPhysiNode { @@ -328,6 +331,7 @@ typedef struct SScanPhysiNode { uint64_t suid; int8_t tableType; SName tableName; + bool groupOrderScan; } SScanPhysiNode; typedef SScanPhysiNode STagScanPhysiNode; @@ -525,7 +529,6 @@ typedef struct SSortPhysiNode { SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode SNodeList* pTargets; - int64_t maxRows; } SSortPhysiNode; typedef SSortPhysiNode SGroupSortPhysiNode; diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 7f9d20a9dd878892e512b170921bdb1794defc52..7747df85956798f6271e06822a4812d03829f89e 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -138,6 +138,8 @@ int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname); int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal); +void streamStateReloadInfo(SStreamState* pState, TSKEY ts); + /***compare func **/ typedef struct SStateChekpoint { diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 73c88fae8d9e0cec33831d1640b8299e0fcaa89d..066f83fbcbb96b1df73d50982c0ba2702bc2b296 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -44,11 +44,9 @@ enum { TASK_STATUS__DROPPING, TASK_STATUS__FAIL, TASK_STATUS__STOP, - TASK_STATUS__WAIT_DOWNSTREAM, - TASK_STATUS__RECOVER_PREPARE, - TASK_STATUS__RECOVER1, - TASK_STATUS__RECOVER2, - TASK_STATUS__PAUSE, + TASK_STATUS__SCAN_HISTORY, // stream task scan history data by using tsdbread in the stream scanner + TASK_STATUS__HALT, // pause, but not be manipulated by user command + TASK_STATUS__PAUSE, // pause }; enum { @@ -133,7 +131,6 @@ typedef struct { // ref data block, for delete typedef struct { int8_t type; - int64_t ver; SSDataBlock* pBlock; } SStreamRefDataBlock; @@ -203,13 +200,11 @@ static FORCE_INLINE void streamQueueProcessFail(SStreamQueue* queue) { atomic_store_8(&queue->status, STREAM_QUEUE__FAILED); } -void* streamQueueNextItem(SStreamQueue* queue); +void* streamQueueNextItem(SStreamQueue* pQueue); SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type); void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit); -SStreamDataSubmit* streamSubmitBlockClone(SStreamDataSubmit* pSubmit); - typedef struct { char* qmsg; void* pExecutor; // not applicable to encoder and decoder @@ -251,7 +246,7 @@ typedef struct { int8_t reserved; } STaskSinkFetch; -typedef struct { +typedef struct SStreamChildEpInfo { int32_t nodeId; int32_t childId; int32_t taskId; @@ -271,32 +266,63 @@ typedef struct SCheckpointInfo { } SCheckpointInfo; typedef struct SStreamStatus { - int8_t taskStatus; - int8_t schedStatus; - int8_t keepTaskStatus; + int8_t taskStatus; + int8_t downstreamReady; // downstream tasks are all ready now, if this flag is set + int8_t schedStatus; + int8_t keepTaskStatus; + bool transferState; + int8_t timerActive; // timer is active + int8_t pauseAllowed; // allowed task status to be set to be paused } SStreamStatus; -struct SStreamTask { - SStreamId id; - int32_t totalLevel; - int8_t taskLevel; - int8_t outputType; - int16_t dispatchMsgType; - SStreamStatus status; - int32_t selfChildId; - int32_t nodeId; // vgroup id - SEpSet epSet; - SCheckpointInfo chkInfo; - STaskExec exec; - int8_t fillHistory; // fill history - int64_t ekey; // end ts key - int64_t endVer; // end version - - // children info - SArray* childEpInfo; // SArray - int32_t nextCheckId; - SArray* checkpointInfo; // SArray +typedef struct SHistDataRange { + SVersionRange range; + STimeWindow window; +} SHistDataRange; + +typedef struct SSTaskBasicInfo { + int32_t nodeId; // vgroup id or snode id + SEpSet epSet; + int32_t selfChildId; + int32_t totalLevel; + int8_t taskLevel; + int8_t fillHistory; // is fill history task or not +} SSTaskBasicInfo; + +typedef struct SDispatchMsgInfo { + void* pData; // current dispatch data + int16_t msgType; // dispatch msg type + int32_t retryCount; // retry send data count + int64_t blockingTs; // output blocking timestamp +} SDispatchMsgInfo; + +typedef struct { + int8_t type; + int8_t status; + SStreamQueue* queue; +} STaskOutputInfo; + +typedef struct { + int64_t init; + int64_t step1Start; + int64_t step2Start; +} STaskTimestamp; +struct SStreamTask { + SStreamId id; + SSTaskBasicInfo info; + STaskOutputInfo outputInfo; + SDispatchMsgInfo msgInfo; + SStreamStatus status; + SCheckpointInfo chkInfo; + STaskExec exec; + SHistDataRange dataRange; + SStreamId historyTaskId; + SStreamId streamTaskId; + SArray* pUpstreamEpInfoList; // SArray, // children info + int32_t nextCheckId; + SArray* checkpointInfo; // SArray + STaskTimestamp tsInfo; // output union { STaskDispatcherFixedEp fixedEpDispatcher; @@ -307,46 +333,49 @@ struct SStreamTask { }; int8_t inputStatus; - int8_t outputStatus; SStreamQueue* inputQueue; - SStreamQueue* outputQueue; // trigger int8_t triggerStatus; int64_t triggerParam; - void* timer; + void* schedTimer; + void* launchTaskTimer; SMsgCb* pMsgCb; // msg handle SStreamState* pState; // state backend + SArray* pRspMsgList; + TdThreadMutex lock; // the followings attributes don't be serialized - int32_t recoverTryingDownstream; - int32_t recoverWaitingUpstream; + int32_t notReadyTasks; + int32_t numOfWaitingUpstream; int64_t checkReqId; SArray* checkReqIds; // shuffle int32_t refCnt; int64_t checkpointingId; int32_t checkpointAlignCnt; + int32_t transferStateAlignCnt; struct SStreamMeta* pMeta; SSHashObj* pNameMap; }; // meta typedef struct SStreamMeta { - char* path; - TDB* db; - TTB* pTaskDb; - TTB* pCheckpointDb; - SHashObj* pTasks; - SArray* pTaskList; // SArray - void* ahandle; - TXN* txn; - FTaskExpand* expandFunc; - int32_t vgId; - SRWLatch lock; - int32_t walScanCounter; - void* streamBackend; - int64_t streamBackendRid; - SHashObj* pTaskBackendUnique; + char* path; + TDB* db; + TTB* pTaskDb; + TTB* pCheckpointDb; + SHashObj* pTasks; + SArray* pTaskList; // SArray + void* ahandle; + TXN* txn; + FTaskExpand* expandFunc; + int32_t vgId; + SRWLatch lock; + int32_t walScanCounter; + void* streamBackend; + int64_t streamBackendRid; + SHashObj* pTaskBackendUnique; + TdThreadMutex backendMutex; } SStreamMeta; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); @@ -431,16 +460,19 @@ typedef struct { SMsgHead msgHead; int64_t streamId; int32_t taskId; -} SStreamRecoverStep1Req, SStreamRecoverStep2Req; + int8_t igUntreated; +} SStreamScanHistoryReq; typedef struct { int64_t streamId; - int32_t taskId; + int32_t upstreamTaskId; + int32_t downstreamTaskId; + int32_t upstreamNodeId; int32_t childId; -} SStreamRecoverFinishReq; +} SStreamScanHistoryFinishReq, SStreamTransferReq; -int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq); -int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq); +int32_t tEncodeStreamScanHistoryFinishReq(SEncoder* pEncoder, const SStreamScanHistoryFinishReq* pReq); +int32_t tDecodeStreamScanHistoryFinishReq(SDecoder* pDecoder, SStreamScanHistoryFinishReq* pReq); typedef struct { int64_t streamId; @@ -496,6 +528,17 @@ int32_t tDecodeSStreamCheckpointReq(SDecoder* pDecoder, SStreamCheckpointReq* pR int32_t tEncodeSStreamCheckpointRsp(SEncoder* pEncoder, const SStreamCheckpointRsp* pRsp); int32_t tDecodeSStreamCheckpointRsp(SDecoder* pDecoder, SStreamCheckpointRsp* pRsp); +typedef struct { + int64_t streamId; + int32_t upstreamTaskId; + int32_t upstreamNodeId; + int32_t downstreamId; + int32_t downstreamNode; +} SStreamCompleteHistoryMsg; + +int32_t tEncodeCompleteHistoryDataMsg(SEncoder* pEncoder, const SStreamCompleteHistoryMsg* pReq); +int32_t tDecodeCompleteHistoryDataMsg(SDecoder* pDecoder, SStreamCompleteHistoryMsg* pReq); + typedef struct { int64_t streamId; int32_t downstreamTaskId; @@ -509,14 +552,14 @@ typedef struct { SArray* checkpointVer; // SArray } SStreamRecoverDownstreamRsp; -int32_t tEncodeSStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq); -int32_t tDecodeSStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq); +int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq); +int32_t tDecodeStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq); -int32_t tEncodeSStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp); -int32_t tDecodeSStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp); +int32_t tEncodeStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp); +int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp); -int32_t tEncodeSStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq); -int32_t tDecodeSStreamTaskRecoverReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq); +int32_t tEncodeSStreamTaskScanHistoryReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq); +int32_t tDecodeSStreamTaskScanHistoryReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq); int32_t tEncodeSStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamRecoverDownstreamRsp* pRsp); int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamRecoverDownstreamRsp* pRsp); @@ -525,16 +568,17 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); void tDeleteStreamRetrieveReq(SStreamRetrieveReq* pReq); -void tDeleteStreamDispatchReq(SStreamDispatchReq* pReq); +int32_t tInitStreamDispatchReq(SStreamDispatchReq* pReq, const SStreamTask* pTask, int32_t vgId, int32_t numOfBlocks, + int64_t dstTaskId); +void tDeleteStreamDispatchReq(SStreamDispatchReq* pReq); -int32_t streamSetupTrigger(SStreamTask* pTask); +int32_t streamSetupScheduleTrigger(SStreamTask* pTask); int32_t streamProcessRunReq(SStreamTask* pTask); int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg, bool exec); int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code); int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg); -// int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); void streamTaskInputFail(SStreamTask* pTask); int32_t streamTryExec(SStreamTask* pTask); @@ -542,48 +586,74 @@ int32_t streamSchedExec(SStreamTask* pTask); int32_t streamTaskOutputResultBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); bool streamTaskShouldStop(const SStreamStatus* pStatus); bool streamTaskShouldPause(const SStreamStatus* pStatus); +bool streamTaskIsIdle(const SStreamTask* pTask); +int32_t streamTaskEndScanWAL(SStreamTask* pTask); +SStreamChildEpInfo * streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId); int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz); +char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); + // recover and fill history -int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version); -int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version); +void streamTaskCheckDownstreamTasks(SStreamTask* pTask); +int32_t streamTaskDoCheckDownstreamTasks(SStreamTask* pTask); +int32_t streamTaskLaunchScanHistory(SStreamTask* pTask); int32_t streamTaskCheckStatus(SStreamTask* pTask); -int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version); +int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp, + SRpcHandleInfo* pRpcInfo, int32_t taskId); +int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp); +int32_t streamLaunchFillHistoryTask(SStreamTask* pTask); +int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask); +int32_t streamStartRecoverTask(SStreamTask* pTask, int8_t igUntreated); +bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer); + +bool streamTaskRecoverScanStep1Finished(SStreamTask* pTask); +bool streamTaskRecoverScanStep2Finished(SStreamTask* pTask); +int32_t streamTaskRecoverSetAllStepFinished(SStreamTask* pTask); // common -int32_t streamSetParamForRecover(SStreamTask* pTask); -int32_t streamRestoreParam(SStreamTask* pTask); -int32_t streamSetStatusNormal(SStreamTask* pTask); +int32_t streamSetParamForScanHistory(SStreamTask* pTask); +int32_t streamRestoreParam(SStreamTask* pTask); +int32_t streamSetStatusNormal(SStreamTask* pTask); +const char* streamGetTaskStatusStr(int32_t status); +void streamTaskPause(SStreamTask* pTask); +void streamTaskResume(SStreamTask* pTask); +void streamTaskHalt(SStreamTask* pTask); +void streamTaskResumeFromHalt(SStreamTask* pTask); +void streamTaskDisablePause(SStreamTask* pTask); +void streamTaskEnablePause(SStreamTask* pTask); + // source level -int32_t streamSourceRecoverPrepareStep1(SStreamTask* pTask, int64_t ver); -int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamRecoverStep1Req* pReq); -int32_t streamSourceRecoverScanStep1(SStreamTask* pTask); -int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* pReq); -int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver); -int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask); +int32_t streamSetParamForStreamScannerStep1(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow); +int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow); +int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated); +int32_t streamSourceScanHistoryData(SStreamTask* pTask); +int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask); + +int32_t streamDispatchTransferStateMsg(SStreamTask* pTask); + // agg level -int32_t streamAggRecoverPrepare(SStreamTask* pTask); -// int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask); -int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId); +int32_t streamTaskScanHistoryPrepare(SStreamTask* pTask); +int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq *pReq, SRpcHandleInfo* pRpcInfo); +int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask); +// stream task meta void streamMetaInit(); void streamMetaCleanup(); SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId); void streamMetaClose(SStreamMeta* streamMeta); -int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); -int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); -int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen); -int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta); - +// save to b-tree meta store +int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); +int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); +int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded); +int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int32_t taskId); +int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); // todo remove it SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); -void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); int32_t streamMetaBegin(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); -int32_t streamMetaRollBack(SStreamMeta* pMeta); int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver); // checkpoint @@ -591,6 +661,11 @@ int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, int32_t streamProcessCheckpointReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointReq* pReq); int32_t streamProcessCheckpointRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointRsp* pRsp); +int32_t streamTaskReleaseState(SStreamTask* pTask); +int32_t streamTaskReloadState(SStreamTask* pTask); +int32_t streamAlignTransferState(SStreamTask* pTask); + + #ifdef __cplusplus } #endif diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index 0dbacf6c9f1a52eac62966206ad0b95d52c0620f..b2255013ca5de8246f94eecd13dae4e382c1cd32 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -28,11 +28,10 @@ extern "C" { #endif typedef struct SStreamFileState SStreamFileState; - typedef SList SStreamSnapshot; SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, - GetTsFun fp, void* pFile, TSKEY delMark); + GetTsFun fp, void* pFile, TSKEY delMark, const char* id); void streamFileStateDestroy(SStreamFileState* pFileState); void streamFileStateClear(SStreamFileState* pFileState); bool needClearDiskBuff(SStreamFileState* pFileState); @@ -50,6 +49,7 @@ int32_t recoverSnapshot(SStreamFileState* pFileState); int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list); int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark); int32_t streamFileStateGeSelectRowSize(SStreamFileState* pFileState); +void streamFileStateReloadInfo(SStreamFileState* pFileState, TSKEY ts); #ifdef __cplusplus } diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 2a0a4b0f63ae70c95423a0fc59cf7200417ebd10..88ccf562c72eb346fd7901865f9d44229e082c74 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -239,29 +239,31 @@ typedef struct SSyncState { ESyncState state; bool restored; bool canRead; + SyncTerm term; + int64_t roleTimeMs; } SSyncState; -int32_t syncInit(); -void syncCleanUp(); -int64_t syncOpen(SSyncInfo* pSyncInfo); -int32_t syncStart(int64_t rid); -void syncStop(int64_t rid); -void syncPreStop(int64_t rid); -void syncPostStop(int64_t rid); -int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq); -int32_t syncIsCatchUp(int64_t rid); +int32_t syncInit(); +void syncCleanUp(); +int64_t syncOpen(SSyncInfo* pSyncInfo); +int32_t syncStart(int64_t rid); +void syncStop(int64_t rid); +void syncPreStop(int64_t rid); +void syncPostStop(int64_t rid); +int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq); +int32_t syncIsCatchUp(int64_t rid); ESyncRole syncGetRole(int64_t rid); -int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); -int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); -int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex); -int32_t syncEndSnapshot(int64_t rid); -int32_t syncLeaderTransfer(int64_t rid); -int32_t syncStepDown(int64_t rid, SyncTerm newTerm); -bool syncIsReadyForRead(int64_t rid); -bool syncSnapshotSending(int64_t rid); -bool syncSnapshotRecving(int64_t rid); -int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq); -int32_t syncForceBecomeFollower(SSyncNode* ths, const SRpcMsg* pRpcMsg); +int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); +int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); +int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex); +int32_t syncEndSnapshot(int64_t rid); +int32_t syncLeaderTransfer(int64_t rid); +int32_t syncStepDown(int64_t rid, SyncTerm newTerm); +bool syncIsReadyForRead(int64_t rid); +bool syncSnapshotSending(int64_t rid); +bool syncSnapshotRecving(int64_t rid); +int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq); +int32_t syncForceBecomeFollower(SSyncNode* ths, const SRpcMsg* pRpcMsg); SSyncState syncGetState(int64_t rid); void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 622cd615b8e14a82d1eb940986cb70ad9a086992..509f8dc9e8418ebac4b8d44e5d5e199fd3c27463 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -69,6 +69,13 @@ void tfsUpdateSize(STfs *pTfs); */ SDiskSize tfsGetSize(STfs *pTfs); +/** + * @brief Get the number of disks at level of multi-tier storage. + * + * @param pTfs + * @return int32_t + */ +int32_t tfsGetDisksAtLevel(STfs *pTfs, int32_t level); /** * @brief Get level of multi-tier storage. * @@ -123,6 +130,15 @@ int32_t tfsMkdir(STfs *pTfs, const char *rname); */ int32_t tfsMkdirAt(STfs *pTfs, const char *rname, SDiskID diskId); +/** + * @brief Recursive make directory at all levels in tfs. + * + * @param pTfs The fs object. + * @param rname The rel name of directory. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t tfsMkdirRecur(STfs *pTfs, const char *rname); + /** * @brief Recursive create directories in tfs. * @@ -160,7 +176,17 @@ int32_t tfsRmdir(STfs *pTfs, const char *rname); * @param nrname The rel name of new file. * @return int32_t 0 for success, -1 for failure. */ -int32_t tfsRename(STfs *pTfs, const char *orname, const char *nrname); +int32_t tfsRename(STfs *pTfs, int32_t diskPrimary, const char *orname, const char *nrname); + +/** + * @brief Search fname in level of tfs + * + * @param pTfs The fs object. + * @param level The level to search on + * @param fname The relative file name to be searched + * @param int32_t diskId for successs, -1 for failure + */ +int32_t tfsSearch(STfs *pTfs, int32_t level, const char *fname); /** * @brief Init file object in tfs. diff --git a/include/os/os.h b/include/os/os.h index 309a977ff6b7e6d500d93cba0ee487cc7befbd9e..ac1a750b78cc01109821a39f24da5d4453abf8d8 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -53,6 +53,7 @@ extern "C" { #else #include #include +#include #if defined(_TD_X86_) #include #endif diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index a8ccb67bfb1bb345150a2d7ee90d776e7b251aa3..a6a3655a5516748a8cb75ac453c21948636888a9 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -35,8 +35,9 @@ typedef struct { bool taosCheckSystemIsLittleEnd(); void taosGetSystemInfo(); +int64_t taosGetOsUptime(); int32_t taosGetEmail(char *email, int32_t maxLen); -int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen); +int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t maxLen); int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores); int32_t taosGetCpuCores(float *numOfCores); void taosGetCpuUsage(double *cpu_system, double *cpu_engine); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index f37402c18c0ebe8cbe9cac00f18d9880fe90e286..2b8c72b829fe6c75189b0e9b38f8052abba784c9 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -416,7 +416,7 @@ int32_t* taosGetErrno(); // #define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) // 2.x // #define TSDB_CODE_VND_ACTION_NEED_REPROCESS. TAOS_DEF_ERROR_CODE(0, 0x0502) // 2.x #define TSDB_CODE_VND_INVALID_VGROUP_ID TAOS_DEF_ERROR_CODE(0, 0x0503) -// #define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) // 2.x +#define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) // #define TSDB_CODE_VND_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0505) // 2.x // #define TSDB_CODE_VND_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0506) // 2.x // #define TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR TAOS_DEF_ERROR_CODE(0, 0x0507) // 2.x @@ -706,6 +706,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_TIMELINE_QUERY TAOS_DEF_ERROR_CODE(0, 0x2666) #define TSDB_CODE_PAR_INVALID_OPTR_USAGE TAOS_DEF_ERROR_CODE(0, 0x2667) #define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2668) +#define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED TAOS_DEF_ERROR_CODE(0, 0x2669) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner @@ -756,9 +757,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RSMA_INVALID_SCHEMA TAOS_DEF_ERROR_CODE(0, 0x3153) #define TSDB_CODE_RSMA_STREAM_STATE_OPEN TAOS_DEF_ERROR_CODE(0, 0x3154) #define TSDB_CODE_RSMA_STREAM_STATE_COMMIT TAOS_DEF_ERROR_CODE(0, 0x3155) -#define TSDB_CODE_RSMA_FS_REF TAOS_DEF_ERROR_CODE(0, 0x3156) -#define TSDB_CODE_RSMA_FS_SYNC TAOS_DEF_ERROR_CODE(0, 0x3157) -#define TSDB_CODE_RSMA_FS_UPDATE TAOS_DEF_ERROR_CODE(0, 0x3158) +#define TSDB_CODE_RSMA_FS_SYNC TAOS_DEF_ERROR_CODE(0, 0x3156) +#define TSDB_CODE_RSMA_RESULT TAOS_DEF_ERROR_CODE(0, 0x3157) //index #define TSDB_CODE_INDEX_REBUILDING TAOS_DEF_ERROR_CODE(0, 0x3200) diff --git a/include/util/tarray.h b/include/util/tarray.h index a93c6953700fcc5f8861a2161e9f0b8bd1e97e56..f56c9e3a1761303961aa9e946e7063049a48baf9 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -22,7 +22,7 @@ extern "C" { #endif -#define TARRAY_MIN_SIZE 8 +#define TARRAY_MIN_SIZE 4 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) #define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) @@ -138,7 +138,7 @@ size_t taosArrayGetSize(const SArray* pArray); * @param index * @param pData */ -void* taosArrayInsert(SArray* pArray, size_t index, void* pData); +void* taosArrayInsert(SArray* pArray, size_t index, const void* pData); /** * set data in array @@ -204,9 +204,9 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)); void* taosArrayDestroy(SArray* pArray); -void taosArrayDestroyP(SArray* pArray, FDelete fp); +void taosArrayDestroyP(SArray* pArray, FDelete fp); -void taosArrayDestroyEx(SArray* pArray, FDelete fp); +void taosArrayDestroyEx(SArray* pArray, FDelete fp); void taosArraySwap(SArray* a, SArray* b); diff --git a/include/util/tarray2.h b/include/util/tarray2.h new file mode 100644 index 0000000000000000000000000000000000000000..cd49e647895e1571b6b70ad46408420d6972a4b7 --- /dev/null +++ b/include/util/tarray2.h @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "talgo.h" + +#ifndef _TD_UTIL_TARRAY2_H_ +#define _TD_UTIL_TARRAY2_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// a: a +// e: element +// ep: element pointer +// cmp: compare function +// idx: index +// cb: callback function + +#define TARRAY2(TYPE) \ + struct { \ + int32_t size; \ + int32_t capacity; \ + TYPE *data; \ + } + +typedef void (*TArray2Cb)(void *); + +#define TARRAY2_SIZE(a) ((a)->size) +#define TARRAY2_CAPACITY(a) ((a)->capacity) +#define TARRAY2_DATA(a) ((a)->data) +#define TARRAY2_GET(a, i) ((a)->data[i]) +#define TARRAY2_GET_PTR(a, i) ((a)->data + i) +#define TARRAY2_FIRST(a) ((a)->data[0]) +#define TARRAY2_LAST(a) ((a)->data[(a)->size - 1]) +#define TARRAY2_DATA_LEN(a) ((a)->size * sizeof(((a)->data[0]))) + +static FORCE_INLINE int32_t tarray2_make_room(void *arr, int32_t expSize, int32_t eleSize) { + TARRAY2(void) *a = arr; + + int32_t capacity = (a->capacity > 0) ? (a->capacity << 1) : 32; + while (capacity < expSize) { + capacity <<= 1; + } + void *p = taosMemoryRealloc(a->data, capacity * eleSize); + if (p == NULL) return TSDB_CODE_OUT_OF_MEMORY; + a->capacity = capacity; + a->data = p; + return 0; +} + +static FORCE_INLINE int32_t tarray2InsertBatch(void *arr, int32_t idx, const void *elePtr, int32_t numEle, + int32_t eleSize) { + TARRAY2(uint8_t) *a = arr; + + int32_t ret = 0; + if (a->size + numEle > a->capacity) { + ret = tarray2_make_room(a, a->size + numEle, eleSize); + } + if (ret == 0) { + if (idx < a->size) { + memmove(a->data + (idx + numEle) * eleSize, a->data + idx * eleSize, (a->size - idx) * eleSize); + } + memcpy(a->data + idx * eleSize, elePtr, numEle * eleSize); + a->size += numEle; + } + return ret; +} + +static FORCE_INLINE void *tarray2Search(void *arr, const void *elePtr, int32_t eleSize, __compar_fn_t compar, + int32_t flag) { + TARRAY2(void) *a = arr; + return taosbsearch(elePtr, a->data, a->size, eleSize, compar, flag); +} + +static FORCE_INLINE int32_t tarray2SearchIdx(void *arr, const void *elePtr, int32_t eleSize, __compar_fn_t compar, + int32_t flag) { + TARRAY2(void) *a = arr; + void *p = taosbsearch(elePtr, a->data, a->size, eleSize, compar, flag); + if (p == NULL) { + return -1; + } else { + return (int32_t)(((uint8_t *)p - (uint8_t *)a->data) / eleSize); + } +} + +static FORCE_INLINE int32_t tarray2SortInsert(void *arr, const void *elePtr, int32_t eleSize, __compar_fn_t compar) { + TARRAY2(void) *a = arr; + int32_t idx = tarray2SearchIdx(arr, elePtr, eleSize, compar, TD_GT); + return tarray2InsertBatch(arr, idx < 0 ? a->size : idx, elePtr, 1, eleSize); +} + +#define TARRAY2_INIT_EX(a, size_, capacity_, data_) \ + do { \ + (a)->size = (size_); \ + (a)->capacity = (capacity_); \ + (a)->data = (data_); \ + } while (0) + +#define TARRAY2_INIT(a) TARRAY2_INIT_EX(a, 0, 0, NULL) + +#define TARRAY2_CLEAR(a, cb) \ + do { \ + if ((cb) && (a)->size > 0) { \ + TArray2Cb cb_ = (TArray2Cb)(cb); \ + for (int32_t i = 0; i < (a)->size; ++i) { \ + cb_((a)->data + i); \ + } \ + } \ + (a)->size = 0; \ + } while (0) + +#define TARRAY2_DESTROY(a, cb) \ + do { \ + TARRAY2_CLEAR(a, cb); \ + if ((a)->data) { \ + taosMemoryFree((a)->data); \ + (a)->data = NULL; \ + } \ + (a)->capacity = 0; \ + } while (0) + +#define TARRAY2_INSERT_PTR(a, idx, ep) tarray2InsertBatch(a, idx, ep, 1, sizeof((a)->data[0])) +#define TARRAY2_APPEND_PTR(a, ep) tarray2InsertBatch(a, (a)->size, ep, 1, sizeof((a)->data[0])) +#define TARRAY2_APPEND_BATCH(a, ep, n) tarray2InsertBatch(a, (a)->size, ep, n, sizeof((a)->data[0])) +#define TARRAY2_APPEND(a, e) TARRAY2_APPEND_PTR(a, &(e)) + +// return (TYPE *) +#define TARRAY2_SEARCH(a, ep, cmp, flag) tarray2Search(a, ep, sizeof(((a)->data[0])), (__compar_fn_t)cmp, flag) + +#define TARRAY2_SEARCH_IDX(a, ep, cmp, flag) tarray2SearchIdx(a, ep, sizeof(((a)->data[0])), (__compar_fn_t)cmp, flag) + +#define TARRAY2_SORT_INSERT(a, e, cmp) tarray2SortInsert(a, &(e), sizeof(((a)->data[0])), (__compar_fn_t)cmp) +#define TARRAY2_SORT_INSERT_P(a, ep, cmp) tarray2SortInsert(a, ep, sizeof(((a)->data[0])), (__compar_fn_t)cmp) + +#define TARRAY2_REMOVE(a, idx, cb) \ + do { \ + if ((idx) < (a)->size) { \ + if (cb) { \ + TArray2Cb cb_ = (TArray2Cb)(cb); \ + cb_((a)->data + (idx)); \ + } \ + if ((idx) < (a)->size - 1) { \ + memmove((a)->data + (idx), (a)->data + (idx) + 1, sizeof((*(a)->data)) * ((a)->size - (idx)-1)); \ + } \ + (a)->size--; \ + } \ + } while (0) + +#define TARRAY2_FOREACH(a, e) for (int32_t __i = 0; __i < (a)->size && ((e) = (a)->data[__i], 1); __i++) +#define TARRAY2_FOREACH_REVERSE(a, e) for (int32_t __i = (a)->size - 1; __i >= 0 && ((e) = (a)->data[__i], 1); __i--) +#define TARRAY2_FOREACH_PTR(a, ep) for (int32_t __i = 0; __i < (a)->size && ((ep) = &(a)->data[__i], 1); __i++) +#define TARRAY2_FOREACH_PTR_REVERSE(a, ep) \ + for (int32_t __i = (a)->size - 1; __i >= 0 && ((ep) = &(a)->data[__i], 1); __i--) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_TARRAY2_H_*/ diff --git a/include/util/tconfig.h b/include/util/tconfig.h index ca3c5eb0ebd35867464c5c02851a9761e111d250..ed8eedb7e574fd7f5e682f45eebb57df3328ee31 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -50,11 +50,17 @@ typedef enum { CFG_DTYPE_TIMEZONE } ECfgDataType; +typedef enum { + CFG_SCOPE_SERVER, + CFG_SCOPE_CLIENT, + CFG_SCOPE_BOTH +} ECfgScopeType; + typedef struct SConfigItem { ECfgSrcType stype; ECfgDataType dtype; - bool tsc; - char *name; + int8_t scope; + char *name; union { bool bval; float fval; @@ -92,20 +98,21 @@ int32_t cfgGetSize(SConfig *pCfg); SConfigItem *cfgGetItem(SConfig *pCfg, const char *name); int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype); -int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, bool tsc); -int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, bool tsc); -int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, bool tsc); -int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, bool tsc); -int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc); -int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc); -int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal); -int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal); -int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope); +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope); +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope); +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, int8_t scope); +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); const char *cfgStypeStr(ECfgSrcType type); const char *cfgDtypeStr(ECfgDataType type); void cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); +void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); diff --git a/include/util/tdef.h b/include/util/tdef.h index 69b012ecea6227dac152fe74ad8cadf6f83e47e0..3ee35b3b6b86da562434101f7ca3f539a52797aa 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -191,16 +191,16 @@ typedef enum ELogicConditionType { #define TSDB_MAX_COLUMNS 4096 #define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns -#define TSDB_NODE_NAME_LEN 64 -#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string -#define TSDB_CGROUP_LEN 193 // it is a null-terminated string -#define TSDB_OFFSET_LEN 64 // it is a null-terminated string -#define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string -#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 65 -#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) -#define TSDB_PRIVILEDGE_CONDITION_LEN 200 +#define TSDB_NODE_NAME_LEN 64 +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string +#define TSDB_CGROUP_LEN 193 // it is a null-terminated string +#define TSDB_OFFSET_LEN 64 // it is a null-terminated string +#define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string +#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string +#define TSDB_DB_NAME_LEN 65 +#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_PRIVILEDGE_CONDITION_LEN 200 #define TSDB_FUNC_NAME_LEN 65 #define TSDB_FUNC_COMMENT_LEN 1024 * 1024 @@ -249,15 +249,15 @@ typedef enum ELogicConditionType { #define TSDB_LABEL_LEN 8 #define TSDB_JOB_STATUS_LEN 32 -#define TSDB_CLUSTER_ID_LEN 40 -#define TSDB_FQDN_LEN 128 -#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) -#define TSDB_IPv4ADDR_LEN 16 -#define TSDB_FILENAME_LEN 128 -#define TSDB_SHOW_SQL_LEN 2048 +#define TSDB_CLUSTER_ID_LEN 40 +#define TSDB_FQDN_LEN 128 +#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) +#define TSDB_IPv4ADDR_LEN 16 +#define TSDB_FILENAME_LEN 128 +#define TSDB_SHOW_SQL_LEN 2048 #define TSDB_SHOW_SCHEMA_JSON_LEN TSDB_MAX_COLUMNS * 256 -#define TSDB_SLOW_QUERY_SQL_LEN 512 -#define TSDB_SHOW_SUBQUERY_LEN 1000 +#define TSDB_SLOW_QUERY_SQL_LEN 512 +#define TSDB_SHOW_SUBQUERY_LEN 1000 #define TSDB_TRANS_STAGE_LEN 12 #define TSDB_TRANS_TYPE_LEN 16 @@ -370,7 +370,7 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_DB_SCHEMALESS TSDB_DB_SCHEMALESS_OFF #define TSDB_MIN_STT_TRIGGER 1 #define TSDB_MAX_STT_TRIGGER 16 -#define TSDB_DEFAULT_SST_TRIGGER 1 +#define TSDB_DEFAULT_SST_TRIGGER 2 #define TSDB_MIN_HASH_PREFIX (2 - TSDB_TABLE_NAME_LEN) #define TSDB_MAX_HASH_PREFIX (TSDB_TABLE_NAME_LEN - 2) #define TSDB_DEFAULT_HASH_PREFIX 0 @@ -379,8 +379,8 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_HASH_SUFFIX 0 #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 -#define TSDB_REP_DEF_DB_WAL_RET_PERIOD 0 -#define TSDB_REPS_DEF_DB_WAL_RET_PERIOD 0 +#define TSDB_REP_DEF_DB_WAL_RET_PERIOD 3600 +#define TSDB_REPS_DEF_DB_WAL_RET_PERIOD 3600 #define TSDB_DB_MIN_WAL_RETENTION_SIZE -1 #define TSDB_REP_DEF_DB_WAL_RET_SIZE 0 #define TSDB_REPS_DEF_DB_WAL_RET_SIZE 0 @@ -410,10 +410,10 @@ typedef enum ELogicConditionType { #define TSDB_EXPLAIN_RESULT_ROW_SIZE (16 * 1024) #define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN" -#define TSDB_MAX_FIELD_LEN 65519 // 16384:65519 -#define TSDB_MAX_BINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 -#define TSDB_MAX_NCHAR_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 -#define TSDB_MAX_GEOMETRY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 +#define TSDB_MAX_FIELD_LEN 65519 // 16384:65519 +#define TSDB_MAX_BINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 +#define TSDB_MAX_NCHAR_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 +#define TSDB_MAX_GEOMETRY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 #define PRIMARYKEY_TIMESTAMP_COL_ID 1 #define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) @@ -492,6 +492,7 @@ enum { #define TSDB_CONFIG_OPTION_LEN 32 #define TSDB_CONFIG_VALUE_LEN 64 +#define TSDB_CONFIG_SCOPE_LEN 8 #define TSDB_CONFIG_NUMBER 8 #define QUERY_ID_SIZE 20 diff --git a/include/util/theap.h b/include/util/theap.h index 8ddeeb28a43c107a0986d76beecebe0f0aa894d0..b795db6aea921b64e584eefc9a87018cfa38ff28 100644 --- a/include/util/theap.h +++ b/include/util/theap.h @@ -77,7 +77,7 @@ PriorityQueueNode* taosPQTop(PriorityQueue* pq); size_t taosPQSize(PriorityQueue* pq); -void taosPQPush(PriorityQueue* pq, const PriorityQueueNode* node); +PriorityQueueNode* taosPQPush(PriorityQueue* pq, const PriorityQueueNode* node); void taosPQPop(PriorityQueue* pq); @@ -89,7 +89,13 @@ void taosBQSetFn(BoundedQueue* q, pq_comp_fn fn); void destroyBoundedQueue(BoundedQueue* q); -void taosBQPush(BoundedQueue* q, PriorityQueueNode* n); +/* + * Push one node into BQ + * @retval NULL if n is upper than top node in q, and n is not freed + * @retval the pushed Node if pushing succeeded + * @note if maxSize exceeded, the original highest node is popped and freed with deleteFn + * */ +PriorityQueueNode* taosBQPush(BoundedQueue* q, PriorityQueueNode* n); PriorityQueueNode* taosBQTop(BoundedQueue* q); diff --git a/include/util/tlist.h b/include/util/tlist.h index c684e90a3318444b706720dee0b64ba220437aee..0924c133b9c77b5717ff30386f49fe63c9e506dd 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -241,6 +241,54 @@ void tdListNodeGetData(SList *list, SListNode *node, void *target); void tdListInitIter(SList *list, SListIter *pIter, TD_LIST_DIRECTION_T direction); SListNode *tdListNext(SListIter *pIter); +// macros ==================================================================================== + +// q: for queue +// n: for node +// m: for member + +#define LISTD(TYPE) \ + struct { \ + TYPE *next, *prev; \ + } + +#define LISTD_NEXT(n, m) ((n)->m.next) +#define LISTD_PREV(n, m) ((n)->m.prev) +#define LISTD_INIT(q, m) (LISTD_NEXT(q, m) = LISTD_PREV(q, m) = (q)) +#define LISTD_HEAD(q, m) (LISTD_NEXT(q, m)) +#define LISTD_TAIL(q, m) (LISTD_PREV(q, m)) +#define LISTD_PREV_NEXT(n, m) (LISTD_NEXT(LISTD_PREV(n, m), m)) +#define LISTD_NEXT_PREV(n, m) (LISTD_PREV(LISTD_NEXT(n, m), m)) + +#define LISTD_INSERT_HEAD(q, n, m) \ + do { \ + LISTD_NEXT(n, m) = LISTD_NEXT(q, m); \ + LISTD_PREV(n, m) = (q); \ + LISTD_NEXT_PREV(n, m) = (n); \ + LISTD_NEXT(q, m) = (n); \ + } while (0) + +#define LISTD_INSERT_TAIL(q, n, m) \ + do { \ + LISTD_NEXT(n, m) = (q); \ + LISTD_PREV(n, m) = LISTD_PREV(q, m); \ + LISTD_PREV_NEXT(n, m) = (n); \ + LISTD_PREV(q, m) = (n); \ + } while (0) + +#define LISTD_REMOVE(n, m) \ + do { \ + LISTD_PREV_NEXT(n, m) = LISTD_NEXT(n, m); \ + LISTD_NEXT_PREV(n, m) = LISTD_PREV(n, m); \ + } while (0) + +#define LISTD_FOREACH(q, n, m) for ((n) = LISTD_HEAD(q, m); (n) != (q); (n) = LISTD_NEXT(n, m)) +#define LISTD_FOREACH_REVERSE(q, n, m) for ((n) = LISTD_TAIL(q, m); (n) != (q); (n) = LISTD_PREV(n, m)) +#define LISTD_FOREACH_SAFE(q, n, t, m) \ + for ((n) = LISTD_HEAD(q, m), (t) = LISTD_NEXT(n, m); (n) != (q); (n) = (t), (t) = LISTD_NEXT(n, m)) +#define LISTD_FOREACH_REVERSE_SAFE(q, n, t, m) \ + for ((n) = LISTD_TAIL(q, m), (t) = LISTD_PREV(n, m); (n) != (q); (n) = (t), (t) = LISTD_PREV(n, m)) + #ifdef __cplusplus } #endif diff --git a/include/util/trbtree.h b/include/util/trbtree.h index e2264194401bee65a22f0301f0339cb76a8d7356..8353a91f0acc3b5ced557ef594572f7db53c64fd 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -39,7 +39,7 @@ void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z); SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey); SRBTreeNode *tRBTreeDropMin(SRBTree *pTree); SRBTreeNode *tRBTreeDropMax(SRBTree *pTree); -SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode); +SRBTreeNode *tRBTreeGet(const SRBTree *pTree, const SRBTreeNode *pKeyNode); // SRBTreeIter ============================================= #define tRBTreeIterCreate(tree, ascend) \ @@ -67,9 +67,9 @@ struct SRBTree { }; struct SRBTreeIter { - int8_t asc; - SRBTree *pTree; - SRBTreeNode *pNode; + int8_t asc; + const SRBTree *pTree; + SRBTreeNode *pNode; }; #ifdef __cplusplus diff --git a/include/util/tutil.h b/include/util/tutil.h index 7a59aa170aa0f0011ec55cbb5507047c6c5d2a21..a2cfa4cfe56f0c5a6021327f6461bde812ebc46e 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -29,7 +29,7 @@ extern "C" { int32_t strdequote(char *src); size_t strtrim(char *src); char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote); -TdUcs4* wcsnchr(const TdUcs4* haystack, TdUcs4 needle, size_t len); +TdUcs4 *wcsnchr(const TdUcs4 *haystack, TdUcs4 needle, size_t len); char **strsplit(char *src, const char *delim, int32_t *num); char *strtolower(char *dst, const char *src); @@ -37,11 +37,11 @@ char *strntolower(char *dst, const char *src, int32_t n); char *strntolower_s(char *dst, const char *src, int32_t n); int64_t strnatoi(char *num, int32_t len); -size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize); -size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize); +size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize); +size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize); -char *strbetween(char *string, char *begin, char *end); -char *paGetToken(char *src, char **token, int32_t *tokenLen); +char *strbetween(char *string, char *begin, char *end); +char *paGetToken(char *src, char **token, int32_t *tokenLen); int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]); int32_t taosHexStrToByteArray(char hexstr[], char bytes[]); @@ -81,12 +81,13 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix, int32_t suffix) { - if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) || prefix * suffix < 0) { + if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) || + prefix * suffix < 0) { return MurmurHash3_32(tbname, tblen); } else if (prefix > 0 || suffix > 0) { return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix); } else { - char tbName[TSDB_TABLE_FNAME_LEN]; + char tbName[TSDB_TABLE_FNAME_LEN]; int32_t offset = 0; if (prefix < 0) { offset = -1 * prefix; @@ -94,20 +95,33 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, } if (suffix < 0) { strncpy(tbName + offset, tbname + tblen + suffix, -1 * suffix); - offset += -1 *suffix; + offset += -1 * suffix; } return MurmurHash3_32(tbName, offset); } } #define TSDB_CHECK_CODE(CODE, LINO, LABEL) \ - if (CODE) { \ - LINO = __LINE__; \ - goto LABEL; \ + do { \ + if ((CODE)) { \ + LINO = __LINE__; \ + goto LABEL; \ + } \ + } while (0) + +#define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, ERRNO) \ + if ((ptr) == NULL) { \ + (CODE) = (ERRNO); \ + (LINO) = __LINE__; \ + goto LABEL; \ } +#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) + #define VND_CHECK_CODE(CODE, LINO, LABEL) TSDB_CHECK_CODE(CODE, LINO, LABEL) +#define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr)-offsetof(type, member))) + #ifdef __cplusplus } #endif diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 2159899aa277f550adf76eb7c107bf3da17c9ab1..236cf1f520b903c4d35bef5d51e5108e91cb9d42 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -108,6 +108,9 @@ # time period of keeping log files, in days # logKeepDays 0 +# unit Hour. Latency of data migration +# keepTimeOffset 0 + ############ 3. Debug Flag and levels ############################################# diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 736582dff2595485819d36a6b7b44dccd6440145..aa7caaaba3c7453dea3adb604b0e64e5a60ec231 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -46,9 +46,10 @@ enum { RES_TYPE__TMQ_METADATA, }; -#define SHOW_VARIABLES_RESULT_COLS 2 +#define SHOW_VARIABLES_RESULT_COLS 3 #define SHOW_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) +#define SHOW_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) #define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) #define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 7455a2c1c82466ede861df1eb187765807bc8697..9f9809b2274592fcb9bd75ba32907a040a4a7a5d 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -435,13 +435,16 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { SColumnInfoData infoData = {0}; infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD1_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD2_LEN; taosArrayPush(pBlock->pDataBlock, &infoData); + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD3_LEN; + taosArrayPush(pBlock->pDataBlock, &infoData); + int32_t numOfCfg = taosArrayGetSize(pVars); blockDataEnsureCapacity(pBlock, numOfCfg); @@ -457,6 +460,11 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { STR_WITH_MAXSIZE_TO_VARSTR(value, pInfo->value, TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); colDataSetVal(pColInfo, i, value, false); + + char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(scope, pInfo->scope, TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE); + pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + colDataSetVal(pColInfo, i, scope, false); } pBlock->info.rows = numOfCfg; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index b2c901c800367d7aa1ccfeb0d0c6fb998492c9f2..92dd4ea41960f3d620d947fb9bb4a3085a60f933 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -899,7 +899,7 @@ static void* tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; tDeleteSMqAskEpRsp(&pEpRspWrapper->msg); - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; taosMemoryFreeClear(pRsp->pEpset); @@ -912,7 +912,7 @@ static void* tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { taosMemoryFreeClear(pRsp->pEpset); taosMemoryFree(pRsp->metaRsp.metaRsp); - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) { + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; taosMemoryFreeClear(pRsp->pEpset); @@ -1394,7 +1394,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { strcpy(pRspWrapper->topicName, pParam->topicName); pMsg->pEpSet = NULL; - if (rspType == TMQ_MSG_TYPE__POLL_RSP) { + if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp); @@ -1411,7 +1411,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tDecodeMqMetaRsp(&decoder, &pRspWrapper->metaRsp); tDecoderClear(&decoder); memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead)); - } else if (rspType == TMQ_MSG_TYPE__TAOSX_RSP) { + } else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp); @@ -1896,7 +1896,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; tscError("consumer:0x%" PRIx64 " unexpected rsp from poll, code:%s", tmq->consumerId, tstrerror(terrno)); return NULL; - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); @@ -1984,7 +1984,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { pRspWrapper = tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pollRspWrapper); } - } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) { + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); @@ -2008,18 +2008,22 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { pVg->emptyBlockReceiveTs = taosGetTimestampMs(); pRspWrapper = tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pollRspWrapper); + taosWUnLockLatch(&tmq->lock); + continue; } else { pVg->emptyBlockReceiveTs = 0; // reset the ts - // build rsp - void* pRsp = NULL; - int64_t numOfRows = 0; - if (pollRspWrapper->taosxRsp.createTableNum == 0) { - pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows); - } else { - pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows); - } + } - tmq->totalRows += numOfRows; + // build rsp + void* pRsp = NULL; + int64_t numOfRows = 0; + if (pollRspWrapper->taosxRsp.createTableNum == 0) { + tscError("consumer:0x%" PRIx64" createTableNum should > 0 if rsp type is data_meta", tmq->consumerId); + } else { + pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows); + } + + tmq->totalRows += numOfRows; char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); @@ -2028,11 +2032,9 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); - taosFreeQitem(pollRspWrapper); - taosWUnLockLatch(&tmq->lock); - return pRsp; - } + taosFreeQitem(pollRspWrapper); taosWUnLockLatch(&tmq->lock); + return pRsp; } else { tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 9c6d941172b7ec58737a33f1515da20f122f0c01..356ea2be1cb35be9ae0cedb6061abfa60b3307a3 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -1,4 +1,8 @@ aux_source_directory(src COMMON_SRC) +IF (TD_ENTERPRISE) +LIST(APPEND COMMON_SRC ${TD_ENTERPRISE_DIR}/src/plugins/common/src/tglobal.c) +ENDIF() + add_library(common STATIC ${COMMON_SRC}) if (DEFINED GRANT_CFG_INCLUDE_DIR) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 53692c94a4d906e94bf98515b49c53663fe5bb56..0940fcef6a749489077d45f931f4c27e8800f4ee 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -33,7 +33,7 @@ static const SSysDbTableSchema dnodesSchema[] = { {.name = "support_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, - {.name = "reboot_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, + {.name = "reboot_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, {.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, #ifdef TD_ENTERPRISE {.name = "active_code", .bytes = TSDB_ACTIVE_KEY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, @@ -47,7 +47,7 @@ static const SSysDbTableSchema mnodesSchema[] = { {.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, - {.name = "reboot_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, + {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, }; static const SSysDbTableSchema modulesSchema[] = { @@ -73,7 +73,7 @@ static const SSysDbTableSchema clusterSchema[] = { {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "uptime", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, - {.name = "version", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "version", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "expire_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, }; @@ -102,8 +102,6 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "wal_fsync_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "wal_retention_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "wal_retention_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true}, - {.name = "wal_roll_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "wal_segment_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true}, {.name = "stt_trigger", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "table_prefix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, @@ -160,7 +158,7 @@ static const SSysDbTableSchema streamSchema[] = { static const SSysDbTableSchema streamTaskSchema[] = { {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "task_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "task_id", .bytes = 32, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "node_type", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "node_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "level", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, @@ -273,6 +271,7 @@ static const SSysDbTableSchema variablesSchema[] = { {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "value", .bytes = TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "scope", .bytes = TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema topicSchema[] = { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 96889882b65f75ccb7f7de5095d03286c1a2609d..887a11083153c0daa3bf6f06a36ae49d6c65dab5 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -47,7 +47,6 @@ int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRo } } - int32_t colDataGetRowLength(const SColumnInfoData* pColumnInfoData, int32_t rowIdx) { if (colDataIsNull_s(pColumnInfoData, rowIdx)) return 0; @@ -67,10 +66,6 @@ int32_t colDataGetFullLength(const SColumnInfoData* pColumnInfoData, int32_t num } } -void colDataTrim(SColumnInfoData* pColumnInfoData) { - // TODO -} - int32_t getJsonValueLen(const char* data) { int32_t dataLen = 0; if (*data == TSDB_DATA_TYPE_NULL) { @@ -89,10 +84,6 @@ int32_t getJsonValueLen(const char* data) { return dataLen; } -int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) { - return colDataSetVal(pColumnInfoData, rowIndex, pData, isNull); -} - int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) { if (isNull) { // There is a placehold for each NULL value of binary or nchar type. @@ -174,7 +165,7 @@ int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, } -int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) { +static int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) { if (!IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return TSDB_CODE_SUCCESS; } @@ -641,9 +632,12 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { pStart += colSize; } } else { - memcpy(pStart, pCol->pData, dataSize); + if (dataSize != 0) { + // ubsan reports error if pCol->pData==NULL && dataSize==0 + memcpy(pStart, pCol->pData, dataSize); + } pStart += dataSize; - } + } } return 0; @@ -693,8 +687,10 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { return TSDB_CODE_FAILED; } } - - memcpy(pCol->pData, pStart, colLength); + if (colLength != 0) { + // ubsan reports error if colLength==0 && pCol->pData == 0 + memcpy(pCol->pData, pStart, colLength); + } pStart += colLength; } @@ -882,41 +878,8 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { return 0; } -static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, - int32_t tupleIndex) { - int32_t code = 0; - size_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pDst = &pDstCols[i]; - SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, i); - - if (pSrc->hasNull && colDataIsNull(pSrc, pSrcBlock->info.rows, tupleIndex, pSrcBlock->pBlockAgg[i])) { - code = colDataSetVal(pDst, numOfRows, NULL, true); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else { - char* p = colDataGetData(pSrc, tupleIndex); - code = colDataSetVal(pDst, numOfRows, p, false); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - } - - return TSDB_CODE_SUCCESS; -} - static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, const int32_t* index) { -#if 0 - for (int32_t i = 0; i < pDataBlock->info.rows; ++i) { - int32_t code = doAssignOneTuple(pCols, i, pDataBlock, index[i]); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } -#else + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = &pCols[i]; @@ -941,7 +904,7 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB } } } -#endif + return TSDB_CODE_SUCCESS; } @@ -1101,114 +1064,6 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { return TSDB_CODE_SUCCESS; } -#if 0 -typedef struct SHelper { - int32_t index; - union { - char* pData; - int64_t i64; - double d64; - }; -} SHelper; - -SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* pBlock) { - int32_t sortValLengthPerRow = 0; - int32_t numOfCols = taosArrayGetSize(pOrderInfo); - - for (int32_t i = 0; i < numOfCols; ++i) { - SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->slotId); - pInfo->pColData = pColInfo; - sortValLengthPerRow += pColInfo->info.bytes; - } - - size_t len = sortValLengthPerRow * pBlock->info.rows; - - char* buf = taosMemoryCalloc(1, len); - SHelper* phelper = taosMemoryCalloc(numOfRows, sizeof(SHelper)); - for (int32_t i = 0; i < numOfRows; ++i) { - phelper[i].index = i; - phelper[i].pData = buf + sortValLengthPerRow * i; - } - - int32_t offset = 0; - for (int32_t i = 0; i < numOfCols; ++i) { - SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); - for (int32_t j = 0; j < numOfRows; ++j) { - phelper[j].i64 = *(int32_t*)pInfo->pColData->pData + pInfo->pColData->info.bytes * j; - // memcpy(phelper[j].pData + offset, pInfo->pColData->pData + pInfo->pColData->info.bytes * j, - // pInfo->pColData->info.bytes); - } - - offset += pInfo->pColData->info.bytes; - } - - taosMemoryFree(buf); - return phelper; -} - -int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { - const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*)param; - - SHelper* left = (SHelper*)p1; - SHelper* right = (SHelper*)p2; - - SArray* pInfo = pHelper->orderInfo; - - int32_t offset = 0; - int32_t leftx = *(int32_t*)left->pData; //*(int32_t*)(left->pData + offset); - int32_t rightx = *(int32_t*)right->pData; //*(int32_t*)(right->pData + offset); - - if (leftx == rightx) { - return 0; - } else { - return (leftx < rightx) ? -1 : 1; - } - return 0; -} - -int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) { - // Allocate the additional buffer. - int64_t p0 = taosGetTimestampUs(); - - SSDataBlockSortHelper helper = {.pDataBlock = pDataBlock, .orderInfo = pOrderInfo}; - - uint32_t rows = pDataBlock->info.rows; - SHelper* index = createTupleIndex_rv(rows, helper.orderInfo, pDataBlock); - if (index == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return terrno; - } - - taosqsort(index, rows, sizeof(SHelper), &helper, dataBlockCompar_rv); - - int64_t p1 = taosGetTimestampUs(); - SColumnInfoData* pCols = createHelpColInfoData(pDataBlock); - if (pCols == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return terrno; - } - - int64_t p2 = taosGetTimestampUs(); - - // int32_t code = blockDataAssign(pCols, pDataBlock, index); - // if (code != TSDB_CODE_SUCCESS) { - // terrno = code; - // return code; - // } - - int64_t p3 = taosGetTimestampUs(); - - copyBackToBlock(pDataBlock, pCols); - int64_t p4 = taosGetTimestampUs(); - - printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, - p3 - p2, p4 - p3, rows); - // destroyTupleIndex(index); - return 0; -} -#endif - void blockDataCleanup(SSDataBlock* pDataBlock) { blockDataEmpty(pDataBlock); SDataBlockInfo* pInfo = &pDataBlock->info; @@ -1345,8 +1200,7 @@ void blockDataFreeRes(SSDataBlock* pBlock) { colDataDestroy(pColInfoData); } - taosArrayDestroy(pBlock->pDataBlock); - pBlock->pDataBlock = NULL; + pBlock->pDataBlock = taosArrayDestroy(pBlock->pDataBlock); taosMemoryFreeClear(pBlock->pBlockAgg); memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); } @@ -1361,6 +1215,7 @@ void* blockDataDestroy(SSDataBlock* pBlock) { return NULL; } +// todo remove it int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { dst->info = src->info; dst->info.rows = 0; @@ -1759,16 +1614,6 @@ static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_ if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); - } else { // reset the bitmap value - /*int32_t stopIndex = BitmapLen(n) * 8; - for(int32_t i = n; i < stopIndex; ++i) { - colDataClearNull_f(pColInfoData->nullbitmap, i); - } - - int32_t remain = BitmapLen(total) - BitmapLen(n); - if (remain > 0) { - memset(pColInfoData->nullbitmap+BitmapLen(n), 0, remain); - }*/ } } @@ -1875,32 +1720,6 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { return (void*)buf; } -int32_t tEncodeDataBlocks(void** buf, const SArray* blocks) { - int32_t tlen = 0; - int32_t sz = taosArrayGetSize(blocks); - tlen += taosEncodeFixedI32(buf, sz); - - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pBlock = taosArrayGet(blocks, i); - tlen += tEncodeDataBlock(buf, pBlock); - } - - return tlen; -} - -void* tDecodeDataBlocks(const void* buf, SArray** blocks) { - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - - *blocks = taosArrayInit(sz, sizeof(SSDataBlock)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock pBlock = {0}; - buf = tDecodeDataBlock(buf, &pBlock); - taosArrayPush(*blocks, &pBlock); - } - return (void*)buf; -} - static char* formatTimestamp(char* buf, int64_t val, int precision) { time_t tt; int32_t ms = 0; @@ -1950,101 +1769,6 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } -#if 0 -void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag) { - SArray* dataBlocks = taosArrayInit(1, sizeof(SSDataBlock*)); - taosArrayPush(dataBlocks, &pBlock); - blockDebugShowDataBlocks(dataBlocks, flag); - taosArrayDestroy(dataBlocks); -} - -void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { - char pBuf[128] = {0}; - int32_t sz = taosArrayGetSize(dataBlocks); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); - size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); - - int32_t rows = pDataBlock->info.rows; - printf("%s |block ver %" PRIi64 " |block type %d |child id %d|group id %" PRIu64 "\n", flag, - pDataBlock->info.version, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, - pDataBlock->info.id.groupId); - for (int32_t j = 0; j < rows; j++) { - printf("%s |", flag); - for (int32_t k = 0; k < numOfCols; k++) { - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - if (k == 0) { - printf("cols:%d |", (int32_t)numOfCols); - } - if (colDataIsNull(pColInfoData, rows, j, NULL)) { - printf(" %15s |", "NULL"); - continue; - } - - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); - printf(" %25s |", pBuf); - break; - case TSDB_DATA_TYPE_BOOL: - printf(" %15" PRIi8 " |", *(int8_t*)var); - break; - case TSDB_DATA_TYPE_TINYINT: - printf(" %15" PRIi8 " |", *(int8_t*)var); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf(" %15" PRIi16 " |", *(int16_t*)var); - break; - case TSDB_DATA_TYPE_INT: - printf(" %15d |", *(int32_t*)var); - break; - case TSDB_DATA_TYPE_UTINYINT: - printf(" %15" PRIu8 " |", *(uint8_t*)var); - break; - case TSDB_DATA_TYPE_USMALLINT: - printf(" %15" PRIu16 " |", *(uint16_t*)var); - break; - case TSDB_DATA_TYPE_UINT: - printf(" %15u |", *(uint32_t*)var); - break; - case TSDB_DATA_TYPE_BIGINT: - printf(" %15" PRId64 " |", *(int64_t*)var); - break; - case TSDB_DATA_TYPE_UBIGINT: - printf(" %15" PRIu64 " |", *(uint64_t*)var); - break; - case TSDB_DATA_TYPE_FLOAT: - printf(" %15f |", *(float*)var); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf(" %15lf |", *(double*)var); - break; - case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_GEOMETRY: { - char* pData = colDataGetVarData(pColInfoData, j); - int32_t dataSize = TMIN(sizeof(pBuf) - 1, varDataLen(pData)); - memset(pBuf, 0, dataSize + 1); - strncpy(pBuf, varDataVal(pData), dataSize); - printf(" %15s |", pBuf); - } break; - case TSDB_DATA_TYPE_NCHAR: { - char* pData = colDataGetVarData(pColInfoData, j); - int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); - memset(pBuf, 0, dataSize); - (void)taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); - printf(" %15s |", pBuf); - } break; - default: - break; - } - } - printf("\n"); - } - } -} -#endif - // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) { int32_t size = 2048*1024; @@ -2153,182 +1877,6 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) return dumpBuf; } -/** - * @brief TODO: Assume that the final generated result it less than 3M - * - * @param pReq - * @param pDataBlocks - * @param vgId - * @param suid - * - */ -#if 0 -int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlock, STSchema* pTSchema, int32_t vgId, - tb_uid_t suid) { - int32_t bufSize = sizeof(SSubmitReq); - int32_t sz = 1; - for (int32_t i = 0; i < sz; ++i) { - const SDataBlockInfo* pBlkInfo = &pDataBlock->info; - - int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); - bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(colNum)); - bufSize += sizeof(SSubmitBlk); - } - - *pReq = taosMemoryCalloc(1, bufSize); - if (!(*pReq)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - void* pDataBuf = *pReq; - - int32_t msgLen = sizeof(SSubmitReq); - int32_t numOfBlks = 0; - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - - for (int32_t i = 0; i < sz; ++i) { - int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); - int32_t rows = pDataBlock->info.rows; - - if (colNum <= 1) { - // invalid if only with TS col - continue; - } - - if (rb.nCols != colNum) { - tdSRowSetTpInfo(&rb, colNum, pTSchema->flen); - } - - SSubmitBlk* pSubmitBlk = POINTER_SHIFT(pDataBuf, msgLen); - pSubmitBlk->suid = suid; - pSubmitBlk->uid = pDataBlock->info.id.groupId; - pSubmitBlk->numOfRows = rows; - pSubmitBlk->sversion = pTSchema->version; - - msgLen += sizeof(SSubmitBlk); - int32_t dataLen = 0; - for (int32_t j = 0; j < rows; ++j) { // iterate by row - tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen + dataLen)); // set row buf - bool isStartKey = false; - int32_t offset = 0; - for (int32_t k = 0; k < colNum; ++k) { // iterate by column - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - STColumn* pCol = &pTSchema->columns[k]; - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - if (!isStartKey) { - isStartKey = true; - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, - offset, k); - continue; // offset should keep 0 for next column - - } else if (colDataIsNull_s(pColInfoData, j)) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NULL, NULL, - false, offset, k); - } else { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, - true, offset, k); - } - break; - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY - case TSDB_DATA_TYPE_GEOMETRY: { - if (colDataIsNull_s(pColInfoData, j)) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pColInfoData->info.type, TD_VTYPE_NULL, NULL, - false, offset, k); - } else { - void* data = colDataGetData(pColInfoData, j); - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pColInfoData->info.type, TD_VTYPE_NORM, data, - true, offset, k); - } - break; - } - case TSDB_DATA_TYPE_VARBINARY: - case TSDB_DATA_TYPE_DECIMAL: - case TSDB_DATA_TYPE_BLOB: - case TSDB_DATA_TYPE_JSON: - case TSDB_DATA_TYPE_MEDIUMBLOB: - uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); - break; - default: - if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { - if (colDataIsNull_s(pColInfoData, j)) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NULL, NULL, false, - offset, k); - } else if (pCol->type == pColInfoData->info.type) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, var, true, offset, - k); - } else { - char tv[8] = {0}; - if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { - float v = 0; - GET_TYPED_DATA(v, float, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { - double v = 0; - GET_TYPED_DATA(v, double, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else { - uint64_t v = 0; - GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset, - k); - } - } else { - uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); - } - break; - } - offset += TYPE_BYTES[pCol->type]; // sum/avg would convert to int64_t/uint64_t/double during aggregation - } - tdSRowEnd(&rb); - dataLen += TD_ROW_LEN(rb.pBuf); -#ifdef TD_DEBUG_PRINT_ROW - tdSRowPrint(rb.pBuf, pTSchema, __func__); -#endif - } - - ++numOfBlks; - - pSubmitBlk->dataLen = dataLen; - msgLen += pSubmitBlk->dataLen; - } - - if (numOfBlks > 0) { - (*pReq)->length = msgLen; - - (*pReq)->header.vgId = htonl(vgId); - (*pReq)->header.contLen = htonl(msgLen); - (*pReq)->length = (*pReq)->header.contLen; - (*pReq)->numOfBlocks = htonl(numOfBlks); - SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1); - while (numOfBlks--) { - int32_t dataLen = blk->dataLen; - blk->uid = htobe64(blk->uid); - blk->suid = htobe64(blk->suid); - blk->sversion = htonl(blk->sversion); - blk->dataLen = htonl(blk->dataLen); - blk->schemaLen = htonl(blk->schemaLen); - blk->numOfRows = htonl(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + dataLen); - } - } else { - // no valid rows - taosMemoryFreeClear(*pReq); - } - - return TSDB_CODE_SUCCESS; -} -#endif - int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDataBlock, const STSchema* pTSchema, int64_t uid, int32_t vgId, tb_uid_t suid) { SSubmitReq2* pReq = *ppReq; @@ -2732,3 +2280,149 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { ASSERT(pStart - pData == dataLen); return pStart; } + +void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList) { +// int32_t totalRows = pBlock->info.rows; + int32_t bmLen = BitmapLen(totalRows); + char* pBitmap = NULL; + int32_t maxRows = 0; + + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); + // it is a reserved column for scalar function, and no data in this column yet. + if (pDst->pData == NULL) { + continue; + } + + int32_t numOfRows = 0; + if (IS_VAR_DATA_TYPE(pDst->info.type)) { + int32_t j = 0; + pDst->varmeta.length = 0; + + while (j < totalRows) { + if (pBoolList[j] == 0) { + j += 1; + continue; + } + + if (colDataIsNull_var(pDst, j)) { + colDataSetNull_var(pDst, numOfRows); + } else { + // fix address sanitizer error. p1 may point to memory that will change during realloc of colDataSetVal, first copy it to p2 + char* p1 = colDataGetVarData(pDst, j); + int32_t len = 0; + if (pDst->info.type == TSDB_DATA_TYPE_JSON) { + len = getJsonValueLen(p1); + } else { + len = varDataTLen(p1); + } + char* p2 = taosMemoryMalloc(len); + memcpy(p2, p1, len); + colDataSetVal(pDst, numOfRows, p2, false); + taosMemoryFree(p2); + } + numOfRows += 1; + j += 1; + } + + if (maxRows < numOfRows) { + maxRows = numOfRows; + } + } else { + if (pBitmap == NULL) { + pBitmap = taosMemoryCalloc(1, bmLen); + } + + memcpy(pBitmap, pDst->nullbitmap, bmLen); + memset(pDst->nullbitmap, 0, bmLen); + + int32_t j = 0; + + switch (pDst->info.type) { + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + while (j < totalRows) { + if (pBoolList[j] == 0) { + j += 1; + continue; + } + + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int64_t*)pDst->pData)[numOfRows] = ((int64_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + while (j < totalRows) { + if (pBoolList[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int32_t*)pDst->pData)[numOfRows] = ((int32_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + while (j < totalRows) { + if (pBoolList[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int16_t*)pDst->pData)[numOfRows] = ((int16_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + while (j < totalRows) { + if (pBoolList[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int8_t*)pDst->pData)[numOfRows] = ((int8_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + } + } + + if (maxRows < numOfRows) { + maxRows = numOfRows; + } + } + + pBlock->info.rows = maxRows; + if (pBitmap != NULL) { + taosMemoryFree(pBitmap); + } +} + +int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { + return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock); +} \ No newline at end of file diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 0b121777544b33c918ca5c421e78e1b644f3109b..7c6939635a290adae506c2b712ef209c7a888326 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2245,15 +2245,18 @@ static int32_t tColDataUpdateValue72(SColData *pColData, uint8_t *pData, uint32_ } return 0; } +static FORCE_INLINE int32_t tColDataUpdateNothing(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + return 0; +} static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) = { - {NULL, NULL, NULL}, // 0 - {tColDataUpdateValue10, NULL, tColDataUpdateValue12}, // HAS_NONE - {tColDataUpdateValue20, NULL, NULL}, // HAS_NULL - {tColDataUpdateValue30, NULL, tColDataUpdateValue32}, // HAS_NULL|HAS_NONE - {tColDataUpdateValue40, NULL, tColDataUpdateValue42}, // HAS_VALUE - {tColDataUpdateValue50, NULL, tColDataUpdateValue52}, // HAS_VALUE|HAS_NONE - {tColDataUpdateValue60, NULL, tColDataUpdateValue62}, // HAS_VALUE|HAS_NULL - {tColDataUpdateValue70, NULL, tColDataUpdateValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE + {NULL, NULL, NULL}, // 0 + {tColDataUpdateValue10, tColDataUpdateNothing, tColDataUpdateValue12}, // HAS_NONE + {tColDataUpdateValue20, tColDataUpdateNothing, tColDataUpdateNothing}, // HAS_NULL + {tColDataUpdateValue30, tColDataUpdateNothing, tColDataUpdateValue32}, // HAS_NULL|HAS_NONE + {tColDataUpdateValue40, tColDataUpdateNothing, tColDataUpdateValue42}, // HAS_VALUE + {tColDataUpdateValue50, tColDataUpdateNothing, tColDataUpdateValue52}, // HAS_VALUE|HAS_NONE + {tColDataUpdateValue60, tColDataUpdateNothing, tColDataUpdateValue62}, // HAS_VALUE|HAS_NULL + {tColDataUpdateValue70, tColDataUpdateNothing, tColDataUpdateValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE // VALUE NONE NULL }; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index bacafee34992e2b9d86028bde67d55373c167caa..f06eeb230fc17e6dbfa651b7437db670f883acec 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -34,6 +34,7 @@ char tsFirst[TSDB_EP_LEN] = {0}; char tsSecond[TSDB_EP_LEN] = {0}; char tsLocalFqdn[TSDB_FQDN_LEN] = {0}; char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port +char tsVersionName[16] = "community"; uint16_t tsServerPort = 6030; int32_t tsVersion = 30000000; int32_t tsStatusInterval = 1; // second @@ -54,7 +55,7 @@ int32_t tsNumOfMnodeQueryThreads = 4; int32_t tsNumOfMnodeFetchThreads = 1; int32_t tsNumOfMnodeReadThreads = 1; int32_t tsNumOfVnodeQueryThreads = 4; -float tsRatioOfVnodeStreamThreads = 2.0; +float tsRatioOfVnodeStreamThreads = 4.0; int32_t tsNumOfVnodeFetchThreads = 4; int32_t tsNumOfVnodeRsmaThreads = 2; int32_t tsNumOfQnodeQueryThreads = 4; @@ -76,8 +77,14 @@ int64_t tsVndCommitMaxIntervalMs = 600 * 1000; int64_t tsMndSdbWriteDelta = 200; int64_t tsMndLogRetention = 2000; int8_t tsGrant = 1; +int32_t tsMndGrantMode = 0; bool tsMndSkipGrant = false; +// dnode +int64_t tsDndStart = 0; +int64_t tsDndStartOsUptime = 0; +int64_t tsDndUpTime = 0; + // monitor bool tsEnableMonitor = true; int32_t tsMonitorInterval = 30; @@ -111,7 +118,7 @@ int32_t tmqMaxTopicNum = 20; int32_t tsQueryPolicy = 1; int32_t tsQueryRspPolicy = 0; int64_t tsQueryMaxConcurrentTables = 200; // unit is TSDB_TABLE_NUM_UNIT -bool tsEnableQueryHb = false; +bool tsEnableQueryHb = true; bool tsEnableScience = false; // on taos-cli show float and doulbe with scientific notation if true bool tsTtlChangeOnWrite = false; // ttl delete time changes on last write if true int32_t tsQuerySmaOptimize = 0; @@ -186,6 +193,13 @@ bool tsDeployOnSnode = true; * TSDB_TIME_PRECISION_NANO: 60000000000L */ int64_t tsTickPerMin[] = {60000L, 60000000L, 60000000000L}; +/* + * millisecond by default + * for TSDB_TIME_PRECISION_MILLI: 3600000L + * TSDB_TIME_PRECISION_MICRO: 3600000000L + * TSDB_TIME_PRECISION_NANO: 3600000000000L + */ +int64_t tsTickPerHour[] = {3600000L, 3600000000L, 3600000000000L}; // lossy compress 6 char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty @@ -217,6 +231,7 @@ bool tsDisableStream = false; int64_t tsStreamBufferSize = 128 * 1024 * 1024; int64_t tsCheckpointInterval = 3 * 60 * 60 * 1000; bool tsFilterScalarMode = false; +int32_t tsKeepTimeOffset = 0; // latency of data migration #ifndef _STORAGE int32_t taosSetTfsCfg(SConfig *pCfg) { @@ -290,38 +305,38 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input } int32_t taosAddClientLogCfg(SConfig *pCfg) { - if (cfgAddDir(pCfg, "configDir", configDir, 1) != 0) return -1; - if (cfgAddDir(pCfg, "scriptDir", configDir, 1) != 0) return -1; - if (cfgAddDir(pCfg, "logDir", tsLogDir, 1) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, 1) != 0) return -1; - if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, CFG_SCOPE_CLIENT) != 0) return -1; return 0; } static int32_t taosAddServerLogCfg(SConfig *pCfg) { - if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, 0) != CFG_SCOPE_SERVER) return -1; return 0; } @@ -332,53 +347,52 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { strcpy(defaultFqdn, "localhost"); } - if (cfgAddString(pCfg, "firstEp", "", 1) != 0) return -1; - if (cfgAddString(pCfg, "secondEp", "", 1) != 0) return -1; - if (cfgAddString(pCfg, "fqdn", defaultFqdn, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1; - if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, 1) != 0) return -1; - if (cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, false) != 0) return -1; - if (cfgAddBool(pCfg, "enableScience", tsEnableScience, false) != 0) return -1; - if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1; - if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1; - if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, true) != 0) return -1; - if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, true) != 0) return -1; - if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, true) != 0) return -1; - if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; - if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; - // if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; - // if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1; - if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, true) != 0) return -1; - if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; - if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; - if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1; - if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, 1) != 0) - return -1; - if (cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, true) != 0) return -1; - if (cfgAddString(pCfg, "slowLogScope", "", true) != 0) return -1; + if (cfgAddString(pCfg, "firstEp", "", CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlChildTableName", "", CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT) != 0) return -1; + // if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, CFG_SCOPE_CLIENT) != 0) return -1; + // if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "slowLogScope", "", CFG_SCOPE_CLIENT) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); - if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH) != 0) return -1; tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 100000); - if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1; tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 10000000); - if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); if (tsNumOfTaskQueueThreads >= 10) { tsNumOfTaskQueueThreads = 10; } - if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT) != 0) return -1; return 0; } @@ -386,92 +400,92 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { static int32_t taosAddSystemCfg(SConfig *pCfg) { SysNameInfo info = taosGetSysNameInfo(); - if (cfgAddTimezone(pCfg, "timezone", tsTimezoneStr) != 0) return -1; - if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1; - if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1; - if (cfgAddBool(pCfg, "assert", 1, 1) != 0) return -1; - if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1; - if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1; - - if (cfgAddBool(pCfg, "SSE42", tsSSE42Enable, 0) != 0) return -1; - if (cfgAddBool(pCfg, "AVX", tsAVXEnable, 0) != 0) return -1; - if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, 0) != 0) return -1; - if (cfgAddBool(pCfg, "FMA", tsFMAEnable, 0) != 0) return -1; - if (cfgAddBool(pCfg, "SIMD-builtins", tsSIMDBuiltins, 0) != 0) return -1; - if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, 0) != 0) return -1; - - if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "assert", 1, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1; + + if (cfgAddBool(pCfg, "SSE42", tsSSE42Enable, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "AVX", tsAVXEnable, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "FMA", tsFMAEnable, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "SIMD-builtins", tsSIMDBuiltins, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH) != 0) return -1; + + if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; #if !defined(_ALPINE) - if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; #endif - if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1; - if (cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1; - if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; - if (cfgAddString(pCfg, "os nodename", info.nodename, 1) != 0) return -1; - if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1; - if (cfgAddString(pCfg, "os version", info.version, 1) != 0) return -1; - if (cfgAddString(pCfg, "os machine", info.machine, 1) != 0) return -1; - - if (cfgAddString(pCfg, "version", version, 1) != 0) return -1; - if (cfgAddString(pCfg, "compatible_version", compatible_version, 1) != 0) return -1; - if (cfgAddString(pCfg, "gitinfo", gitinfo, 1) != 0) return -1; - if (cfgAddString(pCfg, "buildinfo", buildinfo, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "os sysname", info.sysname, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "os nodename", info.nodename, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "os release", info.release, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "os version", info.version, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "os machine", info.machine, CFG_SCOPE_BOTH) != 0) return -1; + + if (cfgAddString(pCfg, "version", version, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "compatible_version", compatible_version, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "gitinfo", gitinfo, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "buildinfo", buildinfo, CFG_SCOPE_BOTH) != 0) return -1; return 0; } static int32_t taosAddServerCfg(SConfig *pCfg) { - if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1; + if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfSupportVnodes = tsNumOfCores * 2; tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); - if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, 0) != 0) return -1; - - if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, 0) != 0) return -1; - if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER) != 0) return -1; + + if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); - if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH) != 0) return -1; tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 10000); - if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1; tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 1000000); - if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsNumOfRpcSessions, 20, 1000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsNumOfRpcSessions, 20, 1000000, CFG_SCOPE_BOTH) != 0) return -1; tsNumOfCommitThreads = tsNumOfCores / 2; tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfMnodeReadThreads = tsNumOfCores / 8; tsNumOfMnodeReadThreads = TRANGE(tsNumOfMnodeReadThreads, 1, 4); - if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfVnodeQueryThreads = tsNumOfCores * 2; tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 100, 0) != 0) return -1; + if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 100, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfVnodeFetchThreads = tsNumOfCores / 4; tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfVnodeRsmaThreads = tsNumOfCores; tsNumOfVnodeRsmaThreads = TMAX(tsNumOfVnodeRsmaThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfQnodeQueryThreads = tsNumOfCores * 2; tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 4); - if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 4, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER) != 0) return -1; // tsNumOfQnodeFetchThreads = tsNumOfCores / 2; // tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); @@ -479,66 +493,68 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfSnodeStreamThreads = tsNumOfCores / 4; tsNumOfSnodeStreamThreads = TRANGE(tsNumOfSnodeStreamThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER) != 0) return -1; tsNumOfSnodeWriteThreads = tsNumOfCores / 4; tsNumOfSnodeWriteThreads = TRANGE(tsNumOfSnodeWriteThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER) != 0) return -1; tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); - if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, 0) != 0) + if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; - - if (cfgAddInt64(pCfg, "vndCommitMaxInterval", tsVndCommitMaxIntervalMs, 1000, 1000 * 60 * 60, 0) != 0) return -1; - - if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, 0) != 0) return -1; - if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, 0) != 0) return -1; - if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, 0) != 0) return -1; - - if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1; - if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, 0) != 0) return -1; - if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, 0) != 0) return -1; - - if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, 0) != 0) return -1; - if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, 0) != 0) return -1; - if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, 0) != 0) return -1; - - if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, 1) != 0) return -1; - - if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 100000, 1) != 0) return -1; - if (cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, 0) != 0) return -1; - - if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, 0) != 0) + if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) return -1; + + if (cfgAddInt64(pCfg, "vndCommitMaxInterval", tsVndCommitMaxIntervalMs, 1000, 1000 * 60 * 60, CFG_SCOPE_SERVER) != 0) return -1; + + if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "grantMode", tsMndGrantMode, 0, 10000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER) != 0) return -1; + + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER) != 0) return -1; + + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH) != 0) return -1; + + if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; + + if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER) != 0) return -1; + + if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1; - if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, 0) != 0) return -1; - if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, 0) != 0) return -1; + if (cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "disableStream", tsDisableStream, 0) != 0) return -1; - if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, 0) != 0) return -1; - if (cfgAddInt64(pCfg, "checkpointInterval", tsCheckpointInterval, 0, INT64_MAX, 0) != 0) return -1; + if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt64(pCfg, "checkpointInterval", tsCheckpointInterval, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, 0) != 0) return -1; + if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "keepTimeOffset", tsKeepTimeOffset, 0, 23, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER) != 0) return -1; GRANT_CFG_ADD; return 0; @@ -906,6 +922,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64; tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64; tsMndSkipGrant = cfgGetItem(pCfg, "skipGrant")->bval; + tsMndGrantMode = cfgGetItem(pCfg, "grantMode")->i32; tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs)); @@ -921,6 +938,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsCheckpointInterval = cfgGetItem(pCfg, "checkpointInterval")->i64; tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval; + tsKeepTimeOffset = cfgGetItem(pCfg, "keepTimeOffset")->i32; tsMaxStreamBackendCache = cfgGetItem(pCfg, "maxStreamBackendCache")->i32; tsPQSortMemThreshold = cfgGetItem(pCfg, "pqSortMemThreshold")->i32; @@ -928,6 +946,12 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { return 0; } +#ifndef TD_ENTERPRISE +static int32_t taosSetReleaseCfg(SConfig *pCfg) { return 0; } +#else +int32_t taosSetReleaseCfg(SConfig *pCfg); +#endif + void taosLocalCfgForbiddenToChange(char *name, bool *forbidden) { int32_t len = strlen(name); char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; @@ -1434,6 +1458,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile if (taosSetClientCfg(tsCfg)) return -1; if (taosUpdateServerCfg(tsCfg)) return -1; if (taosSetServerCfg(tsCfg)) return -1; + if (taosSetReleaseCfg(tsCfg)) return -1; if (taosSetTfsCfg(tsCfg) != 0) return -1; } taosSetSystemCfg(tsCfg); @@ -1478,6 +1503,13 @@ void taosCfgDynamicOptions(const char *option, const char *value) { return; } + if (strcasecmp(option, "keepTimeOffset") == 0) { + int32_t newKeepTimeOffset = atoi(value); + uInfo("keepTimeOffset set from %d to %d", tsKeepTimeOffset, newKeepTimeOffset); + tsKeepTimeOffset = newKeepTimeOffset; + return; + } + const char *options[] = { "dDebugFlag", "vDebugFlag", "mDebugFlag", "wDebugFlag", "sDebugFlag", "tsdbDebugFlag", "tqDebugFlag", "fsDebugFlag", "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", "tdbDebugFlag", "tmrDebugFlag", "uDebugFlag", diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index c195f5387cb2e01fb128895465980a2569b4d04a..95a5c27cf16835a60945263c5b4dd282d9ee53b6 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -109,7 +109,7 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t taosGetAppName(tmp, NULL); tjsonAddStringToObject(pJson, "appName", tmp); - if (taosGetOsReleaseName(tmp, sizeof(tmp)) == 0) { + if (taosGetOsReleaseName(tmp, NULL, NULL, sizeof(tmp)) == 0) { tjsonAddStringToObject(pJson, "os", tmp); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8051f4d0bdad348da85ac94f0dce466e6afccc1e..73a0bed48f2df5e085fd219c95a3aeb81d4fe22f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -835,6 +835,7 @@ int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq if (tEncodeBinary(&encoder, pReq->ast, pReq->astLen) < 0) return -1; } if (tEncodeI64(&encoder, pReq->deleteMark) < 0) return -1; + if (tEncodeI64(&encoder, pReq->lastTs) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -884,6 +885,7 @@ int32_t tDeserializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pR if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; } if (tDecodeI64(&decoder, &pReq->deleteMark) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->lastTs) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -1099,6 +1101,9 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tEncodeI64(&encoder, pReq->qload.timeInFetchQueue) < 0) return -1; if (tEncodeI32(&encoder, pReq->statusSeq) < 0) return -1; + if (tEncodeI64(&encoder, pReq->mload.syncTerm) < 0) return -1; + if (tEncodeI64(&encoder, pReq->mload.roleTimeMs) < 0) return -1; + if (tEncodeI8(&encoder, pReq->clusterCfg.ttlChangeOnWrite) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1182,6 +1187,19 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tDecodeI64(&decoder, &pReq->qload.timeInFetchQueue) < 0) return -1; if (tDecodeI32(&decoder, &pReq->statusSeq) < 0) return -1; + + pReq->mload.syncTerm = -1; + pReq->mload.roleTimeMs = 0; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI64(&decoder, &pReq->mload.syncTerm) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->mload.roleTimeMs) < 0) return -1; + } + + pReq->clusterCfg.ttlChangeOnWrite = false; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->clusterCfg.ttlChangeOnWrite) < 0) return -1; + } + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -3496,12 +3514,14 @@ int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesR int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) { if (tEncodeCStr(pEncoder, pInfo->name) < 0) return -1; if (tEncodeCStr(pEncoder, pInfo->value) < 0) return -1; + if (tEncodeCStr(pEncoder, pInfo->scope) < 0) return -1; return 0; } int32_t tDecodeSVariablesInfo(SDecoder *pDecoder, SVariablesInfo *pInfo) { if (tDecodeCStrTo(pDecoder, pInfo->name) < 0) return -1; if (tDecodeCStrTo(pDecoder, pInfo->value) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pInfo->scope) < 0) return -1; return 0; } diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 01a9a245be73750cfa27a16ccf95c66d5f656cd7..3c08714218dfa861958e27080a23a407e7323eb5 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -359,7 +359,11 @@ int mainWindows(int argc, char **argv) { taosCleanupArgs(); if (dmInit() != 0) { - dError("failed to init dnode since %s", terrstr()); + if (terrno == TSDB_CODE_NOT_FOUND) { + dError("failed to init dnode since unsupported platform, please visit https://www.taosdata.com for support"); + } else { + dError("failed to init dnode since %s", terrstr()); + } taosCleanupCfg(); taosCloseLog(); @@ -369,6 +373,8 @@ int mainWindows(int argc, char **argv) { dInfo("start to init service"); dmSetSignalHandle(); + tsDndStart = taosGetTimestampMs(); + tsDndStartOsUptime = taosGetOsUptime(); int32_t code = dmRun(); dInfo("shutting down the service"); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 19982698968c8c91bc3f53edd3868f173b070005..1bce20ff44627c814c9ce677408e7ae3e12ed3ba 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -90,6 +90,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.clusterCfg.statusInterval = tsStatusInterval; req.clusterCfg.checkTime = 0; + req.clusterCfg.ttlChangeOnWrite = tsTtlChangeOnWrite; char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); @@ -265,6 +266,12 @@ int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) { pColInfo = taosArrayGet(pBlock->pDataBlock, c++); colDataSetVal(pColInfo, i, value, false); + char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; + cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen); + varDataSetLen(scope, valueLen); + pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + colDataSetVal(pColInfo, i, scope, false); + numOfRows++; } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 89c394fdd0889a30b737b43f534aec96e3fb3afa..76cb65b53a850b5b597c17806b501c88174380be 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -24,12 +24,16 @@ static void *dmStatusThreadFp(void *param) { const static int16_t TRIM_FREQ = 30; int32_t trimCount = 0; + int32_t upTimeCount = 0; + int64_t upTime = 0; + while (1) { taosMsleep(200); if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; int64_t curTime = taosGetTimestampMs(); - float interval = (curTime - lastTime) / 1000.0f; + if (curTime < lastTime) lastTime = curTime; + float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsStatusInterval) { dmSendStatusReq(pMgmt); lastTime = curTime; @@ -38,6 +42,11 @@ static void *dmStatusThreadFp(void *param) { if (trimCount == 0) { taosMemoryTrim(0); } + + if ((upTimeCount = ((upTimeCount + 1) & 63)) == 0) { + upTime = taosGetOsUptime() - tsDndStartOsUptime; + tsDndUpTime = TMAX(tsDndUpTime, upTime); + } } } @@ -54,7 +63,8 @@ static void *dmMonitorThreadFp(void *param) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; int64_t curTime = taosGetTimestampMs(); - float interval = (curTime - lastTime) / 1000.0f; + if (curTime < lastTime) lastTime = curTime; + float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsMonitorInterval) { (*pMgmt->sendMonitorReportFp)(); lastTime = curTime; diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index c098d546b693b88eee078dcdc6a67a6a606663d0..8206b4e4258a0eacb7eb91b5cde66a4bcb53f3d2 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -76,6 +76,10 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_SCAN_HISTORY_FINISH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_SCAN_HISTORY_FINISH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; code = 0; _OVER: diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index d1dc872f4b38a9fd9260579b4e71fa2c9b131db8..5d08320fab973bec1c8a0bf3f93df9e78c0d4f50 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -46,6 +46,7 @@ typedef struct { int32_t vgId; int32_t vgVersion; int8_t dropped; + int32_t diskPrimary; int32_t toVgId; char path[PATH_MAX + 20]; } SWrapperCfg; @@ -56,6 +57,7 @@ typedef struct { int32_t refCount; int8_t dropped; int8_t disable; + int32_t diskPrimary; int32_t toVgId; char *path; SVnode *pImpl; @@ -81,6 +83,7 @@ typedef struct { } SVnodeThread; // vmInt.c +int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId); SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId); void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode); int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index 769b274f7fbc1b50c0c2131ad9e3a78151591869..da7f4d4a564379cea22dd01e934179aea8bf6851 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -71,6 +71,8 @@ static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg ** if (code < 0) goto _OVER; tjsonGetInt32ValueFromDouble(vnode, "vgVersion", pCfg->vgVersion, code); if (code < 0) goto _OVER; + tjsonGetInt32ValueFromDouble(vnode, "diskPrimary", pCfg->diskPrimary, code); + if (code < 0) goto _OVER; tjsonGetInt32ValueFromDouble(vnode, "toVgId", pCfg->toVgId, code); if (code < 0) goto _OVER; @@ -167,6 +169,7 @@ static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t num if (tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId) < 0) return -1; if (tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped) < 0) return -1; if (tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion) < 0) return -1; + if (tjsonAddDoubleToObject(vnode, "diskPrimary", pVnode->diskPrimary) < 0) return -1; if (pVnode->toVgId && tjsonAddDoubleToObject(vnode, "toVgId", pVnode->toVgId) < 0) return -1; if (tjsonAddItemToArray(vnodes, vnode) < 0) return -1; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 8a5a4e507903ab56202581ae20dd6ec1ac79e51d..bed9a6730326437a8b67c475d4da45c1b5588e71 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -263,32 +263,19 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } - snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId); + wrapperCfg.diskPrimary = vmAllocPrimaryDisk(pMgmt, vnodeCfg.vgId); + int32_t diskPrimary = wrapperCfg.diskPrimary; -#if 0 - if (pMgmt->pTfs) { - if (tfsDirExistAt(pMgmt->pTfs, path, (SDiskID){0})) { - terrno = TSDB_CODE_VND_DIR_ALREADY_EXIST; - dError("vgId:%d, failed to restore vnode since %s", req.vgId, terrstr()); - return -1; - } - } else { - if (taosDirExist(path)) { - terrno = TSDB_CODE_VND_DIR_ALREADY_EXIST; - dError("vgId:%d, failed to restore vnode since %s", req.vgId, terrstr()); - return -1; - } - } -#endif + snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId); - if (vnodeCreate(path, &vnodeCfg, pMgmt->pTfs) < 0) { + if (vnodeCreate(path, &vnodeCfg, diskPrimary, pMgmt->pTfs) < 0) { tFreeSCreateVnodeReq(&req); dError("vgId:%d, failed to create vnode since %s", req.vgId, terrstr()); code = terrno; goto _OVER; } - SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr()); code = terrno; @@ -419,21 +406,23 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { .dropped = pVnode->dropped, .vgId = pVnode->vgId, .vgVersion = pVnode->vgVersion, + .diskPrimary = pVnode->diskPrimary, }; tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path)); vmCloseVnode(pMgmt, pVnode, false); + int32_t diskPrimary = wrapperCfg.diskPrimary; char path[TSDB_FILENAME_LEN] = {0}; snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vgId); dInfo("vgId:%d, start to alter vnode replica at %s", vgId, path); - if (vnodeAlterReplica(path, &req, pMgmt->pTfs) < 0) { + if (vnodeAlterReplica(path, &req, diskPrimary, pMgmt->pTfs) < 0) { dError("vgId:%d, failed to alter vnode at %s since %s", vgId, path, terrstr()); return -1; } dInfo("vgId:%d, begin to open vnode", vgId); - SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr()); return -1; @@ -506,6 +495,7 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { .dropped = pVnode->dropped, .vgId = dstVgId, .vgVersion = pVnode->vgVersion, + .diskPrimary = pVnode->diskPrimary, }; tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path)); @@ -519,19 +509,20 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { dInfo("vgId:%d, close vnode", srcVgId); vmCloseVnode(pMgmt, pVnode, true); + int32_t diskPrimary = wrapperCfg.diskPrimary; char srcPath[TSDB_FILENAME_LEN] = {0}; char dstPath[TSDB_FILENAME_LEN] = {0}; snprintf(srcPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, srcVgId); snprintf(dstPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, dstVgId); dInfo("vgId:%d, alter vnode hashrange at %s", srcVgId, srcPath); - if (vnodeAlterHashRange(srcPath, dstPath, &req, pMgmt->pTfs) < 0) { + if (vnodeAlterHashRange(srcPath, dstPath, &req, diskPrimary, pMgmt->pTfs) < 0) { dError("vgId:%d, failed to alter vnode hashrange since %s", srcVgId, terrstr()); return -1; } dInfo("vgId:%d, open vnode", dstVgId); - SVnode *pImpl = vnodeOpen(dstPath, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(dstPath, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode at %s since %s", dstVgId, dstPath, terrstr()); return -1; @@ -618,21 +609,23 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { .dropped = pVnode->dropped, .vgId = pVnode->vgId, .vgVersion = pVnode->vgVersion, + .diskPrimary = pVnode->diskPrimary, }; tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path)); vmCloseVnode(pMgmt, pVnode, false); + int32_t diskPrimary = wrapperCfg.diskPrimary; char path[TSDB_FILENAME_LEN] = {0}; snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vgId); dInfo("vgId:%d, start to alter vnode replica at %s", vgId, path); - if (vnodeAlterReplica(path, &alterReq, pMgmt->pTfs) < 0) { + if (vnodeAlterReplica(path, &alterReq, diskPrimary, pMgmt->pTfs) < 0) { dError("vgId:%d, failed to alter vnode at %s since %s", vgId, path, terrstr()); return -1; } dInfo("vgId:%d, begin to open vnode", vgId); - SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr()); return -1; @@ -747,9 +740,11 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_RECOVER_FINISH, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_SCAN_HISTORY_FINISH, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_SCAN_HISTORY_FINISH_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TRANSFER_STATE, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK_RSP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index db46ce3ca05ca261b5a396fa647eae7c1ac8a398..94a753062c8f0998c65bf950f700d89e3282eb98 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -15,8 +15,64 @@ #define _DEFAULT_SOURCE #include "vmInt.h" +#include "tfs.h" #include "vnd.h" +int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { + STfs *pTfs = pMgmt->pTfs; + int32_t diskId = 0; + if (!pTfs) { + return diskId; + } + + // search fs + char vnodePath[TSDB_FILENAME_LEN] = {0}; + snprintf(vnodePath, TSDB_FILENAME_LEN - 1, "vnode%svnode%d", TD_DIRSEP, vgId); + char fname[TSDB_FILENAME_LEN] = {0}; + char fnameTmp[TSDB_FILENAME_LEN] = {0}; + snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s", vnodePath, TD_DIRSEP, VND_INFO_FNAME); + snprintf(fnameTmp, TSDB_FILENAME_LEN - 1, "%s%s%s", vnodePath, TD_DIRSEP, VND_INFO_FNAME_TMP); + + diskId = tfsSearch(pTfs, 0, fname); + if (diskId >= 0) { + return diskId; + } + diskId = tfsSearch(pTfs, 0, fnameTmp); + if (diskId >= 0) { + return diskId; + } + + // alloc + int32_t disks[TFS_MAX_DISKS_PER_TIER] = {0}; + int32_t numOfVnodes = 0; + SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); + for (int32_t v = 0; v < numOfVnodes; v++) { + SVnodeObj *pVnode = ppVnodes[v]; + disks[pVnode->diskPrimary] += 1; + } + + int32_t minVal = INT_MAX; + int32_t ndisk = tfsGetDisksAtLevel(pTfs, 0); + diskId = 0; + for (int32_t id = 0; id < ndisk; id++) { + if (minVal > disks[id]) { + minVal = disks[id]; + diskId = id; + } + } + + for (int32_t i = 0; i < numOfVnodes; ++i) { + if (ppVnodes == NULL || ppVnodes[i] == NULL) continue; + vmReleaseVnode(pMgmt, ppVnodes[i]); + } + if (ppVnodes != NULL) { + taosMemoryFree(ppVnodes); + } + + dInfo("vgId:%d, alloc disk:%d of level 0. ndisk:%d, vnodes: %d", vgId, diskId, ndisk, numOfVnodes); + return diskId; +} + SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { SVnodeObj *pVnode = NULL; @@ -52,6 +108,7 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { pVnode->vgId = pCfg->vgId; pVnode->vgVersion = pCfg->vgVersion; + pVnode->diskPrimary = pCfg->diskPrimary; pVnode->refCount = 0; pVnode->dropped = 0; pVnode->path = taosStrdup(pCfg->path); @@ -169,7 +226,8 @@ static int32_t vmRestoreVgroupId(SWrapperCfg *pCfg, STfs *pTfs) { snprintf(srcPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, srcVgId); snprintf(dstPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, dstVgId); - int32_t vgId = vnodeRestoreVgroupId(srcPath, dstPath, srcVgId, dstVgId, pTfs); + int32_t diskPrimary = pCfg->diskPrimary; + int32_t vgId = vnodeRestoreVgroupId(srcPath, dstPath, srcVgId, dstVgId, diskPrimary, pTfs); if (vgId <= 0) { dError("vgId:%d, failed to restore vgroup id. srcPath: %s", pCfg->vgId, srcPath); return -1; @@ -205,11 +263,12 @@ static void *vmOpenVnodeInThread(void *param) { pThread->updateVnodesList = true; } + int32_t diskPrimary = pCfg->diskPrimary; snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId); - SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); + SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { - dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); + dError("vgId:%d, failed to open vnode by thread:%d since %s", pCfg->vgId, pThread->threadIndex, terrstr()); pThread->failed++; continue; } @@ -296,6 +355,7 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { if (pMgmt->state.openVnodes != pMgmt->state.totalVnodes) { dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes); + terrno = TSDB_CODE_VND_INIT_FAILED; return -1; } @@ -518,7 +578,7 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { tmsgReportStartup("vnode-worker", "initialized"); if (vmOpenVnodes(pMgmt) != 0) { - dError("failed to open vnode since %s", terrstr()); + dError("failed to open all vnodes since %s", terrstr()); goto _OVER; } tmsgReportStartup("vnode-vnodes", "initialized"); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 76e2f930273ec4558a1c11ffb5b75e9ba3a60505..247c1729a34b32243d01ac7b80c96ad3b606f1a7 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -92,7 +92,7 @@ static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { const STraceId *trace = &pMsg->info.traceId; dGTrace("vgId:%d, msg:%p get from vnode-stream queue", pVnode->vgId, pMsg); - int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); + int32_t code = vnodeProcessStreamMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType), diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 56bff0c760b042c8dd1bf389ff93ce371df6a9df..3f9c5bbeaf8f7481069e252efe19ac84eeb14e41 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -16,7 +16,33 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" -static SDnode globalDnode = {0}; +#define STR_CASE_CMP(s, d) (0 == strcasecmp((s), (d))) +#define STR_STR_CMP(s, d) (strstr((s), (d))) +#define STR_INT_CMP(s, d, c) (taosStr2Int32(s, 0, 10) c(d)) +#define STR_STR_SIGN ("ia") +#define DM_INIT_MON() \ + do { \ + code = (int32_t)(2147483648 | 298); \ + strncpy(stName, tsVersionName, 64); \ + monCfg.maxLogs = tsMonitorMaxLogs; \ + monCfg.port = tsMonitorPort; \ + monCfg.server = tsMonitorFqdn; \ + monCfg.comp = tsMonitorComp; \ + if (monInit(&monCfg) != 0) { \ + if (terrno != 0) code = terrno; \ + goto _exit; \ + } \ + } while (0) + +#define DM_ERR_RTN(c) \ + do { \ + code = (c); \ + goto _exit; \ + } while (0) + +static SDnode globalDnode = {0}; +static const char *dmOS[10] = {"Ubuntu", "CentOS Linux", "Red Hat", "Debian GNU", "CoreOS", + "FreeBSD", "openSUSE", "SLES", "Fedora", "macOS"}; SDnode *dmInstance() { return &globalDnode; } @@ -37,16 +63,37 @@ static int32_t dmInitSystem() { } static int32_t dmInitMonitor() { + int32_t code = 0; SMonCfg monCfg = {0}; - monCfg.maxLogs = tsMonitorMaxLogs; - monCfg.port = tsMonitorPort; - monCfg.server = tsMonitorFqdn; - monCfg.comp = tsMonitorComp; - if (monInit(&monCfg) != 0) { - dError("failed to init monitor since %s", terrstr()); - return -1; + char reName[64] = {0}; + char stName[64] = {0}; + char ver[64] = {0}; + + DM_INIT_MON(); + + if (STR_STR_CMP(stName, STR_STR_SIGN)) { + DM_ERR_RTN(0); } - return 0; + if (taosGetOsReleaseName(reName, stName, ver, 64) != 0) { + DM_ERR_RTN(code); + } + if (STR_CASE_CMP(stName, dmOS[0])) { + if (STR_INT_CMP(ver, 17, >)) { + DM_ERR_RTN(0); + } + } else if (STR_CASE_CMP(stName, dmOS[1])) { + if (STR_INT_CMP(ver, 6, >)) { + DM_ERR_RTN(0); + } + } else if (STR_STR_CMP(stName, dmOS[2]) || STR_STR_CMP(stName, dmOS[3]) || STR_STR_CMP(stName, dmOS[4]) || + STR_STR_CMP(stName, dmOS[5]) || STR_STR_CMP(stName, dmOS[6]) || STR_STR_CMP(stName, dmOS[7]) || + STR_STR_CMP(stName, dmOS[8]) || STR_STR_CMP(stName, dmOS[9])) { + DM_ERR_RTN(0); + } + +_exit: + if (code) terrno = code; + return code; } static bool dmCheckDiskSpace() { diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 5d6d16ccf84f73a0c45193cc9b15dda3bf59adcc..df54f8abbad86a9c23c96c0b533bbd8f432ae7a3 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -290,6 +290,7 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = TSDB_DEFAULT_USER; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.parent = pDnode; rpcInit.rfp = rpcRfp; diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 2cb41edd7c1d37c8dab6f0e276259e9cc530fea8..e33ffdb372d0c317f5478add89b587bbe91562a9 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -27,7 +27,7 @@ void mndCleanupCluster(SMnode *pMnode); int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len); int64_t mndGetClusterId(SMnode *pMnode); int64_t mndGetClusterCreateTime(SMnode *pMnode); -float mndGetClusterUpTime(SMnode *pMnode); +int64_t mndGetClusterUpTime(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 44dbfe6b12e8afb934ff55528025c24344335593..c83a40c25dc9079f2e07dba2e1c8ca6796228cdb 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -133,6 +133,7 @@ typedef enum { DND_REASON_TIME_ZONE_NOT_MATCH, DND_REASON_LOCALE_NOT_MATCH, DND_REASON_CHARSET_NOT_MATCH, + DND_REASON_TTL_CHANGE_ON_WRITE_NOT_MATCH, DND_REASON_OTHERS } EDndReason; @@ -215,8 +216,9 @@ typedef struct { int64_t createdTime; int64_t updateTime; ESyncState syncState; + SyncTerm syncTerm; bool syncRestore; - int64_t stateStartTime; + int64_t roleTimeMs; SDnodeObj* pDnode; int32_t role; SyncIndex lastIndex; @@ -646,6 +648,14 @@ typedef struct { // SMqSubActionLogEntry* pLogEntry; } SMqRebOutputObj; +typedef struct SStreamConf { + int8_t igExpired; + int8_t trigger; + int8_t fillHistory; + int64_t triggerParam; + int64_t watermark; +} SStreamConf; + typedef struct { char name[TSDB_STREAM_FNAME_LEN]; // ctl @@ -659,12 +669,7 @@ typedef struct { // info int64_t uid; int8_t status; - // config - int8_t igExpired; - int8_t trigger; - int8_t fillHistory; - int64_t triggerParam; - int64_t watermark; + SStreamConf conf; // source and target int64_t sourceDbUid; int64_t targetDbUid; @@ -674,14 +679,18 @@ typedef struct { int64_t targetStbUid; // fixedSinkVg is not applicable for encode and decode - SVgObj fixedSinkVg; + SVgObj fixedSinkVg; int32_t fixedSinkVgId; // 0 for shuffle // transformation char* sql; char* ast; char* physicalPlan; - SArray* tasks; // SArray> + SArray* tasks; // SArray> + + SArray* pHTasksList; // generate the results for already stored ts data + int64_t hTaskUid; // stream task for history ts data + SSchemaWrapper outputSchema; SSchemaWrapper tagSchema; diff --git a/source/dnode/mnode/impl/inc/mndScheduler.h b/source/dnode/mnode/impl/inc/mndScheduler.h index 23085c53eed7ef6234da38f146f962ea96d9fdde..cba52c6b45a14cc6c5fa434279b2c3f1ccecc4ff 100644 --- a/source/dnode/mnode/impl/inc/mndScheduler.h +++ b/source/dnode/mnode/impl/inc/mndScheduler.h @@ -22,15 +22,12 @@ extern "C" { #endif -int32_t mndInitScheduler(SMnode* pMnode); -void mndCleanupScheduler(SMnode* pMnode); int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub); - int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, int64_t watermark, int64_t deleteMark); -int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream); +int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t nextWindowSkey); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 99af413539b850eb4f62808fdf7899ab679933f0..db960d790f708e76ce2921f4ad6b6fefba82e441 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -39,6 +39,7 @@ int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, vo void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst); void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst); +void mndExtractShortDbNameFromDbFullName(const char *stbFullName, char *dst); void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize); const char *mndGetStbStr(const char *src); diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index f9f01c77ed942ee318d36c99487f9f7a65418543..05adc17d64fed3a87e48ac3ec4e4eaaff869a7f8 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -38,8 +38,6 @@ int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); -int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); - int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index aa00580c9385032f88254953d321988669151ecb..4c799e1e1ea4b4ba54811cf3df5ef663afb80e7c 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -20,7 +20,6 @@ #define CLUSTER_VER_NUMBE 1 #define CLUSTER_RESERVE_SIZE 60 -char tsVersionName[16] = "community"; int64_t tsExpireTime = 0; static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); @@ -123,7 +122,7 @@ static int32_t mndGetClusterUpTimeImp(SClusterObj *pCluster) { #endif } -float mndGetClusterUpTime(SMnode *pMnode) { +int64_t mndGetClusterUpTime(SMnode *pMnode) { int64_t upTime = 0; void *pIter = NULL; SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); @@ -132,7 +131,7 @@ float mndGetClusterUpTime(SMnode *pMnode) { mndReleaseCluster(pMnode, pCluster, pIter); } - return upTime / 86400.0f; + return upTime; } static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 87383265d9f2fbcc551fd18457fc1dd785f44f06..1bd629e56f59c4b49bd1027a04ce81ccea166445 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1873,12 +1873,6 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walRetentionSize, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walRollPeriod, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walSegmentSize, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.sstTrigger, false); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 287b39d8c799c157cc113081cd66963fbdb526d6..a8a719edda4d204baf753d7ccb622ed85f50d9a0 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -30,11 +30,11 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->igExpired) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->fillHistory) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->triggerParam) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->watermark) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->conf.igExpired) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->conf.trigger) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->conf.fillHistory) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->conf.triggerParam) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->conf.watermark) < 0) return -1; if (tEncodeI64(pEncoder, pObj->sourceDbUid) < 0) return -1; if (tEncodeI64(pEncoder, pObj->targetDbUid) < 0) return -1; @@ -97,11 +97,11 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->igExpired) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->fillHistory) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->triggerParam) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->watermark) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->conf.igExpired) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->conf.trigger) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->conf.fillHistory) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->conf.triggerParam) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->conf.watermark) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->sourceDbUid) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->targetDbUid) < 0) return -1; @@ -154,18 +154,10 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { return 0; } -void tFreeStreamObj(SStreamObj *pStream) { - taosMemoryFree(pStream->sql); - taosMemoryFree(pStream->ast); - taosMemoryFree(pStream->physicalPlan); - - if (pStream->outputSchema.nCols) { - taosMemoryFree(pStream->outputSchema.pSchema); - } - - int32_t sz = taosArrayGetSize(pStream->tasks); - for (int32_t i = 0; i < sz; i++) { - SArray *pLevel = taosArrayGetP(pStream->tasks, i); +static void* freeStreamTasks(SArray* pTaskLevel) { + int32_t numOfLevel = taosArrayGetSize(pTaskLevel); + for (int32_t i = 0; i < numOfLevel; i++) { + SArray *pLevel = taosArrayGetP(pTaskLevel, i); int32_t taskSz = taosArrayGetSize(pLevel); for (int32_t j = 0; j < taskSz; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); @@ -175,7 +167,20 @@ void tFreeStreamObj(SStreamObj *pStream) { taosArrayDestroy(pLevel); } - taosArrayDestroy(pStream->tasks); + return taosArrayDestroy(pTaskLevel); +} + +void tFreeStreamObj(SStreamObj *pStream) { + taosMemoryFree(pStream->sql); + taosMemoryFree(pStream->ast); + taosMemoryFree(pStream->physicalPlan); + + if (pStream->outputSchema.nCols || pStream->outputSchema.pSchema) { + taosMemoryFree(pStream->outputSchema.pSchema); + } + + pStream->tasks = freeStreamTasks(pStream->tasks); + pStream->pHTasksList = freeStreamTasks(pStream->pHTasksList); // tagSchema.pSchema if (pStream->tagSchema.nCols > 0) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 47029c8df135ca2c7e299dcbbb6194d7ad8809b0..1f566b14c74b1428a8ba9d24b6e44242768315e5 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -41,6 +41,7 @@ static const char *offlineReason[] = { "timezone not match", "locale not match", "charset not match", + "ttl change on write not match" "unknown", }; @@ -70,6 +71,8 @@ static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); +static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t opLen, int32_t *pOutValue); + int32_t mndInitDnode(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_DNODE, @@ -412,6 +415,12 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S return DND_REASON_CHARSET_NOT_MATCH; } + if (pCfg->ttlChangeOnWrite != tsTtlChangeOnWrite) { + mError("dnode:%d, ttlChangeOnWrite:%d inconsistent with cluster:%d", pDnode->id, pCfg->ttlChangeOnWrite, + tsTtlChangeOnWrite); + return DND_REASON_TTL_CHANGE_ON_WRITE_NOT_MATCH; + } + return 0; } @@ -522,13 +531,23 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SMnodeObj *pObj = mndAcquireMnode(pMnode, pDnode->id); if (pObj != NULL) { - if (pObj->syncState != statusReq.mload.syncState || pObj->syncRestore != statusReq.mload.syncRestore) { - mInfo("dnode:%d, mnode syncState from %s to %s, restoreState from %d to %d", pObj->id, syncStr(pObj->syncState), - syncStr(statusReq.mload.syncState), pObj->syncRestore, statusReq.mload.syncRestore); + bool roleChanged = pObj->syncState != statusReq.mload.syncState || + (statusReq.mload.syncTerm != -1 && pObj->syncTerm != statusReq.mload.syncTerm); + bool restoreChanged = pObj->syncRestore != statusReq.mload.syncRestore; + if (roleChanged || restoreChanged) { + mInfo("dnode:%d, mnode syncState from %s to %s, restoreState from %d to %d, syncTerm from %" PRId64 + " to %" PRId64, + pObj->id, syncStr(pObj->syncState), syncStr(statusReq.mload.syncState), pObj->syncRestore, + statusReq.mload.syncRestore, pObj->syncTerm, statusReq.mload.syncTerm); pObj->syncState = statusReq.mload.syncState; pObj->syncRestore = statusReq.mload.syncRestore; - pObj->stateStartTime = taosGetTimestampMs(); + pObj->syncTerm = statusReq.mload.syncTerm; } + + if (roleChanged) { + pObj->roleTimeMs = (statusReq.mload.roleTimeMs != 0) ? statusReq.mload.roleTimeMs : taosGetTimestampMs(); + } + mndReleaseMnode(pMnode, pObj); } @@ -653,6 +672,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg STrans *pTrans = NULL; SDnodeObj *pDnode = NULL; bool cfgAll = pCfgReq->dnodeId == -1; + int32_t iter = 0; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; @@ -660,7 +680,8 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg if (cfgAll) { pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); if (pIter == NULL) break; - } else if(!(pDnode = mndAcquireDnode(pMnode, pCfgReq->dnodeId))) { + ++iter; + } else if (!(pDnode = mndAcquireDnode(pMnode, pCfgReq->dnodeId))) { goto _OVER; } @@ -697,7 +718,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg } if (pTrans && mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - + tsGrantHBInterval = TMIN(TMAX(5, iter / 2), 30); terrno = 0; _OVER: @@ -784,18 +805,22 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { strcpy(info.name, "statusInterval"); snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsStatusInterval); + strcpy(info.scope, "server"); taosArrayPush(rsp.variables, &info); strcpy(info.name, "timezone"); snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsTimezoneStr); + strcpy(info.scope, "both"); taosArrayPush(rsp.variables, &info); strcpy(info.name, "locale"); snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsLocale); + strcpy(info.scope, "both"); taosArrayPush(rsp.variables, &info); strcpy(info.name, "charset"); snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%s", tsCharset); + strcpy(info.scope, "both"); taosArrayPush(rsp.variables, &info); int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp); @@ -858,7 +883,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { code = mndCreateDnode(pMnode, pReq, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; - + tsGrantHBInterval = 5; _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, terrstr()); @@ -948,7 +973,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { goto _OVER; } - mInfo("dnode:%d, start to drop, ep:%s:%d, force:%s, unsafe:%s", + mInfo("dnode:%d, start to drop, ep:%s:%d, force:%s, unsafe:%s", dropReq.dnodeId, dropReq.fqdn, dropReq.port, dropReq.force?"true":"false", dropReq.unsafe?"true":"false"); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; @@ -988,7 +1013,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs()); - + if (isonline && force) { terrno = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE; mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), @@ -1061,6 +1086,20 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, "monitor"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); + } else if (strncasecmp(cfgReq.config, "keeptimeoffset", 14) == 0) { + int32_t optLen = strlen("keeptimeoffset"); + int32_t flag = -1; + int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); + if (code < 0) return code; + + if (flag < 0 || flag > 23) { + mError("dnode:%d, failed to config keepTimeOffset since value:%d. Valid range: [0, 23]", cfgReq.dnodeId, flag); + terrno = TSDB_CODE_INVALID_CFG; + return -1; + } + + strcpy(dcfgReq.config, "keeptimeoffset"); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); #ifdef TD_ENTERPRISE } else if (strncasecmp(cfgReq.config, "activeCode", 10) == 0 || strncasecmp(cfgReq.config, "cActiveCode", 11) == 0) { int8_t opt = strncasecmp(cfgReq.config, "a", 1) == 0 ? DND_ACTIVE_CODE : DND_CONN_ACTIVE_CODE; @@ -1293,3 +1332,28 @@ static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +// get int32_t value from 'SMCfgDnodeReq' +static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t opLen, int32_t *pOutValue) { + terrno = 0; + if (' ' != pMCfgReq->config[opLen] && 0 != pMCfgReq->config[opLen]) { + goto _err; + } + + if (' ' == pMCfgReq->config[opLen]) { + // 'key value' + if (strlen(pMCfgReq->value) != 0) goto _err; + *pOutValue = atoi(pMCfgReq->config + opLen + 1); + } else { + // 'key' 'value' + if (strlen(pMCfgReq->value) == 0) goto _err; + *pOutValue = atoi(pMCfgReq->value); + } + + return 0; + +_err: + mError("dnode:%d, failed to config keeptimeoffset since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); + terrno = TSDB_CODE_INVALID_CFG; + return -1; +} diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index d57053bb5bd1d26827728264277323183b996a3c..62b5cb00e6abd1ff8fd0a567a7679f825f6782c1 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -367,10 +367,10 @@ void dumpStream(SSdb *pSdb, SJson *json) { tjsonAddStringToObject(item, "smaId", i642str(pObj->smaId)); tjsonAddStringToObject(item, "uid", i642str(pObj->uid)); tjsonAddStringToObject(item, "status", i642str(pObj->status)); - tjsonAddStringToObject(item, "igExpired", i642str(pObj->igExpired)); - tjsonAddStringToObject(item, "trigger", i642str(pObj->trigger)); - tjsonAddStringToObject(item, "triggerParam", i642str(pObj->triggerParam)); - tjsonAddStringToObject(item, "watermark", i642str(pObj->watermark)); + tjsonAddStringToObject(item, "igExpired", i642str(pObj->conf.igExpired)); + tjsonAddStringToObject(item, "trigger", i642str(pObj->conf.trigger)); + tjsonAddStringToObject(item, "triggerParam", i642str(pObj->conf.triggerParam)); + tjsonAddStringToObject(item, "watermark", i642str(pObj->conf.watermark)); tjsonAddStringToObject(item, "sourceDbUid", i642str(pObj->sourceDbUid)); tjsonAddStringToObject(item, "targetDbUid", i642str(pObj->targetDbUid)); tjsonAddStringToObject(item, "sourceDb", mndGetDbStr(pObj->sourceDb)); diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index c3e95a2d1f094b80ddc46f16ea3f9bb3069bfff3..efaff7ffc4c95dac9e4d37b59fe004072426cd75 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -542,32 +542,32 @@ int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, i STR_TO_VARSTR(n3, (char *)tNameGetTableName(&stbName)); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)n1, false); + colDataSetVal(pColInfo, numOfRows, (const char *)n1, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)n2, false); + colDataSetVal(pColInfo, numOfRows, (const char *)n2, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)n3, false); + colDataSetVal(pColInfo, numOfRows, (const char *)n3, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&invalid, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&invalid, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pIdx->createdTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pIdx->createdTime, false); char col[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(col, (char *)pIdx->colName); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)col, false); + colDataSetVal(pColInfo, numOfRows, (const char *)col, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); char tag[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(tag, (char *)"tag_index"); - colDataAppend(pColInfo, numOfRows, (const char *)tag, false); + colDataSetVal(pColInfo, numOfRows, (const char *)tag, false); numOfRows++; sdbRelease(pSdb, pIdx); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 381b1e64ed97080a38b3e45e53fe74c18ea3dc15..1071a6cf6ebbe34cf6cd1873f1180ede6c113219 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -804,7 +804,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr if (pObj->id == pMnode->selfDnodeId) { pClusterInfo->first_ep_dnode_id = pObj->id; tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep)); - pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode); + pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; // pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f); tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role)); } else { @@ -890,7 +890,10 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { SSyncState state = syncGetState(pMnode->syncMgmt.sync); pLoad->syncState = state.state; pLoad->syncRestore = state.restored; - mTrace("mnode current syncState is %s, syncRestore:%d", syncStr(pLoad->syncState), pLoad->syncRestore); + pLoad->syncTerm = state.term; + pLoad->roleTimeMs = state.roleTimeMs; + mTrace("mnode current syncState is %s, syncRestore:%d, syncTerm:%" PRId64 " ,roleTimeMs:%" PRId64, + syncStr(pLoad->syncState), pLoad->syncRestore, pLoad->syncTerm, pLoad->roleTimeMs); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 91fe1257d2b2ab736cfd35093a4dd40758004477..4ee2bc159b4697564a0614d8ff72d8235e071e98 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -319,7 +319,7 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p return 0; } -static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, +static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeTypeReq *pAlterMnodeTypeReq, SEpSet *pAlterMnodeTypeEpSet) { int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq); void *pReq = taosMemoryMalloc(contLen); @@ -803,9 +803,17 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB int32_t numOfRows = 0; int32_t cols = 0; SMnodeObj *pObj = NULL; + SMnodeObj *pSelfObj = NULL; ESdbStatus objStatus = 0; char *pWrite; int64_t curMs = taosGetTimestampMs(); + int64_t dummyTimeMs = 0; + + pSelfObj = sdbAcquire(pSdb, SDB_MNODE, &pMnode->selfDnodeId); + if (pSelfObj == NULL) { + mError("mnode:%d, failed to acquire self %s", pMnode->selfDnodeId, terrstr()); + goto _out; + } while (numOfRows < rows) { pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus, true); @@ -825,7 +833,8 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB if (pObj->id == pMnode->selfDnodeId) { snprintf(role, sizeof(role), "%s%s", syncStr(TAOS_SYNC_STATE_LEADER), pMnode->restored ? "" : "*"); } - if (mndIsDnodeOnline(pObj->pDnode, curMs)) { + bool isDnodeOnline = mndIsDnodeOnline(pObj->pDnode, curMs); + if (isDnodeOnline) { tstrncpy(role, syncStr(pObj->syncState), sizeof(role)); if (pObj->syncState == TAOS_SYNC_STATE_LEADER && pObj->id != pMnode->selfDnodeId) { tstrncpy(role, syncStr(TAOS_SYNC_STATE_ERROR), sizeof(role)); @@ -840,7 +849,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB const char *status = "ready"; if (objStatus == SDB_STATUS_CREATING) status = "creating"; if (objStatus == SDB_STATUS_DROPPING) status = "dropping"; - if (!mndIsDnodeOnline(pObj->pDnode, curMs)) status = "offline"; + if (!isDnodeOnline) status = "offline"; char b3[9 + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(b3, status, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -850,7 +859,15 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->stateStartTime, false); + if (pObj->syncTerm != pSelfObj->syncTerm || !isDnodeOnline) { + // state of old term / no status report => use dummyTimeMs + if (pObj->syncTerm > pSelfObj->syncTerm) { + mError("mnode:%d has a newer term:%" PRId64 " than me:%" PRId64, pObj->id, pObj->syncTerm, pSelfObj->syncTerm); + } + colDataSetVal(pColInfo, numOfRows, (const char *)&dummyTimeMs, false); + } else { + colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->roleTimeMs, false); + } numOfRows++; sdbRelease(pSdb, pObj); @@ -858,6 +875,8 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pShow->numOfRows += numOfRows; +_out: + sdbRelease(pSdb, pSelfObj); return numOfRows; } @@ -999,12 +1018,12 @@ static void mndReloadSyncConfig(SMnode *pMnode) { } if (pMnode->syncMgmt.sync > 0) { - mInfo("vgId:1, mnode sync reconfig, totalReplica:%d replica:%d myIndex:%d", + mInfo("vgId:1, mnode sync reconfig, totalReplica:%d replica:%d myIndex:%d", cfg.totalReplicaNum, cfg.replicaNum, cfg.myIndex); for (int32_t i = 0; i < cfg.totalReplicaNum; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; - mInfo("vgId:1, index:%d, ep:%s:%u dnode:%d cluster:%" PRId64 " role:%d", i, pNode->nodeFqdn, pNode->nodePort, + mInfo("vgId:1, index:%d, ep:%s:%u dnode:%d cluster:%" PRId64 " role:%d", i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId, pNode->nodeRole); } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 9647b5eaba5de76618816240e9e48969ac988d3b..524ea1a06bdec94d6f77b9fcb8dcb524b9cdc0a9 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -836,7 +836,7 @@ static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBloc } varDataLen(subStatus) = strlen(&subStatus[VARSTR_HEADER_SIZE]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, curRowIndex, subStatus, false); + colDataSetVal(pColInfo, curRowIndex, subStatus, (varDataLen(subStatus) == 0) ? true : false); char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(sql, pQuery->sql); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index dbac9ec9bbdeea9466200789a0508ed996c7fd20..2aac05b22d2c126780b479a3cbc2fa255d2f3e5f 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -22,10 +22,13 @@ #include "tname.h" #include "tuuid.h" +#define SINK_NODE_LEVEL (0) extern bool tsDeployOnSnode; -static int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup); -static void setFixedDownstreamEpInfo(SStreamTask* pDstTask, const SStreamTask* pTask); +static int32_t setTaskUpstreamEpInfo(const SStreamTask* pTask, SStreamTask* pDownstream); +static int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SArray* pTaskList, SMnode* pMnode, int32_t vgId, + SVgObj* pVgroup, int32_t fillHistory); +static void setFixedDownstreamEpInfo(SStreamTask* pDstTask, const SStreamTask* pTask); int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, int64_t watermark, int64_t deleteMark) { @@ -85,10 +88,10 @@ END: int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) { if (pStream->smaId != 0) { - pTask->outputType = TASK_OUTPUT__SMA; + pTask->outputInfo.type = TASK_OUTPUT__SMA; pTask->smaSink.smaId = pStream->smaId; } else { - pTask->outputType = TASK_OUTPUT__TABLE; + pTask->outputInfo.type = TASK_OUTPUT__TABLE; pTask->tbSink.stbUid = pStream->targetStbUid; memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); @@ -100,18 +103,16 @@ int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) { return 0; } -#define SINK_NODE_LEVEL (0) - -int32_t mndAddDispatcherForInnerTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { +int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SArray* pSinkNodeList, + SStreamTask* pTask) { bool isShuffle = false; if (pStream->fixedSinkVgId == 0) { SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb); if (pDb != NULL && pDb->cfg.numOfVgroups > 1) { - isShuffle = true; - pTask->outputType = TASK_OUTPUT__SHUFFLE_DISPATCH; - pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; + pTask->outputInfo.type = TASK_OUTPUT__SHUFFLE_DISPATCH; + pTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH; if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) { return -1; } @@ -120,7 +121,6 @@ int32_t mndAddDispatcherForInnerTask(SMnode* pMnode, SStreamObj* pStream, SStrea sdbRelease(pMnode->pSdb, pDb); } - SArray* pSinkNodeList = taosArrayGetP(pStream->tasks, SINK_NODE_LEVEL); int32_t numOfSinkNodes = taosArrayGetSize(pSinkNodeList); if (isShuffle) { @@ -133,7 +133,7 @@ int32_t mndAddDispatcherForInnerTask(SMnode* pMnode, SStreamObj* pStream, SStrea for (int32_t j = 0; j < numOfSinkNodes; j++) { SStreamTask* pSinkTask = taosArrayGetP(pSinkNodeList, j); - if (pSinkTask->nodeId == pVgInfo->vgId) { + if (pSinkTask->info.nodeId == pVgInfo->vgId) { pVgInfo->taskId = pSinkTask->id.taskId; break; } @@ -150,11 +150,11 @@ int32_t mndAddDispatcherForInnerTask(SMnode* pMnode, SStreamObj* pStream, SStrea int32_t mndAssignStreamTaskToVgroup(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { int32_t msgLen; - pTask->nodeId = pVgroup->vgId; - pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); + pTask->info.nodeId = pVgroup->vgId; + pTask->info.epSet = mndGetVgroupEpset(pMnode, pVgroup); - plan->execNode.nodeId = pTask->nodeId; - plan->execNode.epSet = pTask->epSet; + plan->execNode.nodeId = pTask->info.nodeId; + plan->execNode.epSet = pTask->info.epSet; if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; @@ -172,14 +172,15 @@ SSnodeObj* mndSchedFetchOneSnode(SMnode* pMnode) { return pObj; } -int32_t mndAssignTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SSnodeObj* pSnode) { +int32_t mndAssignStreamTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SSnodeObj* pSnode) { int32_t msgLen; - pTask->nodeId = SNODE_HANDLE; - pTask->epSet = mndAcquireEpFromSnode(pMnode, pSnode); + pTask->info.nodeId = SNODE_HANDLE; + pTask->info.epSet = mndAcquireEpFromSnode(pMnode, pSnode); plan->execNode.nodeId = SNODE_HANDLE; - plan->execNode.epSet = pTask->epSet; + plan->execNode.epSet = pTask->info.epSet; + mDebug("s-task:0x%x set the agg task to snode:%d", pTask->id.taskId, SNODE_HANDLE); if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { terrno = TSDB_CODE_QRY_INVALID_INPUT; @@ -188,6 +189,7 @@ int32_t mndAssignTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, return 0; } +// todo random choose a node to do compute SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) { void* pIter = NULL; SVgObj* pVgroup = NULL; @@ -205,9 +207,9 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) { } // create sink node for each vgroup. -int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { - SSdb* pSdb = pMnode->pSdb; - void* pIter = NULL; +int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SArray* pTaskList, SStreamObj* pStream, int32_t fillHistory) { + SSdb* pSdb = pMnode->pSdb; + void* pIter = NULL; while (1) { SVgObj* pVgroup = NULL; @@ -221,43 +223,45 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { continue; } - mndAddSinkTaskToStream(pStream, pMnode, pVgroup->vgId, pVgroup); + mndAddSinkTaskToStream(pStream, pTaskList, pMnode, pVgroup->vgId, pVgroup, fillHistory); sdbRelease(pSdb, pVgroup); } return 0; } -int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup) { - SArray* pTaskList = taosArrayGetP(pStream->tasks, SINK_NODE_LEVEL); - - SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SINK, pStream->fillHistory, 0, pTaskList); +int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SArray* pTaskList, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup, + int32_t fillHistory) { + SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SINK, fillHistory, 0, pTaskList); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pTask->nodeId = vgId; - pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); + pTask->info.nodeId = vgId; + pTask->info.epSet = mndGetVgroupEpset(pMnode, pVgroup); mndSetSinkTaskInfo(pStream, pTask); return 0; } -static int32_t mndScheduleFillHistoryStreamTask(SMnode* pMnode, SStreamObj* pStream) { - return 0; -} - -static int32_t addSourceStreamTask(SMnode* pMnode, SVgObj* pVgroup, SArray* pTaskList, SStreamObj* pStream, - SSubplan* plan, uint64_t uid, int8_t taskLevel, int8_t fillHistory, - bool hasExtraSink) { - SStreamTask* pTask = tNewStreamTask(uid, taskLevel, fillHistory, pStream->triggerParam, pTaskList); +static int32_t addSourceStreamTask(SMnode* pMnode, SVgObj* pVgroup, SArray* pTaskList, SArray* pSinkTaskList, + SStreamObj* pStream, SSubplan* plan, uint64_t uid, int8_t fillHistory, + bool hasExtraSink, int64_t firstWindowSkey) { + SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, fillHistory, pStream->conf.triggerParam, pTaskList); if (pTask == NULL) { return terrno; } + // todo set the correct ts, which should be last key of queried table. + STimeWindow* pWindow = &pTask->dataRange.window; + + pWindow->skey = INT64_MIN; + pWindow->ekey = firstWindowSkey - 1; + mDebug("add source task 0x%x window:%" PRId64 " - %" PRId64, pTask->id.taskId, pWindow->skey, pWindow->ekey); + // sink or dispatch if (hasExtraSink) { - mndAddDispatcherForInnerTask(pMnode, pStream, pTask); + mndAddDispatcherForInternalTask(pMnode, pStream, pSinkTaskList, pTask); } else { mndSetSinkTaskInfo(pStream, pTask); } @@ -266,19 +270,24 @@ static int32_t addSourceStreamTask(SMnode* pMnode, SVgObj* pVgroup, SArray* pTas return terrno; } + for(int32_t i = 0; i < taosArrayGetSize(pSinkTaskList); ++i) { + SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, i); + setTaskUpstreamEpInfo(pTask, pSinkTask); + } + return TSDB_CODE_SUCCESS; } -static SStreamChildEpInfo* createStreamTaskEpInfo(SStreamTask* pTask) { +static SStreamChildEpInfo* createStreamTaskEpInfo(const SStreamTask* pTask) { SStreamChildEpInfo* pEpInfo = taosMemoryMalloc(sizeof(SStreamChildEpInfo)); if (pEpInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pEpInfo->childId = pTask->selfChildId; - pEpInfo->epSet = pTask->epSet; - pEpInfo->nodeId = pTask->nodeId; + pEpInfo->childId = pTask->info.selfChildId; + pEpInfo->epSet = pTask->info.epSet; + pEpInfo->nodeId = pTask->info.nodeId; pEpInfo->taskId = pTask->id.taskId; return pEpInfo; @@ -287,226 +296,388 @@ static SStreamChildEpInfo* createStreamTaskEpInfo(SStreamTask* pTask) { void setFixedDownstreamEpInfo(SStreamTask* pDstTask, const SStreamTask* pTask) { STaskDispatcherFixedEp* pDispatcher = &pDstTask->fixedEpDispatcher; pDispatcher->taskId = pTask->id.taskId; - pDispatcher->nodeId = pTask->nodeId; - pDispatcher->epSet = pTask->epSet; + pDispatcher->nodeId = pTask->info.nodeId; + pDispatcher->epSet = pTask->info.epSet; - pDstTask->outputType = TASK_OUTPUT__FIXED_DISPATCH; - pDstTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; + pDstTask->outputInfo.type = TASK_OUTPUT__FIXED_DISPATCH; + pDstTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH; } -int32_t appendToUpstream(SStreamTask* pTask, SStreamTask* pUpstream) { +int32_t setTaskUpstreamEpInfo(const SStreamTask* pTask, SStreamTask* pDownstream) { SStreamChildEpInfo* pEpInfo = createStreamTaskEpInfo(pTask); if (pEpInfo == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - if(pUpstream->childEpInfo == NULL) { - pUpstream->childEpInfo = taosArrayInit(4, POINTER_BYTES); + if (pDownstream->pUpstreamEpInfoList == NULL) { + pDownstream->pUpstreamEpInfoList = taosArrayInit(4, POINTER_BYTES); } - - taosArrayPush(pUpstream->childEpInfo, &pEpInfo); + + taosArrayPush(pDownstream->pUpstreamEpInfoList, &pEpInfo); return TSDB_CODE_SUCCESS; } -int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { - SSdb* pSdb = pMnode->pSdb; +static SArray* addNewTaskList(SArray* pTasksList) { + SArray* pTaskList = taosArrayInit(0, POINTER_BYTES); + taosArrayPush(pTasksList, &pTaskList); + return pTaskList; +} - SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); - if (pPlan == NULL) { +// set the history task id +static void setHTasksId(SArray* pTaskList, const SArray* pHTaskList) { + for (int32_t i = 0; i < taosArrayGetSize(pTaskList); ++i) { + SStreamTask** pStreamTask = taosArrayGet(pTaskList, i); + SStreamTask** pHTask = taosArrayGet(pHTaskList, i); + + (*pStreamTask)->historyTaskId.taskId = (*pHTask)->id.taskId; + (*pStreamTask)->historyTaskId.streamId = (*pHTask)->id.streamId; + + (*pHTask)->streamTaskId.taskId = (*pStreamTask)->id.taskId; + (*pHTask)->streamTaskId.streamId = (*pStreamTask)->id.streamId; + + mDebug("s-task:0x%x related history task:0x%x, level:%d", (*pStreamTask)->id.taskId, (*pHTask)->id.taskId, + (*pHTask)->info.taskLevel); + } +} + +static int32_t addSourceTasksForOneLevelStream(SMnode* pMnode, const SQueryPlan* pPlan, SStreamObj* pStream, + bool hasExtraSink, int64_t nextWindowSkey) { + // create exec stream task, since only one level, the exec task is also the source task + SArray* pTaskList = addNewTaskList(pStream->tasks); + SSdb* pSdb = pMnode->pSdb; + + SArray* pHTaskList = NULL; + if (pStream->conf.fillHistory) { + pHTaskList = addNewTaskList(pStream->pHTasksList); + } + + SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); + if (LIST_LENGTH(inner->pNodeList) != 1) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - int32_t planTotLevel = LIST_LENGTH(pPlan->pSubplans); - pStream->tasks = taosArrayInit(planTotLevel, POINTER_BYTES); - - bool hasExtraSink = false; - bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; - SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb); - if (pDbObj == NULL) { + SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); + if (plan->subplanType != SUBPLAN_TYPE_SCAN) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - bool multiTarget = (pDbObj->cfg.numOfVgroups > 1); - sdbRelease(pSdb, pDbObj); + void* pIter = NULL; + while (1) { + SVgObj* pVgroup; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + if (pIter == NULL) { + break; + } - if (planTotLevel == 2 || externalTargetDB || multiTarget || pStream->fixedSinkVgId) { - SArray* taskOneLevel = taosArrayInit(0, POINTER_BYTES); - taosArrayPush(pStream->tasks, &taskOneLevel); + if (!mndVgroupInDb(pVgroup, pStream->sourceDbUid)) { + sdbRelease(pSdb, pVgroup); + continue; + } - // add extra sink - hasExtraSink = true; - if (pStream->fixedSinkVgId == 0) { - if (mndAddShuffleSinkTasksToStream(pMnode, pStream) < 0) { - // TODO free - return -1; - } - } else { - if (mndAddSinkTaskToStream(pStream, pMnode, pStream->fixedSinkVgId, &pStream->fixedSinkVg) < 0) { - // TODO free - return -1; - } + // new stream task + SArray** pSinkTaskList = taosArrayGet(pStream->tasks, SINK_NODE_LEVEL); + int32_t code = addSourceStreamTask(pMnode, pVgroup, pTaskList, *pSinkTaskList, pStream, plan, pStream->uid, 0, + hasExtraSink, nextWindowSkey); + if (code != TSDB_CODE_SUCCESS) { + sdbRelease(pSdb, pVgroup); + return -1; + } + + if (pStream->conf.fillHistory) { + SArray** pHSinkTaskList = taosArrayGet(pStream->pHTasksList, SINK_NODE_LEVEL); + code = addSourceStreamTask(pMnode, pVgroup, pHTaskList, *pHSinkTaskList, pStream, plan, pStream->hTaskUid, + 1, hasExtraSink, nextWindowSkey); + } + + sdbRelease(pSdb, pVgroup); + if (code != TSDB_CODE_SUCCESS) { + return -1; } } - pStream->totalLevel = planTotLevel + hasExtraSink; + if (pStream->conf.fillHistory) { + setHTasksId(pTaskList, pHTaskList); + } - if (planTotLevel > 1) { - SStreamTask* pInnerTask; - // inner level - { - SArray* taskInnerLevel = taosArrayInit(0, POINTER_BYTES); - taosArrayPush(pStream->tasks, &taskInnerLevel); + return TSDB_CODE_SUCCESS; +} - SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); - SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); - if (plan->subplanType != SUBPLAN_TYPE_MERGE) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; - } +static int32_t doAddSourceTask(SArray* pTaskList, int8_t fillHistory, int64_t uid, SStreamTask* pDownstreamTask, + SMnode* pMnode, SSubplan* pPlan, SVgObj* pVgroup, int64_t nextWindowSkey) { + SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, fillHistory, 0, pTaskList); + if (pTask == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } - pInnerTask = tNewStreamTask(pStream->uid, TASK_LEVEL__AGG, pStream->fillHistory, pStream->triggerParam, taskInnerLevel); - if (pInnerTask == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - qDestroyQueryPlan(pPlan); - return -1; - } + // todo set the correct ts, which should be last key of queried table. + STimeWindow* pWindow = &pTask->dataRange.window; + pWindow->skey = INT64_MIN; + pWindow->ekey = nextWindowSkey - 1; - // dispatch - if (mndAddDispatcherForInnerTask(pMnode, pStream, pInnerTask) < 0) { - qDestroyQueryPlan(pPlan); - return -1; - } + mDebug("s-task:0x%x level:%d set time window:%" PRId64 " - %" PRId64, pTask->id.taskId, pTask->info.taskLevel, + pWindow->skey, pWindow->ekey); - if (tsDeployOnSnode) { - SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode); - if (pSnode == NULL) { - SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); - if (mndAssignStreamTaskToVgroup(pMnode, pInnerTask, plan, pVgroup) < 0) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - return -1; - } - sdbRelease(pSdb, pVgroup); - } else { - if (mndAssignTaskToSnode(pMnode, pInnerTask, plan, pSnode) < 0) { - sdbRelease(pSdb, pSnode); - qDestroyQueryPlan(pPlan); - return -1; - } - } - } else { - SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); - if (mndAssignStreamTaskToVgroup(pMnode, pInnerTask, plan, pVgroup) < 0) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - return -1; - } + // all the source tasks dispatch result to a single agg node. + setFixedDownstreamEpInfo(pTask, pDownstreamTask); + if (mndAssignStreamTaskToVgroup(pMnode, pTask, pPlan, pVgroup) < 0) { + return -1; + } - sdbRelease(pSdb, pVgroup); - } - } + return setTaskUpstreamEpInfo(pTask, pDownstreamTask); +} - // source level - SArray* taskSourceLevel = taosArrayInit(0, POINTER_BYTES); - taosArrayPush(pStream->tasks, &taskSourceLevel); +static int32_t doAddAggTask(uint64_t uid, SArray* pTaskList, SArray* pSinkNodeList, SMnode* pMnode, SStreamObj* pStream, + int32_t fillHistory, SStreamTask** pAggTask) { + *pAggTask = tNewStreamTask(uid, TASK_LEVEL__AGG, fillHistory, pStream->conf.triggerParam, pTaskList); + if (*pAggTask == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } - SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 1); - SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); - if (plan->subplanType != SUBPLAN_TYPE_SCAN) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; + // dispatch + if (mndAddDispatcherForInternalTask(pMnode, pStream, pSinkNodeList, *pAggTask) < 0) { + return -1; + } + + return 0; +} + +static int32_t addAggTask(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, SStreamTask** pAggTask, + SStreamTask** pHAggTask) { + SArray* pAggTaskList = addNewTaskList(pStream->tasks); + SSdb* pSdb = pMnode->pSdb; + + SNodeListNode* pInnerNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); + SSubplan* plan = (SSubplan*)nodesListGetNode(pInnerNode->pNodeList, 0); + if (plan->subplanType != SUBPLAN_TYPE_MERGE) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + + *pAggTask = NULL; + SArray* pSinkNodeList = taosArrayGetP(pStream->tasks, SINK_NODE_LEVEL); + + int32_t code = doAddAggTask(pStream->uid, pAggTaskList, pSinkNodeList, pMnode, pStream, 0, pAggTask); + if (code != TSDB_CODE_SUCCESS) { + return -1; + } + + SVgObj* pVgroup = NULL; + SSnodeObj* pSnode = NULL; + + if (tsDeployOnSnode) { + pSnode = mndSchedFetchOneSnode(pMnode); + if (pSnode == NULL) { + pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); } + } else { + pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); + } - void* pIter = NULL; - while (1) { - SVgObj* pVgroup; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); - if (pIter == NULL) { - break; - } + if (pSnode != NULL) { + code = mndAssignStreamTaskToSnode(pMnode, *pAggTask, plan, pSnode); + } else { + code = mndAssignStreamTaskToVgroup(pMnode, *pAggTask, plan, pVgroup); + } - if (!mndVgroupInDb(pVgroup, pStream->sourceDbUid)) { - sdbRelease(pSdb, pVgroup); - continue; - } + if (pStream->conf.fillHistory) { + SArray* pHAggTaskList = addNewTaskList(pStream->pHTasksList); + SArray* pHSinkNodeList = taosArrayGetP(pStream->pHTasksList, SINK_NODE_LEVEL); - SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SOURCE, pStream->fillHistory, 0, taskSourceLevel); - if (pTask == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + *pHAggTask = NULL; + code = doAddAggTask(pStream->hTaskUid, pHAggTaskList, pHSinkNodeList, pMnode, pStream, pStream->conf.fillHistory, + pHAggTask); + if (code != TSDB_CODE_SUCCESS) { + if (pSnode != NULL) { + sdbRelease(pSdb, pSnode); + } else { sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - sdbCancelFetch(pSdb, pIter); - return -1; } + return code; + } - // all the source tasks dispatch result to a single agg node. - setFixedDownstreamEpInfo(pTask, pInnerTask); + if (pSnode != NULL) { + code = mndAssignStreamTaskToSnode(pMnode, *pHAggTask, plan, pSnode); + } else { + code = mndAssignStreamTaskToVgroup(pMnode, *pHAggTask, plan, pVgroup); + } - if (mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup) < 0) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - sdbCancelFetch(pSdb, pIter); - return -1; - } + setHTasksId(pAggTaskList, pHAggTaskList); + } + + if (pSnode != NULL) { + sdbRelease(pSdb, pSnode); + } else { + sdbRelease(pSdb, pVgroup); + } + + return code; +} + +static int32_t addSourceTasksForMultiLevelStream(SMnode* pMnode, SQueryPlan* pPlan, SStreamObj* pStream, + SStreamTask* pDownstreamTask, SStreamTask* pHDownstreamTask, int64_t nextWindowSkey) { + SArray* pSourceTaskList = addNewTaskList(pStream->tasks); + + SArray* pHSourceTaskList = NULL; + if (pStream->conf.fillHistory) { + pHSourceTaskList = addNewTaskList(pStream->pHTasksList); + } + + SSdb* pSdb = pMnode->pSdb; + SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 1); + SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); + if (plan->subplanType != SUBPLAN_TYPE_SCAN) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + + void* pIter = NULL; + while (1) { + SVgObj* pVgroup; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + if (pIter == NULL) { + break; + } - int32_t code = appendToUpstream(pTask, pInnerTask); + if (!mndVgroupInDb(pVgroup, pStream->sourceDbUid)) { sdbRelease(pSdb, pVgroup); + continue; + } + int32_t code = + doAddSourceTask(pSourceTaskList, 0, pStream->uid, pDownstreamTask, pMnode, plan, pVgroup, nextWindowSkey); + if (code != TSDB_CODE_SUCCESS) { + sdbRelease(pSdb, pVgroup); + terrno = code; + return -1; + } + + if (pStream->conf.fillHistory) { + code = doAddSourceTask(pHSourceTaskList, 1, pStream->hTaskUid, pHDownstreamTask, pMnode, plan, pVgroup, + nextWindowSkey); if (code != TSDB_CODE_SUCCESS) { - terrno = code; - qDestroyQueryPlan(pPlan); - sdbCancelFetch(pSdb, pIter); - return -1; + sdbRelease(pSdb, pVgroup); + return code; } } - } else if (planTotLevel == 1) { - // create exec stream task, since only one level, the exec task is also the source task - SArray* pTaskList = taosArrayInit(0, POINTER_BYTES); - taosArrayPush(pStream->tasks, &pTaskList); - SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); - if (LIST_LENGTH(inner->pNodeList) != 1) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; + sdbRelease(pSdb, pVgroup); + } + + if (pStream->conf.fillHistory) { + setHTasksId(pSourceTaskList, pHSourceTaskList); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t addSinkTasks(SArray* pTasksList, SMnode* pMnode, SStreamObj* pStream, SArray** pCreatedTaskList, + int32_t fillHistory) { + SArray* pSinkTaskList = addNewTaskList(pTasksList); + if (pStream->fixedSinkVgId == 0) { + if (mndAddShuffleSinkTasksToStream(pMnode, pSinkTaskList, pStream, fillHistory) < 0) { + // TODO free return -1; } - - SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); - if (plan->subplanType != SUBPLAN_TYPE_SCAN) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; + } else { + if (mndAddSinkTaskToStream(pStream, pSinkTaskList, pMnode, pStream->fixedSinkVgId, &pStream->fixedSinkVg, + fillHistory) < 0) { + // TODO free return -1; } + } - void* pIter = NULL; - while (1) { - SVgObj* pVgroup; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); - if (pIter == NULL) { - break; - } + *pCreatedTaskList = pSinkTaskList; + return TSDB_CODE_SUCCESS; +} - if (!mndVgroupInDb(pVgroup, pStream->sourceDbUid)) { - sdbRelease(pSdb, pVgroup); - continue; - } +static void setSinkTaskUpstreamInfo(SArray* pTasksList, const SStreamTask* pUpstreamTask) { + SArray* pSinkTaskList = taosArrayGetP(pTasksList, SINK_NODE_LEVEL); + for(int32_t i = 0; i < taosArrayGetSize(pSinkTaskList); ++i) { + SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, i); + setTaskUpstreamEpInfo(pUpstreamTask, pSinkTask); + } +} - // new stream task - int32_t code = addSourceStreamTask(pMnode, pVgroup, pTaskList, pStream, plan, pStream->uid, TASK_LEVEL__SOURCE, pStream->fillHistory, hasExtraSink); - sdbRelease(pSdb, pVgroup); +static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, int64_t nextWindowSkey) { + SSdb* pSdb = pMnode->pSdb; + int32_t numOfPlanLevel = LIST_LENGTH(pPlan->pSubplans); + + bool hasExtraSink = false; + bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; + SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb); + if (pDbObj == NULL) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + + bool multiTarget = (pDbObj->cfg.numOfVgroups > 1); + sdbRelease(pSdb, pDbObj); + pStream->tasks = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES); + pStream->pHTasksList = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES); + + if (numOfPlanLevel == 2 || externalTargetDB || multiTarget || pStream->fixedSinkVgId) { + // add extra sink + hasExtraSink = true; + + SArray* pSinkTaskList = NULL; + int32_t code = addSinkTasks(pStream->tasks, pMnode, pStream, &pSinkTaskList, 0); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // check for fill history + if (pStream->conf.fillHistory) { + SArray* pHSinkTaskList = NULL; + code = addSinkTasks(pStream->pHTasksList, pMnode, pStream, &pHSinkTaskList, 1); if (code != TSDB_CODE_SUCCESS) { - qDestroyQueryPlan(pPlan); - sdbCancelFetch(pSdb, pIter); - return -1; + return code; } + + setHTasksId(pSinkTaskList, pHSinkTaskList); } } - qDestroyQueryPlan(pPlan); + pStream->totalLevel = numOfPlanLevel + hasExtraSink; + + if (numOfPlanLevel > 1) { + SStreamTask* pAggTask = NULL; + SStreamTask* pHAggTask = NULL; + + int32_t code = addAggTask(pStream, pMnode, pPlan, &pAggTask, &pHAggTask); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + setSinkTaskUpstreamInfo(pStream->tasks, pAggTask); + setSinkTaskUpstreamInfo(pStream->pHTasksList, pHAggTask); + + // source level + return addSourceTasksForMultiLevelStream(pMnode, pPlan, pStream, pAggTask, pHAggTask, nextWindowSkey); + } else if (numOfPlanLevel == 1) { + return addSourceTasksForOneLevelStream(pMnode, pPlan, pStream, hasExtraSink, nextWindowSkey); + } + return 0; } +int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t nextWindowSkey) { + SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); + if (pPlan == NULL) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + + int32_t code = doScheduleStream(pStream, pMnode, pPlan, nextWindowSkey); + qDestroyQueryPlan(pPlan); + + return code; +} + int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) { SSdb* pSdb = pMnode->pSdb; SVgObj* pVgroup = NULL; @@ -519,8 +690,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - }else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL){ - SNode *pAst = NULL; + } else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL) { + SNode* pAst = NULL; if (nodesStringToNode(pTopic->ast, &pAst) != 0) { mError("topic:%s, failed to create since %s", pTopic->name, terrstr()); return -1; @@ -535,7 +706,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib nodesDestroyNode(pAst); } - if(pPlan){ + if (pPlan) { int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); if (levelNum != 1) { qDestroyQueryPlan(pPlan); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 889f0d76df2029efcfe8f21acf45a38e2a242549..b84297f6bfd77f33ca2e27b04fd5b2b172a1286c 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -560,20 +560,20 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea streamObj.version = 1; streamObj.sql = taosStrdup(pCreate->sql); streamObj.smaId = smaObj.uid; - streamObj.watermark = pCreate->watermark; + streamObj.conf.watermark = pCreate->watermark; streamObj.deleteMark = pCreate->deleteMark; - streamObj.fillHistory = STREAM_FILL_HISTORY_ON; - streamObj.trigger = STREAM_TRIGGER_WINDOW_CLOSE; - streamObj.triggerParam = pCreate->maxDelay; + streamObj.conf.fillHistory = STREAM_FILL_HISTORY_ON; + streamObj.conf.trigger = STREAM_TRIGGER_WINDOW_CLOSE; + streamObj.conf.triggerParam = pCreate->maxDelay; streamObj.ast = taosStrdup(smaObj.ast); // check the maxDelay - if (streamObj.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) { + if (streamObj.conf.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) { int64_t msInterval = convertTimeFromPrecisionToUnit(pCreate->interval, pDb->cfg.precision, TIME_UNIT_MILLISECOND); - streamObj.triggerParam = msInterval > TSDB_MIN_ROLLUP_MAX_DELAY ? msInterval : TSDB_MIN_ROLLUP_MAX_DELAY; + streamObj.conf.triggerParam = msInterval > TSDB_MIN_ROLLUP_MAX_DELAY ? msInterval : TSDB_MIN_ROLLUP_MAX_DELAY; } - if (streamObj.triggerParam > TSDB_MAX_ROLLUP_MAX_DELAY) { - streamObj.triggerParam = TSDB_MAX_ROLLUP_MAX_DELAY; + if (streamObj.conf.triggerParam > TSDB_MAX_ROLLUP_MAX_DELAY) { + streamObj.conf.triggerParam = TSDB_MAX_ROLLUP_MAX_DELAY; } if (mndAllocSmaVgroup(pMnode, pDb, &streamObj.fixedSinkVg) != 0) { @@ -602,8 +602,8 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea .pAstRoot = pAst, .topicQuery = false, .streamQuery = true, - .triggerType = streamObj.trigger, - .watermark = streamObj.watermark, + .triggerType = streamObj.conf.trigger, + .watermark = streamObj.conf.watermark, .deleteMark = streamObj.deleteMark, }; @@ -638,7 +638,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (mndSetCreateSmaVgroupCommitLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj) != 0) goto _OVER; - if (mndScheduleStream(pMnode, &streamObj) != 0) goto _OVER; + if (mndScheduleStream(pMnode, &streamObj, 1685959190000) != 0) goto _OVER; if (mndPersistStream(pMnode, pTrans, &streamObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -894,11 +894,11 @@ _OVER: } int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { - SSdb *pSdb = pMnode->pSdb; - SSmaObj *pSma = NULL; - void *pIter = NULL; - SVgObj *pVgroup = NULL; - int32_t code = -1; + SSdb *pSdb = pMnode->pSdb; + SSmaObj *pSma = NULL; + void *pIter = NULL; + SVgObj *pVgroup = NULL; + int32_t code = -1; while (1) { pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); @@ -916,12 +916,18 @@ int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p if (pStream != NULL && pStream->smaId == pSma->uid) { if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { mError("stream:%s, failed to drop task since %s", pStream->name, terrstr()); + mndReleaseStream(pMnode, pStream); goto _OVER; } + if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { + mndReleaseStream(pMnode, pStream); goto _OVER; } + + mndReleaseStream(pMnode, pStream); } + if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER; if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; @@ -1283,13 +1289,13 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc STR_TO_VARSTR(col, (char *)""); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)col, false); + colDataSetVal(pColInfo, numOfRows, (const char *)col, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); char tag[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(tag, (char *)"sma_index"); - colDataAppend(pColInfo, numOfRows, (const char *)tag, false); + colDataSetVal(pColInfo, numOfRows, (const char *)tag, false); numOfRows++; sdbRelease(pSdb, pSma); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index e8af02a8288fa0189f8004127831bed25e6bd917..70fd74afc0a80b327265e17d2cb5f1a8d0069769 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1743,6 +1743,7 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa SSchema *pSrcSchema = &pStb->pColumns[i]; memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); pSchema->type = pSrcSchema->type; + pSchema->flags = pSrcSchema->flags; pSchema->colId = pSrcSchema->colId; pSchema->bytes = pSrcSchema->bytes; } @@ -1793,6 +1794,7 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, SSchema *pSrcSchema = &pStb->pColumns[i]; memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); pSchema->type = pSrcSchema->type; + pSchema->flags = pSrcSchema->flags; pSchema->colId = pSrcSchema->colId; pSchema->bytes = pSrcSchema->bytes; } @@ -1802,6 +1804,7 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, SSchema *pSrcSchema = &pStb->pTags[i]; memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); pSchema->type = pSrcSchema->type; + pSchema->flags = pSrcSchema->flags; pSchema->colId = pSrcSchema->colId; pSchema->bytes = pSrcSchema->bytes; } @@ -2506,12 +2509,14 @@ static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { goto _OVER; } - if (0 == strcmp(cfgReq.dbFName, TSDB_INFORMATION_SCHEMA_DB)) { + char dbName[TSDB_DB_NAME_LEN] = {0}; + mndExtractShortDbNameFromDbFullName(cfgReq.dbFName, dbName); + if (0 == strcmp(dbName, TSDB_INFORMATION_SCHEMA_DB)) { mInfo("information_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); if (mndBuildInsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp) != 0) { goto _OVER; } - } else if (0 == strcmp(cfgReq.dbFName, TSDB_PERFORMANCE_SCHEMA_DB)) { + } else if (0 == strcmp(dbName, TSDB_PERFORMANCE_SCHEMA_DB)) { mInfo("performance_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); if (mndBuildPerfsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp) != 0) { goto _OVER; @@ -2680,6 +2685,13 @@ void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst) { tNameGetDbName(&name, dst); } +void mndExtractShortDbNameFromDbFullName(const char *stbFullName, char *dst) { + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&name, dst); +} + void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize) { int32_t pos = -1; int32_t num = 0; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 028c482e6c26997162631b0a25be0aed2d0cf5c1..06ab1bb638eb71c092393280d7e0b8648f30a9b6 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -239,7 +239,7 @@ static void mndShowStreamStatus(char *dst, SStreamObj *pStream) { } static void mndShowStreamTrigger(char *dst, SStreamObj *pStream) { - int8_t trigger = pStream->trigger; + int8_t trigger = pStream->conf.trigger; if (trigger == STREAM_TRIGGER_AT_ONCE) { strcpy(dst, "at once"); } else if (trigger == STREAM_TRIGGER_WINDOW_CLOSE) { @@ -299,13 +299,18 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, pObj->smaId = 0; pObj->uid = mndGenerateUid(pObj->name, strlen(pObj->name)); + + char p[TSDB_STREAM_FNAME_LEN + 32] = {0}; + snprintf(p, tListLen(p), "%s_%s", pObj->name, "fillhistory"); + + pObj->hTaskUid = mndGenerateUid(pObj->name, strlen(pObj->name)); pObj->status = 0; - pObj->igExpired = pCreate->igExpired; - pObj->trigger = pCreate->triggerType; - pObj->triggerParam = pCreate->maxDelay; - pObj->watermark = pCreate->watermark; - pObj->fillHistory = pCreate->fillHistory; + pObj->conf.igExpired = pCreate->igExpired; + pObj->conf.trigger = pCreate->triggerType; + pObj->conf.triggerParam = pCreate->maxDelay; + pObj->conf.watermark = pCreate->watermark; + pObj->conf.fillHistory = pCreate->fillHistory; pObj->deleteMark = pCreate->deleteMark; pObj->igCheckUpdate = pCreate->igUpdate; @@ -387,9 +392,9 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, .pAstRoot = pAst, .topicQuery = false, .streamQuery = true, - .triggerType = pObj->trigger == STREAM_TRIGGER_MAX_DELAY ? STREAM_TRIGGER_WINDOW_CLOSE : pObj->trigger, - .watermark = pObj->watermark, - .igExpired = pObj->igExpired, + .triggerType = pObj->conf.trigger == STREAM_TRIGGER_MAX_DELAY ? STREAM_TRIGGER_WINDOW_CLOSE : pObj->conf.trigger, + .watermark = pObj->conf.watermark, + .igExpired = pObj->conf.igExpired, .deleteMark = pObj->deleteMark, .igCheckUpdate = pObj->igCheckUpdate, }; @@ -428,30 +433,37 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { SEncoder encoder; tEncoderInit(&encoder, NULL, 0); tEncodeStreamTask(&encoder, pTask); + int32_t size = encoder.pos; int32_t tlen = sizeof(SMsgHead) + size; tEncoderClear(&encoder); + void *buf = taosMemoryCalloc(1, tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - ((SMsgHead *)buf)->vgId = htonl(pTask->nodeId); + + ((SMsgHead *)buf)->vgId = htonl(pTask->info.nodeId); + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncoderInit(&encoder, abuf, size); + tEncodeStreamTask(&encoder, pTask); tEncoderClear(&encoder); STransAction action = {0}; action.mTraceId = pTrans->mTraceId; - memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + memcpy(&action.epSet, &pTask->info.epSet, sizeof(SEpSet)); action.pCont = buf; action.contLen = tlen; action.msgType = TDMT_STREAM_TASK_DEPLOY; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(buf); return -1; } + return 0; } @@ -459,14 +471,33 @@ int32_t mndPersistStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStrea int32_t level = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < level; i++) { SArray *pLevel = taosArrayGetP(pStream->tasks, i); - int32_t sz = taosArrayGetSize(pLevel); - for (int32_t j = 0; j < sz; j++) { + + int32_t numOfTasks = taosArrayGetSize(pLevel); + for (int32_t j = 0; j < numOfTasks; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); if (mndPersistTaskDeployReq(pTrans, pTask) < 0) { return -1; } } } + + // persistent stream task for already stored ts data + if (pStream->conf.fillHistory) { + level = taosArrayGetSize(pStream->pHTasksList); + + for (int32_t i = 0; i < level; i++) { + SArray *pLevel = taosArrayGetP(pStream->pHTasksList, i); + + int32_t numOfTasks = taosArrayGetSize(pLevel); + for (int32_t j = 0; j < numOfTasks; j++) { + SStreamTask *pTask = taosArrayGetP(pLevel, j); + if (mndPersistTaskDeployReq(pTrans, pTask) < 0) { + return -1; + } + } + } + } + return 0; } @@ -474,11 +505,13 @@ int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { if (mndPersistStreamTasks(pMnode, pTrans, pStream) < 0) { return -1; } + SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); return -1; } + (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); return 0; } @@ -490,6 +523,7 @@ int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr mndTransDrop(pTrans); return -1; } + (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); return 0; } @@ -603,16 +637,17 @@ _OVER: static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { // vnode - /*if (pTask->nodeId > 0) {*/ + /*if (pTask->info.nodeId > 0) {*/ SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->head.vgId = htonl(pTask->nodeId); + + pReq->head.vgId = htonl(pTask->info.nodeId); pReq->taskId = pTask->id.taskId; STransAction action = {0}; - memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + memcpy(&action.epSet, &pTask->info.epSet, sizeof(SEpSet)); action.pCont = pReq; action.contLen = sizeof(SVDropStreamTaskReq); action.msgType = TDMT_STREAM_TASK_DROP; @@ -734,6 +769,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); goto _OVER; } + mInfo("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name); mndTransSetDbName(pTrans, createStreamReq.sourceDB, streamObj.targetDb); @@ -750,7 +786,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } // schedule stream task for stream obj - if (mndScheduleStream(pMnode, &streamObj) < 0) { + if (mndScheduleStream(pMnode, &streamObj, createStreamReq.lastTs) < 0) { mError("stream:%s, failed to schedule since %s", createStreamReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; @@ -836,7 +872,7 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, con SMStreamDoCheckpointMsg *pMsg) { SStreamCheckpointSourceReq req = {0}; req.checkpointId = pMsg->checkpointId; - req.nodeId = pTask->nodeId; + req.nodeId = pTask->info.nodeId; req.expireTime = -1; req.streamId = pTask->streamId; req.taskId = pTask->taskId; @@ -865,7 +901,7 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, con SMsgHead *pMsgHead = (SMsgHead *)buf; pMsgHead->contLen = htonl(tlen); - pMsgHead->vgId = htonl(pTask->nodeId); + pMsgHead->vgId = htonl(pTask->info.nodeId); tEncoderClear(&encoder); @@ -904,12 +940,12 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { for (int32_t i = 0; i < totLevel; i++) { SArray *pLevel = taosArrayGetP(pStream->tasks, i); SStreamTask *pTask = taosArrayGetP(pLevel, 0); - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { int32_t sz = taosArrayGetSize(pLevel); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); - /*A(pTask->nodeId > 0);*/ - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->nodeId); + /*A(pTask->info.nodeId > 0);*/ + SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); if (pVgObj == NULL) { taosRUnLockLatch(&pStream->lock); mndReleaseStream(pMnode, pStream); @@ -967,8 +1003,6 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; - /*SDbObj *pDb = NULL;*/ - /*SUserObj *pUser = NULL;*/ SMDropStreamReq dropReq = {0}; if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) { @@ -1159,7 +1193,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->watermark, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->conf.watermark, false); char trigger[20 + VARSTR_HEADER_SIZE] = {0}; char trigger2[20] = {0}; @@ -1189,12 +1223,16 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock while (numOfRows < rowsCapacity) { pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream); - if (pShow->pIter == NULL) break; + if (pShow->pIter == NULL) { + break; + } // lock taosRLockLatch(&pStream->lock); + // count task num int32_t sz = taosArrayGetSize(pStream->tasks); + int32_t count = 0; for (int32_t i = 0; i < sz; i++) { SArray *pLevel = taosArrayGetP(pStream->tasks, i); @@ -1204,10 +1242,12 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock if (numOfRows + count > rowsCapacity) { blockDataEnsureCapacity(pBlock, numOfRows + count); } + // add row for each task for (int32_t i = 0; i < sz; i++) { SArray *pLevel = taosArrayGetP(pStream->tasks, i); int32_t levelCnt = taosArrayGetSize(pLevel); + for (int32_t j = 0; j < levelCnt; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); @@ -1217,18 +1257,25 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock // stream name char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName)); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false); // task id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pTask->id.taskId, false); + + char idstr[128] = {0}; + int32_t len = tintToHex(pTask->id.taskId, &idstr[4]); + idstr[2] = '0'; + idstr[3] = 'x'; + varDataSetLen(idstr, len + 2); + colDataSetVal(pColInfo, numOfRows, idstr, false); // node type char nodeType[20 + VARSTR_HEADER_SIZE] = {0}; varDataSetLen(nodeType, 5); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - if (pTask->nodeId > 0) { + if (pTask->info.nodeId > 0) { memcpy(varDataVal(nodeType), "vnode", 5); } else { memcpy(varDataVal(nodeType), "snode", 5); @@ -1237,30 +1284,50 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock // node id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - int64_t nodeId = TMAX(pTask->nodeId, 0); + int64_t nodeId = TMAX(pTask->info.nodeId, 0); colDataSetVal(pColInfo, numOfRows, (const char *)&nodeId, false); // level char level[20 + VARSTR_HEADER_SIZE] = {0}; - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { memcpy(varDataVal(level), "source", 6); varDataSetLen(level, 6); - } else if (pTask->taskLevel == TASK_LEVEL__AGG) { + } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) { memcpy(varDataVal(level), "agg", 3); varDataSetLen(level, 3); - } else if (pTask->taskLevel == TASK_LEVEL__SINK) { + } else if (pTask->info.taskLevel == TASK_LEVEL__SINK) { memcpy(varDataVal(level), "sink", 4); varDataSetLen(level, 4); - } else if (pTask->taskLevel == TASK_LEVEL__SINK) { } + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&level, false); // status char status[20 + VARSTR_HEADER_SIZE] = {0}; - char status2[20] = {0}; - strcpy(status, "normal"); - STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status)); + int8_t taskStatus = atomic_load_8(&pTask->status.taskStatus); + if (taskStatus == TASK_STATUS__NORMAL) { + memcpy(varDataVal(status), "normal", 6); + varDataSetLen(status, 6); + } else if (taskStatus == TASK_STATUS__DROPPING) { + memcpy(varDataVal(status), "dropping", 8); + varDataSetLen(status, 8); + } else if (taskStatus == TASK_STATUS__FAIL) { + memcpy(varDataVal(status), "fail", 4); + varDataSetLen(status, 4); + } else if (taskStatus == TASK_STATUS__STOP) { + memcpy(varDataVal(status), "stop", 4); + varDataSetLen(status, 4); + } else if (taskStatus == TASK_STATUS__SCAN_HISTORY) { + memcpy(varDataVal(status), "history", 7); + varDataSetLen(status, 7); + } else if (taskStatus == TASK_STATUS__HALT) { + memcpy(varDataVal(status), "halt", 4); + varDataSetLen(status, 4); + } else if (taskStatus == TASK_STATUS__PAUSE) { + memcpy(varDataVal(status), "pause", 5); + varDataSetLen(status, 5); + } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&status, false); @@ -1289,10 +1356,10 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->head.vgId = htonl(pTask->nodeId); + pReq->head.vgId = htonl(pTask->info.nodeId); pReq->taskId = pTask->id.taskId; STransAction action = {0}; - memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + memcpy(&action.epSet, &pTask->info.epSet, sizeof(SEpSet)); action.pCont = pReq; action.contLen = sizeof(SVPauseStreamTaskReq); action.msgType = TDMT_STREAM_TASK_PAUSE; @@ -1303,21 +1370,36 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) { return 0; } -int32_t mndPauseAllStreamTasks(STrans *pTrans, SStreamObj *pStream) { - int32_t size = taosArrayGetSize(pStream->tasks); +int32_t mndPauseAllStreamTaskImpl(STrans *pTrans, SArray* tasks) { + int32_t size = taosArrayGetSize(tasks); for (int32_t i = 0; i < size; i++) { - SArray *pTasks = taosArrayGetP(pStream->tasks, i); + SArray *pTasks = taosArrayGetP(tasks, i); int32_t sz = taosArrayGetSize(pTasks); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pTasks, j); - if (pTask->taskLevel != TASK_LEVEL__SINK && mndPauseStreamTask(pTrans, pTask) < 0) { + if (pTask->info.taskLevel != TASK_LEVEL__SINK && mndPauseStreamTask(pTrans, pTask) < 0) { return -1; } + + if (atomic_load_8(&pTask->status.taskStatus) != TASK_STATUS__PAUSE) { + atomic_store_8(&pTask->status.keepTaskStatus, pTask->status.taskStatus); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__PAUSE); + } } } return 0; } +int32_t mndPauseAllStreamTasks(STrans *pTrans, SStreamObj *pStream) { + int32_t code = mndPauseAllStreamTaskImpl(pTrans, pStream->tasks); + if (code != 0) { + return code; + } + // pStream->pHTasksList is null + // code = mndPauseAllStreamTaskImpl(pTrans, pStream->pHTasksList); + return code; +} + static int32_t mndPersistStreamLog(STrans *pTrans, const SStreamObj *pStream, int8_t status) { SStreamObj streamObj = {0}; memcpy(streamObj.name, pStream->name, TSDB_STREAM_FNAME_LEN); @@ -1357,6 +1439,10 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { } } + if (pStream->status == STREAM_STATUS__PAUSE) { + return 0; + } + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { sdbRelease(pMnode->pSdb, pStream); return -1; @@ -1412,11 +1498,11 @@ static int32_t mndResumeStreamTask(STrans *pTrans, SStreamTask *pTask, int8_t ig terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->head.vgId = htonl(pTask->nodeId); + pReq->head.vgId = htonl(pTask->info.nodeId); pReq->taskId = pTask->id.taskId; pReq->igUntreated = igUntreated; STransAction action = {0}; - memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + memcpy(&action.epSet, &pTask->info.epSet, sizeof(SEpSet)); action.pCont = pReq; action.contLen = sizeof(SVResumeStreamTaskReq); action.msgType = TDMT_STREAM_TASK_RESUME; @@ -1434,11 +1520,16 @@ int32_t mndResumeAllStreamTasks(STrans *pTrans, SStreamObj *pStream, int8_t igUn int32_t sz = taosArrayGetSize(pTasks); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pTasks, j); - if (pTask->taskLevel != TASK_LEVEL__SINK && mndResumeStreamTask(pTrans, pTask, igUntreated) < 0) { + if (pTask->info.taskLevel != TASK_LEVEL__SINK && mndResumeStreamTask(pTrans, pTask, igUntreated) < 0) { return -1; } + + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__PAUSE) { + atomic_store_8(&pTask->status.taskStatus, pTask->status.keepTaskStatus); + } } } + // pStream->pHTasksList is null return 0; } @@ -1465,6 +1556,10 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { } } + if (pStream->status != STREAM_STATUS__PAUSE) { + return 0; + } + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { sdbRelease(pMnode->pSdb, pStream); return -1; diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 679fafa28d89e398c1e5900f354925a50c38c06a..ac379a9f9478d579f684263b5a6d73c0f397c922 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -94,7 +94,7 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) { tjsonAddStringToObject(pJson, "instanceId", clusterName); tjsonAddDoubleToObject(pJson, "reportVersion", 1); - if (taosGetOsReleaseName(tmp, sizeof(tmp)) == 0) { + if (taosGetOsReleaseName(tmp, NULL, NULL, sizeof(tmp)) == 0) { tjsonAddStringToObject(pJson, "os", tmp); } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 8b313695a195169023a06a9b944bcaafa5751f0a..f3dded9c76e15f81bea2f7a5ca7313d91910c3a4 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -875,7 +875,7 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p // if (pDb == NULL || pDb->compactStartTime <= 0) { // colDataSetNULL(pColInfo, numOfRows); // } else { - // colDataAppend(pColInfo, numOfRows, (const char *)&pDb->compactStartTime, false); + // colDataSetVal(pColInfo, numOfRows, (const char *)&pDb->compactStartTime, false); // } numOfRows++; diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 678dd34e4a6e12ba3573f4d8103e10bcba754119..7235a56691ce580d8cb67d923245e26fdb53c82f 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -52,30 +52,29 @@ void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) { FAIL: if (pMsg->info.handle == NULL) return; - SRpcMsg rsp = { - .code = code, - .info = pMsg->info, - }; + SRpcMsg rsp = { .code = code, .info = pMsg->info}; tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { - ASSERT(pTask->taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->childEpInfo) != 0); + ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->pUpstreamEpInfoList) != 0); pTask->refCnt = 1; - pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; + pTask->id.idStr = createStreamTaskIdStr(pTask->id.streamId, pTask->id.taskId); - pTask->inputQueue = streamQueueOpen(0); - pTask->outputQueue = streamQueueOpen(0); + pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; + pTask->inputQueue = streamQueueOpen(512 << 10); + pTask->outputInfo.queue = streamQueueOpen(512 << 10); - if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) { + if (pTask->inputQueue == NULL || pTask->outputInfo.queue == NULL) { return -1; } + pTask->tsInfo.init = taosGetTimestampMs(); pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; - pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; + pTask->outputInfo.status = TASK_OUTPUT_STATUS__NORMAL; pTask->pMsgCb = &pSnode->msgCb; pTask->chkInfo.version = ver; pTask->pMeta = pSnode->pMeta; @@ -85,14 +84,19 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { return -1; } - int32_t numOfChildEp = taosArrayGetSize(pTask->childEpInfo); - SReadHandle handle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState }; + int32_t numOfChildEp = taosArrayGetSize(pTask->pUpstreamEpInfoList); + SReadHandle handle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState, .fillHistory = pTask->info.fillHistory }; initStreamStateAPI(&handle.api); pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, 0); ASSERT(pTask->exec.pExecutor); - streamSetupTrigger(pTask); + taosThreadMutexInit(&pTask->lock, NULL); + streamSetupScheduleTrigger(pTask); + + qDebug("snode:%d expand stream task on snode, s-task:%s, checkpoint ver:%" PRId64 " child id:%d, level:%d", SNODE_HANDLE, + pTask->id.idStr, pTask->chkInfo.version, pTask->info.selfChildId, pTask->info.taskLevel); + return 0; } @@ -149,32 +153,42 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { taosMemoryFree(pTask); return -1; } + tDecoderClear(&decoder); - ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); + ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG); // 2.save task taosWLockLatch(&pSnode->pMeta->lock); - code = streamMetaAddDeployedTask(pSnode->pMeta, -1, pTask); + + bool added = false; + code = streamMetaRegisterTask(pSnode->pMeta, -1, pTask, &added); if (code < 0) { taosWUnLockLatch(&pSnode->pMeta->lock); return -1; } + int32_t numOfTasks = streamMetaGetNumOfTasks(pSnode->pMeta); taosWUnLockLatch(&pSnode->pMeta->lock); + qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE, pTask->id.idStr, + streamGetTaskStatusStr(pTask->status.taskStatus), numOfTasks); - // 3.go through recover steps to fill history - if (pTask->fillHistory) { - streamSetParamForRecover(pTask); - streamAggRecoverPrepare(pTask); - } - + streamTaskCheckDownstreamTasks(pTask); return 0; } int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) { SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg; - streamMetaRemoveTask(pSnode->pMeta, pReq->taskId); + qDebug("snode:%d receive msg to drop stream task:0x%x", pSnode->pMeta->vgId, pReq->taskId); + + SStreamTask* pTask = streamMetaAcquireTask(pSnode->pMeta, pReq->taskId); + if (pTask == NULL) { + qError("vgId:%d failed to acquire s-task:0x%x when dropping it", pSnode->pMeta->vgId, pReq->taskId); + return 0; + } + + streamMetaUnregisterTask(pSnode->pMeta, pReq->taskId); + streamMetaReleaseTask(pSnode->pMeta, pTask); return 0; } @@ -255,38 +269,40 @@ int32_t sndProcessTaskRetrieveRsp(SSnode *pSnode, SRpcMsg *pMsg) { } int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { - void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); switch (pMsg->msgType) { - case TDMT_STREAM_TASK_DEPLOY: + case TDMT_STREAM_TASK_DEPLOY: { + void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); return sndProcessTaskDeployReq(pSnode, pReq, len); + } + case TDMT_STREAM_TASK_DROP: - return sndProcessTaskDropReq(pSnode, pReq, len); + return sndProcessTaskDropReq(pSnode, pMsg->pCont, pMsg->contLen); default: ASSERT(0); } return 0; } -int32_t sndProcessTaskRecoverFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { +int32_t sndProcessStreamTaskScanHistoryFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { char *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); // deserialize - SStreamRecoverFinishReq req; + SStreamScanHistoryFinishReq req; SDecoder decoder; tDecoderInit(&decoder, msg, msgLen); - tDecodeSStreamRecoverFinishReq(&decoder, &req); + tDecodeStreamScanHistoryFinishReq(&decoder, &req); tDecoderClear(&decoder); // find task - SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.taskId); + SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.downstreamTaskId); if (pTask == NULL) { return -1; } // do process request - if (streamProcessRecoverFinishReq(pTask, req.childId) < 0) { + if (streamProcessScanHistoryFinishReq(pTask, &req, &pMsg->info) < 0) { streamMetaReleaseTask(pSnode->pMeta, pTask); return -1; } @@ -300,6 +316,102 @@ int32_t sndProcessTaskRecoverFinishRsp(SSnode *pSnode, SRpcMsg *pMsg) { return 0; } +int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) { + char *msgStr = pMsg->pCont; + char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + SStreamTaskCheckReq req; + SDecoder decoder; + + tDecoderInit(&decoder, (uint8_t *)msgBody, msgLen); + tDecodeStreamTaskCheckReq(&decoder, &req); + tDecoderClear(&decoder); + + int32_t taskId = req.downstreamTaskId; + + SStreamTaskCheckRsp rsp = { + .reqId = req.reqId, + .streamId = req.streamId, + .childId = req.childId, + .downstreamNodeId = req.downstreamNodeId, + .downstreamTaskId = req.downstreamTaskId, + .upstreamNodeId = req.upstreamNodeId, + .upstreamTaskId = req.upstreamTaskId, + }; + + SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, taskId); + + if (pTask != NULL) { + rsp.status = streamTaskCheckStatus(pTask); + streamMetaReleaseTask(pSnode->pMeta, pTask); + + const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); + qDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d", + pTask->id.idStr, pStatus, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + } else { + rsp.status = 0; + qDebug("tq recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 + ") from task:0x%x (vgId:%d), rsp status %d", + taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + } + + SEncoder encoder; + int32_t code; + int32_t len; + + tEncodeSize(tEncodeStreamTaskCheckRsp, &rsp, len, code); + if (code < 0) { + qError("vgId:%d failed to encode task check rsp, task:0x%x", pSnode->pMeta->vgId, taskId); + return -1; + } + + void *buf = rpcMallocCont(sizeof(SMsgHead) + len); + ((SMsgHead *)buf)->vgId = htonl(req.upstreamNodeId); + + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncoderInit(&encoder, (uint8_t *)abuf, len); + tEncodeStreamTaskCheckRsp(&encoder, &rsp); + tEncoderClear(&encoder); + + SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info}; + + tmsgSendRsp(&rspMsg); + return 0; +} + +int32_t sndProcessStreamTaskCheckRsp(SSnode* pSnode, SRpcMsg* pMsg) { + char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + + int32_t code; + SStreamTaskCheckRsp rsp; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)pReq, len); + code = tDecodeStreamTaskCheckRsp(&decoder, &rsp); + + if (code < 0) { + tDecoderClear(&decoder); + return -1; + } + + tDecoderClear(&decoder); + qDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", + rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); + + SStreamTask* pTask = streamMetaAcquireTask(pSnode->pMeta, rsp.upstreamTaskId); + if (pTask == NULL) { + qError("tq failed to locate the stream task:0x%x (vgId:%d), it may have been destroyed", rsp.upstreamTaskId, + pSnode->pMeta->vgId); + return -1; + } + + code = streamProcessCheckRsp(pTask, &rsp); + streamMetaReleaseTask(pSnode->pMeta, pTask); + return code; +} + int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_STREAM_TASK_RUN: @@ -312,10 +424,14 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { return sndProcessTaskRetrieveReq(pSnode, pMsg); case TDMT_STREAM_RETRIEVE_RSP: return sndProcessTaskRetrieveRsp(pSnode, pMsg); - case TDMT_STREAM_RECOVER_FINISH: - return sndProcessTaskRecoverFinishReq(pSnode, pMsg); - case TDMT_STREAM_RECOVER_FINISH_RSP: + case TDMT_STREAM_SCAN_HISTORY_FINISH: + return sndProcessStreamTaskScanHistoryFinishReq(pSnode, pMsg); + case TDMT_STREAM_SCAN_HISTORY_FINISH_RSP: return sndProcessTaskRecoverFinishRsp(pSnode, pMsg); + case TDMT_STREAM_TASK_CHECK: + return sndProcessStreamTaskCheckReq(pSnode, pMsg); + case TDMT_STREAM_TASK_CHECK_RSP: + return sndProcessStreamTaskCheckRsp(pSnode, pMsg); default: ASSERT(0); } diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c index f5e924525212c34ffd375454703a91ccc160836f..c046505630251092923189eadbc532e87970e4b6 100644 --- a/source/dnode/snode/src/snodeInitApi.c +++ b/source/dnode/snode/src/snodeInitApi.c @@ -101,6 +101,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateCommit = streamStateCommit; pStore->streamStateDestroy= streamStateDestroy; pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; + pStore->streamStateReloadInfo = streamStateReloadInfo; } void initFunctionStateStore(SFunctionStateStore* pStore) { diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a4dad8f96aa7725b0acc19b24239bc3fa3bd62ee..194ffa16f671b8b63466d3bb1de559b470547de4 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -1,21 +1,18 @@ # vnode add_library(vnode STATIC "") -target_sources( - vnode - PRIVATE - - # vnode - "src/vnd/vnodeOpen.c" - "src/vnd/vnodeBufPool.c" - "src/vnd/vnodeCfg.c" - "src/vnd/vnodeCommit.c" - "src/vnd/vnodeQuery.c" - "src/vnd/vnodeModule.c" - "src/vnd/vnodeSvr.c" - "src/vnd/vnodeSync.c" - "src/vnd/vnodeSnapshot.c" - "src/vnd/vnodeRetention.c" - "src/vnd/vnodeInitApi.c" +set( + VNODE_SOURCE_FILES + "src/vnd/vnodeOpen.c" + "src/vnd/vnodeBufPool.c" + "src/vnd/vnodeCfg.c" + "src/vnd/vnodeCommit.c" + "src/vnd/vnodeQuery.c" + "src/vnd/vnodeModule.c" + "src/vnd/vnodeSvr.c" + "src/vnd/vnodeSync.c" + "src/vnd/vnodeSnapshot.c" + "src/vnd/vnodeRetention.c" + "src/vnd/vnodeInitApi.c" # meta "src/meta/metaOpen.c" @@ -32,30 +29,29 @@ target_sources( # sma "src/sma/smaEnv.c" "src/sma/smaUtil.c" - "src/sma/smaFS.c" "src/sma/smaOpen.c" "src/sma/smaCommit.c" "src/sma/smaRollup.c" "src/sma/smaSnapshot.c" "src/sma/smaTimeRange.c" - # tsdb - "src/tsdb/tsdbCommit.c" - "src/tsdb/tsdbFile.c" - "src/tsdb/tsdbFS.c" - "src/tsdb/tsdbOpen.c" - "src/tsdb/tsdbMemTable.c" - "src/tsdb/tsdbRead.c" - "src/tsdb/tsdbCache.c" - "src/tsdb/tsdbWrite.c" - "src/tsdb/tsdbReaderWriter.c" - "src/tsdb/tsdbUtil.c" - "src/tsdb/tsdbSnapshot.c" - "src/tsdb/tsdbCacheRead.c" - "src/tsdb/tsdbRetention.c" - "src/tsdb/tsdbDiskData.c" - "src/tsdb/tsdbMergeTree.c" - "src/tsdb/tsdbDataIter.c" + # # tsdb + # "src/tsdb/tsdbCommit.c" + # "src/tsdb/tsdbFile.c" + # "src/tsdb/tsdbFS.c" + # "src/tsdb/tsdbOpen.c" + # "src/tsdb/tsdbMemTable.c" + # "src/tsdb/tsdbRead.c" + # "src/tsdb/tsdbCache.c" + # "src/tsdb/tsdbWrite.c" + # "src/tsdb/tsdbReaderWriter.c" + # "src/tsdb/tsdbUtil.c" + # "src/tsdb/tsdbSnapshot.c" + # "src/tsdb/tsdbCacheRead.c" + # "src/tsdb/tsdbRetention.c" + # "src/tsdb/tsdbDiskData.c" + # "src/tsdb/tsdbMergeTree.c" + # "src/tsdb/tsdbDataIter.c" # tq "src/tq/tq.c" @@ -72,6 +68,19 @@ target_sources( "src/tq/tqOffsetSnapshot.c" ) +aux_source_directory("src/tsdb/" TSDB_SOURCE_FILES) +list( + APPEND + VNODE_SOURCE_FILES + ${TSDB_SOURCE_FILES} +) + +target_sources( + vnode + PRIVATE + ${VNODE_SOURCE_FILES} +) + IF (TD_VNODE_PLUGINS) target_sources( vnode diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e84211c7657b4f50bf0d228c5f85376c40635984..a7ce18198dfcd4fe93675a936d43585b83a43d79 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -51,12 +51,14 @@ extern const SVnodeCfg vnodeCfgDefault; int32_t vnodeInit(int32_t nthreads); void vnodeCleanup(); -int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs); -int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs); -int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq, STfs *pTfs); -int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, STfs *pTfs); +int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs *pTfs); +int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t diskPrimary, STfs *pTfs); +int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq, + int32_t diskPrimary, STfs *pTfs); +int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, + int32_t diskPrimary, STfs *pTfs); void vnodeDestroy(const char *path, STfs *pTfs); -SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb); +SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb); void vnodePreClose(SVnode *pVnode); void vnodePostClose(SVnode *pVnode); void vnodeSyncCheckTimeout(SVnode *pVnode); @@ -78,6 +80,7 @@ ESyncRole vnodeGetRole(SVnode *pVnode); int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list); +int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void* arg1), void *arg); void *vnodeGetIdx(void *pVnode); void *vnodeGetIvtIdx(void *pVnode); @@ -96,6 +99,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); +int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit); @@ -125,8 +129,9 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); int32_t metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen); - -int64_t metaGetTbNum(SMeta *pMeta); +bool metaTbInFilterCache(void *pVnode, tb_uid_t suid, int8_t type); +int32_t metaPutTbToFilterCache(void *pVnode, tb_uid_t suid, int8_t type); +int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type); int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); @@ -163,6 +168,27 @@ uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader); void tsdbReaderSetCloseFlag(STsdbReader *pReader); int64_t tsdbGetLastTimestamp(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); +//====================================================================================================================== +int32_t tsdbReaderOpen2(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, + SSDataBlock *pResBlock, void **ppReader, const char *idstr, bool countOnly, + SHashObj **pIgnoreTables); +int32_t tsdbSetTableList2(STsdbReader *pReader, const void *pTableList, int32_t num); +void tsdbReaderSetId2(STsdbReader *pReader, const char *idstr); +void tsdbReaderClose2(STsdbReader *pReader); +int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext); +int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); +void tsdbReleaseDataBlock2(STsdbReader *pReader); +SSDataBlock *tsdbRetrieveDataBlock2(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); +int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond); +int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); +int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle); +void *tsdbGetIdx2(SMeta *pMeta); +void *tsdbGetIvtIdx2(SMeta *pMeta); +uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); +void tsdbReaderSetCloseFlag2(STsdbReader *pReader); +int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); +//====================================================================================================================== + int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); @@ -215,7 +241,7 @@ bool tqNextBlockImpl(STqReader *pReader, const char *idstr); SWalReader* tqGetWalReader(STqReader* pReader); SSDataBlock* tqGetResultBlock (STqReader* pReader); -int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, const char *id); +int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, int64_t maxVer, const char *id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock** pRes, const char* idstr); @@ -235,7 +261,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot); int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData); -int32_t buildSnapContext(SVnode *pVnode, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, +int32_t buildSnapContext(SVnode *pVnode, int64_t snapVersion, int64_t suid, int8_t subType, int8_t withMeta, SSnapContext **ctxRet); int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index c3e8d7ef1dacd2d0c7aaf143f09d3a9de88eec2e..aaf0973b41a6965f4aaeca44877511bbd3c62727 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -105,17 +105,16 @@ struct SRSmaFS { struct SRSmaStat { SSma *pSma; - int64_t commitAppliedVer; // vnode applied version for async commit - int64_t refId; // shared by fetch tasks - volatile int64_t nBufItems; // number of items in queue buffer - SRWLatch lock; // r/w lock for rsma fs(e.g. qtaskinfo) - volatile int32_t nFetchAll; // active number of fetch all - volatile int8_t triggerStat; // shared by fetch tasks - volatile int8_t commitStat; // 0 not in committing, 1 in committing - volatile int8_t delFlag; // 0 no deleted SRSmaInfo, 1 has deleted SRSmaInfo - SRSmaFS fs; // for recovery/snapshot r/w - SHashObj *infoHash; // key: suid, value: SRSmaInfo - tsem_t notEmpty; // has items in queue buffer + int64_t refId; // shared by fetch tasks + volatile int64_t nBufItems; // number of items in queue buffer + SRWLatch lock; // r/w lock for rsma fs(e.g. qtaskinfo) + volatile int32_t nFetchAll; // active number of fetch all + volatile int8_t triggerStat; // shared by fetch tasks + volatile int8_t commitStat; // 0 not in committing, 1 in committing + volatile int8_t delFlag; // 0 no deleted SRSmaInfo, 1 has deleted SRSmaInfo + SRSmaFS fs; // for recovery/snapshot r/w + SHashObj *infoHash; // key: suid, value: SRSmaInfo + tsem_t notEmpty; // has items in queue buffer }; struct SSmaStat { @@ -156,12 +155,9 @@ struct SRSmaInfo { int16_t padding; T_REF_DECLARE() SRSmaInfoItem items[TSDB_RETENTION_L2]; - void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t - STaosQueue *queue; // buffer queue of SubmitReq - STaosQall *qall; // buffer qall of SubmitReq - void *iTaskInfo[TSDB_RETENTION_L2]; // immutable qTaskInfo_t - STaosQueue *iQueue; // immutable buffer queue of SubmitReq - STaosQall *iQall; // immutable buffer qall of SubmitReq + void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t + STaosQueue *queue; // buffer queue of SubmitReq + STaosQall *qall; // buffer qall of SubmitReq }; #define RSMA_INFO_HEAD_LEN offsetof(SRSmaInfo, items) @@ -191,6 +187,12 @@ typedef enum { RSMA_EXEC_COMMIT = 3, // triggered by commit } ERsmaExecType; +#define TD_SMA_LOOPS_CHECK(n, limit) \ + if (++(n) > limit) { \ + sched_yield(); \ + (n) = 0; \ + } + // sma int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType); void tdDestroySmaEnv(SSmaEnv *pSmaEnv); @@ -213,27 +215,12 @@ int32_t smaPreClose(SSma *pSma); // rsma void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); -int32_t tdRSmaFSOpen(SSma *pSma, int64_t version, int8_t rollback); -void tdRSmaFSClose(SRSmaFS *fs); -int32_t tdRSmaFSPrepareCommit(SSma *pSma, SRSmaFS *pFSNew); -int32_t tdRSmaFSCommit(SSma *pSma); -int32_t tdRSmaFSFinishCommit(SSma *pSma); -int32_t tdRSmaFSCopy(SSma *pSma, SRSmaFS *pFS); -int32_t tdRSmaFSTakeSnapshot(SSma *pSma, SRSmaFS *pFS); -int32_t tdRSmaFSRef(SSma *pSma, SRSmaFS *pFS); -void tdRSmaFSUnRef(SSma *pSma, SRSmaFS *pFS); -int32_t tdRSmaFSUpsertQTaskFile(SSma *pSma, SRSmaFS *pFS, SQTaskFile *qTaskFile, int32_t nSize); -int32_t tdRSmaFSRollback(SSma *pSma); int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback); int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type); -int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); +// int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer, int8_t rollback); -void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t suid, int8_t level, int64_t version, char *outputName); -void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t suid, int8_t level, int64_t version, const char *path, - char *outputName); -void tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName); -void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName); +void tdRSmaQTaskInfoGetFullPath(SVnode *pVnode, tb_uid_t suid, int8_t level, STfs *pTfs, char *outputName); static FORCE_INLINE void tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { int32_t ref = T_REF_INC(pRSmaInfo); @@ -244,9 +231,7 @@ static FORCE_INLINE void tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { smaTrace("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); } -void tdRSmaGetFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t suid, - int8_t level, int64_t version, char *outputName); -void tdRSmaGetDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName); +void tdRSmaGetDirName(SVnode *pVnode, STfs *pTfs, bool endWithSep, char *outputName); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index c390cdfd3863ef5dffc9cafab26f287179d00b11..13b991e0386e8419b542f0be8406fa7cdd20254d 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -45,27 +45,10 @@ extern "C" { typedef struct STqOffsetStore STqOffsetStore; // tqPush - -// typedef struct { -// // msg info -// int64_t consumerId; -// int64_t reqOffset; -// int64_t processedVer; -// int32_t epoch; -// // rpc info -// int64_t reqId; -// SRpcHandleInfo rpcInfo; -// tmr_h timerId; -// int8_t tmrStopped; -// // exec -// int8_t inputStatus; -// int8_t execStatus; -// SStreamQueue inputQ; -// SRWLatch lock; -// } STqPushHandle; +#define EXTRACT_DATA_FROM_WAL_ID (-1) +#define STREAM_TASK_STATUS_CHECK_ID (-2) // tqExec - typedef struct { char* qmsg; // SubPlanToString } STqExecCol; @@ -185,11 +168,10 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); // tqStream int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); int32_t tqStreamTasksScanWal(STQ* pTq); +int32_t tqStreamTasksStatusCheck(STQ* pTq); // tq util int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, SStreamRefDataBlock** pRefBlock); -char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); -int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d7694ebfd5abc9b3cc7cf1a36d3936182025118b..75c8eea83a003fbbcce095ebb6e4ace4d5b4a481 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -16,6 +16,9 @@ #ifndef _TD_VNODE_TSDB_H_ #define _TD_VNODE_TSDB_H_ +// #include "../tsdb/tsdbFile2.h" +// #include "../tsdb/tsdbMerge.h" +// #include "../tsdb/tsdbSttFileRW.h" #include "tsimplehash.h" #include "vnodeInt.h" @@ -64,7 +67,6 @@ typedef struct STsdbReadSnap STsdbReadSnap; typedef struct SBlockInfo SBlockInfo; typedef struct SSmaInfo SSmaInfo; typedef struct SBlockCol SBlockCol; -typedef struct SVersionRange SVersionRange; typedef struct SLDataIter SLDataIter; typedef struct SDiskCol SDiskCol; typedef struct SDiskData SDiskData; @@ -76,9 +78,8 @@ typedef struct STsdbFilterInfo STsdbFilterInfo; #define TSDBROW_ROW_FMT ((int8_t)0x0) #define TSDBROW_COL_FMT ((int8_t)0x1) -#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) -#define TSDB_MAX_SUBBLOCKS 8 -#define TSDB_FHDR_SIZE 512 +#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) +#define TSDB_FHDR_SIZE 512 #define VERSION_MIN 0 #define VERSION_MAX INT64_MAX @@ -166,6 +167,7 @@ void tBlockDataDestroy(SBlockData *pBlockData); int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); void tBlockDataReset(SBlockData *pBlockData); int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); +int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema); int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid); int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); void tBlockDataClear(SBlockData *pBlockData); @@ -199,7 +201,7 @@ int32_t tMapDataToArray(SMapData *pMapData, int32_t itemSize, int32_t (*tGetItem // other int32_t tsdbKeyFid(TSKEY key, int32_t minutes, int8_t precision); void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minKey, TSKEY *maxKey); -int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now); +int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t nowSec); int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline); int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); @@ -303,8 +305,11 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== int32_t tsdbTakeReadSnap(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); void tsdbUntakeReadSnap(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive); + +int32_t tsdbTakeReadSnap2(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); +void tsdbUntakeReadSnap2(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive); // tsdbMerge.c ============================================================================================== -int32_t tsdbMerge(STsdb *pTsdb); +int32_t tsdbMerge(void *arg); // tsdbDiskData ============================================================================================== int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder); @@ -363,19 +368,20 @@ typedef struct { } SCacheFlushState; struct STsdb { - char *path; - SVnode *pVnode; - STsdbKeepCfg keepCfg; - TdThreadRwlock rwLock; - SMemTable *mem; - SMemTable *imem; - STsdbFS fs; - SLRUCache *lruCache; - SCacheFlushState flushState; - TdThreadMutex lruMutex; - SLRUCache *biCache; - TdThreadMutex biMutex; - SRocksCache rCache; + char *path; + SVnode *pVnode; + STsdbKeepCfg keepCfg; + TdThreadRwlock rwLock; + SMemTable *mem; + SMemTable *imem; + STsdbFS fs; // old + SLRUCache *lruCache; + SCacheFlushState flushState; + TdThreadMutex lruMutex; + SLRUCache *biCache; + TdThreadMutex biMutex; + struct STFileSystem *pFS; // new + SRocksCache rCache; }; struct TSDBKEY { @@ -383,11 +389,6 @@ struct TSDBKEY { TSKEY ts; }; -struct SVersionRange { - uint64_t minVer; - uint64_t maxVer; -}; - typedef struct SMemSkipListNode SMemSkipListNode; struct SMemSkipListNode { int8_t level; @@ -416,6 +417,7 @@ struct STbData { SDelData *pTail; SMemSkipList sl; STbData *next; + SRBTreeNode rbtn[1]; }; struct SMemTable { @@ -429,11 +431,10 @@ struct SMemTable { TSKEY maxKey; int64_t nRow; int64_t nDel; - struct { - int32_t nTbData; - int32_t nBucket; - STbData **aBucket; - }; + int32_t nTbData; + int32_t nBucket; + STbData **aBucket; + SRBTree tbDataTree[1]; }; struct TSDBROW { @@ -506,7 +507,7 @@ struct SDataBlk { int32_t nRow; int8_t hasDup; int8_t nSubBlock; - SBlockInfo aSubBlock[TSDB_MAX_SUBBLOCKS]; + SBlockInfo aSubBlock[1]; SSmaInfo smaInfo; }; @@ -658,12 +659,19 @@ struct SDelFWriter { uint8_t *aBuf[1]; }; +#include "tarray2.h" +//#include "tsdbFS2.h" +// struct STFileSet; +typedef struct STFileSet STFileSet; +typedef TARRAY2(STFileSet *) TFileSetArray; + struct STsdbReadSnap { - SMemTable *pMem; - SQueryNode *pNode; - SMemTable *pIMem; - SQueryNode *pINode; - STsdbFS fs; + SMemTable *pMem; + SQueryNode *pNode; + SMemTable *pIMem; + SQueryNode *pINode; + TFileSetArray *pfSetArray; + STsdbFS fs; }; struct SDataFWriter { @@ -702,6 +710,7 @@ typedef struct { typedef struct SSttBlockLoadInfo { SBlockData blockData[2]; + void *pSttStatisBlkArray; SArray *aSttBlk; int32_t blockIndex[2]; // to denote the loaded block in the corresponding position. int32_t currentLoadBlockIndex; @@ -710,10 +719,9 @@ typedef struct SSttBlockLoadInfo { STSchema *pSchema; int16_t *colIds; int32_t numOfCols; - bool checkRemainingRow; + bool checkRemainingRow; // todo: no assign value? bool isLast; bool sttBlockLoaded; - int32_t numOfStt; // keep the last access position, this position may be used to reduce the binary times for // starting last block data for a new table @@ -772,60 +780,107 @@ struct SDiskDataBuilder { }; typedef struct SLDataIter { - SRBTreeNode node; - SSttBlk *pSttBlk; - SDataFReader *pReader; - int32_t iStt; - int8_t backward; - int32_t iSttBlk; - int32_t iRow; - SRowInfo rInfo; - uint64_t uid; - STimeWindow timeWindow; - SVersionRange verRange; - SSttBlockLoadInfo *pBlockLoadInfo; - bool ignoreEarlierTs; + SRBTreeNode node; + SSttBlk *pSttBlk; + int32_t iStt; // for debug purpose + int8_t backward; + int32_t iSttBlk; + int32_t iRow; + SRowInfo rInfo; + uint64_t uid; + STimeWindow timeWindow; + SVersionRange verRange; + SSttBlockLoadInfo *pBlockLoadInfo; + bool ignoreEarlierTs; + struct SSttFileReader *pReader; } SLDataIter; #define tMergeTreeGetRow(_t) (&((_t)->pIter->rInfo.row)) int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter *pLDataIter); -void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); -bool tMergeTreeNext(SMergeTree *pMTree); -bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); -void tMergeTreeClose(SMergeTree *pMTree); + +struct SSttFileReader; +typedef int32_t (*_load_tomb_fn)(STsdbReader *pReader, struct SSttFileReader *pSttFileReader, + SSttBlockLoadInfo *pLoadInfo); + +typedef struct { + int8_t backward; + STsdb *pTsdb; + uint64_t suid; + uint64_t uid; + STimeWindow timewindow; + SVersionRange verRange; + bool strictTimeRange; + SArray *pSttFileBlockIterArray; + void *pCurrentFileset; + STSchema *pSchema; + int16_t *pCols; + int32_t numOfCols; + _load_tomb_fn loadTombFn; + void *pReader; + void *idstr; +} SMergeTreeConf; +int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf); + +void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); +bool tMergeTreeNext(SMergeTree *pMTree); +bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); +void tMergeTreeClose(SMergeTree *pMTree); SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, int32_t numOfStt); +SSttBlockLoadInfo *tCreateOneLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols); void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el); void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); +void *destroySttBlockReader(SArray *pLDataIterArray, int64_t *blocks, double *el); // tsdbCache ============================================================================================== +typedef enum { + READ_MODE_COUNT_ONLY = 0x1, + READ_MODE_ALL, +} EReadMode; + +typedef struct STsdbReaderInfo { + uint64_t suid; + STSchema *pSchema; + EReadMode readMode; + uint64_t rowsNum; + STimeWindow window; + SVersionRange verRange; + int16_t order; +} STsdbReaderInfo; + +typedef struct { + SArray *pTombData; +} STableLoadInfo; + +struct SDataFileReader; + typedef struct SCacheRowsReader { - STsdb *pTsdb; - SVersionRange verRange; - TdThreadMutex readerMutex; - SVnode *pVnode; - STSchema *pSchema; - STSchema *pCurrSchema; - uint64_t uid; - uint64_t suid; - char **transferBuf; // todo remove it soon - int32_t numOfCols; - SArray *pCidList; - int32_t *pSlotIds; - int32_t type; - int32_t tableIndex; // currently returned result tables - STableKeyInfo *pTableList; // table id list - int32_t numOfTables; - SSttBlockLoadInfo *pLoadInfo; - SLDataIter *pDataIter; - STsdbReadSnap *pReadSnap; - SDataFReader *pDataFReader; - SDataFReader *pDataFReaderLast; - const char *idstr; - int64_t lastTs; + STsdb *pTsdb; + STsdbReaderInfo info; + TdThreadMutex readerMutex; + SVnode *pVnode; + STSchema *pSchema; + STSchema *pCurrSchema; + uint64_t uid; + char **transferBuf; // todo remove it soon + int32_t numOfCols; + SArray *pCidList; + int32_t *pSlotIds; + int32_t type; + int32_t tableIndex; // currently returned result tables + STableKeyInfo *pTableList; // table id list + int32_t numOfTables; + uint64_t *uidList; + SSHashObj *pTableMap; + SArray *pLDataIterArray; + struct SDataFileReader *pFileReader; + STFileSet *pCurFileSet; + STsdbReadSnap *pReadSnap; + char *idstr; + int64_t lastTs; } SCacheRowsReader; typedef struct { diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index a67f246e73d9656f9c184e99b796edf8ee8dc954..85ef384ea98a79c11c6e7774f9bb797e5de21e8e 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -49,7 +49,8 @@ int32_t vnodeEncodeConfig(const void* pObj, SJson* pJson); int32_t vnodeDecodeConfig(const SJson* pJson, void* pObj); // vnodeModule.c -int32_t vnodeScheduleTask(int32_t (*execute)(void*), void* arg); +int vnodeScheduleTask(int (*execute)(void*), void* arg); +int vnodeScheduleTaskEx(int tpid, int (*execute)(void*), void* arg); // vnodeBufPool.c typedef struct SVBufPoolNode SVBufPoolNode; @@ -86,6 +87,9 @@ void vnodeBufPoolReset(SVBufPool* pPool); void vnodeBufPoolAddToFreeList(SVBufPool* pPool); int32_t vnodeBufPoolRecycle(SVBufPool* pPool); +// vnodeOpen.c +int32_t vnodeGetPrimaryDir(const char* relPath, int32_t diskPrimary, STfs* pTfs, char* buf, size_t bufLen); + // vnodeQuery.c int32_t vnodeQueryOpen(SVnode* pVnode); void vnodeQueryPreClose(SVnode* pVnode); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 2a5cdbe555c21a25e25de9c756aacdefc64f6e9e..cd7704940b3a017dae7d12e15647fc61ca8fd542 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -93,6 +93,7 @@ typedef struct SQueryNode SQueryNode; #define VNODE_BUFPOOL_SEGMENTS 3 #define VND_INFO_FNAME "vnode.json" +#define VND_INFO_FNAME_TMP "vnode_tmp.json" // vnd.h typedef int32_t (*_query_reseek_func_t)(void* pQHandle); @@ -179,7 +180,8 @@ SArray* metaGetSmaTbUids(SMeta* pMeta); void* metaGetIdx(SMeta* pMeta); void* metaGetIvtIdx(SMeta* pMeta); -void metaReaderInit(SMetaReader* pReader, SMeta* pMeta, int32_t flags); +int64_t metaGetTbNum(SMeta* pMeta); +void metaReaderDoInit(SMetaReader* pReader, SMeta* pMeta, int32_t flags); int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); @@ -196,12 +198,12 @@ int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pR int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback); int tsdbClose(STsdb** pTsdb); int32_t tsdbBegin(STsdb* pTsdb); -int32_t tsdbPrepareCommit(STsdb* pTsdb); -int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); +// int32_t tsdbPrepareCommit(STsdb* pTsdb); +// int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); int32_t tsdbCacheCommit(STsdb* pTsdb); int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo); -int32_t tsdbFinishCommit(STsdb* pTsdb); -int32_t tsdbRollbackCommit(STsdb* pTsdb); +// int32_t tsdbFinishCommit(STsdb* pTsdb); +// int32_t tsdbRollbackCommit(STsdb* pTsdb); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); @@ -218,6 +220,7 @@ int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg); int tqUnregisterPushHandle(STQ* pTq, void* pHandle); int tqStartStreamTasks(STQ* pTq); // restore all stream tasks after vnode launching completed. +int tqCheckStreamStatus(STQ* pTq); int tqCommit(STQ*); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); @@ -240,16 +243,16 @@ int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen); +int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, SRpcMsg* pMsg); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskTransferStateReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskScanHistoryFinishReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskScanHistoryFinishRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqCheckLogInWal(STQ* pTq, int64_t version); // sma @@ -318,7 +321,6 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData); // SRSmaSnapWriter ======================================== int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWriter** ppWriter); int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); -int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter); int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback); typedef struct { @@ -385,6 +387,7 @@ struct SVnode { SVState state; SVStatis statis; STfs* pTfs; + int32_t diskPrimary; SMsgCb msgCb; // Buffer Pool diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 078e6ee6af08641f45c4ded20597047159352306..6918634b5daa4268ca401752608b42d5b82ea595 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -66,6 +66,10 @@ struct SMetaCache { SHashObj* pTableEntry; SLRUCache* pResCache; } STbGroupResCache; + + struct STbFilterCache { + SHashObj* pStb; + } STbFilterCache; }; static void entryCacheClose(SMeta* pMeta) { @@ -168,6 +172,12 @@ int32_t metaCacheOpen(SMeta* pMeta) { taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp); taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL); + pCache->STbFilterCache.pStb = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (pCache->STbFilterCache.pStb == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err2; + } + pMeta->pCache = pCache; return code; @@ -193,6 +203,8 @@ void metaCacheClose(SMeta* pMeta) { taosThreadMutexDestroy(&pMeta->pCache->STbGroupResCache.lock); taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry); + taosHashCleanup(pMeta->pCache->STbFilterCache.pStb); + taosMemoryFree(pMeta->pCache); pMeta->pCache = NULL; } @@ -880,3 +892,31 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { metaDebug("vgId:%d suid:%" PRId64 " cached related tb group cleared", vgId, suid); return TSDB_CODE_SUCCESS; } + +bool metaTbInFilterCache(void* pVnode, tb_uid_t suid, int8_t type) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + + if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid))) { + return true; + } + + return false; +} + +int32_t metaPutTbToFilterCache(void* pVnode, tb_uid_t suid, int8_t type) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + + if (type == 0) { + return taosHashPut(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid), NULL, 0); + } + + return 0; +} + +int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + if (type == 0) { + return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb); + } + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 72f4c6b5879ba73791cf72770ce50ea4efededf1..517d9692c75f1831038698f6193eb02f994002a3 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -14,6 +14,7 @@ */ #include "meta.h" +#include "vnd.h" static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); @@ -34,30 +35,27 @@ static void metaCleanup(SMeta **ppMeta); int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { SMeta *pMeta = NULL; int ret; - int slen; + int offset; + char path[TSDB_FILENAME_LEN] = {0}; *ppMeta = NULL; // create handle - if (pVnode->pTfs) { - slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(VNODE_META_DIR) + 3; - } else { - slen = strlen(pVnode->path) + strlen(VNODE_META_DIR) + 2; - } - if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + slen)) == NULL) { + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN); + offset = strlen(path); + snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VNODE_META_DIR); + + if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } metaInitLock(pMeta); + pMeta->path = (char *)&pMeta[1]; - if (pVnode->pTfs) { - sprintf(pMeta->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, - VNODE_META_DIR); - } else { - sprintf(pMeta->path, "%s%s%s", pVnode->path, TD_DIRSEP, VNODE_META_DIR); - } - taosRealPath(pMeta->path, NULL, slen); + strcpy(pMeta->path, path); + taosRealPath(pMeta->path, NULL, strlen(path) + 1); + pMeta->pVnode = pVnode; // create path if not created yet diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 0e380ea0b2f42614d873a7c41cd32645da35ce84..c26bb45c2b51b91ae4e4d9f53d7a895fc832ec9e 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -17,13 +17,13 @@ #include "osMemory.h" #include "tencode.h" -void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta *pAPI) { - SMeta *pMeta = ((SVnode *)pVnode)->pMeta; - metaReaderInit(pReader, pMeta, flags); +void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMeta* pAPI) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + metaReaderDoInit(pReader, pMeta, flags); pReader->pAPI = pAPI; } -void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { +void metaReaderDoInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { memset(pReader, 0, sizeof(*pReader)); pReader->pMeta = pMeta; pReader->flags = flags; @@ -143,7 +143,7 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { int metaGetTableNameByUid(void *pVnode, uint64_t uid, char *tbName) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, ((SVnode *)pVnode)->pMeta, 0); + metaReaderDoInit(&mr, ((SVnode*)pVnode)->pMeta, 0); code = metaReaderGetTableEntryByUid(&mr, uid); if (code < 0) { metaReaderClear(&mr); @@ -159,7 +159,7 @@ int metaGetTableNameByUid(void *pVnode, uint64_t uid, char *tbName) { int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, (SMeta *)meta, 0); + metaReaderDoInit(&mr, (SMeta *)meta, 0); code = metaReaderGetTableEntryByUid(&mr, uid); if (code < 0) { metaReaderClear(&mr); @@ -174,7 +174,7 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, ((SVnode *)pVnode)->pMeta, 0); + metaReaderDoInit(&mr, ((SVnode *)pVnode)->pMeta, 0); SMetaReader *pReader = &mr; @@ -195,7 +195,7 @@ int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid) { int metaGetTableTypeByName(void *pVnode, char *tbName, ETableType *tbType) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, ((SVnode *)pVnode)->pMeta, 0); + metaReaderDoInit(&mr, ((SVnode*)pVnode)->pMeta, 0); code = metaGetTableEntryByName(&mr, tbName); if (code == 0) *tbType = mr.me.type; @@ -215,7 +215,7 @@ int metaReadNext(SMetaReader *pReader) { int metaGetTableTtlByUid(void *meta, uint64_t uid, int64_t *ttlDays) { int code = -1; SMetaReader mr = {0}; - metaReaderInit(&mr, (SMeta *)meta, 0); + metaReaderDoInit(&mr, (SMeta *)meta, 0); code = metaReaderGetTableEntryByUid(&mr, uid); if (code < 0) { goto _exit; @@ -244,9 +244,7 @@ SMTbCursor *metaOpenTbCursor(void *pVnode) { return NULL; } - SVnode *pVnodeObj = pVnode; - // metaReaderInit(&pTbCur->mr, pVnodeObj->pMeta, 0); - + SVnode* pVnodeObj = pVnode; // tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); pTbCur->pMeta = pVnodeObj->pMeta; pTbCur->paused = 1; @@ -277,7 +275,7 @@ void metaPauseTbCursor(SMTbCursor *pTbCur) { } void metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first) { if (pTbCur->paused) { - metaReaderInit(&pTbCur->mr, pTbCur->pMeta, 0); + metaReaderDoInit(&pTbCur->mr, pTbCur->pMeta, 0); tdbTbcOpen(((SMeta *)pTbCur->pMeta)->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); @@ -784,7 +782,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { } SMetaReader mr = {0}; - metaReaderInit(&mr, pMeta, 0); + metaReaderDoInit(&mr, pMeta, 0); int64_t smaId; int smaIdx = 0; STSma *pTSma = NULL; @@ -839,7 +837,7 @@ _err: STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { STSma *pTSma = NULL; SMetaReader mr = {0}; - metaReaderInit(&mr, pMeta, 0); + metaReaderDoInit(&mr, pMeta, 0); if (metaReaderGetTableEntryByUid(&mr, indexUid) < 0) { metaWarn("vgId:%d, failed to get table entry for smaId:%" PRIi64, TD_VID(pMeta->pVnode), indexUid); metaReaderClear(&mr); diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index a49848f4421e1c0e103d2d083713582574aee609..91704f5c7ab1558a5610496757763e053dad570d 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -37,7 +37,7 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { // validate req // save smaIndex - metaReaderInit(&mr, pMeta, 0); + metaReaderDoInit(&mr, pMeta, 0); if (metaReaderGetTableEntryByUidCache(&mr, pCfg->indexUid) == 0) { #if 1 terrno = TSDB_CODE_TSMA_ALREADY_EXIST; diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index f4e930e509cd6406b415ce4296b128f1c72219eb..18190ac533df4ab817e496f02b7bd02c1f4215a6 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -260,7 +260,7 @@ static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo) taosHashPut(suidInfo, &me->uid, sizeof(tb_uid_t), &dataTmp, sizeof(STableInfoForChildTable)); } -int32_t buildSnapContext(SVnode* pVnode, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, +int32_t buildSnapContext(SVnode* pVnode, int64_t snapVersion, int64_t suid, int8_t subType, int8_t withMeta, SSnapContext** ctxRet) { SSnapContext* ctx = taosMemoryCalloc(1, sizeof(SSnapContext)); if (ctx == NULL) return -1; @@ -476,7 +476,7 @@ int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLe if (ctx->index >= taosArrayGetSize(ctx->idList)) { metaDebug("tmqsnap get meta end"); ctx->index = 0; - ctx->queryMeta = false; // change to get data + ctx->queryMeta = 0; // change to get data return 0; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 5abafbe288150154464520e3d4f491862598cd08..632e6dd872be9b7381309653404b88f4dbf5152d 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -712,7 +712,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs } // validate req - metaReaderInit(&mr, pMeta, 0); + metaReaderDoInit(&mr, pMeta, 0); if (metaGetTableEntryByName(&mr, pReq->name) == 0) { if (pReq->type == TSDB_CHILD_TABLE && pReq->ctb.suid != mr.me.ctbEntry.suid) { terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; diff --git a/source/dnode/vnode/src/meta/metaTtl.c b/source/dnode/vnode/src/meta/metaTtl.c index 045a759fad23c1c8cb1986cafb0f8a1b6062de32..45f697258c4e8e599f1d0a2179ce10db03b6f3f5 100644 --- a/source/dnode/vnode/src/meta/metaTtl.c +++ b/source/dnode/vnode/src/meta/metaTtl.c @@ -379,7 +379,7 @@ _out: int ttlMgrFlush(STtlManger *pTtlMgr, TXN *pTxn) { ttlMgrWLock(pTtlMgr); - metaInfo("%s, ttl mgr flush start. dirty uids:%d", pTtlMgr->logPrefix, taosHashGetSize(pTtlMgr->pDirtyUids)); + metaDebug("%s, ttl mgr flush start. dirty uids:%d", pTtlMgr->logPrefix, taosHashGetSize(pTtlMgr->pDirtyUids)); int ret = -1; @@ -433,7 +433,7 @@ int ttlMgrFlush(STtlManger *pTtlMgr, TXN *pTxn) { _out: ttlMgrULock(pTtlMgr); - metaInfo("%s, ttl mgr flush end.", pTtlMgr->logPrefix); + metaDebug("%s, ttl mgr flush end.", pTtlMgr->logPrefix); return ret; } diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 51011ef7916a7f5b4173418a6d67186a8ff7869f..c26157f4b7ab260eda5a0244bbc9c6bc5ee812d1 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -103,18 +103,16 @@ _exit: return code; } -int32_t smaFinishCommit(SSma *pSma) { +extern int32_t tsdbCommitCommit(STsdb *tsdb); +int32_t smaFinishCommit(SSma *pSma) { int32_t code = 0; int32_t lino = 0; SVnode *pVnode = pSma->pVnode; - code = tdRSmaFSFinishCommit(pSma); - TSDB_CHECK_CODE(code, lino, _exit); - - if (VND_RSMA1(pVnode) && (code = tsdbFinishCommit(VND_RSMA1(pVnode))) < 0) { + if (VND_RSMA1(pVnode) && (code = tsdbCommitCommit(VND_RSMA1(pVnode))) < 0) { TSDB_CHECK_CODE(code, lino, _exit); } - if (VND_RSMA2(pVnode) && (code = tsdbFinishCommit(VND_RSMA2(pVnode))) < 0) { + if (VND_RSMA2(pVnode) && (code = tsdbCommitCommit(VND_RSMA2(pVnode))) < 0) { TSDB_CHECK_CODE(code, lino, _exit); } _exit: @@ -133,6 +131,7 @@ _exit: * @param isCommit * @return int32_t */ +extern int32_t tsdbPreCommit(STsdb *tsdb); static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) { int32_t code = 0; int32_t lino = 0; @@ -150,18 +149,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) { atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); if (isCommit) { while (atomic_val_compare_exchange_8(RSMA_COMMIT_STAT(pRSmaStat), 0, 1) != 0) { - ++nLoops; - if (nLoops > 1000) { - sched_yield(); - nLoops = 0; - } - } - - pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; - if (ASSERTS(pRSmaStat->commitAppliedVer >= -1, "commit applied version %" PRIi64 " < -1", - pRSmaStat->commitAppliedVer)) { - code = TSDB_CODE_APP_ERROR; - TSDB_CHECK_CODE(code, lino, _exit); + TD_SMA_LOOPS_CHECK(nLoops, 1000) } } // step 2: wait for all triggered fetch tasks to finish @@ -173,11 +161,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) { } else { smaDebug("vgId:%d, rsma commit%d, fetch tasks are not all finished yet", SMA_VID(pSma), isCommit); } - ++nLoops; - if (nLoops > 1000) { - sched_yield(); - nLoops = 0; - } + TD_SMA_LOOPS_CHECK(nLoops, 1000); } /** @@ -189,49 +173,26 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) { (void *)taosGetSelfPthreadId()); nLoops = 0; while (atomic_load_64(&pRSmaStat->nBufItems) > 0) { - ++nLoops; - if (nLoops > 1000) { - sched_yield(); - nLoops = 0; - } + TD_SMA_LOOPS_CHECK(nLoops, 1000); } if (!isCommit) goto _exit; - smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); + // code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); TSDB_CHECK_CODE(code, lino, _exit); smaInfo("vgId:%d, rsma commit, operator state committed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); -#if 0 // consuming task of qTaskInfo clone - // step 4: swap queue/qall and iQueue/iQall - // lock - taosWLockLatch(SMA_ENV_LOCK(pEnv)); - - void *pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); - - while (pIter) { - SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; - TSWAP(pInfo->iQall, pInfo->qall); - TSWAP(pInfo->iQueue, pInfo->queue); - TSWAP(pInfo->iTaskInfo[0], pInfo->taskInfo[0]); - TSWAP(pInfo->iTaskInfo[1], pInfo->taskInfo[1]); - pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter); - } - - // unlock - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); -#endif + smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); // all rsma results are written completely STsdb *pTsdb = NULL; if ((pTsdb = VND_RSMA1(pSma->pVnode))) { - code = tsdbPrepareCommit(pTsdb); + code = tsdbPreCommit(pTsdb); TSDB_CHECK_CODE(code, lino, _exit); } if ((pTsdb = VND_RSMA2(pSma->pVnode))) { - code = tsdbPrepareCommit(pTsdb); + code = tsdbPreCommit(pTsdb); TSDB_CHECK_CODE(code, lino, _exit); } @@ -248,6 +209,7 @@ _exit: * @param pSma * @return int32_t */ +extern int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info); static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo) { int32_t code = 0; int32_t lino = 0; @@ -258,13 +220,10 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo) { goto _exit; } - code = tdRSmaFSCommit(pSma); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbCommit(VND_RSMA1(pVnode), pInfo); + code = tsdbCommitBegin(VND_RSMA1(pVnode), pInfo); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbCommit(VND_RSMA2(pVnode), pInfo); + code = tsdbCommitBegin(VND_RSMA2(pVnode), pInfo); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -310,20 +269,6 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { continue; } -#if 0 - if (pRSmaInfo->taskInfo[0]) { - if (pRSmaInfo->iTaskInfo[0]) { - SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pRSmaInfo->iTaskInfo[0]; - tdFreeRSmaInfo(pSma, pRSmaInfo, false); - pRSmaInfo->iTaskInfo[0] = NULL; - } - } else { - TSWAP(pRSmaInfo->taskInfo[0], pRSmaInfo->iTaskInfo[0]); - } - - taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); - smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), *pSuid); -#endif } // unlock diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 02766c8076b435e3cdac12885210f533c12e4c6e..04a254fc7a2202827a99f67f82099e4404df9d5d 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -30,7 +30,6 @@ static int32_t tdRsmaStartExecutor(const SSma *pSma); static int32_t tdRsmaStopExecutor(const SSma *pSma); static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); static void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); -static void *tdFreeTSmaStat(STSmaStat *pStat); static void tdDestroyRSmaStat(void *pRSmaStat); /** @@ -63,19 +62,15 @@ int32_t smaInit() { int32_t type = (8 == POINTER_BYTES) ? TSDB_DATA_TYPE_UBIGINT : TSDB_DATA_TYPE_UINT; smaMgmt.refHash = taosHashInit(64, taosGetDefaultHashFunction(type), true, HASH_ENTRY_LOCK); - if (!smaMgmt.refHash) { - taosCloseRef(smaMgmt.rsetId); - atomic_store_8(&smaMgmt.inited, 0); - smaError("failed to init sma tmr hanle since %s", terrstr()); - return TSDB_CODE_FAILED; - } - // init fetch timer handle smaMgmt.tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA"); - if (!smaMgmt.tmrHandle) { + + if (!smaMgmt.refHash || !smaMgmt.tmrHandle) { taosCloseRef(smaMgmt.rsetId); - taosHashCleanup(smaMgmt.refHash); - smaMgmt.refHash = NULL; + if (smaMgmt.refHash) { + taosHashCleanup(smaMgmt.refHash); + smaMgmt.refHash = NULL; + } atomic_store_8(&smaMgmt.inited, 0); smaError("failed to init sma tmr handle since %s", terrstr()); return TSDB_CODE_FAILED; @@ -143,10 +138,6 @@ static int32_t tdNewSmaEnv(SSma *pSma, int8_t smaType, SSmaEnv **ppEnv) { } static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, SSmaEnv **ppEnv) { - if (!ppEnv) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } if (!(*ppEnv)) { if (tdNewSmaEnv(pSma, smaType, ppEnv) != TSDB_CODE_SUCCESS) { @@ -196,10 +187,6 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS int32_t code = 0; int32_t lino = 0; - if (ASSERTS(pSmaStat != NULL, "pSmaStat is NULL")) { - terrno = TSDB_CODE_RSMA_INVALID_ENV; - TSDB_CHECK_CODE(code, lino, _exit); - } if (*pSmaStat) { // no lock return code; // success, return directly @@ -255,15 +242,13 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS taosInitRWLatch(RSMA_FS_LOCK(pRSmaStat)); } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { // TODO - } else { - ASSERTS(0, "unknown smaType:%" PRIi8, smaType); - code = TSDB_CODE_APP_ERROR; - TSDB_CHECK_CODE(code, lino, _exit); } } _exit: if (code) { smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); + } else { + smaDebug("vgId:%d, %s succeed, type:%" PRIi8, SMA_VID(pSma), __func__, smaType); } return code; } @@ -277,12 +262,6 @@ static void tdDestroyTSmaStat(STSmaStat *pStat) { } } -static void *tdFreeTSmaStat(STSmaStat *pStat) { - tdDestroyTSmaStat(pStat); - taosMemoryFreeClear(pStat); - return NULL; -} - static void tdDestroyRSmaStat(void *pRSmaStat) { if (pRSmaStat) { SRSmaStat *pStat = (SRSmaStat *)pRSmaStat; @@ -300,11 +279,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { } else { smaDebug("vgId:%d, rsma fetch tasks are not all finished yet", SMA_VID(pSma)); } - ++nLoops; - if (nLoops > 1000) { - sched_yield(); - nLoops = 0; - } + TD_SMA_LOOPS_CHECK(nLoops, 1000); } // step 3: @@ -313,10 +288,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { // step 4: destroy the rsma info and associated fetch tasks taosHashCleanup(RSMA_INFO_HASH(pStat)); - // step 5: - tdRSmaFSClose(RSMA_FS(pStat)); - - // step 6: free pStat + // step 5: free pStat tsem_destroy(&(pStat->notEmpty)); taosMemoryFreeClear(pStat); } @@ -354,10 +326,7 @@ static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { smaDebug("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " succeed", vid, refId, smaMgmt.rsetId); } } else { - ASSERTS(0, "unknown smaType:%" PRIi8, smaType); - terrno = TSDB_CODE_APP_ERROR; - smaError("%s failed at line %d since %s", __func__, __LINE__, terrstr()); - return -1; + smaError("%s failed at line %d since Unknown type", __func__, __LINE__); } } return 0; @@ -375,11 +344,6 @@ int32_t tdLockSma(SSma *pSma) { } int32_t tdUnLockSma(SSma *pSma) { - if (ASSERTS(SMA_LOCKED(pSma), "pSma %p is not locked:%d", pSma, pSma->locked)) { - terrno = TSDB_CODE_APP_ERROR; - smaError("vgId:%d, failed to unlock since %s", SMA_VID(pSma), tstrerror(terrno)); - return -1; - } pSma->locked = false; int code = taosThreadMutexUnlock(&pSma->mutex); diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c deleted file mode 100644 index 1211ef940585a44effb47aa7f09ee69d36abf5e1..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/sma/smaFS.c +++ /dev/null @@ -1,649 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sma.h" - -// ================================================================================================= - -// static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output); -static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2); - -static FORCE_INLINE int32_t tPutQTaskF(uint8_t *p, SQTaskFile *pFile) { - int32_t n = 0; - - n += tPutI8(p ? p + n : p, pFile->level); - n += tPutI64v(p ? p + n : p, pFile->size); - n += tPutI64v(p ? p + n : p, pFile->suid); - n += tPutI64v(p ? p + n : p, pFile->version); - n += tPutI64v(p ? p + n : p, pFile->mtime); - - return n; -} - -static int32_t tdRSmaFSToBinary(uint8_t *p, SRSmaFS *pFS) { - int32_t n = 0; - uint32_t size = taosArrayGetSize(pFS->aQTaskInf); - - // version - n += tPutI8(p ? p + n : p, 0); - - // SArray - n += tPutU32v(p ? p + n : p, size); - for (uint32_t i = 0; i < size; ++i) { - n += tPutQTaskF(p ? p + n : p, taosArrayGet(pFS->aQTaskInf, i)); - } - - return n; -} - -int32_t tdRSmaGetQTaskF(uint8_t *p, SQTaskFile *pFile) { - int32_t n = 0; - - n += tGetI8(p + n, &pFile->level); - n += tGetI64v(p + n, &pFile->size); - n += tGetI64v(p + n, &pFile->suid); - n += tGetI64v(p + n, &pFile->version); - n += tGetI64v(p + n, &pFile->mtime); - - return n; -} - -static int32_t tsdbBinaryToFS(uint8_t *pData, int64_t nData, SRSmaFS *pFS) { - int32_t code = 0; - int32_t n = 0; - int8_t version = 0; - - // version - n += tGetI8(pData + n, &version); - - // SArray - taosArrayClear(pFS->aQTaskInf); - uint32_t size = 0; - n += tGetU32v(pData + n, &size); - for (uint32_t i = 0; i < size; ++i) { - SQTaskFile qTaskF = {0}; - - int32_t nt = tdRSmaGetQTaskF(pData + n, &qTaskF); - if (nt < 0) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } - - n += nt; - if (taosArrayPush(pFS->aQTaskInf, &qTaskF) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - - if (ASSERTS(n + sizeof(TSCKSUM) == nData, "n:%d + sizeof(TSCKSUM):%d != nData:%d", n, (int32_t)sizeof(TSCKSUM), - nData)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } - -_exit: - return code; -} - -static int32_t tdRSmaSaveFSToFile(SRSmaFS *pFS, const char *fname) { - int32_t code = 0; - int32_t lino = 0; - - // encode to binary - int32_t size = tdRSmaFSToBinary(NULL, pFS) + sizeof(TSCKSUM); - uint8_t *pData = taosMemoryMalloc(size); - if (pData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - tdRSmaFSToBinary(pData, pFS); - taosCalcChecksumAppend(0, pData, size); - - // save to file - TdFilePtr pFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t n = taosWriteFile(pFD, pData, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (taosFsyncFile(pFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - taosCloseFile(&pFD); - -_exit: - if (pData) taosMemoryFree(pData); - if (code) { - smaError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); - } - return code; -} - -static int32_t tdRSmaFSCreate(SRSmaFS *pFS, int32_t size) { - int32_t code = 0; - - pFS->aQTaskInf = taosArrayInit(size, sizeof(SQTaskFile)); - if (pFS->aQTaskInf == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - -_exit: - return code; -} - -static void tdRSmaGetCurrentFName(SSma *pSma, char *current, char *current_t) { - SVnode *pVnode = pSma->pVnode; - if (pVnode->pTfs) { - if (current) { - snprintf(current, TSDB_FILENAME_LEN - 1, "%s%svnode%svnode%d%srsma%sPRESENT", tfsGetPrimaryPath(pVnode->pTfs), - TD_DIRSEP, TD_DIRSEP, TD_VID(pVnode), TD_DIRSEP, TD_DIRSEP); - } - if (current_t) { - snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%svnode%svnode%d%srsma%sPRESENT.t", tfsGetPrimaryPath(pVnode->pTfs), - TD_DIRSEP, TD_DIRSEP, TD_VID(pVnode), TD_DIRSEP, TD_DIRSEP); - } - } else { -#if 0 - if (current) { - snprintf(current, TSDB_FILENAME_LEN - 1, "%s%sPRESENT", pTsdb->path, TD_DIRSEP); - } - if (current_t) { - snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%sPRESENT.t", pTsdb->path, TD_DIRSEP); - } -#endif - } -} - -static int32_t tdRSmaLoadFSFromFile(const char *fname, SRSmaFS *pFS) { - int32_t code = 0; - int32_t lino = 0; - uint8_t *pData = NULL; - - // load binary - TdFilePtr pFD = taosOpenFile(fname, TD_FILE_READ); - if (pFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t size; - if (taosFStatFile(pFD, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - pData = taosMemoryMalloc(size); - if (pData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (taosReadFile(pFD, pData, size) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (!taosCheckChecksumWhole(pData, size)) { - code = TSDB_CODE_FILE_CORRUPTED; - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); - } - - taosCloseFile(&pFD); - - // decode binary - code = tsdbBinaryToFS(pData, size, pFS); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (pData) taosMemoryFree(pData); - if (code) { - smaError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); - } - return code; -} - -static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2) { - const SQTaskFile *q1 = (const SQTaskFile *)p1; - const SQTaskFile *q2 = (const SQTaskFile *)p2; - - if (q1->suid < q2->suid) { - return -1; - } else if (q1->suid > q2->suid) { - return 1; - } - - if (q1->level < q2->level) { - return -1; - } else if (q1->level > q2->level) { - return 1; - } - - if (q1->version < q2->version) { - return -2; - } else if (q1->version > q2->version) { - return 1; - } - - return 0; -} - -static int32_t tdRSmaFSApplyChange(SSma *pSma, SRSmaFS *pFSNew) { - int32_t code = 0; - int32_t lino = 0; - int32_t nRef = 0; - SVnode *pVnode = pSma->pVnode; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SRSmaFS *pFSOld = RSMA_FS(pStat); - int64_t version = pStat->commitAppliedVer; - char fname[TSDB_FILENAME_LEN] = {0}; - - // SQTaskFile - int32_t nNew = taosArrayGetSize(pFSNew->aQTaskInf); - int32_t iNew = 0; - while (iNew < nNew) { - SQTaskFile *pQTaskFNew = TARRAY_GET_ELEM(pFSNew->aQTaskInf, iNew++); - - int32_t idx = taosArraySearchIdx(pFSOld->aQTaskInf, pQTaskFNew, tdQTaskInfCmprFn1, TD_GE); - - if (idx < 0) { - idx = taosArrayGetSize(pFSOld->aQTaskInf); - pQTaskFNew->nRef = 1; - } else { - SQTaskFile *pTaskF = TARRAY_GET_ELEM(pFSOld->aQTaskInf, idx); - int32_t c1 = tdQTaskInfCmprFn1(pQTaskFNew, pTaskF); - if (c1 == 0) { - // utilize the item in pFSOld->qQTaskInf, instead of pFSNew - continue; - } else if (c1 < 0) { - // NOTHING TODO - } else { - code = TSDB_CODE_RSMA_FS_UPDATE; - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - if (taosArrayInsert(pFSOld->aQTaskInf, idx, pQTaskFNew) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - // remove previous version - while (--idx >= 0) { - SQTaskFile *preTaskF = TARRAY_GET_ELEM(pFSOld->aQTaskInf, idx); - int32_t c2 = tdQTaskInfCmprFn1(preTaskF, pQTaskFNew); - if (c2 == 0) { - code = TSDB_CODE_RSMA_FS_UPDATE; - TSDB_CHECK_CODE(code, lino, _exit); - } else if (c2 != -2) { - break; - } - - nRef = atomic_sub_fetch_32(&preTaskF->nRef, 1); - if (nRef <= 0) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), preTaskF->suid, preTaskF->level, preTaskF->version, - tfsGetPrimaryPath(pVnode->pTfs), fname); - (void)taosRemoveFile(fname); - taosArrayRemove(pFSOld->aQTaskInf, idx); - } - } - } - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tdRSmaFSScanAndTryFix(SSma *pSma) { - int32_t code = 0; -#if 0 - int32_t lino = 0; - SVnode *pVnode = pSma->pVnode; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SRSmaFS *pFS = RSMA_FS(pStat); - char fname[TSDB_FILENAME_LEN] = {0}; - char fnameVer[TSDB_FILENAME_LEN] = {0}; - - // SArray - int32_t size = taosArrayGetSize(pFS->aQTaskInf); - for (int32_t i = 0; i < size; ++i) { - SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, i); - - // main.tdb ========= - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, pTaskF->version, - tfsGetPrimaryPath(pVnode->pTfs), fnameVer); - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, -1, tfsGetPrimaryPath(pVnode->pTfs), fname); - - if (taosCheckExistFile(fnameVer)) { - if (taosRenameFile(fnameVer, fname) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - smaDebug("vgId:%d, %s:%d succeed to to rename %s to %s", TD_VID(pVnode), __func__, lino, fnameVer, fname); - } else if (taosCheckExistFile(fname)) { - if (taosRemoveFile(fname) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - smaDebug("vgId:%d, %s:%d succeed to to remove %s", TD_VID(pVnode), __func__, lino, fname); - } - } - - { - // remove those invalid files (todo) - // main.tdb-journal.5 // TDB should handle its clear for kill -9 - } - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); - } -#endif - return code; -} - -// EXPOSED APIS ==================================================================================== - -int32_t tdRSmaFSOpen(SSma *pSma, int64_t version, int8_t rollback) { - int32_t code = 0; - int32_t lino = 0; - SVnode *pVnode = pSma->pVnode; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - - // open handle - code = tdRSmaFSCreate(RSMA_FS(pStat), 0); - TSDB_CHECK_CODE(code, lino, _exit); - - // open impl - char current[TSDB_FILENAME_LEN] = {0}; - char current_t[TSDB_FILENAME_LEN] = {0}; - tdRSmaGetCurrentFName(pSma, current, current_t); - - if (taosCheckExistFile(current)) { - code = tdRSmaLoadFSFromFile(current, RSMA_FS(pStat)); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosCheckExistFile(current_t)) { - if (rollback) { - code = tdRSmaFSRollback(pSma); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - code = tdRSmaFSCommit(pSma); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - } else { - // 1st time open with empty current/qTaskInfoFile - code = tdRSmaSaveFSToFile(RSMA_FS(pStat), current); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // scan and try fix(remove main.db/main.db.xxx and use the one with version) - code = tdRSmaFSScanAndTryFix(pSma); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -void tdRSmaFSClose(SRSmaFS *pFS) { pFS->aQTaskInf = taosArrayDestroy(pFS->aQTaskInf); } - -int32_t tdRSmaFSPrepareCommit(SSma *pSma, SRSmaFS *pFSNew) { - int32_t code = 0; - int32_t lino = 0; - char tfname[TSDB_FILENAME_LEN]; - - tdRSmaGetCurrentFName(pSma, NULL, tfname); - - // generate PRESENT.t - code = tdRSmaSaveFSToFile(pFSNew, tfname); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tdRSmaFSCommit(SSma *pSma) { - int32_t code = 0; - int32_t lino = 0; - SRSmaFS fs = {0}; - - char current[TSDB_FILENAME_LEN] = {0}; - char current_t[TSDB_FILENAME_LEN] = {0}; - tdRSmaGetCurrentFName(pSma, current, current_t); - - if (!taosCheckExistFile(current_t)) { - goto _exit; - } - - // rename the file - if (taosRenameFile(current_t, current) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // load the new FS - code = tdRSmaFSCreate(&fs, 1); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tdRSmaLoadFSFromFile(current, &fs); - TSDB_CHECK_CODE(code, lino, _exit); - - // apply file change - code = tdRSmaFSApplyChange(pSma, &fs); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - tdRSmaFSClose(&fs); - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tdRSmaFSFinishCommit(SSma *pSma) { - int32_t code = 0; - int32_t lino = 0; - SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); - - taosWLockLatch(RSMA_FS_LOCK(pStat)); - code = tdRSmaFSCommit(pSma); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); - } else { - smaInfo("vgId:%d, rsmaFS finish commit", SMA_VID(pSma)); - } - return code; -} - -int32_t tdRSmaFSRollback(SSma *pSma) { - int32_t code = 0; - int32_t lino = 0; - - char current_t[TSDB_FILENAME_LEN] = {0}; - tdRSmaGetCurrentFName(pSma, NULL, current_t); - (void)taosRemoveFile(current_t); - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(errno)); - } - return code; -} - -int32_t tdRSmaFSUpsertQTaskFile(SSma *pSma, SRSmaFS *pFS, SQTaskFile *qTaskFile, int32_t nSize) { - int32_t code = 0; - - for (int32_t i = 0; i < nSize; ++i) { - SQTaskFile *qTaskF = qTaskFile + i; - - int32_t idx = taosArraySearchIdx(pFS->aQTaskInf, qTaskF, tdQTaskInfCmprFn1, TD_GE); - - if (idx < 0) { - idx = taosArrayGetSize(pFS->aQTaskInf); - } else { - SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, idx); - int32_t c = tdQTaskInfCmprFn1(pTaskF, qTaskF); - if (c == 0) { - if (pTaskF->size != qTaskF->size) { - code = TSDB_CODE_RSMA_FS_UPDATE; - smaError("vgId:%d, %s failed at line %d since %s, level:%" PRIi8 ", suid:%" PRIi64 ", version:%" PRIi64 - ", size:%" PRIi64 " != %" PRIi64, - SMA_VID(pSma), __func__, __LINE__, tstrerror(code), pTaskF->level, pTaskF->suid, pTaskF->version, - pTaskF->size, qTaskF->size); - goto _exit; - } - continue; - } - } - - if (!taosArrayInsert(pFS->aQTaskInf, idx, qTaskF)) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - -_exit: - return code; -} - -int32_t tdRSmaFSRef(SSma *pSma, SRSmaFS *pFS) { - int32_t code = 0; - int32_t lino = 0; - int32_t nRef = 0; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SRSmaFS *qFS = RSMA_FS(pStat); - int32_t size = taosArrayGetSize(qFS->aQTaskInf); - - pFS->aQTaskInf = taosArrayInit_s(sizeof(SQTaskFile), size); - if (pFS->aQTaskInf == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - for (int32_t i = 0; i < size; ++i) { - SQTaskFile *qTaskF = (SQTaskFile *)taosArrayGet(qFS->aQTaskInf, i); - nRef = atomic_fetch_add_32(&qTaskF->nRef, 1); - if (nRef <= 0) { - code = TSDB_CODE_RSMA_FS_REF; - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - memcpy(pFS->aQTaskInf->pData, qFS->aQTaskInf->pData, size * sizeof(SQTaskFile)); - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s, nRef %d", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code), - nRef); - } - return code; -} - -void tdRSmaFSUnRef(SSma *pSma, SRSmaFS *pFS) { - int32_t nRef = 0; - char fname[TSDB_FILENAME_LEN]; - SVnode *pVnode = pSma->pVnode; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - int32_t size = taosArrayGetSize(pFS->aQTaskInf); - - for (int32_t i = 0; i < size; ++i) { - SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, i); - - nRef = atomic_sub_fetch_32(&pTaskF->nRef, 1); - if (nRef == 0) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, pTaskF->version, - tfsGetPrimaryPath(pVnode->pTfs), fname); - if (taosRemoveFile(fname) < 0) { - smaWarn("vgId:%d, failed to remove %s since %s", TD_VID(pVnode), fname, tstrerror(TAOS_SYSTEM_ERROR(errno))); - } else { - smaDebug("vgId:%d, success to remove %s", TD_VID(pVnode), fname); - } - } else if (nRef < 0) { - smaWarn("vgId:%d, abnormal unref %s since %s", TD_VID(pVnode), fname, tstrerror(TSDB_CODE_RSMA_FS_REF)); - } - } - - taosArrayDestroy(pFS->aQTaskInf); -} - -int32_t tdRSmaFSTakeSnapshot(SSma *pSma, SRSmaFS *pFS) { - int32_t code = 0; - int32_t lino = 0; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - - taosRLockLatch(RSMA_FS_LOCK(pStat)); - code = tdRSmaFSRef(pSma, pFS); - TSDB_CHECK_CODE(code, lino, _exit); -_exit: - taosRUnLockLatch(RSMA_FS_LOCK(pStat)); - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tdRSmaFSCopy(SSma *pSma, SRSmaFS *pFS) { - int32_t code = 0; - int32_t lino = 0; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SRSmaFS *qFS = RSMA_FS(pStat); - int32_t size = taosArrayGetSize(qFS->aQTaskInf); - - code = tdRSmaFSCreate(pFS, size); - TSDB_CHECK_CODE(code, lino, _exit); - taosArrayAddBatch(pFS->aQTaskInf, qFS->aQTaskInf->pData, size); - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 00000cb12914d7a0fbda6b2218c09c1726e51e17..4dc3e45ffe93ce391317d2f6b2cf3fff13a9c874 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -101,10 +101,6 @@ int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int ty terrno = 0; pKeepCfg->precision = pCfg->precision; switch (type) { - case TSDB_TYPE_TSMA: - ASSERTS(0, "undefined smaType:%d", (int32_t)type); - terrno = TSDB_CODE_APP_ERROR; - break; case TSDB_TYPE_RSMA_L0: SMA_SET_KEEP_CFG(pVnode, 0); break; @@ -115,7 +111,6 @@ int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int ty SMA_SET_KEEP_CFG(pVnode, 2); break; default: - ASSERTS(0, "unknown smaType:%d", (int32_t)type); terrno = TSDB_CODE_APP_ERROR; break; } @@ -189,8 +184,7 @@ int32_t smaClose(SSma *pSma) { */ int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback) { if (!VND_IS_RSMA(pSma->pVnode)) { - terrno = TSDB_CODE_RSMA_INVALID_ENV; - return TSDB_CODE_FAILED; + return TSDB_CODE_RSMA_INVALID_ENV; } return tdRSmaProcessRestoreImpl(pSma, type, committedVer, rollback); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 39aa5c30437a72f6ad26f22f4c2a01aa03a8c7dc..9fd493844877d343e07d9f251b6b0882f25f89e0 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -14,6 +14,7 @@ */ #include "sma.h" +#include "tq.h" #define RSMA_QTASKEXEC_SMOOTH_SIZE (100) // cnt #define RSMA_SUBMIT_BATCH_SIZE (1024) // cnt @@ -30,6 +31,8 @@ SSmaMgmt smaMgmt = { typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; +extern int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now); + static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static void tdUidStoreDestory(STbUidStore *pStore); static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd); @@ -44,7 +47,6 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo); static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid); static void tdRSmaFetchTrigger(void *param, void *tmrId); -static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo); static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables); static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int8_t type, int64_t qTaskFileVer); @@ -64,10 +66,7 @@ static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t l if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { smaDebug("vgId:%d, free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); qDestroyTask(otaskHandle); - } else { - smaDebug("vgId:%d, not free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); } - // TODO: clear files related to qTaskInfo? } /** @@ -95,16 +94,6 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { if (isDeepFree && pInfo->taskInfo[i]) { tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1); - } else { - smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma), - pInfo->suid, i + 1); - } - - if (pInfo->iTaskInfo[i]) { - tdRSmaQTaskInfoFree(&pInfo->iTaskInfo[i], SMA_VID(pSma), i + 1); - } else { - smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty iTaskInfo", - SMA_VID(pSma), pInfo->suid, i + 1); } } if (isDeepFree) { @@ -112,14 +101,14 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { } if (isDeepFree) { - if (pInfo->queue) taosCloseQueue(pInfo->queue); - if (pInfo->qall) taosFreeQall(pInfo->qall); - if (pInfo->iQueue) taosCloseQueue(pInfo->iQueue); - if (pInfo->iQall) taosFreeQall(pInfo->iQall); - pInfo->queue = NULL; - pInfo->qall = NULL; - pInfo->iQueue = NULL; - pInfo->iQall = NULL; + if (pInfo->queue) { + taosCloseQueue(pInfo->queue); + pInfo->queue = NULL; + } + if (pInfo->qall) { + taosFreeQall(pInfo->qall); + pInfo->qall = NULL; + } } taosMemoryFree(pInfo); @@ -129,11 +118,6 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { } static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { - if (ASSERTS(*pStore == NULL, "*pStore:%p != NULL", *pStore)) { - terrno = TSDB_CODE_APP_ERROR; - return TSDB_CODE_FAILED; - } - *pStore = taosMemoryCalloc(1, sizeof(STbUidStore)); if (*pStore == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -260,23 +244,27 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat void *pStreamState = NULL; // set the backend of stream state - tdRSmaQTaskInfoGetFullPathEx(TD_VID(pVnode), pRSmaInfo->suid, idx + 1, tfsGetPrimaryPath(pVnode->pTfs), taskInfDir); + tdRSmaQTaskInfoGetFullPath(pVnode, pRSmaInfo->suid, idx + 1, pVnode->pTfs, taskInfDir); + if (!taosCheckExistFile(taskInfDir)) { char *s = taosStrdup(taskInfDir); - if (taosMulMkDir(taosDirName(s)) != 0) { + if (taosMulMkDir(s) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosMemoryFree(s); return TSDB_CODE_FAILED; } taosMemoryFree(s); } - pStreamState = streamStateOpen(taskInfDir, NULL, true, -1, -1); + + SStreamTask task = {.id.taskId = 0, .id.streamId = 0}; // TODO: assign value + task.pMeta = pVnode->pTq->pStreamMeta; + pStreamState = streamStateOpen(taskInfDir, &task, true, -1, -1); if (!pStreamState) { terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; return TSDB_CODE_FAILED; } - SReadHandle handle = { .vnode = pVnode, .initTqReader = 1, .pStateBackend = pStreamState }; + SReadHandle handle = {.vnode = pVnode, .initTqReader = 1, .pStateBackend = pStreamState}; initStorageAPI(&handle.api); pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode)); @@ -300,11 +288,6 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat pItem->level = idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2; - if (ASSERTS(pItem->level > 0, "pItem level:%" PRIi8 " should > 0", pItem->level)) { - terrno = TSDB_CODE_APP_ERROR; - return TSDB_CODE_FAILED; - } - SRSmaRef rsmaRef = {.refId = pStat->refId, .suid = pRSmaInfo->suid}; taosHashPut(smaMgmt.refHash, &pItem, POINTER_BYTES, &rsmaRef, sizeof(rsmaRef)); @@ -366,25 +349,10 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con pRSmaInfo->pTSchema = pTSchema; pRSmaInfo->suid = suid; T_REF_INIT_VAL(pRSmaInfo, 1); - if (!(pRSmaInfo->queue = taosOpenQueue())) { - goto _err; - } - - if (!(pRSmaInfo->qall = taosAllocateQall())) { - goto _err; - } - if (!(pRSmaInfo->iQueue = taosOpenQueue())) { - goto _err; - } - if (!(pRSmaInfo->iQall = taosAllocateQall())) { - goto _err; - } - - if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) { - goto _err; - } - if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 1) < 0) { + if (!(pRSmaInfo->queue = taosOpenQueue()) || !(pRSmaInfo->qall = taosAllocateQall()) || + tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0 || + tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 1) < 0) { goto _err; } @@ -562,15 +530,12 @@ void *tdUidStoreFree(STbUidStore *pStore) { * @return int32_t */ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { - if (!pReq) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - SSubmitReq2 *pSubmitReq = (SSubmitReq2 *)pReq; - // spin lock for race condition during insert data - if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { - return TSDB_CODE_FAILED; + if (pReq) { + SSubmitReq2 *pSubmitReq = (SSubmitReq2 *)pReq; + // spin lock for race condition during insert data + if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { + return TSDB_CODE_FAILED; + } } return TSDB_CODE_SUCCESS; @@ -592,7 +557,6 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq2 *pMsg, STbUidStore *pStore) { return 0; } -#if 0 /** * @brief retention of rsma1/rsma2 * @@ -608,56 +572,39 @@ int32_t smaDoRetention(SSma *pSma, int64_t now) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { - code = tsdbDoRetention(pSma->pRSmaTsdb[i], now); - if (code) goto _end; + // code = tsdbDoRetention(pSma->pRSmaTsdb[i], now); + // if (code) goto _end; } } _end: return code; } -#endif - -static void tdBlockDataDestroy(SArray *pBlockArr) { - for (int32_t i = 0; i < taosArrayGetSize(pBlockArr); ++i) { - blockDataDestroy(taosArrayGetP(pBlockArr, i)); - } - taosArrayDestroy(pBlockArr); -} static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid) { + int32_t code = 0; + int32_t lino = 0; + SSDataBlock *output = NULL; + SArray *pResList = taosArrayInit(1, POINTER_BYTES); if (pResList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } while (1) { uint64_t ts; bool hasMore = false; - int32_t code = qExecTaskOpt(taskInfo, pResList, &ts, &hasMore, NULL); - if (code < 0) { - if (code == TSDB_CODE_QRY_IN_EXEC) { - break; - } else { - smaError("vgId:%d, qExecTask for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, - pItem->level, terrstr(code)); - goto _err; - } + code = qExecTaskOpt(taskInfo, pResList, &ts, &hasMore, NULL); + if (code == TSDB_CODE_QRY_IN_EXEC) { + code = 0; + break; } + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayGetSize(pResList) == 0) { - if (terrno == 0) { - // smaDebug("vgId:%d, no rsma level %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level); - } else { - smaDebug("vgId:%d, no rsma level %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, terrstr()); - goto _err; - } - break; - } else { - smaDebug("vgId:%d, rsma level %" PRIi8 " data fetched", SMA_VID(pSma), pItem->level); } #if 0 char flag[10] = {0}; @@ -665,28 +612,24 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma blockDebugShowDataBlocks(pResList, flag); #endif for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) { - SSDataBlock *output = taosArrayGetP(pResList, i); - smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%" PRId64, output->info.id.uid, - output->info.id.groupId, output->info.rows); + output = taosArrayGetP(pResList, i); + smaDebug("vgId:%d, result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%" PRIi64, SMA_VID(pSma), + output->info.id.uid, output->info.id.groupId, output->info.rows); STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); SSubmitReq2 *pReq = NULL; // TODO: the schema update should be handled later(TD-17965) if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) { - smaError("vgId:%d, build submit req for rsma table suid:%" PRIu64 ", uid:%" PRIu64 ", level %" PRIi8 - " failed since %s", - SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); - goto _err; + code = terrno ? terrno : TSDB_CODE_RSMA_RESULT; + TSDB_CHECK_CODE(code, lino, _exit); } if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { + code = terrno ? terrno : TSDB_CODE_RSMA_RESULT; tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); - smaError("vgId:%d, process submit req for rsma suid:%" PRIu64 ", uid:%" PRIu64 " level %" PRIi8 - " failed since %s", - SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64, @@ -698,15 +641,18 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } } } - - taosArrayDestroy(pResList); - qCleanExecTaskBlockBuf(taskInfo); - return TSDB_CODE_SUCCESS; - -_err: +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, suid:%" PRIi64 ", level:%" PRIi8 ", uid:%" PRIi64 + ", ver:%" PRIi64, + SMA_VID(pSma), __func__, lino, tstrerror(code), suid, pItem->level, output ? output->info.id.uid : -1, + output ? output->info.version : -1); + } else { + smaDebug("vgId:%d, %s succeed, suid:%" PRIi64 ", level:%" PRIi8, SMA_VID(pSma), __func__, suid, pItem->level); + } taosArrayDestroy(pResList); qCleanExecTaskBlockBuf(taskInfo); - return TSDB_CODE_FAILED; + return code; } /** @@ -800,7 +746,8 @@ static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo, ERsmaExecType type, int8_t level) { int32_t idx = level - 1; - void *qTaskInfo = (type == RSMA_EXEC_COMMIT) ? RSMA_INFO_IQTASK(pInfo, idx) : RSMA_INFO_QTASK(pInfo, idx); + void *qTaskInfo = RSMA_INFO_QTASK(pInfo, idx); + if (!qTaskInfo) { smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, pInfo->suid); @@ -833,109 +780,6 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, return TSDB_CODE_SUCCESS; } -static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t srcTaskInfo, SRSmaParam *param, - tb_uid_t suid, int8_t idx) { - int32_t code = 0; - int32_t lino = 0; - SVnode *pVnode = pSma->pVnode; - char *pOutput = NULL; - int32_t len = 0; - - if (!srcTaskInfo) { - code = TSDB_CODE_INVALID_PTR; - smaWarn("vgId:%d, rsma clone, table %" PRIi64 ", no need since srcTaskInfo is NULL", TD_VID(pVnode), suid); - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len); - TSDB_CHECK_CODE(code, lino, _exit); - - SReadHandle handle = { .vnode = pVnode, .initTqReader = 1 }; - initStorageAPI(&handle.api); - - if (ASSERTS(!dstTaskInfo, "dstTaskInfo:%p is not NULL", dstTaskInfo)) { - code = TSDB_CODE_APP_ERROR; - TSDB_CHECK_CODE(code, lino, _exit); - } - - dstTaskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode)); - if (!dstTaskInfo) { - code = TSDB_CODE_RSMA_QTASKINFO_CREATE; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = qDeserializeTaskStatus(dstTaskInfo, pOutput, len); - TSDB_CHECK_CODE(code, lino, _exit); - - smaDebug("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " succeed", TD_VID(pVnode), suid); - -_exit: - taosMemoryFreeClear(pOutput); - if (code) { - tdRSmaQTaskInfoFree(dstTaskInfo, TD_VID(pVnode), idx + 1); - smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, - terrstr()); - } - return code; -} - -/** - * @brief Clone qTaskInfo of SRSmaInfo - * - * @param pSma - * @param pInfo - * @return int32_t - */ -static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo) { - int32_t code = 0; - int32_t lino = 0; - SRSmaParam *param = NULL; - SMetaReader mr = {0}; - - if (!pInfo) { - return TSDB_CODE_SUCCESS; - } - - metaReaderInit(&mr, SMA_META(pSma), 0); - smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid); - if (metaReaderGetTableEntryByUidCache(&mr, pInfo->suid) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (mr.me.type != TSDB_SUPER_TABLE) { - code = TSDB_CODE_RSMA_INVALID_SCHEMA; - TSDB_CHECK_CODE(code, lino, _exit); - } - if (mr.me.uid != pInfo->suid) { - code = TSDB_CODE_RSMA_INVALID_SCHEMA; - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (TABLE_IS_ROLLUP(mr.me.flags)) { - param = &mr.me.stbEntry.rsmaParam; - for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - if (!pInfo->iTaskInfo[i]) { - continue; - } - code = tdCloneQTaskInfo(pSma, pInfo->taskInfo[i], pInfo->iTaskInfo[i], param, pInfo->suid, i); - TSDB_CHECK_CODE(code, lino, _exit); - } - smaDebug("vgId:%d, rsma clone env success for %" PRIi64, SMA_VID(pSma), pInfo->suid); - } else { - code = TSDB_CODE_RSMA_INVALID_SCHEMA; - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s, suid:%" PRIi64 ", flags:%" PRIi8 ",type:%" PRIi8 ", uid:%" PRIi64, - SMA_VID(pSma), __func__, lino, tstrerror(code), pInfo->suid, mr.me.flags, mr.me.type, mr.me.uid); - } - metaReaderClear(&mr); - return code; -} - /** * @brief During async commit, the SRSmaInfo object would be COW from iRSmaInfoHash and write lock should be applied. * @@ -970,12 +814,7 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } - if (!pRSmaInfo->taskInfo[0]) { - if ((terrno = tdRSmaInfoClone(pSma, pRSmaInfo)) < 0) { - taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); - return NULL; - } - } + tdRefRSmaInfo(pSma, pRSmaInfo); taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); if (ASSERTS(pRSmaInfo->suid == suid, "suid:%" PRIi64 " != %" PRIi64, pRSmaInfo->suid, suid)) { @@ -1116,7 +955,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { } int64_t nRsmaTables = 0; - metaReaderInit(&mr, SMA_META(pSma), 0); + metaReaderDoInit(&mr, SMA_META(pSma), 0); if (!(uidStore.tbUids = taosArrayInit(1024, sizeof(tb_uid_t)))) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); @@ -1187,58 +1026,43 @@ _exit: * N.B. the data would be restored from the unified WAL replay procedure */ int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer, int8_t rollback) { + int32_t code = 0; + int64_t nTables = 0; + // step 1: init env if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TDB_INIT_FAILED; - return TSDB_CODE_FAILED; - } - - // step 2: open SRSmaFS for qTaskFiles - if (tdRSmaFSOpen(pSma, qtaskFileVer, rollback) < 0) { + code = TSDB_CODE_TDB_INIT_FAILED; goto _err; } - // step 3: iterate all stables to restore the rsma env - int64_t nTables = 0; - if (tdRSmaRestoreQTaskInfoInit(pSma, &nTables) < 0) { + // step 2: iterate all stables to restore the rsma env + if ((code = tdRSmaRestoreQTaskInfoInit(pSma, &nTables)) < 0) { goto _err; } - if (nTables <= 0) { - smaDebug("vgId:%d, no need to restore rsma task %" PRIi8 " since no tables", SMA_VID(pSma), type); - return TSDB_CODE_SUCCESS; - } - smaInfo("vgId:%d, restore rsma task %" PRIi8 " from qtaskf %" PRIi64 " succeed", SMA_VID(pSma), type, qtaskFileVer); - return TSDB_CODE_SUCCESS; _err: - smaError("vgId:%d, restore rsma task %" PRIi8 "from qtaskf %" PRIi64 " failed since %s", SMA_VID(pSma), type, - qtaskFileVer, terrstr()); - return TSDB_CODE_FAILED; -} + if (code) { + smaError("vgId:%d, restore rsma task %" PRIi8 "from qtaskf %" PRIi64 " failed since %s", SMA_VID(pSma), type, + qtaskFileVer, tstrerror(code)); + } else { + smaInfo("vgId:%d, restore rsma task %" PRIi8 " from qtaskf %" PRIi64 " succeed, nTables:%" PRIi64, SMA_VID(pSma), + type, qtaskFileVer, nTables); + } + return code; +} +#if 0 int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { - int32_t code = 0; - int32_t lino = 0; - SSma *pSma = pRSmaStat->pSma; - SVnode *pVnode = pSma->pVnode; - SArray *qTaskFArray = NULL; - int64_t version = pRSmaStat->commitAppliedVer; - TdFilePtr pOutFD = NULL; - TdFilePtr pInFD = NULL; - char fname[TSDB_FILENAME_LEN]; - char fnameVer[TSDB_FILENAME_LEN]; - SRSmaFS fs = {0}; + int32_t code = 0; + int32_t lino = 0; + SSma *pSma = pRSmaStat->pSma; + SVnode *pVnode = pSma->pVnode; + SRSmaFS fs = {0}; if (taosHashGetSize(pInfoHash) <= 0) { return TSDB_CODE_SUCCESS; } - qTaskFArray = taosArrayInit(taosHashGetSize(pInfoHash) << 1, sizeof(SQTaskFile)); - if (!qTaskFArray) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - void *infoHash = NULL; while ((infoHash = taosHashIterate(pInfoHash, infoHash))) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; @@ -1256,76 +1080,19 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { } smaDebug("vgId:%d, rsma persist, stream state commit success, table %" PRIi64 ", level %d", TD_VID(pVnode), pRSmaInfo->suid, i + 1); - - // qTaskInfo file - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pRSmaInfo->suid, i + 1, -1, tfsGetPrimaryPath(pVnode->pTfs), fname); - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pRSmaInfo->suid, i + 1, version, tfsGetPrimaryPath(pVnode->pTfs), - fnameVer); - if (taosCheckExistFile(fnameVer)) { - smaWarn("vgId:%d, rsma persist, duplicate file %s exist", TD_VID(pVnode), fnameVer); - } - - pOutFD = taosCreateFile(fnameVer, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - pInFD = taosOpenFile(fname, TD_FILE_READ); - if (pInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t size = 0; - uint32_t mtime = 0; - if (taosFStatFile(pInFD, &size, &mtime) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t offset = 0; - if (taosFSendFile(pOutFD, pInFD, &offset, size) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - smaError("vgId:%d, rsma persist, send qtaskinfo file %s to %s failed since %s", TD_VID(pVnode), fname, - fnameVer, tstrerror(code)); - TSDB_CHECK_CODE(code, lino, _exit); - } - taosCloseFile(&pOutFD); - taosCloseFile(&pInFD); - - SQTaskFile qTaskF = { - .nRef = 1, .level = i + 1, .suid = pRSmaInfo->suid, .version = version, .size = size, .mtime = mtime}; - - taosArrayPush(qTaskFArray, &qTaskF); } } } - // prepare - code = tdRSmaFSCopy(pSma, &fs); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tdRSmaFSUpsertQTaskFile(pSma, &fs, qTaskFArray->pData, taosArrayGetSize(qTaskFArray)); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tdRSmaFSPrepareCommit(pSma, &fs); - TSDB_CHECK_CODE(code, lino, _exit); - _exit: - - taosArrayDestroy(fs.aQTaskInf); - taosArrayDestroy(qTaskFArray); - if (code) { - if (pOutFD) taosCloseFile(&pOutFD); - if (pInFD) taosCloseFile(&pInFD); smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } terrno = code; return code; } - +#endif /** * @brief trigger to get rsma result in async mode * @@ -1346,8 +1113,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { } if (!(pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaRef->refId))) { - smaDebug("rsma fetch task not start since rsma stat already destroyed, rsetId:%d refId:%" PRIi64 ")", - smaMgmt.rsetId, pRSmaRef->refId); // pRSmaRef freed in taosHashRemove + smaWarn("rsma fetch task not start since rsma stat already destroyed, rsetId:%d refId:%" PRIi64 ")", smaMgmt.rsetId, + pRSmaRef->refId); // pRSmaRef freed in taosHashRemove taosHashRemove(smaMgmt.refHash, ¶m, POINTER_BYTES); return; } diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index c00e96a06664db0a60184fdb09e16ee0b68c3d45..e01a33936b5673464ac7d7bdb15019aabdcd1e0b 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -15,9 +15,6 @@ #include "sma.h" -static int32_t rsmaSnapReadQTaskInfo(SRSmaSnapReader* pReader, uint8_t** ppData); -static int32_t rsmaSnapWriteQTaskInfo(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); - // SRSmaSnapReader ======================================== struct SRSmaSnapReader { SSma* pSma; @@ -28,11 +25,6 @@ struct SRSmaSnapReader { // for data file int8_t rsmaDataDone[TSDB_RETENTION_L2]; STsdbSnapReader* pDataReader[TSDB_RETENTION_L2]; - - // for qtaskinfo file - int8_t qTaskDone; - int32_t fsIter; - SQTaskFReader* pQTaskFReader; }; int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapReader** ppReader) { @@ -62,22 +54,6 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapRead } } - // open qtaskinfo - taosRLockLatch(RSMA_FS_LOCK(pStat)); - code = tdRSmaFSRef(pSma, &pReader->fs); - taosRUnLockLatch(RSMA_FS_LOCK(pStat)); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosArrayGetSize(pReader->fs.aQTaskInf) > 0) { - pReader->pQTaskFReader = taosMemoryCalloc(1, sizeof(SQTaskFReader)); - if (!pReader->pQTaskFReader) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - pReader->pQTaskFReader->pSma = pSma; - pReader->pQTaskFReader->version = pReader->ever; - } - *ppReader = pReader; _exit: if (code) { @@ -88,114 +64,6 @@ _exit: return code; } -static int32_t rsmaSnapReadQTaskInfo(SRSmaSnapReader* pReader, uint8_t** ppBuf) { - int32_t code = 0; - int32_t lino = 0; - SVnode* pVnode = pReader->pSma->pVnode; - SQTaskFReader* qReader = pReader->pQTaskFReader; - SRSmaFS* pFS = &pReader->fs; - int64_t n = 0; - uint8_t* pBuf = NULL; - int64_t version = pReader->ever; - char fname[TSDB_FILENAME_LEN]; - - if (!qReader) { - *ppBuf = NULL; - smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, not needed since qTaskReader is NULL", TD_VID(pVnode)); - goto _exit; - } - - if (pReader->fsIter >= taosArrayGetSize(pFS->aQTaskInf)) { - *ppBuf = NULL; - smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, fsIter reach end", TD_VID(pVnode)); - goto _exit; - } - - while (pReader->fsIter < taosArrayGetSize(pFS->aQTaskInf)) { - SQTaskFile* qTaskF = taosArrayGet(pFS->aQTaskInf, pReader->fsIter++); - if (qTaskF->version != version) { - continue; - } - - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), qTaskF->suid, qTaskF->level, version, tfsGetPrimaryPath(pVnode->pTfs), - fname); - if (!taosCheckExistFile(fname)) { - smaError("vgId:%d, vnode snapshot rsma reader for qtaskinfo, table %" PRIi64 ", level %" PRIi8 - ", version %" PRIi64 " failed since %s not exist", - TD_VID(pVnode), qTaskF->suid, qTaskF->level, version, fname); - code = TSDB_CODE_RSMA_FS_SYNC; - TSDB_CHECK_CODE(code, lino, _exit); - } - - TdFilePtr fp = taosOpenFile(fname, TD_FILE_READ); - if (!fp) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - qReader->pReadH = fp; - qReader->level = qTaskF->level; - qReader->suid = qTaskF->suid; - } - - if (!qReader->pReadH) { - *ppBuf = NULL; - smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, not needed since readh is NULL", TD_VID(pVnode)); - goto _exit; - } - - int64_t size = 0; - if (taosFStatFile(qReader->pReadH, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // seek - if (taosLSeekFile(qReader->pReadH, 0, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (*ppBuf) { - *ppBuf = taosMemoryRealloc(*ppBuf, sizeof(SSnapDataHdr) + size); - } else { - *ppBuf = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); - } - if (!(*ppBuf)) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - // read - n = taosReadFile(qReader->pReadH, POINTER_SHIFT(*ppBuf, sizeof(SSnapDataHdr)), size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } else if (n != size) { - code = TSDB_CODE_FILE_CORRUPTED; - TSDB_CHECK_CODE(code, lino, _exit); - } - - smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo, version:%" PRIi64 ", size:%" PRIi64, TD_VID(pVnode), version, - size); - - SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppBuf); - pHdr->type = SNAP_DATA_QTASK; - pHdr->flag = qReader->level; - pHdr->index = qReader->suid; - pHdr->size = size; - -_exit: - if (qReader) taosCloseFile(&qReader->pReadH); - - if (code) { - *ppBuf = NULL; - smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); - } else { - smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo succeed", TD_VID(pVnode)); - } - return code; -} - int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; int32_t lino = 0; @@ -223,18 +91,6 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { } } - // read qtaskinfo file - if (!pReader->qTaskDone) { - smaInfo("vgId:%d, vnode snapshot rsma qtaskinfo not done", SMA_VID(pReader->pSma)); - code = rsmaSnapReadQTaskInfo(pReader, ppData); - TSDB_CHECK_CODE(code, lino, _exit); - if (*ppData) { - goto _exit; - } else { - pReader->qTaskDone = 1; - } - } - _exit: if (code) { smaError("vgId:%d, vnode snapshot rsma read failed since %s", SMA_VID(pReader->pSma), tstrerror(code)); @@ -249,9 +105,6 @@ int32_t rsmaSnapReaderClose(SRSmaSnapReader** ppReader) { int32_t code = 0; SRSmaSnapReader* pReader = *ppReader; - tdRSmaFSUnRef(pReader->pSma, &pReader->fs); - taosMemoryFreeClear(pReader->pQTaskFReader); - for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pReader->pDataReader[i]) { tsdbSnapReaderClose(&pReader->pDataReader[i]); @@ -299,10 +152,6 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWrit } } - // qtaskinfo - code = tdRSmaFSCopy(pSma, &pWriter->fs); - TSDB_CHECK_CODE(code, lino, _exit); - // snapWriter *ppWriter = pWriter; _exit: @@ -316,22 +165,6 @@ _exit: return code; } -int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter) { - int32_t code = 0; - int32_t lino = 0; - - if (pWriter) { - code = tdRSmaFSPrepareCommit(pWriter->pSma, &pWriter->fs); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pWriter->pSma), __func__, lino, tstrerror(code)); - } - return code; -} - int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { int32_t code = 0; int32_t lino = 0; @@ -340,7 +173,6 @@ int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { SSmaEnv* pEnv = NULL; SRSmaStat* pStat = NULL; SRSmaSnapWriter* pWriter = *ppWriter; - const char* primaryPath = NULL; char fname[TSDB_FILENAME_LEN] = {0}; char fnameVer[TSDB_FILENAME_LEN] = {0}; TdFilePtr pOutFD = NULL; @@ -354,7 +186,6 @@ int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { pVnode = pSma->pVnode; pEnv = SMA_RSMA_ENV(pSma); pStat = (SRSmaStat*)SMA_ENV_STAT(pEnv); - primaryPath = tfsGetPrimaryPath(pVnode->pTfs); // rsma1/rsma2 for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { @@ -364,61 +195,6 @@ int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { } } - // qtaskinfo - if (rollback) { - tdRSmaFSRollback(pSma); - // remove qTaskFiles - } else { - // sendFile from fname.Ver to fname - SRSmaFS* pFS = &pWriter->fs; - int32_t size = taosArrayGetSize(pFS->aQTaskInf); - for (int32_t i = 0; i < size; ++i) { - SQTaskFile* pTaskF = TARRAY_GET_ELEM(pFS->aQTaskInf, i); - if (pTaskF->version == pWriter->ever) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, pTaskF->version, primaryPath, fnameVer); - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, -1, primaryPath, fname); - - pInFD = taosOpenFile(fnameVer, TD_FILE_READ); - if (pInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - pOutFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t size = 0; - if (taosFStatFile(pInFD, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t offset = 0; - if (taosFSendFile(pOutFD, pInFD, &offset, size) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - smaError("vgId:%d, vnode snapshot rsma writer, send qtaskinfo file %s to %s failed since %s", TD_VID(pVnode), - fnameVer, fname, tstrerror(code)); - TSDB_CHECK_CODE(code, lino, _exit); - } - taosCloseFile(&pOutFD); - taosCloseFile(&pInFD); - } - } - - // lock - taosWLockLatch(RSMA_FS_LOCK(pStat)); - code = tdRSmaFSCommit(pSma); - if (code) { - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); - goto _exit; - } - // unlock - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); - } - // rsma restore code = tdRSmaRestore(pWriter->pSma, RSMA_RESTORE_SYNC, pWriter->ever, rollback); TSDB_CHECK_CODE(code, lino, _exit); @@ -450,8 +226,6 @@ int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) } else if (pHdr->type == SNAP_DATA_RSMA2) { pHdr->type = SNAP_DATA_TSDB; code = tsdbSnapWrite(pWriter->pDataWriter[1], pHdr); - } else if (pHdr->type == SNAP_DATA_QTASK) { - code = rsmaSnapWriteQTaskInfo(pWriter, pData, nData); } else { code = TSDB_CODE_RSMA_FS_SYNC; } @@ -466,68 +240,3 @@ _exit: } return code; } - -static int32_t rsmaSnapWriteQTaskInfo(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - int32_t lino = 0; - SSma* pSma = pWriter->pSma; - SVnode* pVnode = pSma->pVnode; - char fname[TSDB_FILENAME_LEN]; - TdFilePtr fp = NULL; - SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; - - fname[0] = '\0'; - - if (pHdr->size != (nData - sizeof(SSnapDataHdr))) { - code = TSDB_CODE_RSMA_FS_SYNC; - TSDB_CHECK_CODE(code, lino, _exit); - } - - SQTaskFile qTaskFile = { - .nRef = 1, .level = pHdr->flag, .suid = pHdr->index, .version = pWriter->ever, .size = pHdr->size}; - - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pHdr->index, pHdr->flag, qTaskFile.version, - tfsGetPrimaryPath(pVnode->pTfs), fname); - - fp = taosCreateFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (!fp) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - int64_t contLen = taosWriteFile(fp, pHdr->data, pHdr->size); - if (contLen != pHdr->size) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - uint32_t mtime = 0; - if (taosFStatFile(fp, NULL, &mtime) != 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - qTaskFile.mtime = mtime; - } - - if (taosFsyncFile(fp) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); - } - - taosCloseFile(&fp); - - code = tdRSmaFSUpsertQTaskFile(pSma, &pWriter->fs, &qTaskFile, 1); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - if (fp) { - (void)taosRemoveFile(fname); - } - smaError("vgId:%d, %s failed at line %d since %s, file:%s", TD_VID(pVnode), __func__, lino, tstrerror(code), fname); - } else { - smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo %s succeed", TD_VID(pVnode), fname); - } - - return code; -} diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 84cd44e837dc1edab8a130dc7598d9e4e7bab115..0c37008344054d9af74894cc4825c53d079e2b3e 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -29,27 +29,21 @@ int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) { - smaError("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + smaError("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(code)); } return code; } int32_t tdProcessTSmaCreate(SSma *pSma, int64_t version, const char *msg) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = tdProcessTSmaCreateImpl(pSma, version, msg); - if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) { - smaWarn("vgId:%d, create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } return code; } int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days)) < 0) { - smaWarn("vgId:%d, get tsma days failed since %s", pCfg->vgId, tstrerror(terrno)); - } - smaDebug("vgId:%d, get tsma days %d", pCfg->vgId, *days); + int32_t code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days); + return code; } @@ -63,19 +57,22 @@ int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t * * @return int32_t */ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { + int32_t code = 0; + int32_t lino = 0; SDecoder coder = {0}; tDecoderInit(&coder, pCont, contLen); STSma tsma = {0}; if (tDecodeSVCreateTSmaReq(&coder, &tsma) < 0) { - terrno = TSDB_CODE_MSG_DECODE_ERROR; - goto _err; + code = TSDB_CODE_MSG_DECODE_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); } + STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg; int64_t sInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND); if (sInterval <= 0) { *days = pTsdbCfg->days; - return 0; + goto _exit; } int64_t records = pTsdbCfg->days * 60 / sInterval; if (records >= SMA_STORAGE_SPLIT_FACTOR) { @@ -94,11 +91,14 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c *days = pTsdbCfg->days; } } +_exit: + if (code) { + smaWarn("vgId:%d, failed at line %d to get tsma days %d since %s", pCfg->vgId, lino, *days, tstrerror(code)); + } else { + smaDebug("vgId:%d, succeed to get tsma days %d", pCfg->vgId, *days); + } tDecoderClear(&coder); - return 0; -_err: - tDecoderClear(&coder); - return -1; + return code; } /** @@ -157,6 +157,8 @@ _exit: int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema, SSchemaWrapper *pTagSchemaWrapper, bool createTb, int64_t suid, const char *stbFullName, SBatchDeleteReq *pDeleteReq, void **ppData, int32_t *pLen) { + int32_t code = 0; + int32_t lino = 0; void *pBuf = NULL; int32_t len = 0; SSubmitReq2 *pReq = NULL; @@ -166,21 +168,14 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * int32_t sz = taosArrayGetSize(pBlocks); - if (!(tagArray = taosArrayInit(1, sizeof(STagVal)))) { - goto _end; - } - - if (!(createTbArray = taosArrayInit(sz, POINTER_BYTES))) { - goto _end; - } - - if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } + tagArray = taosArrayInit(1, sizeof(STagVal)); + createTbArray = taosArrayInit(sz, POINTER_BYTES); + pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)); + pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)); - if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { - goto _end; + if(!tagArray || !createTbArray || !pReq || !pReq->aSubmitTbData) { + code = terrno == TSDB_CODE_SUCCESS ? TSDB_CODE_OUT_OF_MEMORY : terrno; + TSDB_CHECK_CODE(code, lino, _exit); } // create table req @@ -194,7 +189,8 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * } if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { - goto _end; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); }; // don't move to the end of loop as to destroy in the end of func when error occur @@ -223,8 +219,8 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * STag *pTag = NULL; tTagNew(tagArray, 1, false, &pTag); if (pTag == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } pCreateTbReq->ctb.pTag = (uint8_t *)pTag; @@ -259,7 +255,8 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * SSubmitTbData tbData = {0}; if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow *)))) { - goto _end; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } tbData.suid = suid; tbData.uid = 0; // uid is assigned by vnode @@ -272,7 +269,8 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { taosArrayDestroy(tbData.aRowP); - goto _end; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } for (int32_t j = 0; j < rows; ++j) { @@ -298,9 +296,9 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * } } SRow *pRow = NULL; - if ((terrno = tRowBuild(pVals, (STSchema *)pTSchema, &pRow)) < 0) { + if ((code = tRowBuild(pVals, (STSchema *)pTSchema, &pRow)) < 0) { tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); - goto _end; + TSDB_CHECK_CODE(code, lino, _exit); } taosArrayPush(tbData.aRowP, &pRow); } @@ -309,25 +307,27 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * } // encode - tEncodeSize(tEncodeSubmitReq, pReq, len, terrno); - if (TSDB_CODE_SUCCESS == terrno) { + tEncodeSize(tEncodeSubmitReq, pReq, len, code); + if (TSDB_CODE_SUCCESS == code) { SEncoder encoder; len += sizeof(SSubmitReq2Msg); - pBuf = rpcMallocCont(len); - if (NULL == pBuf) { - goto _end; + if (!(pBuf = rpcMallocCont(len))) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } + ((SSubmitReq2Msg *)pBuf)->header.vgId = TD_VID(pVnode); ((SSubmitReq2Msg *)pBuf)->header.contLen = htonl(len); ((SSubmitReq2Msg *)pBuf)->version = htobe64(1); tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); if (tEncodeSubmitReq(&encoder, pReq) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - /*vError("failed to encode submit req since %s", terrstr());*/ + tEncoderClear(&encoder); + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } tEncoderClear(&encoder); } -_end: +_exit: taosArrayDestroy(createTbArray); taosArrayDestroy(tagArray); taosArrayDestroy(pVals); @@ -336,14 +336,15 @@ _end: taosMemoryFree(pReq); } - if (terrno != 0) { + if (code) { rpcFreeCont(pBuf); taosArrayDestroy(pDeleteReq->deleteReqs); - return TSDB_CODE_FAILED; + smaWarn("vgId:%d, failed at line %d since %s", TD_VID(pVnode), lino, tstrerror(code)); + } else { + if (ppData) *ppData = pBuf; + if (pLen) *pLen = len; } - if (ppData) *ppData = pBuf; - if (pLen) *pLen = len; - return TSDB_CODE_SUCCESS; + return code; } static int32_t tsmaProcessDelReq(SSma *pSma, int64_t indexUid, SBatchDeleteReq *pDelReq) { @@ -391,22 +392,18 @@ _exit: * @return int32_t */ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { + int32_t code = 0; + int32_t lino = 0; const SArray *pDataBlocks = (const SArray *)msg; - if (!pDataBlocks) { - terrno = TSDB_CODE_TSMA_INVALID_PTR; - smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is NULL", SMA_VID(pSma)); - return TSDB_CODE_FAILED; - } if (taosArrayGetSize(pDataBlocks) <= 0) { - terrno = TSDB_CODE_TSMA_INVALID_PARA; - smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is empty", SMA_VID(pSma)); - return TSDB_CODE_FAILED; + code = TSDB_CODE_TSMA_INVALID_PARA; + TSDB_CHECK_CODE(code, lino, _exit); } if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != 0) { - terrno = TSDB_CODE_TSMA_INIT_FAILED; - return TSDB_CODE_FAILED; + code = TSDB_CODE_TSMA_INIT_FAILED; + TSDB_CHECK_CODE(code, lino, _exit); } SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); @@ -414,49 +411,43 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char STSmaStat *pTsmaStat = NULL; if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) { - terrno = TSDB_CODE_TSMA_INVALID_ENV; - return TSDB_CODE_FAILED; + code = TSDB_CODE_TSMA_INVALID_ENV; + TSDB_CHECK_CODE(code, lino, _exit); } pTsmaStat = SMA_STAT_TSMA(pStat); if (!pTsmaStat->pTSma) { + terrno = 0; STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); if (!pTSma) { - smaError("vgId:%d, failed to get STSma while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), - indexUid, tstrerror(terrno)); - goto _err; + code = terrno ? terrno : TSDB_CODE_TSMA_INVALID_PTR; + TSDB_CHECK_CODE(code, lino, _exit); } pTsmaStat->pTSma = pTSma; pTsmaStat->pTSchema = metaGetTbTSchema(SMA_META(pSma), pTSma->dstTbUid, -1, 1); if (!pTsmaStat->pTSchema) { - smaError("vgId:%d, failed to get STSchema while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), - indexUid, tstrerror(terrno)); - goto _err; + code = terrno ? terrno : TSDB_CODE_TSMA_INVALID_PTR; + TSDB_CHECK_CODE(code, lino, _exit); } } - if (pTsmaStat->pTSma->indexUid != indexUid) { - terrno = TSDB_CODE_APP_ERROR; - smaError("vgId:%d, tsma insert for smaIndex %" PRIi64 "(!=%" PRIi64 ") failed since %s", SMA_VID(pSma), indexUid, - pTsmaStat->pTSma->indexUid, tstrerror(terrno)); - goto _err; + if (ASSERTS(pTsmaStat->pTSma->indexUid == indexUid, "indexUid:%" PRIi64 " != %" PRIi64, pTsmaStat->pTSma->indexUid, + indexUid)) { + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); } SBatchDeleteReq deleteReq = {0}; void *pSubmitReq = NULL; int32_t contLen = 0; - if (smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, - pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, - &contLen) < 0) { - smaError("vgId:%d, failed to gen submit msg while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), - indexUid, tstrerror(terrno)); - goto _err; - } + code = smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, + pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, &contLen); + TSDB_CHECK_CODE(code, lino, _exit); if ((terrno = tsmaProcessDelReq(pSma, indexUid, &deleteReq)) != 0) { - goto _err; + goto _exit; } #if 0 @@ -474,13 +465,13 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char .contLen = contLen, }; - if (tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg) < 0) { - smaError("vgId:%d, failed to put SubmitReq msg while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), - indexUid, tstrerror(terrno)); - goto _err; - } + code = tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg); + TSDB_CHECK_CODE(code, lino, _exit); - return TSDB_CODE_SUCCESS; -_err: - return TSDB_CODE_FAILED; +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, smaIndex:%" PRIi64, SMA_VID(pSma), __func__, lino, + tstrerror(code), indexUid); + } + return code; } diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 7c538280e52ed127ff2815623a2afa00c126de7f..e45cbac3292a39f08074a75df8ab5c113deb6ac3 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -14,107 +14,34 @@ */ #include "sma.h" +#include "vnd.h" -#define TD_QTASKINFO_FNAME_PREFIX "main.tdb" - -void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t suid, int8_t level, int64_t version, char *outputName) { - tdRSmaGetFileName(vgId, NULL, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, suid, level, version, outputName); -} - -void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t suid, int8_t level, int64_t version, const char *path, - char *outputName) { - tdRSmaGetFileName(vgId, path, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, suid, level, version, outputName); -} - -void tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName) { - tdRSmaGetDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); - int32_t rsmaLen = strlen(outputName); - snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi8, level); -} - -void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName) { - tdRSmaGetDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); +void tdRSmaQTaskInfoGetFullPath(SVnode *pVnode, tb_uid_t suid, int8_t level, STfs *pTfs, char *outputName) { + tdRSmaGetDirName(pVnode, pTfs, true, outputName); int32_t rsmaLen = strlen(outputName); snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi8 "%s%" PRIi64, level, TD_DIRSEP, suid); } -void tdRSmaGetFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t suid, - int8_t level, int64_t version, char *outputName) { - if (level >= 0 && suid > 0) { - if (version >= 0) { - if (pdname) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s.%" PRIi64, pdname, - TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname, - version); - } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s.%" PRIi64, TD_DIRSEP, - vgId, TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname, version); - } - } else { - if (pdname) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s", pdname, - TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname); - } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s", TD_DIRSEP, vgId, - TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname); - } - } - } else { - if (version >= 0) { - if (pdname) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s%" PRIi64, pdname, TD_DIRSEP, TD_DIRSEP, - vgId, TD_DIRSEP, dname, TD_DIRSEP, vgId, fname, version); - } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s%" PRIi64, TD_DIRSEP, vgId, TD_DIRSEP, dname, - TD_DIRSEP, vgId, fname, version); - } - } else { - if (pdname) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, - TD_DIRSEP, dname, TD_DIRSEP, vgId, fname); - } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, - TD_DIRSEP, vgId, fname); - } - } - } -} +void tdRSmaGetDirName(SVnode *pVnode, STfs *pTfs, bool endWithSep, char *outputName) { + int32_t offset = 0; -void tdRSmaGetDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName) { - if (pdname) { - if (endWithSep) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, - dname, TD_DIRSEP); - } else { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, - dname); - } - } else { - if (endWithSep) { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP); - } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, dname); - } - } + // vnode + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pTfs, outputName, TSDB_FILENAME_LEN); + offset = strlen(outputName); + + // rsma + snprintf(outputName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s%s", TD_DIRSEP, VNODE_RSMA_DIR, + (endWithSep ? TD_DIRSEP : "")); } // smaXXXUtil ================ -void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) { - void *pResult = taosAcquireRef(rsetId, refId); - if (!pResult) { - smaWarn("rsma acquire ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, terrstr()); - } else { - smaTrace("rsma acquire ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); - } - return pResult; -} +void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) { return taosAcquireRef(rsetId, refId); } int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId) { if (taosReleaseRef(rsetId, refId) < 0) { smaWarn("rsma release ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, terrstr()); return TSDB_CODE_FAILED; } - smaTrace("rsma release ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 89ed3ca1c7b8557fe1905fb32874a92632d54c65..db9ef7805faebe17394857de9b83d0e9cc4f22da 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -18,8 +18,6 @@ // 0: not init // 1: already inited // 2: wait to be inited or cleaup -#define WAL_READ_TASKS_ID (-1) - static int32_t tqInitialize(STQ* pTq); static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_STATUS_EXEC == pHandle->status; } @@ -158,6 +156,29 @@ void tqClose(STQ* pTq) { taosMemoryFree(pTq); } +static bool hasStreamTaskInTimer(SStreamMeta* pMeta) { + bool inTimer = false; + + taosWLockLatch(&pMeta->lock); + + void* pIter = NULL; + while(1) { + pIter = taosHashIterate(pMeta->pTasks, pIter); + if (pIter == NULL) { + break; + } + + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->status.timerActive == 1) { + inTimer = true; + } + } + + taosWUnLockLatch(&pMeta->lock); + + return inTimer; +} + void tqNotifyClose(STQ* pTq) { if (pTq != NULL) { taosWLockLatch(&pTq->pStreamMeta->lock); @@ -170,68 +191,81 @@ void tqNotifyClose(STQ* pTq) { } SStreamTask* pTask = *(SStreamTask**)pIter; - tqDebug("vgId:%d s-task:%s set dropping flag", pTq->pStreamMeta->vgId, pTask->id.idStr); + tqDebug("vgId:%d s-task:%s set closing flag", pTq->pStreamMeta->vgId, pTask->id.idStr); pTask->status.taskStatus = TASK_STATUS__STOP; int64_t st = taosGetTimestampMs(); qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS); + int64_t el = taosGetTimestampMs() - st; tqDebug("vgId:%d s-task:%s is closed in %" PRId64 " ms", pTq->pStreamMeta->vgId, pTask->id.idStr, el); } taosWUnLockLatch(&pTq->pStreamMeta->lock); - } -} - -static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, - int64_t consumerId, int32_t type) { - int32_t len = 0; - int32_t code = 0; - - if (type == TMQ_MSG_TYPE__POLL_RSP) { - tEncodeSize(tEncodeMqDataRsp, pRsp, len, code); - } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { - tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code); - } - - if (code < 0) { - return -1; - } - - int32_t tlen = sizeof(SMqRspHead) + len; - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - return -1; - } - ((SMqRspHead*)buf)->mqMsgType = type; - ((SMqRspHead*)buf)->epoch = epoch; - ((SMqRspHead*)buf)->consumerId = consumerId; + tqDebug("vgId:%d start to check all tasks", pTq->pStreamMeta->vgId); - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + int64_t st = taosGetTimestampMs(); - SEncoder encoder = {0}; - tEncoderInit(&encoder, abuf, len); + while(hasStreamTaskInTimer(pTq->pStreamMeta)) { + tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pTq->pStreamMeta->vgId); + taosMsleep(100); + } - if (type == TMQ_MSG_TYPE__POLL_RSP) { - tEncodeMqDataRsp(&encoder, pRsp); - } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { - tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp); + int64_t el = taosGetTimestampMs() - st; + tqDebug("vgId:%d all stream tasks are not in timer, continue close, elapsed time:%"PRId64" ms", pTq->pStreamMeta->vgId, el); } - - tEncoderClear(&encoder); - - SRpcMsg rsp = { - .info = *pRpcHandleInfo, - .pCont = buf, - .contLen = tlen, - .code = 0, - }; - - tmsgSendRsp(&rsp); - return 0; } +//static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, +// int64_t consumerId, int32_t type) { +// int32_t len = 0; +// int32_t code = 0; +// +// if (type == TMQ_MSG_TYPE__POLL_DATA_RSP) { +// tEncodeSize(tEncodeMqDataRsp, pRsp, len, code); +// } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { +// tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code); +// } +// +// if (code < 0) { +// return -1; +// } +// +// int32_t tlen = sizeof(SMqRspHead) + len; +// void* buf = rpcMallocCont(tlen); +// if (buf == NULL) { +// return -1; +// } +// +// ((SMqRspHead*)buf)->mqMsgType = type; +// ((SMqRspHead*)buf)->epoch = epoch; +// ((SMqRspHead*)buf)->consumerId = consumerId; +// +// void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); +// +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, abuf, len); +// +// if (type == TMQ_MSG_TYPE__POLL_DATA_RSP) { +// tEncodeMqDataRsp(&encoder, pRsp); +// } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { +// tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp); +// } +// +// tEncoderClear(&encoder); +// +// SRpcMsg rsp = { +// .info = *pRpcHandleInfo, +// .pCont = buf, +// .contLen = tlen, +// .code = 0, +// }; +// +// tmsgSendRsp(&rsp); +// return 0; +//} + int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { SMqPollReq req = {0}; if (tDeserializeSMqPollReq(pHandle->msg->pCont, pHandle->msg->contLen, &req) < 0) { @@ -244,7 +278,7 @@ int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { tqInitDataRsp(&dataRsp, &req); dataRsp.blockNum = 0; dataRsp.rspOffset = dataRsp.reqOffset; - tqSendDataRsp(pHandle, pHandle->msg, &req, &dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); + tqSendDataRsp(pHandle, pHandle->msg, &req, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); tDeleteMqDataRsp(&dataRsp); return 0; } @@ -855,29 +889,45 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->refCnt = 1; pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputQueue = streamQueueOpen(512 << 10); - pTask->outputQueue = streamQueueOpen(512 << 10); + pTask->outputInfo.queue = streamQueueOpen(512 << 10); - if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) { + if (pTask->inputQueue == NULL || pTask->outputInfo.queue == NULL) { + tqError("s-task:%s failed to prepare the input/output queue, initialize task failed", pTask->id.idStr); return -1; } + pTask->tsInfo.init = taosGetTimestampMs(); pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; - pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; + pTask->outputInfo.status = TASK_OUTPUT_STATUS__NORMAL; pTask->pMsgCb = &pTq->pVnode->msgCb; pTask->pMeta = pTq->pStreamMeta; + + // backup the initial status, and set it to be TASK_STATUS__INIT pTask->chkInfo.version = ver; pTask->chkInfo.currentVer = ver; - // expand executor - pTask->status.taskStatus = (pTask->fillHistory) ? TASK_STATUS__WAIT_DOWNSTREAM : TASK_STATUS__NORMAL; + pTask->dataRange.range.maxVer = ver; + pTask->dataRange.range.minVer = ver; + + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + SStreamTask* pSateTask = pTask; + SStreamTask task = {0}; + if (pTask->info.fillHistory) { + task.id = pTask->streamTaskId; + task.pMeta = pTask->pMeta; + pSateTask = &task; + } - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); + pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pSateTask, false, -1, -1); if (pTask->pState == NULL) { return -1; } - SReadHandle handle = {.vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState}; + SReadHandle handle = {.vnode = pTq->pVnode, + .initTqReader = 1, + .pStateBackend = pTask->pState, + .fillHistory = pTask->info.fillHistory, + .winRange = pTask->dataRange.window}; initStorageAPI(&handle.api); pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); @@ -886,29 +936,39 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { } qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); - } else if (pTask->taskLevel == TASK_LEVEL__AGG) { - pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); + } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) { + SStreamTask* pSateTask = pTask; + SStreamTask task = {0}; + if (pTask->info.fillHistory) { + task.id = pTask->streamTaskId; + task.pMeta = pTask->pMeta; + pSateTask = &task; + } + pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pSateTask, false, -1, -1); if (pTask->pState == NULL) { return -1; } - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo); - SReadHandle handle = {.vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState}; + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->pUpstreamEpInfoList); + SReadHandle handle = {.vnode = NULL, + .numOfVgroups = numOfVgroups, + .pStateBackend = pTask->pState, + .fillHistory = pTask->info.fillHistory, + .winRange = pTask->dataRange.window}; initStorageAPI(&handle.api); pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); if (pTask->exec.pExecutor == NULL) { return -1; } - qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); } // sink - if (pTask->outputType == TASK_OUTPUT__SMA) { + if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { pTask->smaSink.vnode = pTq->pVnode; pTask->smaSink.smaSink = smaHandleRes; - } else if (pTask->outputType == TASK_OUTPUT__TABLE) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline; @@ -928,15 +988,24 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { tSimpleHashSetFreeFp(pTask->tbSink.pTblInfo, freePtr); } - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { SWalFilterCond cond = {.deleteMsg = 1}; // delete msg also extract from wal files pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond); } - streamSetupTrigger(pTask); + // reset the task status from unfinished transaction + if (pTask->status.taskStatus == TASK_STATUS__PAUSE) { + tqWarn("s-task:%s reset task status to be normal, kept in meta status: Paused", pTask->id.idStr); + pTask->status.taskStatus = TASK_STATUS__NORMAL; + } + + taosThreadMutexInit(&pTask->lock, NULL); + streamSetupScheduleTrigger(pTask); - tqInfo("vgId:%d expand stream task, s-task:%s, checkpoint ver:%" PRId64 " child id:%d, level:%d", vgId, - pTask->id.idStr, pTask->chkInfo.version, pTask->selfChildId, pTask->taskLevel); + tqInfo("vgId:%d expand stream task, s-task:%s, checkpoint ver:%" PRId64 + " child id:%d, level:%d, fill-history:%d, trigger:%" PRId64 " ms, disable pause", + vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->info.selfChildId, pTask->info.taskLevel, + pTask->info.fillHistory, pTask->triggerParam); // next valid version will add one pTask->chkInfo.version += 1; @@ -952,10 +1021,11 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); - tDecodeSStreamTaskCheckReq(&decoder, &req); + tDecodeStreamTaskCheckReq(&decoder, &req); tDecoderClear(&decoder); - int32_t taskId = req.downstreamTaskId; + int32_t taskId = req.downstreamTaskId; + SStreamTaskCheckRsp rsp = { .reqId = req.reqId, .streamId = req.streamId, @@ -967,54 +1037,32 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { }; SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - if (pTask != NULL) { rsp.status = streamTaskCheckStatus(pTask); streamMetaReleaseTask(pTq->pStreamMeta, pTask); - tqDebug("s-task:%s recv task check req(reqId:0x%" PRIx64 - ") %d at node %d task status:%d, check req from task %d at node %d, rsp status %d", - pTask->id.idStr, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, pTask->status.taskStatus, - rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); + tqDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d", + pTask->id.idStr, pStatus, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); } else { rsp.status = 0; - tqDebug("tq recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 - ") %d at node %d, check req from task:0x%x at node %d, rsp status %d", - taskId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, - rsp.status); + tqDebug("tq recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp status %d", + taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); } - SEncoder encoder; - int32_t code; - int32_t len; - - tEncodeSize(tEncodeSStreamTaskCheckRsp, &rsp, len, code); - if (code < 0) { - tqError("vgId:%d failed to encode task check rsp, task:0x%x", pTq->pStreamMeta->vgId, taskId); - return -1; - } - - void* buf = rpcMallocCont(sizeof(SMsgHead) + len); - ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId); - - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncoderInit(&encoder, (uint8_t*)abuf, len); - tEncodeSStreamTaskCheckRsp(&encoder, &rsp); - tEncoderClear(&encoder); - - SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info}; - - tmsgSendRsp(&rspMsg); - return 0; + return streamSendCheckRsp(pTq->pStreamMeta, &req, &rsp, &pMsg->info, taskId); } -int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { +int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t sversion, SRpcMsg* pMsg) { + char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + int32_t code; SStreamTaskCheckRsp rsp; SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - code = tDecodeSStreamTaskCheckRsp(&decoder, &rsp); + tDecoderInit(&decoder, (uint8_t*)pReq, len); + code = tDecodeStreamTaskCheckRsp(&decoder, &rsp); if (code < 0) { tDecoderClear(&decoder); @@ -1022,17 +1070,18 @@ int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t sversion, char* msg, int32 } tDecoderClear(&decoder); - tqDebug("tq recv task check rsp(reqId:0x%" PRIx64 ") %d at node %d check req from task:0x%x at node %d, status %d", - rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); + tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", + rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, rsp.upstreamTaskId); if (pTask == NULL) { - tqError("tq failed to locate the stream task:0x%x vgId:%d, it may have been destroyed", rsp.upstreamTaskId, + tqError("tq failed to locate the stream task:0x%x (vgId:%d), it may have been destroyed", rsp.upstreamTaskId, pTq->pStreamMeta->vgId); + terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; return -1; } - code = streamProcessTaskCheckRsp(pTask, &rsp, sversion); + code = streamProcessCheckRsp(pTask, &rsp); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return code; } @@ -1065,188 +1114,308 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms tDecoderClear(&decoder); + SStreamMeta* pStreamMeta = pTq->pStreamMeta; + // 2.save task, use the newest commit version as the initial start version of stream task. - taosWLockLatch(&pTq->pStreamMeta->lock); - code = streamMetaAddDeployedTask(pTq->pStreamMeta, sversion, pTask); - int32_t numOfTasks = streamMetaGetNumOfTasks(pTq->pStreamMeta); + int32_t taskId = pTask->id.taskId; + bool added = false; + + taosWLockLatch(&pStreamMeta->lock); + code = streamMetaRegisterTask(pStreamMeta, sversion, pTask, &added); + int32_t numOfTasks = streamMetaGetNumOfTasks(pStreamMeta); + if (code < 0) { - tqError("vgId:%d failed to add s-task:%s, total:%d", vgId, pTask->id.idStr, numOfTasks); - taosWUnLockLatch(&pTq->pStreamMeta->lock); + tqError("vgId:%d failed to add s-task:0x%x, total:%d", vgId, pTask->id.taskId, numOfTasks); + tFreeStreamTask(pTask); + taosWUnLockLatch(&pStreamMeta->lock); return -1; } - taosWUnLockLatch(&pTq->pStreamMeta->lock); + // not added into meta store + if (!added) { + tqWarn("vgId:%d failed to add s-task:0x%x, already exists in meta store", vgId, taskId); + tFreeStreamTask(pTask); + pTask = NULL; + } + + taosWUnLockLatch(&pStreamMeta->lock); + + tqDebug("vgId:%d s-task:0x%x is deployed and add into meta, numOfTasks:%d", vgId, taskId, numOfTasks); - // 3.go through recover steps to fill history - if (pTask->fillHistory) { - streamTaskCheckDownstream(pTask, sversion); + // 3. It's an fill history task, do nothing. wait for the main task to start it + SStreamTask* p = streamMetaAcquireTask(pStreamMeta, taskId); + if (p != NULL) { // reset the downstreamReady flag. + streamTaskCheckDownstreamTasks(p); } - tqDebug("vgId:%d s-task:%s is deployed and add meta from mnd, status:%d, total:%d", vgId, pTask->id.idStr, - pTask->status.taskStatus, numOfTasks); + streamMetaReleaseTask(pStreamMeta, p); return 0; } -int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { - int32_t code; - char* msg = pMsg->pCont; - int32_t msgLen = pMsg->contLen; +int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { + SStreamScanHistoryReq* pReq = (SStreamScanHistoryReq*)pMsg->pCont; + SStreamMeta* pMeta = pTq->pStreamMeta; - SStreamRecoverStep1Req* pReq = (SStreamRecoverStep1Req*)msg; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); + int32_t code = TSDB_CODE_SUCCESS; + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->taskId); if (pTask == NULL) { + tqError("vgId:%d failed to acquire stream task:0x%x during stream recover, task may have been destroyed", + pMeta->vgId, pReq->taskId); return -1; } - // check param - int64_t fillVer1 = pTask->chkInfo.version; - if (fillVer1 <= 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; - } - - // do recovery step 1 - tqDebug("s-task:%s start non-blocking recover stage(step 1) scan", pTask->id.idStr); - int64_t st = taosGetTimestampMs(); + // do recovery step1 + const char* id = pTask->id.idStr; + const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); + tqDebug("s-task:%s start scan-history stage(step 1), status:%s", id, pStatus); - streamSourceRecoverScanStep1(pTask); - if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { - tqDebug("s-task:%s is dropped, abort recover in step1", pTask->id.idStr); + if (pTask->tsInfo.step1Start == 0) { + ASSERT(pTask->status.pauseAllowed == false); + pTask->tsInfo.step1Start = taosGetTimestampMs(); + if (pTask->info.fillHistory == 1) { + streamTaskEnablePause(pTask); + } + } else { + tqDebug("s-task:%s resume from paused, start ts:%"PRId64, pTask->id.idStr, pTask->tsInfo.step1Start); + } - streamMetaReleaseTask(pTq->pStreamMeta, pTask); + // we have to continue retrying to successfully execute the scan history task. + int8_t schedStatus = atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE, + TASK_SCHED_STATUS__WAITING); + if (schedStatus != TASK_SCHED_STATUS__INACTIVE) { + tqError( + "s-task:%s failed to start scan-history in first stream time window since already started, unexpected " + "sched-status:%d", + id, schedStatus); return 0; } - double el = (taosGetTimestampMs() - st) / 1000.0; - tqDebug("s-task:%s non-blocking recover stage(step 1) ended, elapsed time:%.2fs", pTask->id.idStr, el); - - // build msg to launch next step - SStreamRecoverStep2Req req; - code = streamBuildSourceRecover2Req(pTask, &req); - if (code < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; + if (pTask->info.fillHistory == 1) { + ASSERT(pTask->status.pauseAllowed == true); } - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { + streamSourceScanHistoryData(pTask); + if (pTask->status.taskStatus == TASK_STATUS__PAUSE) { + double el = (taosGetTimestampMs() - pTask->tsInfo.step1Start) / 1000.0; + tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs, sched-status:%d", pTask->id.idStr, el, + TASK_SCHED_STATUS__INACTIVE); + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); return 0; } - // serialize msg - int32_t len = sizeof(SStreamRecoverStep1Req); + // the following procedure should be executed, no matter status is stop/pause or not + double el = (taosGetTimestampMs() - pTask->tsInfo.step1Start) / 1000.0; + tqDebug("s-task:%s scan-history stage(step 1) ended, elapsed time:%.2fs", id, el); - void* serializedReq = rpcMallocCont(len); - if (serializedReq == NULL) { - tqError("s-task:%s failed to prepare the step2 stage, out of memory", pTask->id.idStr); - return -1; - } + if (pTask->info.fillHistory) { + SVersionRange* pRange = NULL; + SStreamTask* pStreamTask = NULL; + bool done = false; + + // 1. get the related stream task + pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.taskId); + if (pStreamTask == NULL) { + // todo delete this task, if the related stream task is dropped + qError("failed to find s-task:0x%x, it may have been destroyed, drop fill-history task:%s", + pTask->streamTaskId.taskId, pTask->id.idStr); + + tqDebug("s-task:%s fill-history task set status to be dropping", id); + + streamMetaUnregisterTask(pMeta, pTask->id.taskId); + streamMetaReleaseTask(pMeta, pTask); + return -1; + } + + ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE); + + // 2. it cannot be paused, when the stream task in TASK_STATUS__SCAN_HISTORY status. Let's wait for the + // stream task get ready for scan history data + while (pStreamTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) { + tqDebug( + "s-task:%s level:%d related stream task:%s(status:%s) not ready for halt, wait for it and recheck in 100ms", + id, pTask->info.taskLevel, pStreamTask->id.idStr, streamGetTaskStatusStr(pStreamTask->status.taskStatus)); + taosMsleep(100); + } + + // now we can stop the stream task execution + streamTaskHalt(pStreamTask); + + tqDebug("s-task:%s level:%d sched-status:%d is halt by fill-history task:%s", pStreamTask->id.idStr, + pStreamTask->info.taskLevel, pStreamTask->status.schedStatus, id); + + // if it's an source task, extract the last version in wal. + pRange = &pTask->dataRange.range; + int64_t latestVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader); + done = streamHistoryTaskSetVerRangeStep2(pTask, latestVer); - memcpy(serializedReq, &req, len); + if (done) { + pTask->tsInfo.step2Start = taosGetTimestampMs(); + streamTaskEndScanWAL(pTask); + streamMetaReleaseTask(pMeta, pTask); + } else { + STimeWindow* pWindow = &pTask->dataRange.window; + tqDebug("s-task:%s level:%d verRange:%" PRId64 " - %" PRId64 " window:%" PRId64 "-%" PRId64 + ", do secondary scan-history from WAL after halt the related stream task:%s", + id, pTask->info.taskLevel, pRange->minVer, pRange->maxVer, pWindow->skey, pWindow->ekey, + pStreamTask->id.idStr); + ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__WAITING); + + pTask->tsInfo.step2Start = taosGetTimestampMs(); + streamSetParamForStreamScannerStep2(pTask, pRange, pWindow); + + int64_t dstVer = pTask->dataRange.range.minVer - 1; + + pTask->chkInfo.currentVer = dstVer; + walReaderSetSkipToVersion(pTask->exec.pWalReader, dstVer); + tqDebug("s-task:%s wal reader start scan WAL verRange:%" PRId64 "-%" PRId64 ", set sched-status:%d", id, dstVer, + pTask->dataRange.range.maxVer, TASK_SCHED_STATUS__INACTIVE); + + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + + // set the fill-history task to be normal + if (pTask->info.fillHistory == 1) { + streamSetStatusNormal(pTask); + } + + // 4. 1) transfer the ownership of executor state, 2) update the scan data range for source task. + // 5. resume the related stream task. + streamMetaReleaseTask(pMeta, pTask); + streamMetaReleaseTask(pMeta, pStreamTask); - // dispatch msg - tqDebug("s-task:%s start recover block stage", pTask->id.idStr); + tqStartStreamTasks(pTq); + } + } else { + // todo update the chkInfo version for current task. + // this task has an associated history stream task, so we need to scan wal from the end version of + // history scan. The current version of chkInfo.current is not updated during the history scan + STimeWindow* pWindow = &pTask->dataRange.window; + + if (pTask->historyTaskId.taskId == 0) { + *pWindow = (STimeWindow){INT64_MIN, INT64_MAX}; + tqDebug( + "s-task:%s scan-history in stream time window completed, no related fill-history task, reset the time " + "window:%" PRId64 " - %" PRId64, + id, pWindow->skey, pWindow->ekey); + qResetStreamInfoTimeWindow(pTask->exec.pExecutor); + } else { + // when related fill-history task exists, update the fill-history time window only when the + // state transfer is completed. + tqDebug( + "s-task:%s scan-history in stream time window completed, now start to handle data from WAL, start " + "ver:%" PRId64 ", window:%" PRId64 " - %" PRId64, + id, pTask->chkInfo.currentVer, pWindow->skey, pWindow->ekey); + } + + // notify the downstream agg tasks that upstream tasks are ready to processing the WAL data, update the + code = streamTaskScanHistoryDataComplete(pTask); + streamMetaReleaseTask(pMeta, pTask); + + // when all source task complete to scan history data in stream time window, they are allowed to handle stream data + // at the same time. + return code; + } - SRpcMsg rpcMsg = { - .code = 0, .contLen = len, .msgType = TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, .pCont = serializedReq}; - tmsgPutToQueue(&pTq->pVnode->msgCb, WRITE_QUEUE, &rpcMsg); return 0; } -int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - int32_t code = 0; +// notify the downstream tasks to transfer executor state after handle all history blocks. +int32_t tqProcessTaskTransferStateReq(STQ* pTq, SRpcMsg* pMsg) { + char* pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); - SStreamRecoverStep2Req* pReq = (SStreamRecoverStep2Req*)msg; + SStreamTransferReq req = {0}; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)pReq, len); + int32_t code = tDecodeStreamScanHistoryFinishReq(&decoder, &req); + tDecoderClear(&decoder); + + tqDebug("vgId:%d start to process transfer state msg, from s-task:0x%x", pTq->pStreamMeta->vgId, req.downstreamTaskId); + + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.downstreamTaskId); if (pTask == NULL) { + tqError("failed to find task:0x%x, it may have been dropped already. process transfer state failed", req.downstreamTaskId); return -1; } - // do recovery step 2 - int64_t st = taosGetTimestampMs(); - tqDebug("s-task:%s start step2 recover, ts:%" PRId64, pTask->id.idStr, st); - - code = streamSourceRecoverScanStep2(pTask, sversion); - if (code < 0) { + int32_t remain = streamAlignTransferState(pTask); + if (remain > 0) { + tqDebug("s-task:%s receive upstream transfer state msg, remain:%d", pTask->id.idStr, remain); streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; + return 0; } - qDebug("s-task:%s set start wal scan start ver:%"PRId64, pTask->id.idStr, sversion); + // transfer the ownership of executor state + tqDebug("s-task:%s all upstream tasks send transfer msg, open transfer state flag", pTask->id.idStr); + ASSERT(pTask->streamTaskId.taskId != 0 && pTask->info.fillHistory == 1); - walReaderSeekVer(pTask->exec.pWalReader, sversion); - pTask->chkInfo.currentVer = sversion; + pTask->status.transferState = true; - if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return 0; - } + streamSchedExec(pTask); + streamMetaReleaseTask(pTq->pStreamMeta, pTask); + return 0; +} - // restore param - code = streamRestoreParam(pTask); - if (code < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; - } +int32_t tqProcessTaskScanHistoryFinishReq(STQ* pTq, SRpcMsg* pMsg) { + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - // set status normal - tqDebug("s-task:%s blocking stage completed, set the status to be normal", pTask->id.idStr); - code = streamSetStatusNormal(pTask); - if (code < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; - } + // deserialize + SStreamScanHistoryFinishReq req = {0}; - double el = (taosGetTimestampMs() - st) / 1000.0; - tqDebug("s-task:%s step2 recover finished, el:%.2fs", pTask->id.idStr, el); + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + tDecodeStreamScanHistoryFinishReq(&decoder, &req); + tDecoderClear(&decoder); - // dispatch recover finish req to all related downstream task - code = streamDispatchRecoverFinishReq(pTask); - if (code < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.downstreamTaskId); + if (pTask == NULL) { + tqError("vgId:%d process scan history finish msg, failed to find task:0x%x, it may be destroyed", + pTq->pStreamMeta->vgId, req.downstreamTaskId); return -1; } - atomic_store_8(&pTask->fillHistory, 0); - streamMetaSaveTask(pTq->pStreamMeta, pTask); + tqDebug("s-task:%s receive scan-history finish msg from task:0x%x", pTask->id.idStr, req.upstreamTaskId); + int32_t code = streamProcessScanHistoryFinishReq(pTask, &req, &pMsg->info); streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return 0; + return code; } -int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg) { +int32_t tqProcessTaskScanHistoryFinishRsp(STQ* pTq, SRpcMsg* pMsg) { char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); // deserialize - SStreamRecoverFinishReq req; + SStreamCompleteHistoryMsg req = {0}; SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - tDecodeSStreamRecoverFinishReq(&decoder, &req); + tDecodeCompleteHistoryDataMsg(&decoder, &req); tDecoderClear(&decoder); - // find task - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.taskId); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.upstreamTaskId); if (pTask == NULL) { + tqError("vgId:%d process scan history finish rsp, failed to find task:0x%x, it may be destroyed", + pTq->pStreamMeta->vgId, req.upstreamTaskId); return -1; } - // do process request - if (streamProcessRecoverFinishReq(pTask, req.childId) < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; + + int32_t remain = atomic_sub_fetch_32(&pTask->notReadyTasks, 1); + if (remain > 0) { + tqDebug("s-task:%s scan-history finish rsp received from downstream task:0x%x, remain:%d not send finish rsp", + pTask->id.idStr, req.downstreamId, remain); + } else { + tqDebug( + "s-task:%s scan-history finish rsp received from downstream task:0x%x, all downstream tasks rsp scan-history " + "completed msg", pTask->id.idStr, req.downstreamId); + streamProcessScanHistoryFinishRsp(pTask); } streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } -int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg) { - // - return 0; -} - int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, SStreamRefDataBlock** pRefBlock) { SDecoder* pCoder = &(SDecoder){0}; SDeleteRes* pRes = &(SDeleteRes){0}; @@ -1306,22 +1475,28 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { int32_t taskId = pReq->taskId; int32_t vgId = TD_VID(pTq->pVnode); - if (taskId == WAL_READ_TASKS_ID) { // all tasks are extracted submit data from the wal + if (taskId == STREAM_TASK_STATUS_CHECK_ID) { + tqStreamTasksStatusCheck(pTq); + return 0; + } + + if (taskId == EXTRACT_DATA_FROM_WAL_ID) { // all tasks are extracted submit data from the wal tqStreamTasksScanWal(pTq); return 0; } SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); if (pTask != NULL) { - if (pTask->status.taskStatus == TASK_STATUS__NORMAL) { - tqDebug("vgId:%d s-task:%s start to process block from wal, last chk point:%" PRId64, vgId, pTask->id.idStr, + // even in halt status, the data in inputQ must be processed + int8_t st = pTask->status.taskStatus; + if (st == TASK_STATUS__NORMAL || st == TASK_STATUS__SCAN_HISTORY/* || st == TASK_STATUS__SCAN_HISTORY_WAL*/) { + tqDebug("vgId:%d s-task:%s start to process block from inputQ, last chk point:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.version); streamProcessRunReq(pTask); } else { - if (streamTaskShouldPause(&pTask->status)) { - atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); - } - tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr); + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, + pTask->id.idStr, streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus); } streamMetaReleaseTask(pTq->pStreamMeta, pTask); @@ -1375,33 +1550,72 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskDropReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; tqDebug("vgId:%d receive msg to drop stream task:0x%x", TD_VID(pTq->pVnode), pReq->taskId); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); + if (pTask == NULL) { + tqError("vgId:%d failed to acquire s-task:0x%x when dropping it", pTq->pStreamMeta->vgId, pReq->taskId); + return 0; + } - streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId); + streamMetaUnregisterTask(pTq->pStreamMeta, pReq->taskId); + streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)msg; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); - if (pTask) { - tqDebug("vgId:%d s-task:%s set pause flag", pTq->pStreamMeta->vgId, pTask->id.idStr); - atomic_store_8(&pTask->status.keepTaskStatus, pTask->status.taskStatus); - atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__PAUSE); - streamMetaReleaseTask(pTq->pStreamMeta, pTask); + + SStreamMeta* pMeta = pTq->pStreamMeta; + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->taskId); + if (pTask == NULL) { + tqError("vgId:%d failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, + pReq->taskId); + + // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active + return TSDB_CODE_SUCCESS; } - return 0; + + tqDebug("s-task:%s receive pause msg from mnode", pTask->id.idStr); + streamTaskPause(pTask); + + SStreamTask* pHistoryTask = NULL; + if (pTask->historyTaskId.taskId != 0) { + pHistoryTask = streamMetaAcquireTask(pMeta, pTask->historyTaskId.taskId); + if (pHistoryTask == NULL) { + tqError("vgId:%d failed to acquire fill-history task:0x%x, it may have been dropped already. Pause success", + pMeta->vgId, pTask->historyTaskId.taskId); + + streamMetaReleaseTask(pMeta, pTask); + + // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active + return TSDB_CODE_SUCCESS; + } + + tqDebug("s-task:%s fill-history task handle paused along with related stream task", pHistoryTask->id.idStr); + streamTaskPause(pHistoryTask); + } + + streamMetaReleaseTask(pMeta, pTask); + if (pHistoryTask != NULL) { + streamMetaReleaseTask(pMeta, pHistoryTask); + } + + return TSDB_CODE_SUCCESS; } -int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - SVResumeStreamTaskReq* pReq = (SVResumeStreamTaskReq*)msg; +int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion, int8_t igUntreated) { + int32_t vgId = pTq->pStreamMeta->vgId; + if (pTask == NULL) { + return -1; + } - int32_t vgId = pTq->pStreamMeta->vgId; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); - if (pTask) { - atomic_store_8(&pTask->status.taskStatus, pTask->status.keepTaskStatus); + // todo: handle the case: resume from halt to pause/ from halt to normal/ from pause to normal + streamTaskResume(pTask); + int32_t level = pTask->info.taskLevel; + int8_t status = pTask->status.taskStatus; + if (status == TASK_STATUS__NORMAL || status == TASK_STATUS__SCAN_HISTORY) { // no lock needs to secure the access of the version - if (pReq->igUntreated && pTask->taskLevel == TASK_LEVEL__SOURCE) { + if (igUntreated && level == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) { // discard all the data when the stream task is suspended. walReaderSetSkipToVersion(pTask->exec.pWalReader, sversion); tqDebug("vgId:%d s-task:%s resume to exec, prev paused version:%" PRId64 ", start from vnode ver:%" PRId64 @@ -1412,33 +1626,53 @@ int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms vgId, pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); } - if (pTask->taskLevel == TASK_LEVEL__SOURCE && taosQueueItemSize(pTask->inputQueue->queue) == 0) { + if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) { + streamStartRecoverTask(pTask, igUntreated); + } else if (level == TASK_LEVEL__SOURCE && (taosQueueItemSize(pTask->inputQueue->queue) == 0)) { tqStartStreamTasks(pTq); } else { streamSchedExec(pTask); } - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - } else { - tqError("vgId:%d failed to find the s-task:0x%x for resume stream task", vgId, pReq->taskId); } + streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } +int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { + SVResumeStreamTaskReq* pReq = (SVResumeStreamTaskReq*)msg; + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); + int32_t code = tqProcessTaskResumeImpl(pTq, pTask, sversion, pReq->igUntreated); + if (code != 0) { + return code; + } + + SStreamTask* pHistoryTask = streamMetaAcquireTask(pTq->pStreamMeta, pTask->historyTaskId.taskId); + if (pHistoryTask) { + code = tqProcessTaskResumeImpl(pTq, pHistoryTask, sversion, pReq->igUntreated); + } + + return code; +} + int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { - char* msgStr = pMsg->pCont; - char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + SDecoder decoder; + SStreamRetrieveReq req; - SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecodeStreamRetrieveReq(&decoder, &req); tDecoderClear(&decoder); + int32_t taskId = req.dstTaskId; SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); + if (pTask) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamProcessRetrieveReq(pTask, &req, &rsp); + streamMetaReleaseTask(pTq->pStreamMeta, pTask); tDeleteStreamRetrieveReq(&req); return 0; @@ -1471,9 +1705,8 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { tDecoderClear(&decoder); int32_t taskId = req.taskId; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - if (pTask) { + if (pTask != NULL) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamProcessDispatchMsg(pTask, &req, &rsp, false); streamMetaReleaseTask(pTq->pStreamMeta, pTask); @@ -1481,18 +1714,22 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { taosFreeQitem(pMsg); return 0; } else { + tDeleteStreamDispatchReq(&req); } code = TSDB_CODE_STREAM_TASK_NOT_EXIST; FAIL: - if (pMsg->info.handle == NULL) return -1; + if (pMsg->info.handle == NULL) { + tqError("s-task:0x%x vgId:%d msg handle is null, abort enqueue dispatch msg", pTq->pStreamMeta->vgId, taskId); + return -1; + } SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp)); if (pRspHead == NULL) { SRpcMsg rsp = {.code = TSDB_CODE_OUT_OF_MEMORY, .info = pMsg->info}; - tqDebug("send dispatch error rsp, code: %x", code); + tqError("s-task:0x%x send dispatch error rsp, code:%s", taskId, tstrerror(code)); tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); @@ -1508,9 +1745,10 @@ FAIL: pRsp->downstreamTaskId = htonl(req.taskId); pRsp->inputStatus = TASK_OUTPUT_STATUS__NORMAL; - SRpcMsg rsp = { - .code = code, .info = pMsg->info, .contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp), .pCont = pRspHead}; - tqDebug("send dispatch error rsp, code: %x", code); + int32_t len = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); + SRpcMsg rsp = { .code = code, .info = pMsg->info, .contLen = len, .pCont = pRspHead}; + tqError("s-task:0x%x send dispatch error rsp, code:%s", taskId, tstrerror(code)); + tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); @@ -1519,43 +1757,3 @@ FAIL: int32_t tqCheckLogInWal(STQ* pTq, int64_t sversion) { return sversion <= pTq->walLogLastVer; } -int32_t tqStartStreamTasks(STQ* pTq) { - int32_t vgId = TD_VID(pTq->pVnode); - SStreamMeta* pMeta = pTq->pStreamMeta; - - taosWLockLatch(&pMeta->lock); - - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - if (numOfTasks == 0) { - tqInfo("vgId:%d no stream tasks exist", vgId); - taosWUnLockLatch(&pMeta->lock); - return 0; - } - - pMeta->walScanCounter += 1; - - if (pMeta->walScanCounter > 1) { - tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScanCounter); - taosWUnLockLatch(&pMeta->lock); - return 0; - } - - SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); - if (pRunReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); - taosWUnLockLatch(&pMeta->lock); - return -1; - } - - tqDebug("vgId:%d create msg to start wal scan to launch stream tasks, numOfTasks:%d", vgId, numOfTasks); - pRunReq->head.vgId = vgId; - pRunReq->streamId = 0; - pRunReq->taskId = WAL_READ_TASKS_ID; - - SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; - tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); - taosWUnLockLatch(&pMeta->lock); - - return 0; -} diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 06af53d453cb4e189b82bcaff9c17c0e81936ed5..5ccf4c825bbeef6cf12c911ea54352311e53f739 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -35,7 +35,10 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v tqProcessSubmitReqForSubscribe(pTq); } + taosRLockLatch(&pTq->pStreamMeta->lock); int32_t numOfTasks = streamMetaGetNumOfTasks(pTq->pStreamMeta); + taosRUnLockLatch(&pTq->pStreamMeta->lock); + tqDebug("handle submit, restore:%d, size:%d", pTq->pVnode->restored, numOfTasks); // push data for stream processing: diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 4299cd471a0cdaab2f509ccc26c99168d4cf6fc4..9b8f1781cb47182156d7834adb1cbb2c5fbf2050 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -114,7 +114,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) { } SMetaReader mr = {0}; - metaReaderInit(&mr, pHandle->execHandle.pTqReader->pVnodeMeta, 0); + metaReaderDoInit(&mr, pHandle->execHandle.pTqReader->pVnodeMeta, 0); if (metaGetTableEntryByName(&mr, req.tbName) < 0) { metaReaderClear(&mr); @@ -216,9 +216,9 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea code = 0; goto END; } else { - if (pHandle->fetchMeta) { + if (pHandle->fetchMeta != WITH_DATA) { SWalCont* pHead = &((*ppCkHead)->head); - if (IS_META_MSG(pHead->msgType)) { + if (IS_META_MSG(pHead->msgType) && !(pHead->msgType == TDMT_VND_DELETE && pHandle->fetchMeta == ONLY_META)) { code = walFetchBody(pHandle->pWalReader, ppCkHead); if (code < 0) { *fetchOffset = offset; @@ -302,13 +302,17 @@ int32_t tqReaderSeek(STqReader* pReader, int64_t ver, const char* id) { return 0; } -int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, const char* id) { +int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, const char* id) { int32_t code = walNextValidMsg(pReader); if (code != TSDB_CODE_SUCCESS) { return code; } int64_t ver = pReader->pHead->head.version; + if (ver > maxVer) { + tqDebug("maxVer in WAL:%"PRId64" reached current:%"PRId64", do not scan wal anymore, %s", maxVer, ver, id); + return TSDB_CODE_SUCCESS; + } if (pReader->pHead->head.msgType == TDMT_VND_SUBMIT) { void* pBody = POINTER_SHIFT(pReader->pHead->head.body, sizeof(SSubmitReq2Msg)); @@ -336,6 +340,7 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, const char* id) { int32_t len = pReader->pHead->head.bodyLen - sizeof(SMsgHead); extractDelDataBlock(pBody, len, ver, (SStreamRefDataBlock**)pItem); + tqDebug("s-task:%s delete msg extract from WAL, len:%d, ver:%"PRId64, id, len, ver); } else { ASSERT(0); } @@ -1109,7 +1114,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { int32_t code = qUpdateTableListForStreamScanner(pTask->exec.pExecutor, tbUidList, isAdd); if (code != 0) { tqError("vgId:%d, s-task:%s update qualified table error for stream task", vgId, pTask->id.idStr); diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index fe80f486918413390ee7916fb97fe07c58a1b80d..c3e7d03e4397f5d21c1866e3d2117646cfe5cf2e 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -16,6 +16,7 @@ #include "tq.h" static int32_t createStreamTaskRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle); +static int32_t doSetOffsetForWalReader(SStreamTask *pTask, int32_t vgId); // this function should be executed by stream threads. // extract submit block from WAL, and add them into the input queue for the sources tasks. @@ -37,9 +38,7 @@ int32_t tqStreamTasksScanWal(STQ* pTq) { if (shouldIdle) { taosWLockLatch(&pMeta->lock); - pMeta->walScanCounter -= 1; - times = pMeta->walScanCounter; - + times = (--pMeta->walScanCounter); ASSERT(pMeta->walScanCounter >= 0); if (pMeta->walScanCounter <= 0) { @@ -57,7 +56,117 @@ int32_t tqStreamTasksScanWal(STQ* pTq) { return 0; } -static int32_t doSetOffsetForWalReader(SStreamTask *pTask, int32_t vgId) { +int32_t tqStreamTasksStatusCheck(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + SStreamMeta* pMeta = pTq->pStreamMeta; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + tqDebug("vgId:%d start to check all (%d) stream tasks downstream status", vgId, numOfTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + SArray* pTaskList = NULL; + taosWLockLatch(&pMeta->lock); + pTaskList = taosArrayDup(pMeta->pTaskList, NULL); + taosWUnLockLatch(&pMeta->lock); + + for (int32_t i = 0; i < numOfTasks; ++i) { + int32_t* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, *pTaskId); + if (pTask == NULL) { + continue; + } + + if (pTask->info.fillHistory == 1) { + tqDebug("s-task:%s fill-history task, wait for related stream task:0x%x to launch it", pTask->id.idStr, + pTask->streamTaskId.taskId); + continue; + } + + streamTaskDoCheckDownstreamTasks(pTask); + streamMetaReleaseTask(pMeta, pTask); + } + + taosArrayDestroy(pTaskList); + return 0; +} + +int32_t tqCheckStreamStatus(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + SStreamMeta* pMeta = pTq->pStreamMeta; + + taosWLockLatch(&pMeta->lock); + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + if (numOfTasks == 0) { + tqInfo("vgId:%d no stream tasks exist", vgId); + taosWUnLockLatch(&pMeta->lock); + return 0; + } + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); + taosWUnLockLatch(&pMeta->lock); + return -1; + } + + tqDebug("vgId:%d check for stream tasks status, numOfTasks:%d", vgId, numOfTasks); + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = STREAM_TASK_STATUS_CHECK_ID; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); + taosWUnLockLatch(&pMeta->lock); + + return 0; +} + +int32_t tqStartStreamTasks(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + SStreamMeta* pMeta = pTq->pStreamMeta; + + taosWLockLatch(&pMeta->lock); + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + if (numOfTasks == 0) { + tqInfo("vgId:%d no stream tasks exist", vgId); + taosWUnLockLatch(&pMeta->lock); + return 0; + } + + pMeta->walScanCounter += 1; + + if (pMeta->walScanCounter > 1) { + tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScanCounter); + taosWUnLockLatch(&pMeta->lock); + return 0; + } + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); + taosWUnLockLatch(&pMeta->lock); + return -1; + } + + tqDebug("vgId:%d create msg to start wal scan to launch stream tasks, numOfTasks:%d", vgId, numOfTasks); + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = EXTRACT_DATA_FROM_WAL_ID; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); + taosWUnLockLatch(&pMeta->lock); + + return 0; +} + +int32_t doSetOffsetForWalReader(SStreamTask *pTask, int32_t vgId) { // seek the stored version and extract data from WAL int64_t firstVer = walReaderGetValidFirstVer(pTask->exec.pWalReader); if (pTask->chkInfo.currentVer < firstVer) { @@ -100,9 +209,20 @@ static int32_t doSetOffsetForWalReader(SStreamTask *pTask, int32_t vgId) { return TSDB_CODE_SUCCESS; } +static void checkForFillHistoryVerRange(SStreamTask* pTask, int64_t ver) { + if ((pTask->info.fillHistory == 1) && ver > pTask->dataRange.range.maxVer) { + qWarn("s-task:%s fill-history scan WAL, currentVer:%" PRId64 " reach the maximum ver:%" PRId64 + ", not scan wal anymore, set the transfer state flag", + pTask->id.idStr, ver, pTask->dataRange.range.maxVer); + pTask->status.transferState = true; + + /*int32_t code = */streamSchedExec(pTask); + } +} + int32_t createStreamTaskRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { *pScanIdle = true; - bool noNewDataInWal = true; + bool noDataInWal = true; int32_t vgId = pStreamMeta->vgId; int32_t numOfTasks = taosArrayGetSize(pStreamMeta->pTaskList); @@ -129,15 +249,24 @@ int32_t createStreamTaskRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { } int32_t status = pTask->status.taskStatus; - if (pTask->taskLevel != TASK_LEVEL__SOURCE) { -// tqTrace("s-task:%s level:%d not source task, no need to start", pTask->id.idStr, pTask->taskLevel); + + // non-source or fill-history tasks don't need to response the WAL scan action. + if ((pTask->info.taskLevel != TASK_LEVEL__SOURCE) || (pTask->status.downstreamReady == 0)) { + streamMetaReleaseTask(pStreamMeta, pTask); + continue; + } + + if (status != TASK_STATUS__NORMAL) { + tqDebug("s-task:%s not ready for new submit block from wal, status:%s", pTask->id.idStr, streamGetTaskStatusStr(status)); streamMetaReleaseTask(pStreamMeta, pTask); continue; } - if (streamTaskShouldStop(&pTask->status) || status == TASK_STATUS__RECOVER_PREPARE || - status == TASK_STATUS__WAIT_DOWNSTREAM || streamTaskShouldPause(&pTask->status)) { - tqDebug("s-task:%s not ready for new submit block from wal, status:%d", pTask->id.idStr, status); + if ((pTask->info.fillHistory == 1) && pTask->status.transferState) { + ASSERT(status == TASK_STATUS__NORMAL); + // the maximum version of data in the WAL has reached already, the step2 is done + tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr, + pTask->dataRange.range.maxVer); streamMetaReleaseTask(pStreamMeta, pTask); continue; } @@ -157,39 +286,49 @@ int32_t createStreamTaskRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { continue; } - // append the data for the stream + int32_t numOfItemsInQ = taosQueueItemSize(pTask->inputQueue->queue); + int64_t maxVer = (pTask->info.fillHistory == 1)? pTask->dataRange.range.maxVer:INT64_MAX; + SStreamQueueItem* pItem = NULL; - code = extractMsgFromWal(pTask->exec.pWalReader, (void**) &pItem, pTask->id.idStr); - if (code != TSDB_CODE_SUCCESS) { // failed, continue - streamMetaReleaseTask(pStreamMeta, pTask); - continue; - } + code = extractMsgFromWal(pTask->exec.pWalReader, (void**) &pItem, maxVer, pTask->id.idStr); - // delete ignore - if (pItem == NULL) { + if ((code != TSDB_CODE_SUCCESS || pItem == NULL) && (numOfItemsInQ == 0)) { // failed, continue + checkForFillHistoryVerRange(pTask, walReaderGetCurrentVer(pTask->exec.pWalReader)); streamMetaReleaseTask(pStreamMeta, pTask); continue; } - noNewDataInWal = false; + if (pItem != NULL) { + noDataInWal = false; + code = tAppendDataToInputQueue(pTask, pItem); + if (code == TSDB_CODE_SUCCESS) { + int64_t ver = walReaderGetCurrentVer(pTask->exec.pWalReader); + pTask->chkInfo.currentVer = ver; + checkForFillHistoryVerRange(pTask, ver); + tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr, ver); + } else { + tqError("s-task:%s append input queue failed, too many in inputQ, ver:%" PRId64, pTask->id.idStr, + pTask->chkInfo.currentVer); + } + } - code = tqAddInputBlockNLaunchTask(pTask, pItem); - if (code == TSDB_CODE_SUCCESS) { - pTask->chkInfo.currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader); - tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr, - pTask->chkInfo.currentVer); - } else { - tqError("s-task:%s append input queue failed, ver:%" PRId64, pTask->id.idStr, pTask->chkInfo.currentVer); + if ((code == TSDB_CODE_SUCCESS) || (numOfItemsInQ > 0)) { + code = streamSchedExec(pTask); + if (code != TSDB_CODE_SUCCESS) { + streamMetaReleaseTask(pStreamMeta, pTask); + return -1; + } } streamMetaReleaseTask(pStreamMeta, pTask); } // all wal are checked, and no new data available in wal. - if (noNewDataInWal) { + if (noDataInWal) { *pScanIdle = true; } taosArrayDestroy(pTaskList); return 0; } + diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 1ff78c586f747ad74bce7d752818bddd90a1de67..cbe3ffee9e9fa5f62bad48fc0df07bfedf2ea610 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -48,7 +48,7 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, int32_t n) { SMetaReader mr = {0}; - metaReaderInit(&mr, pTq->pVnode->pMeta, 0); + metaReaderDoInit(&mr, pTq->pVnode->pMeta, 0); // TODO add reference to gurantee success if (metaReaderGetTableEntryByUidCache(&mr, uid) < 0) { @@ -215,19 +215,15 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR taosArrayClear(pSchemas); SSubmitTbData* pSubmitTbDataRet = NULL; if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet) < 0) { - if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; + if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_table; } if (pRsp->withTbName) { int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { - taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); - taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); - pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); - pSchemas = taosArrayInit(0, sizeof(void*)); - continue; + goto loop_table; } } - if (pHandle->fetchMeta && pSubmitTbDataRet->pCreateTbReq != NULL) { + if (pHandle->fetchMeta != WITH_DATA && pSubmitTbDataRet->pCreateTbReq != NULL) { if (pRsp->createTableNum == 0) { pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); @@ -237,7 +233,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR uint32_t len = 0; tEncodeSize(tEncodeSVCreateTbReq, pSubmitTbDataRet->pCreateTbReq, len, code); if (TSDB_CODE_SUCCESS != code) { - continue; + goto loop_table; } void* createReq = taosMemoryCalloc(1, len); SEncoder encoder = {0}; @@ -246,7 +242,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (code < 0) { tEncoderClear(&encoder); taosMemoryFree(createReq); - continue; + goto loop_table; } taosArrayPush(pRsp->createTableLen, &len); @@ -255,6 +251,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR tEncoderClear(&encoder); } + if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL){ + goto loop_table; + } for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { SSDataBlock* pBlock = taosArrayGet(pBlocks, i); tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock), @@ -265,6 +264,12 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR taosArrayPush(pRsp->blockSchema, &pSW); pRsp->blockNum++; } + continue; + loop_table: + taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); + taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); + pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); + pSchemas = taosArrayInit(0, sizeof(void*)); } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { STqReader* pReader = pExec->pTqReader; @@ -274,19 +279,15 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR taosArrayClear(pSchemas); SSubmitTbData* pSubmitTbDataRet = NULL; if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet) < 0) { - if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; + if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_db; } if (pRsp->withTbName) { int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { - taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); - taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); - pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); - pSchemas = taosArrayInit(0, sizeof(void*)); - continue; + goto loop_db; } } - if (pHandle->fetchMeta && pSubmitTbDataRet->pCreateTbReq != NULL) { + if (pHandle->fetchMeta != WITH_DATA && pSubmitTbDataRet->pCreateTbReq != NULL) { if (pRsp->createTableNum == 0) { pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); @@ -296,7 +297,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR uint32_t len = 0; tEncodeSize(tEncodeSVCreateTbReq, pSubmitTbDataRet->pCreateTbReq, len, code); if (TSDB_CODE_SUCCESS != code) { - continue; + goto loop_db; } void* createReq = taosMemoryCalloc(1, len); SEncoder encoder = {0}; @@ -305,7 +306,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (code < 0) { tEncoderClear(&encoder); taosMemoryFree(createReq); - continue; + goto loop_db; } taosArrayPush(pRsp->createTableLen, &len); @@ -314,6 +315,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR tEncoderClear(&encoder); } + if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL){ + goto loop_db; + } for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { SSDataBlock* pBlock = taosArrayGet(pBlocks, i); tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock), @@ -324,6 +328,12 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR taosArrayPush(pRsp->blockSchema, &pSW); pRsp->blockNum++; } + continue; + loop_db: + taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); + taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); + pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); + pSchemas = taosArrayInit(0, sizeof(void*)); } } taosArrayDestroy(pBlocks); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 650f62828f34c29189f2b7674d54446cc52c37b1..b22650d2498c17677607a99fce0dd4cba312d230 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -309,7 +309,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d tbData.uid = pTableSinkInfo->uid; } else { SMetaReader mr = {0}; - metaReaderInit(&mr, pVnode->pMeta, 0); + metaReaderDoInit(&mr, pVnode->pMeta, 0); if (metaGetTableEntryByName(&mr, ctbName) < 0) { metaReaderClear(&mr); taosMemoryFree(pTableSinkInfo); @@ -412,7 +412,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d if (k == 0) { SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); void* colData = colDataGetData(pColData, j); - tqDebug("tq sink pipe2, row %d, col %d ts %" PRId64, j, k, *(int64_t*)colData); + tqTrace("tq sink pipe2, row %d, col %d ts %" PRId64, j, k, *(int64_t*)colData); } if (IS_SET_NULL(pCol)) { SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 4320995306d8f5ae47916f0f5ed67bff784abcb0..906208424658727e17926d0d8c685d9818ff994a 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -20,27 +20,6 @@ static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId); -char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { - char buf[128] = {0}; - sprintf(buf, "0x%" PRIx64 "-0x%x", streamId, taskId); - return taosStrdup(buf); -} - -int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem) { - int32_t code = tAppendDataToInputQueue(pTask, pQueueItem); - if (code < 0) { - tqError("s-task:%s failed to put into queue, too many", pTask->id.idStr); - return -1; - } - - if (streamSchedExec(pTask) < 0) { - tqError("stream task:%d failed to be launched, code:%s", pTask->id.taskId, tstrerror(terrno)); - return -1; - } - - return TSDB_CODE_SUCCESS; -} - int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq) { pRsp->reqOffset = pReq->reqOffset; @@ -123,28 +102,17 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { walRefLastVer(pTq->pVnode->pWal, pHandle->pRef); - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pRequest); - - tqOffsetResetToLog(&dataRsp.rspOffset, pHandle->pRef->refVer + 1); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, - pHandle->subKey, vgId, dataRsp.rspOffset.version); - int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); - tDeleteMqDataRsp(&dataRsp); - - *pBlockReturned = true; - return code; - } else { - STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pRequest); - tqOffsetResetToLog(&taosxRsp.rspOffset, pHandle->pRef->refVer + 1); - int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); - tDeleteSTaosxRsp(&taosxRsp); - - *pBlockReturned = true; - return code; - } + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pRequest); + + tqOffsetResetToLog(&dataRsp.rspOffset, pHandle->pRef->refVer + 1); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, + pHandle->subKey, vgId, dataRsp.rspOffset.version); + int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + tDeleteMqDataRsp(&dataRsp); + + *pBlockReturned = true; + return code; } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", @@ -192,8 +160,9 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, } taosWUnLockLatch(&pTq->lock); } + setRequestVersion(&dataRsp.reqOffset, pOffset->version); - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); end : { char buf[TSDB_OFFSET_LEN] = {0}; @@ -237,7 +206,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid, taosxRsp.rspOffset.ts); if (taosxRsp.blockNum > 0) { - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } else { *offset = taosxRsp.rspOffset; @@ -268,7 +237,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead, pRequest->reqId) < 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); setRequestVersion(&taosxRsp.reqOffset, offset->version); - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } @@ -281,7 +250,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (totalRows > 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); setRequestVersion(&taosxRsp.reqOffset, offset->version); - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } @@ -311,7 +280,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (totalRows >= 4096 || taosxRsp.createTableNum > 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer + 1); setRequestVersion(&taosxRsp.reqOffset, offset->version); - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } else { fetchVer++; @@ -408,9 +377,9 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* int32_t len = 0; int32_t code = 0; - if (type == TMQ_MSG_TYPE__POLL_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { + if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { tEncodeSize(tEncodeMqDataRsp, pRsp, len, code); - } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { + } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code); } @@ -432,9 +401,9 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* SEncoder encoder = {0}; tEncoderInit(&encoder, abuf, len); - if (type == TMQ_MSG_TYPE__POLL_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { + if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { tEncodeMqDataRsp(&encoder, pRsp); - } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { + } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index f4e888919e825ce0601099dc0a64e1cf2ef828b3..32429ff8c6eb6ff96b9877d4d88066df7fababd1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -13,6 +13,9 @@ * along with this program. If not, see . */ #include "tsdb.h" +#include "tsdbDataFileRW.h" +#include "tsdbReadUtil.h" +#include "vnd.h" #define ROCKS_BATCH_SIZE (4096) @@ -58,16 +61,10 @@ typedef struct { static void tsdbGetRocksPath(STsdb *pTsdb, char *path) { SVnode *pVnode = pTsdb->pVnode; - if (pVnode->pTfs) { - if (path) { - snprintf(path, TSDB_FILENAME_LEN, "%s%s%s%scache.rdb", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP, - pTsdb->path, TD_DIRSEP); - } - } else { - if (path) { - snprintf(path, TSDB_FILENAME_LEN, "%s%scache.rdb", pTsdb->path, TD_DIRSEP); - } - } + vnodeGetPrimaryDir(pTsdb->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN); + + int32_t offset = strlen(path); + snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%scache.rdb", TD_DIRSEP); } static const char *myCmpName(void *state) { @@ -1031,55 +1028,7 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache return code; } -/* -int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { - int32_t code = 0; - SLRUCache *pCache = pTsdb->lruCache; - SArray *pCidList = pr->pCidList; - int num_keys = TARRAY_SIZE(pCidList); - - for (int i = 0; i < num_keys; ++i) { - SLastCol *pLastCol = NULL; - int16_t cid = *(int16_t *)taosArrayGet(pCidList, i); - - SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid}; - LRUHandle *h = taosLRUCacheLookup(pCache, key, ROCKS_KEY_LEN); - if (!h) { - taosThreadMutexLock(&pTsdb->lruMutex); - h = taosLRUCacheLookup(pCache, key, ROCKS_KEY_LEN); - if (!h) { - pLastCol = tsdbCacheLoadCol(pTsdb, pr, pr->pSlotIds[i], uid, cid, ltype); - - size_t charge = sizeof(*pLastCol); - if (IS_VAR_DATA_TYPE(pLastCol->colVal.type)) { - charge += pLastCol->colVal.value.nData; - } - - LRUStatus status = taosLRUCacheInsert(pCache, key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, &h, - TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } - } - - taosThreadMutexUnlock(&pTsdb->lruMutex); - } - - pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h); - - SLastCol lastCol = *pLastCol; - reallocVarData(&lastCol.colVal); - - if (h) { - taosLRUCacheRelease(pCache, h, false); - } - taosArrayPush(pLastArray, &lastCol); - } - - return code; -} -*/ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) { int32_t code = 0; // fetch schema @@ -1111,6 +1060,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE char **values_list = taosMemoryCalloc(num_keys * 2, sizeof(char *)); size_t *values_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t)); char **errs = taosMemoryCalloc(num_keys * 2, sizeof(char *)); + taosThreadMutexLock(&pTsdb->lruMutex); taosThreadMutexLock(&pTsdb->rCache.rMutex); rocksMayWrite(pTsdb, true, false, false); rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys * 2, (const char *const *)keys_list, @@ -1140,7 +1090,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE rocksdb_free(values_list[i]); rocksdb_free(values_list[i + num_keys]); - taosThreadMutexLock(&pTsdb->lruMutex); + // taosThreadMutexLock(&pTsdb->lruMutex); LRUHandle *h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[i], klen); if (h) { @@ -1162,7 +1112,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE } taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen); - taosThreadMutexUnlock(&pTsdb->lruMutex); + // taosThreadMutexUnlock(&pTsdb->lruMutex); } for (int i = 0; i < num_keys; ++i) { taosMemoryFree(keys_list[i]); @@ -1174,6 +1124,8 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE rocksMayWrite(pTsdb, true, false, true); + taosThreadMutexUnlock(&pTsdb->lruMutex); + _exit: taosMemoryFree(pTSchema); @@ -1314,62 +1266,7 @@ int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { return code; } -/* -int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { - int32_t code = 0; - char key[32] = {0}; - int keyLen = 0; - - // getTableCacheKey(uid, "lr", key, &keyLen); - getTableCacheKey(uid, 0, key, &keyLen); - LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (h) { - SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); - bool invalidate = false; - int16_t nCol = taosArrayGetSize(pLast); - - for (int16_t iCol = 0; iCol < nCol; ++iCol) { - SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); - if (eKey >= tTsVal->ts) { - invalidate = true; - break; - } - } - - if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); - } - } - - // getTableCacheKey(uid, "l", key, &keyLen); - getTableCacheKey(uid, 1, key, &keyLen); - h = taosLRUCacheLookup(pCache, key, keyLen); - if (h) { - SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); - bool invalidate = false; - int16_t nCol = taosArrayGetSize(pLast); - - for (int16_t iCol = 0; iCol < nCol; ++iCol) { - SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); - if (eKey >= tTsVal->ts) { - invalidate = true; - break; - } - } - - if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); - } - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); - } - return code; -} -*/ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup) { int32_t code = 0; STSRow *cacheRow = NULL; @@ -1610,7 +1507,7 @@ static tb_uid_t getTableSuidByUid(tb_uid_t uid, STsdb *pTsdb) { tb_uid_t suid = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0); + metaReaderDoInit(&mr, pTsdb->pVnode->pMeta, 0); if (metaReaderGetTableEntryByUidCache(&mr, uid) < 0) { metaReaderClear(&mr); // table not esist return 0; @@ -1694,533 +1591,549 @@ _err: } return code; } -/* -static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t uid, SDelIdx *pDelIdx) { - int32_t code = 0; - SArray *pDelIdxArray = NULL; - // SMapData delIdxMap; - pDelIdxArray = taosArrayInit(32, sizeof(SDelIdx)); - SDelIdx idx = {.suid = suid, .uid = uid}; - - // tMapDataReset(&delIdxMap); - code = tsdbReadDelIdx(pDelFReader, pDelIdxArray); - if (code) goto _err; - - // code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx); - SDelIdx *pIdx = taosArraySearch(pDelIdxArray, &idx, tCmprDelIdx, TD_EQ); +static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsReader *pReader, void *pFileReader, + bool isFile) { + int32_t code = 0; + uint64_t *uidList = pReader->uidList; + int32_t numOfTables = pReader->numOfTables; + int64_t suid = pReader->info.suid; - *pDelIdx = *pIdx; + for (int i = 0, j = 0; i < pTombBlkArray->size && j < numOfTables; ++i) { + STombBlk *pTombBlk = &pTombBlkArray->data[i]; + if (pTombBlk->maxTbid.suid < suid || (pTombBlk->maxTbid.suid == suid && pTombBlk->maxTbid.uid < uidList[0])) { + continue; + } -_err: - if (pDelIdxArray) { - taosArrayDestroy(pDelIdxArray); - } - return code; -} -*/ -typedef enum { - SFSLASTNEXTROW_FS, - SFSLASTNEXTROW_FILESET, - SFSLASTNEXTROW_BLOCKDATA, - SFSLASTNEXTROW_BLOCKROW -} SFSLASTNEXTROWSTATES; + if (pTombBlk->minTbid.suid > suid || + (pTombBlk->minTbid.suid == suid && pTombBlk->minTbid.uid > uidList[numOfTables - 1])) { + break; + } -typedef struct { - SFSLASTNEXTROWSTATES state; // [input] - STsdb *pTsdb; // [input] - STSchema *pTSchema; // [input] - tb_uid_t suid; - tb_uid_t uid; - int32_t nFileSet; - int32_t iFileSet; - SArray *aDFileSet; - SDataFReader **pDataFReader; - TSDBROW row; - - bool checkRemainingRow; - SMergeTree mergeTree; - SMergeTree *pMergeTree; - SSttBlockLoadInfo *pLoadInfo; - SLDataIter *pDataIter; - int64_t lastTs; -} SFSLastNextRowIter; - -static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, - int nCols) { - SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter; - int32_t code = 0; - bool checkRemainingRow = true; + STombBlock block = {0}; + code = isFile ? tsdbDataFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block) + : tsdbSttFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - switch (state->state) { - case SFSLASTNEXTROW_FS: - state->nFileSet = taosArrayGetSize(state->aDFileSet); - state->iFileSet = state->nFileSet; + uint64_t uid = uidList[j]; + STableLoadInfo *pInfo = *(STableLoadInfo **)tSimpleHashGet(pReader->pTableMap, &uid, sizeof(uid)); + if (pInfo->pTombData == NULL) { + pInfo->pTombData = taosArrayInit(4, sizeof(SDelData)); + } - case SFSLASTNEXTROW_FILESET: { - SDFileSet *pFileSet = NULL; - _next_fileset: - if (state->pMergeTree != NULL) { - tMergeTreeClose(state->pMergeTree); - state->pMergeTree = NULL; + STombRecord record = {0}; + bool finished = false; + for (int32_t k = 0; k < TARRAY2_SIZE(block.suid); ++k) { + code = tTombBlockGet(&block, k, &record); + if (code != TSDB_CODE_SUCCESS) { + finished = true; + break; } - if (--state->iFileSet >= 0) { - pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); - } else { - *ppRow = NULL; - return code; + if (record.suid < suid) { + continue; + } + if (record.suid > suid) { + finished = true; + break; } - if (*state->pDataFReader == NULL || (*state->pDataFReader)->pSet->fid != pFileSet->fid) { - if (*state->pDataFReader != NULL) { - tsdbDataFReaderClose(state->pDataFReader); + bool newTable = false; + if (uid < record.uid) { + while (j < numOfTables && uidList[j] < record.uid) { + ++j; + newTable = true; + } - resetLastBlockLoadInfo(state->pLoadInfo); + if (j >= numOfTables) { + finished = true; + break; } - code = tsdbDataFReaderOpen(state->pDataFReader, state->pTsdb, pFileSet); - if (code) goto _err; + uid = uidList[j]; } - int nTmpCols = nCols; - bool hasTs = false; - if (aCols[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { - --nTmpCols; - hasTs = true; - } - for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) { - state->pLoadInfo[i].colIds = hasTs ? aCols + 1 : aCols; - state->pLoadInfo[i].numOfCols = nTmpCols; - state->pLoadInfo[i].isLast = isLast; + if (record.uid < uid) { + continue; } - tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid, - &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX}, - &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true, - state->pDataIter); - state->pMergeTree = &state->mergeTree; - state->state = SFSLASTNEXTROW_BLOCKROW; - } - case SFSLASTNEXTROW_BLOCKROW: { - if (nCols != state->pLoadInfo->numOfCols) { - for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) { - state->pLoadInfo[i].numOfCols = nCols; - state->pLoadInfo[i].checkRemainingRow = state->checkRemainingRow; - } - } - bool hasVal = tMergeTreeNext(&state->mergeTree); - if (!hasVal) { - if (tMergeTreeIgnoreEarlierTs(&state->mergeTree)) { - *pIgnoreEarlierTs = true; - *ppRow = NULL; - return code; + if (newTable) { + pInfo = *(STableLoadInfo **)tSimpleHashGet(pReader->pTableMap, &uid, sizeof(uid)); + if (pInfo->pTombData == NULL) { + pInfo->pTombData = taosArrayInit(4, sizeof(SDelData)); } - state->state = SFSLASTNEXTROW_FILESET; - goto _next_fileset; } - state->row = *tMergeTreeGetRow(&state->mergeTree); - *ppRow = &state->row; - if (TSDBROW_TS(&state->row) <= state->lastTs) { - *pIgnoreEarlierTs = true; - *ppRow = NULL; - return code; + if (record.version <= pReader->info.verRange.maxVer) { + /* + tsdbError("tomb xx load/cache: vgId:%d fid:%d commit %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records", + TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid); + */ + SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; + taosArrayPush(pInfo->pTombData, &delData); } + } - *pIgnoreEarlierTs = false; - /* - if (!hasVal) { - state->state = SFSLASTNEXTROW_FILESET; - } - */ - if (!state->checkRemainingRow) { - state->checkRemainingRow = true; - } + tTombBlockDestroy(&block); + + if (finished) { return code; } - default: - ASSERT(0); - break; } -_err: - /*if (state->pDataFReader) { - tsdbDataFReaderClose(&state->pDataFReader); - state->pDataFReader = NULL; - }*/ - if (state->pMergeTree != NULL) { - tMergeTreeClose(state->pMergeTree); - state->pMergeTree = NULL; - } + return TSDB_CODE_SUCCESS; +} - *ppRow = NULL; +static int32_t loadDataTomb(SCacheRowsReader *pReader, SDataFileReader *pFileReader) { + int32_t code = 0; - return code; + const TTombBlkArray *pBlkArray = NULL; + code = tsdbDataFileReadTombBlk(pFileReader, &pBlkArray); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + return loadTombFromBlk(pBlkArray, pReader, pFileReader, true); } -int32_t clearNextRowFromFSLast(void *iter) { - SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter; - int32_t code = 0; +static int32_t loadSttTomb(STsdbReader *pTsdbReader, SSttFileReader *pSttFileReader, SSttBlockLoadInfo *pLoadInfo) { + int32_t code = 0; - if (!state) { + SCacheRowsReader *pReader = (SCacheRowsReader *)pTsdbReader; + + const TTombBlkArray *pBlkArray = NULL; + code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray); + if (code != TSDB_CODE_SUCCESS) { return code; } - /* - if (state->pDataFReader) { - tsdbDataFReaderClose(&state->pDataFReader); - state->pDataFReader = NULL; + + return loadTombFromBlk(pBlkArray, pReader, pSttFileReader, false); +} + +typedef struct { + SMergeTree mergeTree; + SMergeTree *pMergeTree; +} SFSLastIter; + +static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, + tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, int16_t *aCols, int nCols) { + int32_t code = 0; + + int64_t loadBlocks = 0; + double elapse = 0; + pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, &loadBlocks, &elapse); + pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + + SMergeTreeConf conf = { + .uid = uid, + .suid = suid, + .pTsdb = pTsdb, + .timewindow = (STimeWindow){.skey = lastTs, .ekey = TSKEY_MAX}, + .verRange = (SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, + .strictTimeRange = false, + .pSchema = pTSchema, + .pCurrentFileset = pFileSet, + .backward = 1, + .pSttFileBlockIterArray = pr->pLDataIterArray, + .pCols = aCols, + .numOfCols = nCols, + .loadTombFn = loadSttTomb, + .pReader = pr, + .idstr = pr->idstr, + }; + + code = tMergeTreeOpen2(&iter->mergeTree, &conf); + if (code != TSDB_CODE_SUCCESS) { + return -1; + } + + iter->pMergeTree = &iter->mergeTree; + + return code; +} + +static int32_t lastIterClose(SFSLastIter **iter) { + int32_t code = 0; + + if ((*iter)->pMergeTree) { + tMergeTreeClose((*iter)->pMergeTree); + (*iter)->pMergeTree = NULL; } - */ - if (state->pMergeTree != NULL) { - tMergeTreeClose(state->pMergeTree); - state->pMergeTree = NULL; + + *iter = NULL; + + return code; +} + +static int32_t lastIterNext(SFSLastIter *iter, TSDBROW **ppRow) { + int32_t code = 0; + + bool hasVal = tMergeTreeNext(iter->pMergeTree); + if (!hasVal) { + *ppRow = NULL; + return code; } + *ppRow = tMergeTreeGetRow(iter->pMergeTree); + return code; } typedef enum SFSNEXTROWSTATES { SFSNEXTROW_FS, SFSNEXTROW_FILESET, + SFSNEXTROW_INDEXLIST, + SFSNEXTROW_BRINBLOCK, + SFSNEXTROW_BRINRECORD, SFSNEXTROW_BLOCKDATA, - SFSNEXTROW_BLOCKROW + SFSNEXTROW_BLOCKROW, + SFSNEXTROW_NEXTSTTROW } SFSNEXTROWSTATES; +struct CacheNextRowIter; + typedef struct SFSNextRowIter { - SFSNEXTROWSTATES state; // [input] - STsdb *pTsdb; // [input] - SBlockIdx *pBlockIdxExp; // [input] - STSchema *pTSchema; // [input] - tb_uid_t suid; - tb_uid_t uid; - int32_t nFileSet; - int32_t iFileSet; - SArray *aDFileSet; - SDataFReader **pDataFReader; - SArray *aBlockIdx; - LRUHandle *aBlockIdxHandle; - SBlockIdx *pBlockIdx; - SMapData blockMap; - int32_t nBlock; - int32_t iBlock; - SDataBlk block; - SBlockData blockData; - SBlockData *pBlockData; - int32_t nRow; - int32_t iRow; - TSDBROW row; - SSttBlockLoadInfo *pLoadInfo; - int64_t lastTs; + SFSNEXTROWSTATES state; // [input] + STsdb *pTsdb; // [input] + SBlockIdx *pBlockIdxExp; // [input] + STSchema *pTSchema; // [input] + tb_uid_t suid; + tb_uid_t uid; + int32_t nFileSet; + int32_t iFileSet; + STFileSet *pFileSet; + TFileSetArray *aDFileSet; + SArray *pIndexList; + int32_t iBrinIndex; + SBrinBlock brinBlock; + int32_t iBrinRecord; + SBrinRecord brinRecord; + SBlockData blockData; + SBlockData *pBlockData; + int32_t nRow; + int32_t iRow; + TSDBROW row; + int64_t lastTs; + SFSLastIter lastIter; + SFSLastIter *pLastIter; + int8_t lastEmpty; + TSDBROW *pLastRow; + SRow *pTSRow; + SRowMerger rowMerger; + SCacheRowsReader *pr; + struct CacheNextRowIter *pRowIter; } SFSNextRowIter; +static void clearLastFileSet(SFSNextRowIter *state); + static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, int nCols) { SFSNextRowIter *state = (SFSNextRowIter *)iter; int32_t code = 0; - bool checkRemainingRow = true; - switch (state->state) { - case SFSNEXTROW_FS: - // state->aDFileSet = state->pTsdb->pFS->cState->aDFileSet; - state->nFileSet = taosArrayGetSize(state->aDFileSet); - state->iFileSet = state->nFileSet; - - state->pBlockData = NULL; - - case SFSNEXTROW_FILESET: { - SDFileSet *pFileSet = NULL; - _next_fileset: - if (--state->iFileSet >= 0) { - pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); - } else { - // tBlockDataDestroy(&state->blockData, 1); - if (state->pBlockData) { - tBlockDataDestroy(state->pBlockData); - state->pBlockData = NULL; - } + if (SFSNEXTROW_FS == state->state) { + state->nFileSet = TARRAY2_SIZE(state->aDFileSet); + state->iFileSet = state->nFileSet; - *ppRow = NULL; - return code; - } + state->state = SFSNEXTROW_FILESET; + } + + if (SFSNEXTROW_FILESET == state->state) { + _next_fileset: + if (--state->iFileSet < 0) { + clearLastFileSet(state); + + *ppRow = NULL; + return code; + } else { + state->pFileSet = TARRAY2_GET(state->aDFileSet, state->iFileSet); + } + + STFileObj **pFileObj = state->pFileSet->farr; + if (pFileObj[0] != NULL || pFileObj[3] != NULL) { + if (state->pFileSet != state->pr->pCurFileSet) { + SDataFileReaderConfig conf = {.tsdb = state->pTsdb, .szPage = state->pTsdb->pVnode->config.tsdbPageSize}; + const char *filesName[4] = {0}; + if (pFileObj[0] != NULL) { + conf.files[0].file = *pFileObj[0]->f; + conf.files[0].exist = true; + filesName[0] = pFileObj[0]->fname; + + conf.files[1].file = *pFileObj[1]->f; + conf.files[1].exist = true; + filesName[1] = pFileObj[1]->fname; + + conf.files[2].file = *pFileObj[2]->f; + conf.files[2].exist = true; + filesName[2] = pFileObj[2]->fname; + } - if (*state->pDataFReader == NULL || (*state->pDataFReader)->pSet->fid != pFileSet->fid) { - if (*state->pDataFReader != NULL) { - tsdbDataFReaderClose(state->pDataFReader); + if (pFileObj[3] != NULL) { + conf.files[3].exist = true; + conf.files[3].file = *pFileObj[3]->f; + filesName[3] = pFileObj[3]->fname; + } - // resetLastBlockLoadInfo(state->pLoadInfo); + code = tsdbDataFileReaderOpen(filesName, &conf, &state->pr->pFileReader); + if (code != TSDB_CODE_SUCCESS) { + goto _err; } - code = tsdbDataFReaderOpen(state->pDataFReader, state->pTsdb, pFileSet); - if (code) goto _err; + state->pr->pCurFileSet = state->pFileSet; + + loadDataTomb(state->pr, state->pr->pFileReader); } - // tMapDataReset(&state->blockIdxMap); - /* - if (!state->aBlockIdx) { - state->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (!state->pIndexList) { + state->pIndexList = taosArrayInit(1, sizeof(SBrinBlk)); } else { - taosArrayClear(state->aBlockIdx); + taosArrayClear(state->pIndexList); } - code = tsdbReadBlockIdx(*state->pDataFReader, state->aBlockIdx); - if (code) goto _err; - */ - int32_t code = tsdbCacheGetBlockIdx(state->pTsdb->biCache, *state->pDataFReader, &state->aBlockIdxHandle); - if (code != TSDB_CODE_SUCCESS || state->aBlockIdxHandle == NULL) { + const TBrinBlkArray *pBlkArray = NULL; + + int32_t code = tsdbDataFileReadBrinBlk(state->pr->pFileReader, &pBlkArray); + if (code != TSDB_CODE_SUCCESS) { goto _err; } - state->aBlockIdx = (SArray *)taosLRUCacheValue(state->pTsdb->biCache, state->aBlockIdxHandle); - /* if (state->pBlockIdx) { */ - /* } */ - /* code = tMapDataSearch(&state->blockIdxMap, state->pBlockIdxExp, tGetBlockIdx, tCmprBlockIdx, - * &state->blockIdx); - */ - state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ); - if (!state->pBlockIdx) { - tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + for (int i = TARRAY2_SIZE(pBlkArray) - 1; i >= 0; --i) { + SBrinBlk *pBrinBlk = &pBlkArray->data[i]; + if (state->suid >= pBrinBlk->minTbid.suid && state->suid <= pBrinBlk->maxTbid.suid) { + if (state->uid >= pBrinBlk->minTbid.uid && state->uid <= pBrinBlk->maxTbid.uid) { + taosArrayPush(state->pIndexList, pBrinBlk); + } + } else if (state->suid > pBrinBlk->maxTbid.suid || + (state->suid == pBrinBlk->maxTbid.suid && state->uid > pBrinBlk->maxTbid.uid)) { + break; + } + } - state->aBlockIdxHandle = NULL; - state->aBlockIdx = NULL; - /* - tsdbDataFReaderClose(state->pDataFReader); - *state->pDataFReader = NULL; - resetLastBlockLoadInfo(state->pLoadInfo);*/ + int indexSize = TARRAY_SIZE(state->pIndexList); + if (indexSize <= 0) { + clearLastFileSet(state); + state->state = SFSNEXTROW_FILESET; goto _next_fileset; } - tMapDataReset(&state->blockMap); - /* - if (state->blockMap.pData != NULL) { - tMapDataClear(&state->blockMap); - } - */ - code = tsdbReadDataBlk(*state->pDataFReader, state->pBlockIdx, &state->blockMap); - if (code) goto _err; + state->state = SFSNEXTROW_INDEXLIST; + state->iBrinIndex = indexSize; + } - state->nBlock = state->blockMap.nItem; - state->iBlock = state->nBlock - 1; + if (state->pFileSet != state->pr->pCurFileSet) { + state->pr->pCurFileSet = state->pFileSet; + } - if (!state->pBlockData) { - state->pBlockData = &state->blockData; + code = lastIterOpen(&state->lastIter, state->pFileSet, state->pTsdb, state->pTSchema, state->suid, state->uid, + state->pr, state->lastTs, aCols, nCols); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - code = tBlockDataCreate(&state->blockData); - if (code) goto _err; - } + code = lastIterNext(&state->lastIter, &state->pLastRow); + if (code != TSDB_CODE_SUCCESS) { + goto _err; } - case SFSNEXTROW_BLOCKDATA: - _next_datablock: - if (state->iBlock >= 0) { - SDataBlk block = {0}; - bool skipBlock = true; - int inputColIndex = 0; - tDataBlkReset(&block); - tBlockDataReset(state->pBlockData); + if (!state->pLastRow) { + state->lastEmpty = 1; - tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk); - if (block.maxKey.ts <= state->lastTs) { - *pIgnoreEarlierTs = true; + if (SFSNEXTROW_INDEXLIST != state->state) { + clearLastFileSet(state); + goto _next_fileset; + } + } else { + state->lastEmpty = 0; - tBlockDataDestroy(state->pBlockData); - state->pBlockData = NULL; + if (SFSNEXTROW_INDEXLIST != state->state) { + state->state = SFSNEXTROW_NEXTSTTROW; - *ppRow = NULL; - return code; - } - *pIgnoreEarlierTs = false; - tBlockDataReset(state->pBlockData); - TABLEID tid = {.suid = state->suid, .uid = state->uid}; - int nTmpCols = nCols; - bool hasTs = false; - if (aCols[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { - --nTmpCols; - skipBlock = false; - hasTs = true; - } - code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, hasTs ? aCols + 1 : aCols, nTmpCols); - if (code) goto _err; + *ppRow = state->pLastRow; + state->pLastRow = NULL; + return code; + } + } - code = tsdbReadDataBlock(*state->pDataFReader, &block, state->pBlockData); - if (code) goto _err; + state->pLastIter = &state->lastIter; + } - for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) { - SColData *pColData = &state->pBlockData->aColData[colIndex]; + if (SFSNEXTROW_NEXTSTTROW == state->state) { + code = lastIterNext(&state->lastIter, &state->pLastRow); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - if (isLast && (pColData->flag & HAS_VALUE)) { - skipBlock = false; - break; - } /*else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { - skipBlock = false; - break; - }*/ - } + if (!state->pLastRow) { + if (state->pLastIter) { + lastIterClose(&state->pLastIter); + } - if (!isLast) { - skipBlock = false; - } + clearLastFileSet(state); + state->state = SFSNEXTROW_FILESET; + goto _next_fileset; + } else { + *ppRow = state->pLastRow; + state->pLastRow = NULL; + return code; + } + } - if (skipBlock) { - if (--state->iBlock < 0) { - tsdbDataFReaderClose(state->pDataFReader); - *state->pDataFReader = NULL; - // resetLastBlockLoadInfo(state->pLoadInfo); + if (SFSNEXTROW_INDEXLIST == state->state) { + SBrinBlk *pBrinBlk = NULL; + _next_brinindex: + if (--state->iBrinIndex < 0) { + if (state->pLastRow) { + state->state = SFSNEXTROW_NEXTSTTROW; + *ppRow = state->pLastRow; + state->pLastRow = NULL; + return code; + } - if (state->aBlockIdx) { - // taosArrayDestroy(state->aBlockIdx); - tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + clearLastFileSet(state); + goto _next_fileset; + } else { + pBrinBlk = taosArrayGet(state->pIndexList, state->iBrinIndex); + } - state->aBlockIdxHandle = NULL; - state->aBlockIdx = NULL; - } + code = tsdbDataFileReadBrinBlock(state->pr->pFileReader, pBrinBlk, &state->brinBlock); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - state->state = SFSNEXTROW_FILESET; - goto _next_fileset; - } else { - goto _next_datablock; - } - } + state->iBrinRecord = BRIN_BLOCK_SIZE(&state->brinBlock) - 1; + state->state = SFSNEXTROW_BRINBLOCK; + } - state->nRow = state->blockData.nRow; - state->iRow = state->nRow - 1; + if (SFSNEXTROW_BRINBLOCK == state->state) { + _next_brinrecord: + if (state->iBrinRecord < 0) { // empty brin block, goto _next_brinindex + tBrinBlockClear(&state->brinBlock); + goto _next_brinindex; + } + code = tBrinBlockGet(&state->brinBlock, state->iBrinRecord, &state->brinRecord); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - state->state = SFSNEXTROW_BLOCKROW; - checkRemainingRow = false; - } - case SFSNEXTROW_BLOCKROW: { - if (checkRemainingRow) { - bool skipBlock = true; - int inputColIndex = 0; - if (aCols[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { - ++inputColIndex; - } - for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) { - SColData *pColData = &state->pBlockData->aColData[colIndex]; - int16_t cid = pColData->cid; - - if (inputColIndex < nCols && cid == aCols[inputColIndex]) { - if (isLast && (pColData->flag & HAS_VALUE)) { - skipBlock = false; - break; - } /*else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { - skipBlock = false; - break; - }*/ - - ++inputColIndex; - } - } + SBrinRecord *pRecord = &state->brinRecord; + if (pRecord->uid != state->uid) { + // TODO: goto next brin block early + --state->iBrinRecord; + goto _next_brinrecord; + } - if (!isLast) { - skipBlock = false; - } + state->state = SFSNEXTROW_BRINRECORD; + } - if (skipBlock) { - if (--state->iBlock < 0) { - tsdbDataFReaderClose(state->pDataFReader); - *state->pDataFReader = NULL; - // resetLastBlockLoadInfo(state->pLoadInfo); + if (SFSNEXTROW_BRINRECORD == state->state) { + SBrinRecord *pRecord = &state->brinRecord; - if (state->aBlockIdx) { - // taosArrayDestroy(state->aBlockIdx); - tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + if (!state->pBlockData) { + state->pBlockData = &state->blockData; + code = tBlockDataCreate(&state->blockData); + if (code) goto _err; + } else { + tBlockDataReset(state->pBlockData); + } - state->aBlockIdxHandle = NULL; - state->aBlockIdx = NULL; - } + if (aCols[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { + --nCols; + ++aCols; + } + code = tsdbDataFileReadBlockDataByColumn(state->pr->pFileReader, pRecord, state->pBlockData, state->pTSchema, aCols, + nCols); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - state->state = SFSNEXTROW_FILESET; - goto _next_fileset; - } else { - goto _next_datablock; - } - } - } + state->nRow = state->blockData.nRow; + state->iRow = state->nRow - 1; - if (state->iRow >= 0) { - state->row = tsdbRowFromBlockData(state->pBlockData, state->iRow); - *ppRow = &state->row; + state->state = SFSNEXTROW_BLOCKROW; + } - if (--state->iRow < 0) { - state->state = SFSNEXTROW_BLOCKDATA; - if (--state->iBlock < 0) { - tsdbDataFReaderClose(state->pDataFReader); - *state->pDataFReader = NULL; - // resetLastBlockLoadInfo(state->pLoadInfo); + if (SFSNEXTROW_BLOCKROW == state->state) { + if (state->iRow < 0) { + --state->iBrinRecord; + goto _next_brinrecord; + } - if (state->aBlockIdx) { - // taosArrayDestroy(state->aBlockIdx); - tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + state->row = tsdbRowFromBlockData(state->pBlockData, state->iRow); + if (!state->pLastIter) { + *ppRow = &state->row; + --state->iRow; + return code; + } - state->aBlockIdxHandle = NULL; - state->aBlockIdx = NULL; - } + if (!state->pLastRow) { + // get next row from fslast and process with fs row, --state->Row if select fs row + code = lastIterNext(&state->lastIter, &state->pLastRow); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + } - state->state = SFSNEXTROW_FILESET; - } - } + if (!state->pLastRow) { + if (state->pLastIter) { + lastIterClose(&state->pLastIter); } + *ppRow = &state->row; + --state->iRow; return code; } - default: - ASSERT(0); - break; - } -_err: - /* - if (*state->pDataFReader) { - tsdbDataFReaderClose(state->pDataFReader); - *state->pDataFReader = NULL; - resetLastBlockLoadInfo(state->pLoadInfo); - }*/ - if (state->aBlockIdx) { - // taosArrayDestroy(state->aBlockIdx); - tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); - - state->aBlockIdxHandle = NULL; - state->aBlockIdx = NULL; - } - if (state->pBlockData) { - tBlockDataDestroy(state->pBlockData); - state->pBlockData = NULL; - } + // process state->pLastRow & state->row + TSKEY rowTs = TSDBROW_TS(&state->row); + TSKEY lastRowTs = TSDBROW_TS(state->pLastRow); + if (lastRowTs > rowTs) { + *ppRow = state->pLastRow; + state->pLastRow = NULL; + return code; + } else if (lastRowTs < rowTs) { + *ppRow = &state->row; + --state->iRow; + return code; + } else { + // TODO: merge rows and *ppRow = mergedRow + SRowMerger *pMerger = &state->rowMerger; + tsdbRowMergerInit(pMerger, state->pTSchema); - *ppRow = NULL; + code = tsdbRowMergerAdd(pMerger, &state->row, state->pTSchema); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + code = tsdbRowMergerAdd(pMerger, state->pLastRow, state->pTSchema); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - return code; -} + if (state->pTSRow) { + taosMemoryFree(state->pTSRow); + state->pTSRow = NULL; + } -int32_t clearNextRowFromFS(void *iter) { - int32_t code = 0; + code = tsdbRowMergerGetRow(pMerger, &state->pTSRow); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - SFSNextRowIter *state = (SFSNextRowIter *)iter; - if (!state) { - return code; - } - /* - if (state->pDataFReader) { - tsdbDataFReaderClose(&state->pDataFReader); - state->pDataFReader = NULL; - }*/ - if (state->aBlockIdx) { - // taosArrayDestroy(state->aBlockIdx); - tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + state->row = tsdbRowFromTSRow(TSDBROW_VERSION(&state->row), state->pTSRow); + *ppRow = &state->row; + --state->iRow; - state->aBlockIdxHandle = NULL; - state->aBlockIdx = NULL; - } - if (state->pBlockData) { - // tBlockDataDestroy(&state->blockData, 1); - tBlockDataDestroy(state->pBlockData); - state->pBlockData = NULL; - } + tsdbRowMergerClear(pMerger); - if (state->blockMap.pData != NULL) { - tMapDataClear(&state->blockMap); + return code; + } } +_err: + clearLastFileSet(state); + + *ppRow = NULL; + return code; } @@ -2234,8 +2147,6 @@ typedef struct SMemNextRowIter { STbData *pMem; // [input] STbDataIter iter; // mem buffer skip list iterator int64_t lastTs; - // bool iterOpened; - // TSDBROW *curRow; } SMemNextRowIter; static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, @@ -2286,45 +2197,6 @@ _err: return code; } -/* static int32_t tsRowFromTsdbRow(STSchema *pTSchema, TSDBROW *pRow, STSRow **ppRow) { */ -/* int32_t code = 0; */ - -/* SColVal *pColVal = &(SColVal){0}; */ - -/* if (pRow->type == 0) { */ -/* *ppRow = tdRowDup(pRow->pTSRow); */ -/* } else { */ -/* SArray *pArray = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); */ -/* if (pArray == NULL) { */ -/* code = TSDB_CODE_OUT_OF_MEMORY; */ -/* goto _exit; */ -/* } */ - -/* TSDBKEY key = TSDBROW_KEY(pRow); */ -/* STColumn *pTColumn = &pTSchema->columns[0]; */ -/* *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts}); */ - -/* if (taosArrayPush(pArray, pColVal) == NULL) { */ -/* code = TSDB_CODE_OUT_OF_MEMORY; */ -/* goto _exit; */ -/* } */ - -/* for (int16_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { */ -/* tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); */ -/* if (taosArrayPush(pArray, pColVal) == NULL) { */ -/* code = TSDB_CODE_OUT_OF_MEMORY; */ -/* goto _exit; */ -/* } */ -/* } */ - -/* code = tdSTSRowNew(pArray, pTSchema, ppRow); */ -/* if (code) goto _exit; */ -/* } */ - -/* _exit: */ -/* return code; */ -/* } */ - static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { bool deleted = false; while (*iSkyline > 0) { @@ -2370,103 +2242,122 @@ typedef struct { _next_row_clear_fn_t nextRowClearFn; } TsdbNextRowState; -typedef struct { - SArray *pSkyline; - int64_t iSkyline; - - SBlockIdx idx; - SMemNextRowIter memState; - SMemNextRowIter imemState; - SFSLastNextRowIter fsLastState; - SFSNextRowIter fsState; - TSDBROW memRow, imemRow, fsLastRow, fsRow; - - TsdbNextRowState input[4]; - STsdb *pTsdb; +typedef struct CacheNextRowIter { + SArray *pMemDelData; + SArray *pSkyline; + int64_t iSkyline; + SBlockIdx idx; + SMemNextRowIter memState; + SMemNextRowIter imemState; + SFSNextRowIter fsState; + TSDBROW memRow, imemRow, fsLastRow, fsRow; + TsdbNextRowState input[3]; + SCacheRowsReader *pr; + STsdb *pTsdb; } CacheNextRowIter; -static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, - SSttBlockLoadInfo *pLoadInfo, SLDataIter *pLDataIter, STsdbReadSnap *pReadSnap, - SDataFReader **pDataFReader, SDataFReader **pDataFReaderLast, int64_t lastTs) { - int code = 0; +int32_t clearNextRowFromFS(void *iter) { + int32_t code = 0; - STbData *pMem = NULL; - if (pReadSnap->pMem) { - pMem = tsdbGetTbDataFromMemTable(pReadSnap->pMem, suid, uid); + SFSNextRowIter *state = (SFSNextRowIter *)iter; + if (!state) { + return code; } - STbData *pIMem = NULL; - if (pReadSnap->pIMem) { - pIMem = tsdbGetTbDataFromMemTable(pReadSnap->pIMem, suid, uid); + if (state->pLastIter) { + lastIterClose(&state->pLastIter); } - pIter->pTsdb = pTsdb; + if (state->pBlockData) { + tBlockDataDestroy(state->pBlockData); + state->pBlockData = NULL; + } - pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); + if (state->pTSRow) { + taosMemoryFree(state->pTSRow); + state->pTSRow = NULL; + } - SDelFile *pDelFile = pReadSnap->fs.pDelFile; - if (pDelFile) { - SDelFReader *pDelFReader; + if (state->pRowIter->pSkyline) { + taosArrayDestroy(state->pRowIter->pSkyline); + state->pRowIter->pSkyline = NULL; + } - code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb); - if (code) goto _err; + return code; +} - SArray *pDelIdxArray = taosArrayInit(32, sizeof(SDelIdx)); +static void clearLastFileSet(SFSNextRowIter *state) { + if (state->pLastIter) { + lastIterClose(&state->pLastIter); + } - code = tsdbReadDelIdx(pDelFReader, pDelIdxArray); - if (code) { - taosArrayDestroy(pDelIdxArray); - tsdbDelFReaderClose(&pDelFReader); - goto _err; - } + if (state->pBlockData) { + tBlockDataDestroy(state->pBlockData); + state->pBlockData = NULL; + } - SDelIdx *delIdx = taosArraySearch(pDelIdxArray, &(SDelIdx){.suid = suid, .uid = uid}, tCmprDelIdx, TD_EQ); + if (state->pr->pFileReader) { + tsdbDataFileReaderClose(&state->pr->pFileReader); + state->pr->pFileReader = NULL; - code = getTableDelSkyline(pMem, pIMem, pDelFReader, delIdx, pIter->pSkyline); - if (code) { - taosArrayDestroy(pDelIdxArray); - tsdbDelFReaderClose(&pDelFReader); - goto _err; + state->pr->pCurFileSet = NULL; + } + + if (state->pTSRow) { + taosMemoryFree(state->pTSRow); + state->pTSRow = NULL; + } + + if (state->pRowIter->pSkyline) { + taosArrayDestroy(state->pRowIter->pSkyline); + state->pRowIter->pSkyline = NULL; + + void *pe = NULL; + int32_t iter = 0; + while ((pe = tSimpleHashIterate(state->pr->pTableMap, pe, &iter)) != NULL) { + STableLoadInfo *pInfo = *(STableLoadInfo **)pe; + pInfo->pTombData = taosArrayDestroy(pInfo->pTombData); } + } +} - taosArrayDestroy(pDelIdxArray); - tsdbDelFReaderClose(&pDelFReader); - } else { - code = getTableDelSkyline(pMem, pIMem, NULL, NULL, pIter->pSkyline); - if (code) goto _err; +static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, + SArray *pLDataIterArray, STsdbReadSnap *pReadSnap, int64_t lastTs, + SCacheRowsReader *pr) { + int code = 0; + + STbData *pMem = NULL; + if (pReadSnap->pMem) { + pMem = tsdbGetTbDataFromMemTable(pReadSnap->pMem, suid, uid); } - pIter->iSkyline = taosArrayGetSize(pIter->pSkyline) - 1; + STbData *pIMem = NULL; + if (pReadSnap->pIMem) { + pIMem = tsdbGetTbDataFromMemTable(pReadSnap->pIMem, suid, uid); + } - pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; + pIter->pTsdb = pTsdb; + + pIter->pMemDelData = NULL; - pIter->fsLastState.state = (SFSLASTNEXTROWSTATES)SFSNEXTROW_FS; - pIter->fsLastState.pTsdb = pTsdb; - pIter->fsLastState.aDFileSet = pReadSnap->fs.aDFileSet; - pIter->fsLastState.pTSchema = pTSchema; - pIter->fsLastState.suid = suid; - pIter->fsLastState.uid = uid; - pIter->fsLastState.pLoadInfo = pLoadInfo; - pIter->fsLastState.pDataFReader = pDataFReaderLast; - pIter->fsLastState.lastTs = lastTs; - pIter->fsLastState.pDataIter = pLDataIter; + loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer); + pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; + + pIter->fsState.pRowIter = pIter; pIter->fsState.state = SFSNEXTROW_FS; pIter->fsState.pTsdb = pTsdb; - pIter->fsState.aDFileSet = pReadSnap->fs.aDFileSet; + pIter->fsState.aDFileSet = pReadSnap->pfSetArray; pIter->fsState.pBlockIdxExp = &pIter->idx; pIter->fsState.pTSchema = pTSchema; pIter->fsState.suid = suid; pIter->fsState.uid = uid; - pIter->fsState.pLoadInfo = pLoadInfo; - pIter->fsState.pDataFReader = pDataFReader; pIter->fsState.lastTs = lastTs; + pIter->fsState.pr = pr; pIter->input[0] = (TsdbNextRowState){&pIter->memRow, true, false, false, &pIter->memState, getNextRowFromMem, NULL}; pIter->input[1] = (TsdbNextRowState){&pIter->imemRow, true, false, false, &pIter->imemState, getNextRowFromMem, NULL}; - pIter->input[2] = (TsdbNextRowState){ - &pIter->fsLastRow, false, true, false, &pIter->fsLastState, getNextRowFromFSLast, clearNextRowFromFSLast}; - pIter->input[3] = + pIter->input[2] = (TsdbNextRowState){&pIter->fsRow, false, true, false, &pIter->fsState, getNextRowFromFS, clearNextRowFromFS}; if (pMem) { @@ -2485,7 +2376,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->input[1].next = true; } - return code; + pIter->pr = pr; _err: return code; } @@ -2493,7 +2384,7 @@ _err: static int32_t nextRowIterClose(CacheNextRowIter *pIter) { int code = 0; - for (int i = 0; i < 4; ++i) { + for (int i = 0; i < 3; ++i) { if (pIter->input[i].nextRowClearFn) { pIter->input[i].nextRowClearFn(pIter->input[i].iter); } @@ -2503,6 +2394,10 @@ static int32_t nextRowIterClose(CacheNextRowIter *pIter) { taosArrayDestroy(pIter->pSkyline); } + if (pIter->pMemDelData) { + taosArrayDestroy(pIter->pMemDelData); + } + _err: return code; } @@ -2512,7 +2407,7 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI int16_t *aCols, int nCols) { int code = 0; for (;;) { - for (int i = 0; i < 4; ++i) { + for (int i = 0; i < 3; ++i) { if (pIter->input[i].next && !pIter->input[i].stop) { code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs, isLast, aCols, nCols); @@ -2525,10 +2420,10 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI } } - if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop && pIter->input[3].stop) { + if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop) { *ppRow = NULL; - *pIgnoreEarlierTs = (pIter->input[0].ignoreEarlierTs || pIter->input[1].ignoreEarlierTs || - pIter->input[2].ignoreEarlierTs || pIter->input[3].ignoreEarlierTs); + *pIgnoreEarlierTs = + (pIter->input[0].ignoreEarlierTs || pIter->input[1].ignoreEarlierTs || pIter->input[2].ignoreEarlierTs); return code; } @@ -2538,7 +2433,7 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI int nMax = 0; TSKEY maxKey = TSKEY_MIN; - for (int i = 0; i < 4; ++i) { + for (int i = 0; i < 3; ++i) { if (!pIter->input[i].stop && pIter->input[i].pRow != NULL) { TSDBKEY key = TSDBROW_KEY(pIter->input[i].pRow); @@ -2564,6 +2459,21 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI for (int i = 0; i < nMax; ++i) { TSDBKEY maxKey1 = TSDBROW_KEY(max[i]); + if (!pIter->pSkyline) { + pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); + + uint64_t uid = pIter->idx.uid; + STableLoadInfo *pInfo = *(STableLoadInfo **)tSimpleHashGet(pIter->pr->pTableMap, &uid, sizeof(uid)); + SArray *pTombData = pInfo->pTombData; + if (pTombData) { + taosArrayAddAll(pTombData, pIter->pMemDelData); + + code = tsdbBuildDeleteSkyline(pTombData, 0, (int32_t)(TARRAY_SIZE(pTombData) - 1), pIter->pSkyline); + } + + pIter->iSkyline = taosArrayGetSize(pIter->pSkyline) - 1; + } + bool deleted = tsdbKeyDeleted(&maxKey1, pIter->pSkyline, &pIter->iSkyline); if (!deleted) { iMerge[nMerge] = iMax[i]; @@ -2634,322 +2544,7 @@ static int32_t updateTSchema(int32_t sversion, SCacheRowsReader *pReader, uint64 } taosMemoryFreeClear(pReader->pCurrSchema); - return metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pCurrSchema); -} - -static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppColArray, SCacheRowsReader *pr) { - STSchema *pTSchema = pr->pSchema; // metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - int16_t nLastCol = pTSchema->numOfCols; - int16_t iCol = 0; - int16_t noneCol = 0; - bool setNoneCol = false; - bool hasRow = false; - bool ignoreEarlierTs = false; - SArray *pColArray = NULL; - SColVal *pColVal = &(SColVal){0}; - - int32_t code = initLastColArray(pTSchema, &pColArray); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - - TSKEY lastRowTs = TSKEY_MAX; - - CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader, - &pr->pDataFReaderLast, pr->lastTs); - - do { - TSDBROW *pRow = NULL; - nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, NULL, 0); - - if (!pRow) { - break; - } - - hasRow = true; - - int32_t sversion = TSDBROW_SVERSION(pRow); - if (sversion != -1) { - code = updateTSchema(sversion, pr, uid); - if (TSDB_CODE_SUCCESS != code) { - goto _err; - } - pTSchema = pr->pCurrSchema; - } - int16_t nCol = pTSchema->numOfCols; - - TSKEY rowTs = TSDBROW_TS(pRow); - - if (lastRowTs == TSKEY_MAX) { - lastRowTs = rowTs; - STColumn *pTColumn = &pTSchema->columns[0]; - - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs}); - taosArraySet(pColArray, 0, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}); - - for (iCol = 1; iCol < nCol; ++iCol) { - if (iCol >= nLastCol) { - break; - } - SLastCol *pCol = taosArrayGet(pColArray, iCol); - if (pCol->colVal.cid != pTSchema->columns[iCol].colId) { - continue; - } - tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); - - *pCol = (SLastCol){.ts = lastRowTs, .colVal = *pColVal}; - if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) { - pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData); - if (pCol->colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); - } - - if (COL_VAL_IS_NONE(pColVal) && !setNoneCol) { - noneCol = iCol; - setNoneCol = true; - } - } - if (!setNoneCol) { - // done, goto return pColArray - break; - } else { - continue; - } - } - - if ((rowTs < lastRowTs)) { - // done, goto return pColArray - break; - } - - // merge into pColArray - setNoneCol = false; - for (iCol = noneCol; iCol < nCol; ++iCol) { - // high version's column value - SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, iCol); - - tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); - if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { - SLastCol lastCol = {.ts = rowTs, .colVal = *pColVal}; - if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pColArray, iCol); - taosMemoryFree(pLastCol->colVal.value.pData); - - lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData); - } - - taosArraySet(pColArray, iCol, &lastCol); - } else if (COL_VAL_IS_NONE(tColVal) && COL_VAL_IS_NONE(pColVal) && !setNoneCol) { - noneCol = iCol; - setNoneCol = true; - } - } - } while (setNoneCol); - - // build the result ts row here - *dup = false; - // if (taosArrayGetSize(pColArray) != nCol) { - //*ppColArray = NULL; - // taosArrayDestroy(pColArray); - //} else { - if (!hasRow) { - if (ignoreEarlierTs) { - taosArrayDestroy(pColArray); - pColArray = NULL; - } else { - taosArrayClear(pColArray); - } - } - *ppColArray = pColArray; - //} - - nextRowIterClose(&iter); - // taosMemoryFreeClear(pTSchema); - return code; - -_err: - nextRowIterClose(&iter); - taosArrayDestroy(pColArray); - // taosMemoryFreeClear(pTSchema); - return code; -} - -static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr) { - STSchema *pTSchema = pr->pSchema; // metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - int16_t nLastCol = pTSchema->numOfCols; - int16_t noneCol = 0; - bool setNoneCol = false; - bool hasRow = false; - bool ignoreEarlierTs = false; - SArray *pColArray = NULL; - SColVal *pColVal = &(SColVal){0}; - int16_t nCols = nLastCol; - - int32_t code = initLastColArray(pTSchema, &pColArray); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - SArray *aColArray = taosArrayInit(nCols, sizeof(int16_t)); - if (NULL == aColArray) { - taosArrayDestroy(pColArray); - - return TSDB_CODE_OUT_OF_MEMORY; - } - for (int i = 1; i < pTSchema->numOfCols; ++i) { - taosArrayPush(aColArray, &pTSchema->columns[i].colId); - } - - TSKEY lastRowTs = TSKEY_MAX; - - CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader, - &pr->pDataFReaderLast, pr->lastTs); - - do { - TSDBROW *pRow = NULL; - nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); - - if (!pRow) { - break; - } - - hasRow = true; - - int32_t sversion = TSDBROW_SVERSION(pRow); - if (sversion != -1) { - code = updateTSchema(sversion, pr, uid); - if (TSDB_CODE_SUCCESS != code) { - goto _err; - } - pTSchema = pr->pCurrSchema; - } - int16_t nCol = pTSchema->numOfCols; - - TSKEY rowTs = TSDBROW_TS(pRow); - - if (lastRowTs == TSKEY_MAX) { - lastRowTs = rowTs; - STColumn *pTColumn = &pTSchema->columns[0]; - - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs}); - taosArraySet(pColArray, 0, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}); - - for (int16_t iCol = 1; iCol < nCol; ++iCol) { - if (iCol >= nLastCol) { - break; - } - SLastCol *pCol = taosArrayGet(pColArray, iCol); - if (pCol->colVal.cid != pTSchema->columns[iCol].colId) { - continue; - } - tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); - - *pCol = (SLastCol){.ts = lastRowTs, .colVal = *pColVal}; - if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) { - pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData); - if (pCol->colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); - } - - if (!COL_VAL_IS_VALUE(pColVal)) { - if (!setNoneCol) { - noneCol = iCol; - setNoneCol = true; - } - } else { - int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ); - taosArrayRemove(aColArray, aColIndex); - } - } - if (!setNoneCol) { - // done, goto return pColArray - break; - } else { - continue; - } - } - - // merge into pColArray - setNoneCol = false; - for (int16_t iCol = noneCol; iCol < nCol; ++iCol) { - if (iCol >= nLastCol) { - break; - } - // high version's column value - SLastCol *lastColVal = (SLastCol *)taosArrayGet(pColArray, iCol); - if (lastColVal->colVal.cid != pTSchema->columns[iCol].colId) { - continue; - } - SColVal *tColVal = &lastColVal->colVal; - - tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); - if (!COL_VAL_IS_VALUE(tColVal) && COL_VAL_IS_VALUE(pColVal)) { - SLastCol lastCol = {.ts = rowTs, .colVal = *pColVal}; - if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pColArray, iCol); - taosMemoryFree(pLastCol->colVal.value.pData); - - lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData); - } - - taosArraySet(pColArray, iCol, &lastCol); - int32_t aColIndex = taosArraySearchIdx(aColArray, &lastCol.colVal.cid, compareInt16Val, TD_EQ); - taosArrayRemove(aColArray, aColIndex); - } else if (!COL_VAL_IS_VALUE(tColVal) && !COL_VAL_IS_VALUE(pColVal) && !setNoneCol) { - noneCol = iCol; - setNoneCol = true; - } - } - } while (setNoneCol); - - // if (taosArrayGetSize(pColArray) <= 0) { - //*ppLastArray = NULL; - // taosArrayDestroy(pColArray); - //} else { - if (!hasRow) { - if (ignoreEarlierTs) { - taosArrayDestroy(pColArray); - pColArray = NULL; - } else { - taosArrayClear(pColArray); - } - } - *ppLastArray = pColArray; - //} - - nextRowIterClose(&iter); - taosArrayDestroy(aColArray); - // taosMemoryFreeClear(pTSchema); - return code; - -_err: - nextRowIterClose(&iter); - // taosMemoryFreeClear(pTSchema); - *ppLastArray = NULL; - taosArrayDestroy(pColArray); - taosArrayDestroy(aColArray); - return code; + return metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, sversion, &pReader->pCurrSchema); } static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, @@ -2981,8 +2576,7 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader, - &pr->pDataFReaderLast, pr->lastTs); + nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->info.suid, pr->pLDataIterArray, pr->pReadSnap, pr->lastTs, pr); do { TSDBROW *pRow = NULL; @@ -3151,8 +2745,7 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader, - &pr->pDataFReaderLast, pr->lastTs); + nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->info.suid, pr->pLDataIterArray, pr->pReadSnap, pr->lastTs, pr); do { TSDBROW *pRow = NULL; @@ -3241,92 +2834,6 @@ _err: return code; } -int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **handle) { - int32_t code = 0; - char key[32] = {0}; - int keyLen = 0; - - // getTableCacheKeyS(uid, "lr", key, &keyLen); - getTableCacheKey(uid, 0, key, &keyLen); - LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (!h) { - STsdb *pTsdb = pr->pVnode->pTsdb; - taosThreadMutexLock(&pTsdb->lruMutex); - - h = taosLRUCacheLookup(pCache, key, keyLen); - if (!h) { - SArray *pArray = NULL; - bool dup = false; // which is always false for now - code = mergeLastRow(uid, pTsdb, &dup, &pArray, pr); - // if table's empty or error or ignore ignore earlier ts, set handle NULL and return - if (code < 0 || pArray == NULL) { - if (!dup && pArray) { - taosArrayDestroy(pArray); - } - - taosThreadMutexUnlock(&pTsdb->lruMutex); - - *handle = NULL; - - return 0; - } - - size_t charge = pArray->capacity * pArray->elemSize + sizeof(*pArray); - _taos_lru_deleter_t deleter = deleteTableCacheLast; - LRUStatus status = - taosLRUCacheInsert(pCache, key, keyLen, pArray, charge, deleter, &h, TAOS_LRU_PRIORITY_LOW, NULL); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } - } - taosThreadMutexUnlock(&pTsdb->lruMutex); - } - - *handle = h; - - return code; -} - -int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **handle) { - int32_t code = 0; - char key[32] = {0}; - int keyLen = 0; - - // getTableCacheKeyS(uid, "l", key, &keyLen); - getTableCacheKey(uid, 1, key, &keyLen); - LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (!h) { - STsdb *pTsdb = pr->pVnode->pTsdb; - taosThreadMutexLock(&pTsdb->lruMutex); - - h = taosLRUCacheLookup(pCache, key, keyLen); - if (!h) { - SArray *pLastArray = NULL; - code = mergeLast(uid, pTsdb, &pLastArray, pr); - // if table's empty or error or ignore ignore earlier ts, set handle NULL and return - if (code < 0 || pLastArray == NULL) { - taosThreadMutexUnlock(&pTsdb->lruMutex); - - *handle = NULL; - return 0; - } - - size_t charge = pLastArray->capacity * pLastArray->elemSize + sizeof(*pLastArray); - _taos_lru_deleter_t deleter = deleteTableCacheLast; - LRUStatus status = - taosLRUCacheInsert(pCache, key, keyLen, pLastArray, charge, deleter, &h, TAOS_LRU_PRIORITY_LOW, NULL); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } - } - taosThreadMutexUnlock(&pTsdb->lruMutex); - } - - *handle = h; - - return code; -} - int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h) { int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 6138b1f7b40184be649a78543d8958fbfe80516f..f17041e98b2ee57ca0c0348e281fbc5301bd134e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -17,6 +17,7 @@ #include "tarray.h" #include "tcommon.h" #include "tsdb.h" +#include "tsdbDataFileRW.h" #define HASTYPE(_type, _t) (((_type) & (_t)) == (_t)) @@ -124,11 +125,29 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf pReader->numOfTables = numOfTables; pReader->lastTs = INT64_MIN; - resetLastBlockLoadInfo(pReader->pLoadInfo); + int64_t blocks; + double elapse; + pReader->pLDataIterArray = destroySttBlockReader(pReader->pLDataIterArray, &blocks, &elapse); + pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); return TSDB_CODE_SUCCESS; } +static int32_t uidComparFunc(const void* p1, const void* p2) { + uint64_t pu1 = *(uint64_t*)p1; + uint64_t pu2 = *(uint64_t*)p2; + if (pu1 == pu2) { + return 0; + } else { + return (pu1 < pu2) ? -1 : 1; + } +} + +static void freeTableInfoFunc(void* param) { + void** p = (void**)param; + taosMemoryFreeClear(*p); +} + int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols, SArray* pCidList, int32_t* pSlotIds, uint64_t suid, void** pReader, const char* idstr) { *pReader = NULL; @@ -140,11 +159,11 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, p->type = type; p->pVnode = pVnode; p->pTsdb = p->pVnode->pTsdb; - p->verRange = (SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}; + p->info.verRange = (SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}; + p->info.suid = suid; p->numOfCols = numOfCols; p->pCidList = pCidList; p->pSlotIds = pSlotIds; - p->suid = suid; if (numOfTables == 0) { *pReader = p; @@ -154,6 +173,27 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, p->pTableList = pTableIdList; p->numOfTables = numOfTables; + p->pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + if (p->pTableMap == NULL) { + tsdbCacherowsReaderClose(p); + return TSDB_CODE_OUT_OF_MEMORY; + } + p->uidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); + if (p->uidList == NULL) { + tsdbCacherowsReaderClose(p); + return TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t i = 0; i < numOfTables; ++i) { + uint64_t uid = p->pTableList[i].uid; + p->uidList[i] = uid; + STableLoadInfo* pInfo = taosMemoryCalloc(1, sizeof(STableLoadInfo)); + tSimpleHashPut(p->pTableMap, &uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); + } + + tSimpleHashSetFreeFp(p->pTableMap, freeTableInfoFunc); + + taosSort(p->uidList, numOfTables, sizeof(uint64_t), uidComparFunc); + int32_t code = setTableSchema(p, suid, idstr); if (code != TSDB_CODE_SUCCESS) { tsdbCacherowsReaderClose(p); @@ -178,14 +218,8 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, SVnodeCfg* pCfg = &((SVnode*)pVnode)->config; int32_t numOfStt = pCfg->sttTrigger; - p->pLoadInfo = tCreateLastBlockLoadInfo(p->pSchema, NULL, 0, numOfStt); - if (p->pLoadInfo == NULL) { - tsdbCacherowsReaderClose(p); - return TSDB_CODE_OUT_OF_MEMORY; - } - - p->pDataIter = taosMemoryCalloc(pCfg->sttTrigger, sizeof(SLDataIter)); - if (p->pDataIter == NULL) { + p->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + if (p->pLDataIterArray == NULL) { tsdbCacherowsReaderClose(p); return TSDB_CODE_OUT_OF_MEMORY; } @@ -214,14 +248,34 @@ void* tsdbCacherowsReaderClose(void* pReader) { taosMemoryFree(p->pSchema); } - taosMemoryFree(p->pDataIter); taosMemoryFree(p->pCurrSchema); - destroyLastBlockLoadInfo(p->pLoadInfo); + int64_t loadBlocks = 0; + double elapse = 0; + destroySttBlockReader(p->pLDataIterArray, &loadBlocks, &elapse); + + if (p->pFileReader) { + tsdbDataFileReaderClose(&p->pFileReader); + p->pFileReader = NULL; + } taosMemoryFree((void*)p->idstr); taosThreadMutexDestroy(&p->readerMutex); + if (p->pTableMap) { + void* pe = NULL; + int32_t iter = 0; + while ((pe = tSimpleHashIterate(p->pTableMap, pe, &iter)) != NULL) { + STableLoadInfo* pInfo = *(STableLoadInfo**)pe; + pInfo->pTombData = taosArrayDestroy(pInfo->pTombData); + } + + tSimpleHashCleanup(p->pTableMap); + } + if (p->uidList) { + taosMemoryFree(p->uidList); + } + taosMemoryFree(pReader); return NULL; } @@ -298,12 +352,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } taosThreadMutexLock(&pr->readerMutex); - code = tsdbTakeReadSnap((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap); + code = tsdbTakeReadSnap2((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap); if (code != TSDB_CODE_SUCCESS) { goto _end; } - pr->pDataFReader = NULL; - pr->pDataFReaderLast = NULL; int8_t ltype = (pr->type & CACHESCAN_RETRIEVE_LAST) >> 3; @@ -424,11 +476,13 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } _end: - tsdbDataFReaderClose(&pr->pDataFReaderLast); - tsdbDataFReaderClose(&pr->pDataFReader); + tsdbUntakeReadSnap2((STsdbReader*)pr, pr->pReadSnap, true); + + int64_t loadBlocks = 0; + double elapse = 0; + pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, &loadBlocks, &elapse); + pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); - resetLastBlockLoadInfo(pr->pLoadInfo); - tsdbUntakeReadSnap((STsdbReader*)pr, pr->pReadSnap, true); taosThreadMutexUnlock(&pr->readerMutex); if (pRes != NULL) { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index b440d5188356652b8b32396ab239553bea4eba62..6376f375eaa584501997550be659131ca2ea52ca 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -562,7 +562,8 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } - tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); + code = tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); + TSDB_CHECK_CODE(code, lino, _exit); wSet.diskId = did; wSet.nSttF = 1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c new file mode 100644 index 0000000000000000000000000000000000000000..0639cd91a5ad4f611cc13fbad5923a8df762832a --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -0,0 +1,631 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbCommit2.h" + +// extern dependencies +typedef struct { + STsdb *tsdb; + TFileSetArray *fsetArr; + TFileOpArray fopArray[1]; + + // SSkmInfo skmTb[1]; + // SSkmInfo skmRow[1]; + + int32_t minutes; + int8_t precision; + int32_t minRow; + int32_t maxRow; + int8_t cmprAlg; + int32_t sttTrigger; + int32_t szPage; + int64_t compactVersion; + + struct { + int64_t cid; + int64_t now; + TSKEY nextKey; + TSKEY maxDelKey; + int32_t fid; + int32_t expLevel; + SDiskID did; + TSKEY minKey; + TSKEY maxKey; + STFileSet *fset; + TABLEID tbid[1]; + bool hasTSData; + } ctx[1]; + + // reader + SSttFileReader *sttReader; + + // iter + TTsdbIterArray dataIterArray[1]; + SIterMerger *dataIterMerger; + TTsdbIterArray tombIterArray[1]; + SIterMerger *tombIterMerger; + + // writer + SFSetWriter *writer; +} SCommitter2; + +static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + + SFSetWriterConfig config = { + .tsdb = committer->tsdb, + .toSttOnly = true, + .compactVersion = committer->compactVersion, + .minRow = committer->minRow, + .maxRow = committer->maxRow, + .szPage = committer->szPage, + .cmprAlg = committer->cmprAlg, + .fid = committer->ctx->fid, + .cid = committer->ctx->cid, + .did = committer->ctx->did, + .level = 0, + }; + + if (committer->sttTrigger == 1) { + config.toSttOnly = false; + + if (committer->ctx->fset) { + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ftype++) { + if (committer->ctx->fset->farr[ftype] != NULL) { + config.files[ftype].exist = true; + config.files[ftype].file = committer->ctx->fset->farr[ftype]->f[0]; + } + } + } + } + + code = tsdbFSetWriterOpen(&config, &committer->writer); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbCommitCloseWriter(SCommitter2 *committer) { + return tsdbFSetWriterClose(&committer->writer, 0, committer->fopArray); +} + +static int32_t tsdbCommitTSData(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + int64_t numOfRow = 0; + SMetaInfo info; + + committer->ctx->hasTSData = false; + + committer->ctx->tbid->suid = 0; + committer->ctx->tbid->uid = 0; + for (SRowInfo *row; (row = tsdbIterMergerGetData(committer->dataIterMerger)) != NULL;) { + if (row->uid != committer->ctx->tbid->uid) { + committer->ctx->tbid->suid = row->suid; + committer->ctx->tbid->uid = row->uid; + + if (metaGetInfo(committer->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) { + code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } + } + + int64_t ts = TSDBROW_TS(&row->row); + if (ts > committer->ctx->maxKey) { + committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts); + code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } + + committer->ctx->hasTSData = true; + numOfRow++; + + code = tsdbFSetWriteRow(committer->writer, row); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbIterMergerNext(committer->dataIterMerger); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->fid, numOfRow); + } + return code; +} + +static int32_t tsdbCommitTombData(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + int64_t numRecord = 0; + SMetaInfo info; + + if (committer->ctx->fset == NULL && !committer->ctx->hasTSData) { + if (committer->ctx->maxKey < committer->ctx->maxDelKey) { + committer->ctx->nextKey = committer->ctx->maxKey + 1; + } else { + committer->ctx->nextKey = TSKEY_MAX; + } + return 0; + } + + committer->ctx->tbid->suid = 0; + committer->ctx->tbid->uid = 0; + for (STombRecord *record; (record = tsdbIterMergerGetTombRecord(committer->tombIterMerger));) { + if (record->uid != committer->ctx->tbid->uid) { + committer->ctx->tbid->suid = record->suid; + committer->ctx->tbid->uid = record->uid; + + if (metaGetInfo(committer->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) { + code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } + } + + if (record->ekey < committer->ctx->minKey) { + goto _next; + } else if (record->skey > committer->ctx->maxKey) { + committer->ctx->maxKey = TMIN(record->skey, committer->ctx->maxKey); + goto _next; + } + + TSKEY maxKey = committer->ctx->maxKey; + if (record->ekey > committer->ctx->maxKey) { + maxKey = committer->ctx->maxKey + 1; + } + + if (record->ekey > committer->ctx->maxKey && committer->ctx->nextKey > maxKey) { + committer->ctx->nextKey = maxKey; + } + + record->skey = TMAX(record->skey, committer->ctx->minKey); + record->ekey = TMIN(record->ekey, maxKey); + + numRecord++; + code = tsdbFSetWriteTombRecord(committer->writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + + _next: + code = tsdbIterMergerNext(committer->tombIterMerger); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode), committer->ctx->fid, + numRecord); + } + return code; +} + +static int32_t tsdbCommitOpenReader(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(committer->sttReader == NULL); + + if (committer->ctx->fset == NULL // + || committer->sttTrigger > 1 // + || TARRAY2_SIZE(committer->ctx->fset->lvlArr) == 0 // + ) { + return 0; + } + + ASSERT(TARRAY2_SIZE(committer->ctx->fset->lvlArr) == 1); + + SSttLvl *lvl = TARRAY2_FIRST(committer->ctx->fset->lvlArr); + + ASSERT(lvl->level == 0); + + if (TARRAY2_SIZE(lvl->fobjArr) == 0) { + return 0; + } + + ASSERT(TARRAY2_SIZE(lvl->fobjArr) == 1); + + STFileObj *fobj = TARRAY2_FIRST(lvl->fobjArr); + + SSttFileReaderConfig config = { + .tsdb = committer->tsdb, + .szPage = committer->szPage, + .file = fobj->f[0], + }; + code = tsdbSttFileReaderOpen(fobj->fname, &config, &committer->sttReader); + TSDB_CHECK_CODE(code, lino, _exit); + + STFileOp op = { + .optype = TSDB_FOP_REMOVE, + .fid = fobj->f->fid, + .of = fobj->f[0], + }; + + code = TARRAY2_APPEND(committer->fopArray, op); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbCommitCloseReader(SCommitter2 *committer) { return tsdbSttFileReaderClose(&committer->sttReader); } + +static int32_t tsdbCommitOpenIter(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(TARRAY2_SIZE(committer->dataIterArray) == 0); + ASSERT(committer->dataIterMerger == NULL); + ASSERT(TARRAY2_SIZE(committer->tombIterArray) == 0); + ASSERT(committer->tombIterMerger == NULL); + + STsdbIter *iter; + STsdbIterConfig config = {0}; + + // mem data iter + config.type = TSDB_ITER_TYPE_MEMT; + config.memt = committer->tsdb->imem; + config.from->ts = committer->ctx->minKey; + config.from->version = VERSION_MIN; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(committer->dataIterArray, iter); + TSDB_CHECK_CODE(code, lino, _exit); + + // mem tomb iter + config.type = TSDB_ITER_TYPE_MEMT_TOMB; + config.memt = committer->tsdb->imem; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(committer->tombIterArray, iter); + TSDB_CHECK_CODE(code, lino, _exit); + + // STT + if (committer->sttReader) { + // data iter + config.type = TSDB_ITER_TYPE_STT; + config.sttReader = committer->sttReader; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(committer->dataIterArray, iter); + TSDB_CHECK_CODE(code, lino, _exit); + + // tomb iter + config.type = TSDB_ITER_TYPE_STT_TOMB; + config.sttReader = committer->sttReader; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(committer->tombIterArray, iter); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // open merger + code = tsdbIterMergerOpen(committer->dataIterArray, &committer->dataIterMerger, false); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbIterMergerOpen(committer->tombIterArray, &committer->tombIterMerger, true); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbCommitCloseIter(SCommitter2 *committer) { + tsdbIterMergerClose(&committer->tombIterMerger); + tsdbIterMergerClose(&committer->dataIterMerger); + TARRAY2_CLEAR(committer->tombIterArray, tsdbIterClose); + TARRAY2_CLEAR(committer->dataIterArray, tsdbIterClose); + return 0; +} + +static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + STsdb *tsdb = committer->tsdb; + + committer->ctx->fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision); + committer->ctx->expLevel = tsdbFidLevel(committer->ctx->fid, &tsdb->keepCfg, committer->ctx->now); + tsdbFidKeyRange(committer->ctx->fid, committer->minutes, committer->precision, &committer->ctx->minKey, + &committer->ctx->maxKey); + code = tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did); + TSDB_CHECK_CODE(code, lino, _exit); + tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did); + STFileSet fset = {.fid = committer->ctx->fid}; + committer->ctx->fset = &fset; + STFileSet **fsetPtr = TARRAY2_SEARCH(committer->fsetArr, &committer->ctx->fset, tsdbTFileSetCmprFn, TD_EQ); + committer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr; + committer->ctx->tbid->suid = 0; + committer->ctx->tbid->uid = 0; + + ASSERT(TARRAY2_SIZE(committer->dataIterArray) == 0); + ASSERT(committer->dataIterMerger == NULL); + ASSERT(committer->writer == NULL); + + code = tsdbCommitOpenReader(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCommitOpenIter(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCommitOpenWriter(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + // reset nextKey + committer->ctx->nextKey = TSKEY_MAX; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode), + __func__, committer->ctx->fid, committer->ctx->minKey, committer->ctx->maxKey, committer->ctx->expLevel); + } + return 0; +} + +static int32_t tsdbCommitFileSetEnd(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbCommitCloseWriter(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCommitCloseIter(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCommitCloseReader(committer); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid); + } + return code; +} + +static int32_t tsdbCommitFileSet(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + + // fset commit start + code = tsdbCommitFileSetBegin(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + // commit fset + code = tsdbCommitTSData(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCommitTombData(committer); + TSDB_CHECK_CODE(code, lino, _exit); + + // fset commit end + code = tsdbCommitFileSetEnd(committer); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid); + } + return code; +} + +static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + + memset(committer, 0, sizeof(committer[0])); + + committer->tsdb = tsdb; + code = tsdbFSCreateCopySnapshot(tsdb->pFS, &committer->fsetArr); + TSDB_CHECK_CODE(code, lino, _exit); + committer->minutes = tsdb->keepCfg.days; + committer->precision = tsdb->keepCfg.precision; + committer->minRow = info->info.config.tsdbCfg.minRows; + committer->maxRow = info->info.config.tsdbCfg.maxRows; + committer->cmprAlg = info->info.config.tsdbCfg.compression; + committer->sttTrigger = info->info.config.sttTrigger; + committer->szPage = info->info.config.tsdbPageSize; + committer->compactVersion = INT64_MAX; + committer->ctx->cid = tsdbFSAllocEid(tsdb->pFS); + committer->ctx->now = taosGetTimestampSec(); + + committer->ctx->nextKey = tsdb->imem->minKey; + if (tsdb->imem->nDel > 0) { + SRBTreeIter iter[1] = {tRBTreeIterCreate(tsdb->imem->tbDataTree, 1)}; + + for (SRBTreeNode *node = tRBTreeIterNext(iter); node; node = tRBTreeIterNext(iter)) { + STbData *tbData = TCONTAINER_OF(node, STbData, rbtn); + + for (SDelData *delData = tbData->pHead; delData; delData = delData->pNext) { + if (delData->sKey < committer->ctx->nextKey) { + committer->ctx->nextKey = delData->sKey; + } + } + } + } + + committer->ctx->maxDelKey = TSKEY_MIN; + TSKEY minKey = TSKEY_MAX; + TSKEY maxKey = TSKEY_MIN; + if (TARRAY2_SIZE(committer->fsetArr) > 0) { + STFileSet *fset = TARRAY2_LAST(committer->fsetArr); + tsdbFidKeyRange(fset->fid, committer->minutes, committer->precision, &minKey, &committer->ctx->maxDelKey); + + fset = TARRAY2_FIRST(committer->fsetArr); + tsdbFidKeyRange(fset->fid, committer->minutes, committer->precision, &minKey, &maxKey); + } + + if (committer->ctx->nextKey < TMIN(tsdb->imem->minKey, minKey)) { + committer->ctx->nextKey = TMIN(tsdb->imem->minKey, minKey); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + } + return code; +} + +static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) { + int32_t code = 0; + int32_t lino = 0; + + if (eno == 0) { + code = tsdbFSEditBegin(committer->tsdb->pFS, committer->fopArray, TSDB_FEDIT_COMMIT); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + // TODO + ASSERT(0); + } + + ASSERT(committer->writer == NULL); + ASSERT(committer->dataIterMerger == NULL); + ASSERT(committer->tombIterMerger == NULL); + TARRAY2_DESTROY(committer->dataIterArray, NULL); + TARRAY2_DESTROY(committer->tombIterArray, NULL); + TARRAY2_DESTROY(committer->fopArray, NULL); + tsdbFSDestroyCopySnapshot(&committer->fsetArr); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, lino, + tstrerror(code), committer->ctx->cid); + } else { + tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->cid); + } + return code; +} + +int32_t tsdbPreCommit(STsdb *tsdb) { + taosThreadRwlockWrlock(&tsdb->rwLock); + ASSERT(tsdb->imem == NULL); + tsdb->imem = tsdb->mem; + tsdb->mem = NULL; + taosThreadRwlockUnlock(&tsdb->rwLock); + return 0; +} + +int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) { + if (!tsdb) return 0; + + int32_t code = 0; + int32_t lino = 0; + + SMemTable *imem = tsdb->imem; + int64_t nRow = imem->nRow; + int64_t nDel = imem->nDel; + + if (nRow == 0 && nDel == 0) { + taosThreadRwlockWrlock(&tsdb->rwLock); + tsdb->imem = NULL; + taosThreadRwlockUnlock(&tsdb->rwLock); + tsdbUnrefMemTable(imem, NULL, true); + } else { + SCommitter2 committer[1]; + + code = tsdbOpenCommitter(tsdb, info, committer); + TSDB_CHECK_CODE(code, lino, _exit); + + while (committer->ctx->nextKey != TSKEY_MAX) { + code = tsdbCommitFileSet(committer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbCloseCommitter(committer, code); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbInfo("vgId:%d %s done, nRow:%" PRId64 " nDel:%" PRId64, TD_VID(tsdb->pVnode), __func__, nRow, nDel); + } + return code; +} + +int32_t tsdbCommitCommit(STsdb *tsdb) { + int32_t code = 0; + int32_t lino = 0; + + if (tsdb->imem == NULL) goto _exit; + + SMemTable *pMemTable = tsdb->imem; + taosThreadRwlockWrlock(&tsdb->rwLock); + code = tsdbFSEditCommit(tsdb->pFS); + if (code) { + taosThreadRwlockUnlock(&tsdb->rwLock); + TSDB_CHECK_CODE(code, lino, _exit); + } + tsdb->imem = NULL; + taosThreadRwlockUnlock(&tsdb->rwLock); + tsdbUnrefMemTable(pMemTable, NULL, true); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else { + tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + } + return code; +} + +int32_t tsdbCommitAbort(STsdb *pTsdb) { + int32_t code = 0; + int32_t lino = 0; + + if (pTsdb->imem == NULL) goto _exit; + + code = tsdbFSEditAbort(pTsdb->pFS); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + } + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.h b/source/dnode/vnode/src/tsdb/tsdbCommit2.h new file mode 100644 index 0000000000000000000000000000000000000000..41f72f345b4575f90c0632b857e0d0eae6f89a7a --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRW.h" +#include "tsdbFS2.h" +#include "tsdbFSetRW.h" +#include "tsdbIter.h" +#include "tsdbSttFileRW.h" + +#ifndef _TSDB_COMMIT_H_ +#define _TSDB_COMMIT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_COMMIT_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c new file mode 100644 index 0000000000000000000000000000000000000000..9d07bcb446285696f4ebb6085028075c7ce12b6f --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -0,0 +1,1686 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRW.h" + +extern int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, + TTombBlkArray *tombBlkArray, uint8_t **bufArr); +extern int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize); + +// SDataFileReader ============================================= +struct SDataFileReader { + SDataFileReaderConfig config[1]; + + uint8_t *bufArr[5]; + + struct { + bool headFooterLoaded; + bool tombFooterLoaded; + bool brinBlkLoaded; + bool tombBlkLoaded; + } ctx[1]; + + STsdbFD *fd[TSDB_FTYPE_MAX]; + + SHeadFooter headFooter[1]; + STombFooter tombFooter[1]; + TBrinBlkArray brinBlkArray[1]; + TTombBlkArray tombBlkArray[1]; +}; + +static int32_t tsdbDataFileReadHeadFooter(SDataFileReader *reader) { + if (reader->ctx->headFooterLoaded) return 0; + + int32_t code = 0; + int32_t lino = 0; + + int32_t ftype = TSDB_FTYPE_HEAD; + if (reader->fd[ftype]) { + code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(SHeadFooter), + (uint8_t *)reader->headFooter, sizeof(SHeadFooter)); + TSDB_CHECK_CODE(code, lino, _exit); + } + + reader->ctx->headFooterLoaded = true; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileReadTombFooter(SDataFileReader *reader) { + if (reader->ctx->tombFooterLoaded) return 0; + + int32_t code = 0; + int32_t lino = 0; + + int32_t ftype = TSDB_FTYPE_TOMB; + if (reader->fd[ftype]) { + code = tsdbReadFile(reader->fd[ftype], reader->config->files[ftype].file.size - sizeof(STombFooter), + (uint8_t *)reader->tombFooter, sizeof(STombFooter)); + TSDB_CHECK_CODE(code, lino, _exit); + } + reader->ctx->tombFooterLoaded = true; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReaderOpen(const char *fname[], const SDataFileReaderConfig *config, SDataFileReader **reader) { + int32_t code = 0; + int32_t lino = 0; + + reader[0] = taosMemoryCalloc(1, sizeof(**reader)); + if (reader[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + reader[0]->config[0] = config[0]; + if (reader[0]->config->bufArr == NULL) { + reader[0]->config->bufArr = reader[0]->bufArr; + } + + if (fname) { + for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { + if (fname[i]) { + code = tsdbOpenFile(fname[i], config->szPage, TD_FILE_READ, &reader[0]->fd[i]); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } else { + for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { + if (config->files[i].exist) { + char fname1[TSDB_FILENAME_LEN]; + tsdbTFileName(config->tsdb, &config->files[i].file, fname1); + code = tsdbOpenFile(fname1, config->szPage, TD_FILE_READ, &reader[0]->fd[i]); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReaderClose(SDataFileReader **reader) { + if (reader[0] == NULL) return 0; + + TARRAY2_DESTROY(reader[0]->tombBlkArray, NULL); + TARRAY2_DESTROY(reader[0]->brinBlkArray, NULL); + +#if 0 + TARRAY2_DESTROY(reader[0]->dataBlkArray, NULL); + TARRAY2_DESTROY(reader[0]->blockIdxArray, NULL); +#endif + + for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { + if (reader[0]->fd[i]) { + tsdbCloseFile(&reader[0]->fd[i]); + } + } + + for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->bufArr); ++i) { + tFree(reader[0]->bufArr[i]); + } + + taosMemoryFree(reader[0]); + reader[0] = NULL; + return 0; +} + +int32_t tsdbDataFileReadBrinBlk(SDataFileReader *reader, const TBrinBlkArray **brinBlkArray) { + int32_t code = 0; + int32_t lino = 0; + + if (!reader->ctx->brinBlkLoaded) { + code = tsdbDataFileReadHeadFooter(reader); + TSDB_CHECK_CODE(code, lino, _exit); + + if (reader->headFooter->brinBlkPtr->size > 0) { + void *data = taosMemoryMalloc(reader->headFooter->brinBlkPtr->size); + if (data == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], reader->headFooter->brinBlkPtr->offset, data, + reader->headFooter->brinBlkPtr->size); + if (code) { + taosMemoryFree(data); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int32_t size = reader->headFooter->brinBlkPtr->size / sizeof(SBrinBlk); + TARRAY2_INIT_EX(reader->brinBlkArray, size, size, data); + } else { + TARRAY2_INIT(reader->brinBlkArray); + } + + reader->ctx->brinBlkLoaded = true; + } + brinBlkArray[0] = reader->brinBlkArray; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinBlk, SBrinBlock *brinBlock) { + int32_t code = 0; + int32_t lino = 0; + + code = tRealloc(&reader->config->bufArr[0], brinBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_HEAD], brinBlk->dp->offset, reader->config->bufArr[0], brinBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + int32_t size = 0; + tBrinBlockClear(brinBlock); + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); i++) { + code = tsdbDecmprData(reader->config->bufArr[0] + size, brinBlk->size[i], TSDB_DATA_TYPE_BIGINT, brinBlk->cmprAlg, + &reader->config->bufArr[1], brinBlk->numRec * sizeof(int64_t), &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND_BATCH(&brinBlock->dataArr1[i], reader->config->bufArr[1], brinBlk->numRec); + TSDB_CHECK_CODE(code, lino, _exit); + + size += brinBlk->size[i]; + } + + for (int32_t i = 0, j = ARRAY_SIZE(brinBlock->dataArr1); i < ARRAY_SIZE(brinBlock->dataArr2); i++, j++) { + code = tsdbDecmprData(reader->config->bufArr[0] + size, brinBlk->size[j], TSDB_DATA_TYPE_INT, brinBlk->cmprAlg, + &reader->config->bufArr[1], brinBlk->numRec * sizeof(int32_t), &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND_BATCH(&brinBlock->dataArr2[i], reader->config->bufArr[1], brinBlk->numRec); + TSDB_CHECK_CODE(code, lino, _exit); + + size += brinBlk->size[j]; + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReadBlockData(SDataFileReader *reader, const SBrinRecord *record, SBlockData *bData) { + int32_t code = 0; + int32_t lino = 0; + + code = tRealloc(&reader->config->bufArr[0], record->blockSize); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockSize); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tDecmprBlockData(reader->config->bufArr[0], record->blockSize, bData, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRecord *record, SBlockData *bData, + STSchema *pTSchema, int16_t cids[], int32_t ncid) { + int32_t code = 0; + int32_t lino = 0; + + code = tBlockDataInit(bData, (TABLEID *)record, pTSchema, cids, ncid); + TSDB_CHECK_CODE(code, lino, _exit); + + // uid + version + tskey + code = tRealloc(&reader->config->bufArr[0], record->blockKeySize); + TSDB_CHECK_CODE(code, lino, _exit); + + code = + tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset, reader->config->bufArr[0], record->blockKeySize); + TSDB_CHECK_CODE(code, lino, _exit); + + // hdr + SDiskDataHdr hdr[1]; + int32_t size = 0; + + size += tGetDiskDataHdr(reader->config->bufArr[0] + size, hdr); + + ASSERT(hdr->delimiter == TSDB_FILE_DLMT); + ASSERT(record->uid == hdr->uid); + + bData->nRow = hdr->nRow; + + // uid + ASSERT(hdr->uid); + + // version + code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szVer, TSDB_DATA_TYPE_BIGINT, hdr->cmprAlg, + (uint8_t **)&bData->aVersion, sizeof(int64_t) * hdr->nRow, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + size += hdr->szVer; + + // ts + code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr->cmprAlg, + (uint8_t **)&bData->aTSKEY, sizeof(TSKEY) * hdr->nRow, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + size += hdr->szKey; + + ASSERT(size == record->blockKeySize); + + // other columns + if (bData->nColData > 0) { + if (hdr->szBlkCol > 0) { + code = tRealloc(&reader->config->bufArr[0], hdr->szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize, + reader->config->bufArr[0], hdr->szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + } + + SBlockCol bc[1] = {{.cid = 0}}; + SBlockCol *blockCol = bc; + + size = 0; + for (int32_t i = 0; i < bData->nColData; i++) { + SColData *colData = tBlockDataGetColDataByIdx(bData, i); + + while (blockCol && blockCol->cid < colData->cid) { + if (size < hdr->szBlkCol) { + size += tGetBlockCol(reader->config->bufArr[0] + size, blockCol); + } else { + ASSERT(size == hdr->szBlkCol); + blockCol = NULL; + } + } + + if (blockCol == NULL || blockCol->cid > colData->cid) { + for (int32_t iRow = 0; iRow < hdr->nRow; iRow++) { + code = tColDataAppendValue(colData, &COL_VAL_NONE(colData->cid, colData->type)); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + ASSERT(blockCol->type == colData->type); + ASSERT(blockCol->flag && blockCol->flag != HAS_NONE); + + if (blockCol->flag == HAS_NULL) { + for (int32_t iRow = 0; iRow < hdr->nRow; iRow++) { + code = tColDataAppendValue(colData, &COL_VAL_NULL(blockCol->cid, blockCol->type)); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + int32_t size1 = blockCol->szBitmap + blockCol->szOffset + blockCol->szValue; + + code = tRealloc(&reader->config->bufArr[1], size1); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], + record->blockOffset + record->blockKeySize + hdr->szBlkCol + blockCol->offset, + reader->config->bufArr[1], size1); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDecmprColData(reader->config->bufArr[1], blockCol, hdr->cmprAlg, hdr->nRow, colData, + &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReadBlockSma(SDataFileReader *reader, const SBrinRecord *record, + TColumnDataAggArray *columnDataAggArray) { + int32_t code = 0; + int32_t lino = 0; + + TARRAY2_CLEAR(columnDataAggArray, NULL); + if (record->smaSize > 0) { + code = tRealloc(&reader->config->bufArr[0], record->smaSize); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_SMA], record->smaOffset, reader->config->bufArr[0], record->smaSize); + TSDB_CHECK_CODE(code, lino, _exit); + + // decode sma data + int32_t size = 0; + while (size < record->smaSize) { + SColumnDataAgg sma[1]; + + size += tGetColumnDataAgg(reader->config->bufArr[0] + size, sma); + + code = TARRAY2_APPEND_PTR(columnDataAggArray, sma); + TSDB_CHECK_CODE(code, lino, _exit); + } + ASSERT(size == record->smaSize); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReadTombBlk(SDataFileReader *reader, const TTombBlkArray **tombBlkArray) { + int32_t code = 0; + int32_t lino = 0; + + if (!reader->ctx->tombBlkLoaded) { + code = tsdbDataFileReadTombFooter(reader); + TSDB_CHECK_CODE(code, lino, _exit); + + if (reader->tombFooter->tombBlkPtr->size > 0) { + void *data = taosMemoryMalloc(reader->tombFooter->tombBlkPtr->size); + if (data == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], reader->tombFooter->tombBlkPtr->offset, data, + reader->tombFooter->tombBlkPtr->size); + if (code) { + taosMemoryFree(data); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int32_t size = reader->tombFooter->tombBlkPtr->size / sizeof(STombBlk); + TARRAY2_INIT_EX(reader->tombBlkArray, size, size, data); + } else { + TARRAY2_INIT(reader->tombBlkArray); + } + + reader->ctx->tombBlkLoaded = true; + } + tombBlkArray[0] = reader->tombBlkArray; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileReadTombBlock(SDataFileReader *reader, const STombBlk *tombBlk, STombBlock *tData) { + int32_t code = 0; + int32_t lino = 0; + + code = tRealloc(&reader->config->bufArr[0], tombBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_TOMB], tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + int32_t size = 0; + tTombBlockClear(tData); + for (int32_t i = 0; i < ARRAY_SIZE(tData->dataArr); ++i) { + code = tsdbDecmprData(reader->config->bufArr[0] + size, tombBlk->size[i], TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, + &reader->config->bufArr[1], sizeof(int64_t) * tombBlk->numRec, &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND_BATCH(&tData->dataArr[i], reader->config->bufArr[1], tombBlk->numRec); + TSDB_CHECK_CODE(code, lino, _exit); + + size += tombBlk->size[i]; + } + ASSERT(size == tombBlk->dp->size); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +// SDataFileWriter ============================================= +struct SDataFileWriter { + SDataFileWriterConfig config[1]; + + SSkmInfo skmTb[1]; + SSkmInfo skmRow[1]; + uint8_t *bufArr[5]; + + struct { + bool opened; + SDataFileReader *reader; + + // for ts data + TABLEID tbid[1]; + bool tbHasOldData; + + const TBrinBlkArray *brinBlkArray; + int32_t brinBlkArrayIdx; + SBrinBlock brinBlock[1]; + int32_t brinBlockIdx; + SBlockData blockData[1]; + int32_t blockDataIdx; + // for tomb data + bool hasOldTomb; + const TTombBlkArray *tombBlkArray; + int32_t tombBlkArrayIdx; + STombBlock tombBlock[1]; + int32_t tombBlockIdx; + } ctx[1]; + + STFile files[TSDB_FTYPE_MAX]; + STsdbFD *fd[TSDB_FTYPE_MAX]; + + SHeadFooter headFooter[1]; + STombFooter tombFooter[1]; + + TBrinBlkArray brinBlkArray[1]; + SBrinBlock brinBlock[1]; + SBlockData blockData[1]; + + TTombBlkArray tombBlkArray[1]; + STombBlock tombBlock[1]; +}; + +static int32_t tsdbDataFileWriterCloseAbort(SDataFileWriter *writer) { + ASSERT(0); + return 0; +} + +static int32_t tsdbDataFileWriterDoClose(SDataFileWriter *writer) { + if (writer->ctx->reader) { + tsdbDataFileReaderClose(&writer->ctx->reader); + } + + tTombBlockDestroy(writer->tombBlock); + TARRAY2_DESTROY(writer->tombBlkArray, NULL); + tBlockDataDestroy(writer->blockData); + tBrinBlockDestroy(writer->brinBlock); + TARRAY2_DESTROY(writer->brinBlkArray, NULL); + + tTombBlockDestroy(writer->ctx->tombBlock); + tBlockDataDestroy(writer->ctx->blockData); + tBrinBlockDestroy(writer->ctx->brinBlock); + + for (int32_t i = 0; i < ARRAY_SIZE(writer->bufArr); ++i) { + tFree(writer->bufArr[i]); + } + + tDestroyTSchema(writer->skmRow->pTSchema); + tDestroyTSchema(writer->skmTb->pTSchema); + return 0; +} + +static int32_t tsdbDataFileWriterDoOpenReader(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { + if (writer->config->files[i].exist) { + SDataFileReaderConfig config[1] = {{ + .tsdb = writer->config->tsdb, + .szPage = writer->config->szPage, + .bufArr = writer->config->bufArr, + }}; + + for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { + config->files[i].exist = writer->config->files[i].exist; + if (config->files[i].exist) { + config->files[i].file = writer->config->files[i].file; + } + } + + code = tsdbDataFileReaderOpen(NULL, config, &writer->ctx->reader); + TSDB_CHECK_CODE(code, lino, _exit); + break; + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + int32_t ftype; + + if (!writer->config->skmTb) writer->config->skmTb = writer->skmTb; + if (!writer->config->skmRow) writer->config->skmRow = writer->skmRow; + if (!writer->config->bufArr) writer->config->bufArr = writer->bufArr; + + // open reader + code = tsdbDataFileWriterDoOpenReader(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + // .head + ftype = TSDB_FTYPE_HEAD; + writer->files[ftype] = (STFile){ + .type = ftype, + .did = writer->config->did, + .fid = writer->config->fid, + .cid = writer->config->cid, + .size = 0, + }; + + // .data + ftype = TSDB_FTYPE_DATA; + if (writer->config->files[ftype].exist) { + writer->files[ftype] = writer->config->files[ftype].file; + } else { + writer->files[ftype] = (STFile){ + .type = ftype, + .did = writer->config->did, + .fid = writer->config->fid, + .cid = writer->config->cid, + .size = 0, + }; + } + + // .sma + ftype = TSDB_FTYPE_SMA; + if (writer->config->files[ftype].exist) { + writer->files[ftype] = writer->config->files[ftype].file; + } else { + writer->files[ftype] = (STFile){ + .type = ftype, + .did = writer->config->did, + .fid = writer->config->fid, + .cid = writer->config->cid, + .size = 0, + }; + } + + // .tomb + ftype = TSDB_FTYPE_TOMB; + writer->files[ftype] = (STFile){ + .type = ftype, + .did = writer->config->did, + .fid = writer->config->fid, + .cid = writer->config->cid, + .size = 0, + }; + + writer->ctx->opened = true; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize, + TBrinBlkArray *brinBlkArray, uint8_t **bufArr) { + if (BRIN_BLOCK_SIZE(brinBlock) == 0) return 0; + + int32_t code; + + // get SBrinBlk + SBrinBlk brinBlk[1] = { + { + .dp[0] = + { + .offset = *fileSize, + .size = 0, + }, + .minTbid = + { + .suid = TARRAY2_FIRST(brinBlock->suid), + .uid = TARRAY2_FIRST(brinBlock->uid), + }, + .maxTbid = + { + .suid = TARRAY2_LAST(brinBlock->suid), + .uid = TARRAY2_LAST(brinBlock->uid), + }, + .minVer = TARRAY2_FIRST(brinBlock->minVer), + .maxVer = TARRAY2_FIRST(brinBlock->minVer), + .numRec = BRIN_BLOCK_SIZE(brinBlock), + .cmprAlg = cmprAlg, + }, + }; + + for (int32_t i = 1; i < BRIN_BLOCK_SIZE(brinBlock); i++) { + if (brinBlk->minVer > TARRAY2_GET(brinBlock->minVer, i)) { + brinBlk->minVer = TARRAY2_GET(brinBlock->minVer, i); + } + if (brinBlk->maxVer < TARRAY2_GET(brinBlock->maxVer, i)) { + brinBlk->maxVer = TARRAY2_GET(brinBlock->maxVer, i); + } + } + + // write to file + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); i++) { + code = tsdbCmprData((uint8_t *)TARRAY2_DATA(brinBlock->dataArr1 + i), TARRAY2_DATA_LEN(brinBlock->dataArr1 + i), + TSDB_DATA_TYPE_BIGINT, brinBlk->cmprAlg, &bufArr[0], 0, &brinBlk->size[i], &bufArr[1]); + if (code) return code; + + code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[i]); + if (code) return code; + + brinBlk->dp->size += brinBlk->size[i]; + *fileSize += brinBlk->size[i]; + } + + for (int32_t i = 0, j = ARRAY_SIZE(brinBlock->dataArr1); i < ARRAY_SIZE(brinBlock->dataArr2); i++, j++) { + code = tsdbCmprData((uint8_t *)TARRAY2_DATA(brinBlock->dataArr2 + i), TARRAY2_DATA_LEN(brinBlock->dataArr2 + i), + TSDB_DATA_TYPE_INT, brinBlk->cmprAlg, &bufArr[0], 0, &brinBlk->size[j], &bufArr[1]); + if (code) return code; + + code = tsdbWriteFile(fd, *fileSize, bufArr[0], brinBlk->size[j]); + if (code) return code; + + brinBlk->dp->size += brinBlk->size[j]; + *fileSize += brinBlk->size[j]; + } + +#if 0 + SBrinRecord record; + for (int32_t i = 0; i < BRIN_BLOCK_SIZE(brinBlock); i++) { + tBrinBlockGet(brinBlock, i, &record); + tsdbInfo("write brin block, block num:%04d, idx:%04d suid:%ld, uid:%ld, offset:%ld, numRow:%d, count:%d", + TARRAY2_SIZE(brinBlkArray), i, record.suid, record.uid, record.blockOffset, record.numRow, record.count); + } +#endif + + // append to brinBlkArray + code = TARRAY2_APPEND_PTR(brinBlkArray, brinBlk); + if (code) return code; + + tBrinBlockClear(brinBlock); + + return 0; +} + +static int32_t tsdbDataFileWriteBrinBlock(SDataFileWriter *writer) { + if (BRIN_BLOCK_SIZE(writer->brinBlock) == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteBrinBlock(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlock, writer->config->cmprAlg, + &writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->config->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriteBrinRecord(SDataFileWriter *writer, const SBrinRecord *record) { + int32_t code = 0; + int32_t lino = 0; + + code = tBrinBlockPut(writer->brinBlock, record); + TSDB_CHECK_CODE(code, lino, _exit); + + if (BRIN_BLOCK_SIZE(writer->brinBlock) >= writer->config->maxRow) { + code = tsdbDataFileWriteBrinBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData *bData) { + if (bData->nRow == 0) return 0; + + ASSERT(bData->uid); + + int32_t code = 0; + int32_t lino = 0; + + SBrinRecord record[1] = {{ + .suid = bData->suid, + .uid = bData->uid, + .firstKey = bData->aTSKEY[0], + .firstKeyVer = bData->aVersion[0], + .lastKey = bData->aTSKEY[bData->nRow - 1], + .lastKeyVer = bData->aVersion[bData->nRow - 1], + .minVer = bData->aVersion[0], + .maxVer = bData->aVersion[0], + .blockOffset = writer->files[TSDB_FTYPE_DATA].size, + .smaOffset = writer->files[TSDB_FTYPE_SMA].size, + .blockSize = 0, + .blockKeySize = 0, + .smaSize = 0, + .numRow = bData->nRow, + .count = 1, + }}; + + for (int32_t i = 1; i < bData->nRow; ++i) { + if (bData->aTSKEY[i] != bData->aTSKEY[i - 1]) { + record->count++; + } + if (bData->aVersion[i] < record->minVer) { + record->minVer = bData->aVersion[i]; + } + if (bData->aVersion[i] > record->maxVer) { + record->maxVer = bData->aVersion[i]; + } + } + + // to .data file + int32_t sizeArr[5] = {0}; + + code = tCmprBlockData(bData, writer->config->cmprAlg, NULL, NULL, writer->config->bufArr, sizeArr); + TSDB_CHECK_CODE(code, lino, _exit); + + record->blockKeySize = sizeArr[3] + sizeArr[2]; + record->blockSize = sizeArr[0] + sizeArr[1] + record->blockKeySize; + + for (int32_t i = 3; i >= 0; --i) { + if (sizeArr[i]) { + code = tsdbWriteFile(writer->fd[TSDB_FTYPE_DATA], writer->files[TSDB_FTYPE_DATA].size, writer->config->bufArr[i], + sizeArr[i]); + TSDB_CHECK_CODE(code, lino, _exit); + writer->files[TSDB_FTYPE_DATA].size += sizeArr[i]; + } + } + + // to .sma file + for (int32_t i = 0; i < bData->nColData; ++i) { + SColData *colData = bData->aColData + i; + if ((!colData->smaOn) || ((colData->flag & HAS_VALUE) == 0)) continue; + + SColumnDataAgg sma[1] = {{.colId = colData->cid}}; + tColDataCalcSMA[colData->type](colData, &sma->sum, &sma->max, &sma->min, &sma->numOfNull); + + int32_t size = tPutColumnDataAgg(NULL, sma); + + code = tRealloc(&writer->config->bufArr[0], record->smaSize + size); + TSDB_CHECK_CODE(code, lino, _exit); + + tPutColumnDataAgg(writer->config->bufArr[0] + record->smaSize, sma); + record->smaSize += size; + } + + if (record->smaSize > 0) { + code = tsdbWriteFile(writer->fd[TSDB_FTYPE_SMA], record->smaOffset, writer->config->bufArr[0], record->smaSize); + TSDB_CHECK_CODE(code, lino, _exit); + writer->files[TSDB_FTYPE_SMA].size += record->smaSize; + } + + // append SBrinRecord + code = tsdbDataFileWriteBrinRecord(writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + + tBlockDataClear(bData); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriteDataBlk(SDataFileWriter *writer, const TDataBlkArray *dataBlkArray) { + if (TARRAY2_SIZE(dataBlkArray) == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + int32_t ftype = TSDB_FTYPE_HEAD; + SBlockIdx blockIdx[1] = {{ + .suid = writer->ctx->tbid->suid, + .uid = writer->ctx->tbid->uid, + .offset = writer->files[ftype].size, + .size = TARRAY2_DATA_LEN(dataBlkArray), + }}; + + code = + tsdbWriteFile(writer->fd[ftype], blockIdx->offset, (const uint8_t *)TARRAY2_DATA(dataBlkArray), blockIdx->size); + TSDB_CHECK_CODE(code, lino, _exit); + writer->files[ftype].size += blockIdx->size; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteTSRow(SDataFileWriter *writer, TSDBROW *row) { + int32_t code = 0; + int32_t lino = 0; + + // update/append + if (row->type == TSDBROW_ROW_FMT) { + code = tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, TSDBROW_SVERSION(row), writer->config->skmRow); + TSDB_CHECK_CODE(code, lino, _exit); + } + + TSDBKEY key[1]; + if (row->type == TSDBROW_ROW_FMT) { + key->ts = row->pTSRow->ts; + key->version = row->version; + } else { + key->ts = row->pBlockData->aTSKEY[row->iRow]; + key->version = row->pBlockData->aVersion[row->iRow]; + } + if (key->version <= writer->config->compactVersion // + && writer->blockData->nRow > 0 // + && writer->blockData->aTSKEY[writer->blockData->nRow - 1] == key->ts // + ) { + code = tBlockDataUpdateRow(writer->blockData, row, writer->config->skmRow->pTSchema); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + if (writer->blockData->nRow >= writer->config->maxRow) { + code = tsdbDataFileDoWriteBlockData(writer, writer->blockData); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tBlockDataAppendRow(writer->blockData, row, writer->config->skmRow->pTSchema, writer->ctx->tbid->uid); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const TSDBKEY *key) { + if (writer->ctx->tbHasOldData == false) return 0; + + int32_t code = 0; + int32_t lino = 0; + + for (;;) { + for (;;) { + // SBlockData + for (; writer->ctx->blockDataIdx < writer->ctx->blockData->nRow; writer->ctx->blockDataIdx++) { + if (key->ts < writer->ctx->blockData->aTSKEY[writer->ctx->blockDataIdx] // + || (key->ts == writer->ctx->blockData->aTSKEY[writer->ctx->blockDataIdx] && + key->version < writer->ctx->blockData->aVersion[writer->ctx->blockDataIdx])) { + goto _exit; + } else { + TSDBROW row = tsdbRowFromBlockData(writer->ctx->blockData, writer->ctx->blockDataIdx); + code = tsdbDataFileDoWriteTSRow(writer, &row); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + // SBrinBlock + if (writer->ctx->brinBlockIdx >= BRIN_BLOCK_SIZE(writer->ctx->brinBlock)) { + break; + } + + for (; writer->ctx->brinBlockIdx < BRIN_BLOCK_SIZE(writer->ctx->brinBlock); writer->ctx->brinBlockIdx++) { + if (TARRAY2_GET(writer->ctx->brinBlock->uid, writer->ctx->brinBlockIdx) != writer->ctx->tbid->uid) { + writer->ctx->tbHasOldData = false; + goto _exit; + } + + if (key->ts < TARRAY2_GET(writer->ctx->brinBlock->firstKey, writer->ctx->brinBlockIdx) // + || (key->ts == TARRAY2_GET(writer->ctx->brinBlock->firstKey, writer->ctx->brinBlockIdx) && + key->version < TARRAY2_GET(writer->ctx->brinBlock->firstKeyVer, writer->ctx->brinBlockIdx))) { + goto _exit; + } else { + SBrinRecord record[1]; + tBrinBlockGet(writer->ctx->brinBlock, writer->ctx->brinBlockIdx, record); + if (key->ts > record->lastKey || (key->ts == record->lastKey && key->version > record->maxVer)) { + if (writer->blockData->nRow > 0) { + code = tsdbDataFileDoWriteBlockData(writer, writer->blockData); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDataFileWriteBrinRecord(writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDataFileReadBlockData(writer->ctx->reader, record, writer->ctx->blockData); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->blockDataIdx = 0; + writer->ctx->brinBlockIdx++; + break; + } + } + } + } + + // SBrinBlk + if (writer->ctx->brinBlkArrayIdx >= TARRAY2_SIZE(writer->ctx->brinBlkArray)) { + writer->ctx->brinBlkArray = NULL; + writer->ctx->tbHasOldData = false; + goto _exit; + } + + for (; writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray); writer->ctx->brinBlkArrayIdx++) { + const SBrinBlk *brinBlk = TARRAY2_GET_PTR(writer->ctx->brinBlkArray, writer->ctx->brinBlkArrayIdx); + + if (brinBlk->minTbid.uid != writer->ctx->tbid->uid) { + writer->ctx->tbHasOldData = false; + goto _exit; + } + + code = tsdbDataFileReadBrinBlock(writer->ctx->reader, brinBlk, writer->ctx->brinBlock); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->brinBlockIdx = 0; + writer->ctx->brinBlkArrayIdx++; + break; + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteTSData(SDataFileWriter *writer, TSDBROW *row) { + int32_t code = 0; + int32_t lino = 0; + + if (writer->ctx->tbHasOldData) { + TSDBKEY key[1]; + if (row->type == TSDBROW_ROW_FMT) { + key->ts = row->pTSRow->ts; + key->version = row->version; + } else { + key->ts = row->pBlockData->aTSKEY[row->iRow]; + key->version = row->pBlockData->aVersion[row->iRow]; + } + + code = tsdbDataFileDoWriteTableOldData(writer, key); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDataFileDoWriteTSRow(writer, row); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriteTableDataEnd(SDataFileWriter *writer) { + if (writer->ctx->tbid->uid == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + if (writer->ctx->tbHasOldData) { + TSDBKEY key = { + .ts = TSKEY_MAX, + .version = VERSION_MAX, + }; + + code = tsdbDataFileDoWriteTableOldData(writer, &key); + TSDB_CHECK_CODE(code, lino, _exit); + + ASSERT(writer->ctx->tbHasOldData == false); + } + + code = tsdbDataFileDoWriteBlockData(writer, writer->blockData); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriteTableDataBegin(SDataFileWriter *writer, const TABLEID *tbid) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(writer->ctx->blockDataIdx == writer->ctx->blockData->nRow); + ASSERT(writer->blockData->nRow == 0); + + SMetaInfo info; + bool drop = false; + TABLEID tbid1[1]; + writer->ctx->tbHasOldData = false; + while (writer->ctx->brinBlkArray) { // skip data of previous table + for (; writer->ctx->brinBlockIdx < BRIN_BLOCK_SIZE(writer->ctx->brinBlock); writer->ctx->brinBlockIdx++) { + TABLEID tbid2[1] = {{ + .suid = TARRAY2_GET(writer->ctx->brinBlock->suid, writer->ctx->brinBlockIdx), + .uid = TARRAY2_GET(writer->ctx->brinBlock->uid, writer->ctx->brinBlockIdx), + }}; + + if (tbid2->uid == tbid->uid) { + writer->ctx->tbHasOldData = true; + goto _begin; + } else if (tbid2->suid > tbid->suid || (tbid2->suid == tbid->suid && tbid2->uid > tbid->uid)) { + goto _begin; + } else { + if (tbid2->uid != writer->ctx->tbid->uid) { + if (drop && tbid1->uid == tbid2->uid) { + continue; + } else if (metaGetInfo(writer->config->tsdb->pVnode->pMeta, tbid2->uid, &info, NULL) != 0) { + drop = true; + *tbid1 = *tbid2; + continue; + } else { + drop = false; + writer->ctx->tbid[0] = *tbid2; + } + } + + SBrinRecord record[1]; + tBrinBlockGet(writer->ctx->brinBlock, writer->ctx->brinBlockIdx, record); + + code = tsdbDataFileWriteBrinRecord(writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + if (writer->ctx->brinBlkArrayIdx >= TARRAY2_SIZE(writer->ctx->brinBlkArray)) { + writer->ctx->brinBlkArray = NULL; + break; + } + + for (; writer->ctx->brinBlkArrayIdx < TARRAY2_SIZE(writer->ctx->brinBlkArray); writer->ctx->brinBlkArrayIdx++) { + const SBrinBlk *brinBlk = TARRAY2_GET_PTR(writer->ctx->brinBlkArray, writer->ctx->brinBlkArrayIdx); + + code = tsdbDataFileReadBrinBlock(writer->ctx->reader, brinBlk, writer->ctx->brinBlock); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->brinBlockIdx = 0; + writer->ctx->brinBlkArrayIdx++; + break; + } + } + +_begin: + writer->ctx->tbid[0] = *tbid; + + if (tbid->uid == INT64_MAX) goto _exit; + + code = tsdbUpdateSkmTb(writer->config->tsdb, tbid, writer->config->skmTb); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tBlockDataInit(writer->blockData, writer->ctx->tbid, writer->config->skmTb->pTSchema, NULL, 0); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer) { + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer)); + if (code) return code; + *fileSize += sizeof(*footer); + return 0; +} + +static int32_t tsdbDataFileWriteHeadFooter(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteHeadFooter(writer->fd[TSDB_FTYPE_HEAD], &writer->files[TSDB_FTYPE_HEAD].size, writer->headFooter); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteTombBlock(SDataFileWriter *writer) { + if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg, + &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->config->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteTombBlk(SDataFileWriter *writer) { + ASSERT(TARRAY2_SIZE(writer->tombBlkArray) > 0); + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteTombBlk(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlkArray, writer->tombFooter->tombBlkPtr, + &writer->files[TSDB_FTYPE_TOMB].size); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize) { + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer)); + if (code) return code; + *fileSize += sizeof(*footer); + return 0; +} + +static int32_t tsdbDataFileWriteTombFooter(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteTombFooter(writer->fd[TSDB_FTYPE_TOMB], writer->tombFooter, &writer->files[TSDB_FTYPE_TOMB].size); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STombRecord *record) { + int32_t code = 0; + int32_t lino = 0; + + while (writer->ctx->hasOldTomb) { + for (; writer->ctx->tombBlockIdx < TOMB_BLOCK_SIZE(writer->ctx->tombBlock); writer->ctx->tombBlockIdx++) { + STombRecord record1[1]; + tTombBlockGet(writer->ctx->tombBlock, writer->ctx->tombBlockIdx, record1); + + int32_t c = tTombRecordCompare(record, record1); + if (c < 0) { + break; + } else if (c > 0) { + code = tTombBlockPut(writer->tombBlock, record1); + TSDB_CHECK_CODE(code, lino, _exit); + + if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) { + code = tsdbDataFileDoWriteTombBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + ASSERT(0); + } + } + + if (writer->ctx->tombBlkArrayIdx >= TARRAY2_SIZE(writer->ctx->tombBlkArray)) { + writer->ctx->hasOldTomb = false; + break; + } + + for (; writer->ctx->tombBlkArrayIdx < TARRAY2_SIZE(writer->ctx->tombBlkArray); ++writer->ctx->tombBlkArrayIdx) { + const STombBlk *tombBlk = TARRAY2_GET_PTR(writer->ctx->tombBlkArray, writer->ctx->tombBlkArrayIdx); + + code = tsdbDataFileReadTombBlock(writer->ctx->reader, tombBlk, writer->ctx->tombBlock); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->ctx->tombBlockIdx = 0; + writer->ctx->tombBlkArrayIdx++; + break; + } + } + +_write: + if (record->suid == INT64_MAX) goto _exit; + + code = tTombBlockPut(writer->tombBlock, record); + TSDB_CHECK_CODE(code, lino, _exit); + + if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) { + code = tsdbDataFileDoWriteTombBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize) { + ASSERT(TARRAY2_SIZE(brinBlkArray) > 0); + ptr->offset = *fileSize; + ptr->size = TARRAY2_DATA_LEN(brinBlkArray); + + int32_t code = tsdbWriteFile(fd, ptr->offset, (uint8_t *)TARRAY2_DATA(brinBlkArray), ptr->size); + if (code) return code; + + *fileSize += ptr->size; + return 0; +} + +static int32_t tsdbDataFileWriteBrinBlk(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteBrinBlk(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlkArray, writer->headFooter->brinBlkPtr, + &writer->files[TSDB_FTYPE_HEAD].size); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArray *opArr) { + int32_t code = 0; + int32_t lino = 0; + + int32_t ftype; + STFileOp op; + + if (writer->fd[TSDB_FTYPE_HEAD]) { + TABLEID tbid[1] = {{ + .suid = INT64_MAX, + .uid = INT64_MAX, + }}; + + code = tsdbDataFileWriteTableDataEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteTableDataBegin(writer, tbid); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteBrinBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteBrinBlk(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteHeadFooter(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + // .head + ftype = TSDB_FTYPE_HEAD; + if (writer->config->files[ftype].exist) { + op = (STFileOp){ + .optype = TSDB_FOP_REMOVE, + .fid = writer->config->fid, + .of = writer->config->files[ftype].file, + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = writer->config->fid, + .nf = writer->files[ftype], + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + // .data + ftype = TSDB_FTYPE_DATA; + if (!writer->config->files[ftype].exist) { + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = writer->config->fid, + .nf = writer->files[ftype], + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } else if (writer->config->files[ftype].file.size != writer->files[ftype].size) { + op = (STFileOp){ + .optype = TSDB_FOP_MODIFY, + .fid = writer->config->fid, + .of = writer->config->files[ftype].file, + .nf = writer->files[ftype], + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // .sma + ftype = TSDB_FTYPE_SMA; + if (!writer->config->files[ftype].exist) { + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = writer->config->fid, + .nf = writer->files[ftype], + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } else if (writer->config->files[ftype].file.size != writer->files[ftype].size) { + op = (STFileOp){ + .optype = TSDB_FOP_MODIFY, + .fid = writer->config->fid, + .of = writer->config->files[ftype].file, + .nf = writer->files[ftype], + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + if (writer->fd[TSDB_FTYPE_TOMB]) { + STombRecord record[1] = {{ + .suid = INT64_MAX, + .uid = INT64_MAX, + .version = INT64_MAX, + }}; + + code = tsdbDataFileDoWriteTombRecord(writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileDoWriteTombBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileDoWriteTombBlk(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteTombFooter(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + ftype = TSDB_FTYPE_TOMB; + if (writer->config->files[ftype].exist) { + op = (STFileOp){ + .optype = TSDB_FOP_REMOVE, + .fid = writer->config->fid, + .of = writer->config->files[ftype].file, + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = writer->config->fid, + .nf = writer->files[ftype], + }; + code = TARRAY2_APPEND(opArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + + for (int32_t i = 0; i < TSDB_FTYPE_MAX; ++i) { + if (writer->fd[i]) { + code = tsdbFsyncFile(writer->fd[i]); + TSDB_CHECK_CODE(code, lino, _exit); + tsdbCloseFile(&writer->fd[i]); + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDataFileWriterOpenDataFD(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + int32_t ftypes[] = {TSDB_FTYPE_HEAD, TSDB_FTYPE_DATA, TSDB_FTYPE_SMA}; + + for (int32_t i = 0; i < ARRAY_SIZE(ftypes); ++i) { + int32_t ftype = ftypes[i]; + + char fname[TSDB_FILENAME_LEN]; + int32_t flag = TD_FILE_READ | TD_FILE_WRITE; + + if (writer->files[ftype].size == 0) { + flag |= (TD_FILE_CREATE | TD_FILE_TRUNC); + } + + tsdbTFileName(writer->config->tsdb, &writer->files[ftype], fname); + code = tsdbOpenFile(fname, writer->config->szPage, flag, &writer->fd[ftype]); + TSDB_CHECK_CODE(code, lino, _exit); + + if (writer->files[ftype].size == 0) { + uint8_t hdr[TSDB_FHDR_SIZE] = {0}; + + code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->files[ftype].size += TSDB_FHDR_SIZE; + } + } + + if (writer->ctx->reader) { + code = tsdbDataFileReadBrinBlk(writer->ctx->reader, &writer->ctx->brinBlkArray); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileWriterOpen(const SDataFileWriterConfig *config, SDataFileWriter **writer) { + writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); + if (!writer[0]) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->config[0] = config[0]; + return 0; +} + +int32_t tsdbDataFileWriterClose(SDataFileWriter **writer, bool abort, TFileOpArray *opArr) { + if (writer[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + if (writer[0]->ctx->opened) { + if (abort) { + code = tsdbDataFileWriterCloseAbort(writer[0]); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDataFileWriterCloseCommit(writer[0], opArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + tsdbDataFileWriterDoClose(writer[0]); + } + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileWriteRow(SDataFileWriter *writer, SRowInfo *row) { + int32_t code = 0; + int32_t lino = 0; + + if (!writer->ctx->opened) { + code = tsdbDataFileWriterDoOpen(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (writer->fd[TSDB_FTYPE_HEAD] == NULL) { + code = tsdbDataFileWriterOpenDataFD(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (row->uid != writer->ctx->tbid->uid) { + code = tsdbDataFileWriteTableDataEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteTableDataBegin(writer, (TABLEID *)row); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDataFileDoWriteTSData(writer, &row->row); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData) { + if (bData->nRow == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + ASSERT(bData->uid); + + if (!writer->ctx->opened) { + code = tsdbDataFileWriterDoOpen(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (writer->fd[TSDB_FTYPE_DATA] == NULL) { + code = tsdbDataFileWriterOpenDataFD(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (bData->uid != writer->ctx->tbid->uid) { + code = tsdbDataFileWriteTableDataEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriteTableDataBegin(writer, (TABLEID *)bData); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (writer->ctx->tbHasOldData) { + TSDBKEY key = { + .ts = bData->aTSKEY[0], + .version = bData->aVersion[0], + }; + + code = tsdbDataFileDoWriteTableOldData(writer, &key); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (!writer->ctx->tbHasOldData // + && writer->blockData->nRow == 0 // + ) { + code = tsdbDataFileDoWriteBlockData(writer, bData); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + for (int32_t i = 0; i < bData->nRow; ++i) { + TSDBROW row[1] = {tsdbRowFromBlockData(bData, i)}; + code = tsdbDataFileDoWriteTSData(writer, row); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileFlush(SDataFileWriter *writer) { + ASSERT(writer->ctx->opened); + + if (writer->blockData->nRow == 0) return 0; + if (writer->ctx->tbHasOldData) return 0; + + return tsdbDataFileDoWriteBlockData(writer, writer->blockData); +} + +static int32_t tsdbDataFileWriterOpenTombFD(SDataFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + char fname[TSDB_FILENAME_LEN]; + int32_t ftype = TSDB_FTYPE_TOMB; + + ASSERT(writer->files[ftype].size == 0); + + int32_t flag = (TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + + tsdbTFileName(writer->config->tsdb, writer->files + ftype, fname); + code = tsdbOpenFile(fname, writer->config->szPage, flag, &writer->fd[ftype]); + TSDB_CHECK_CODE(code, lino, _exit); + + uint8_t hdr[TSDB_FHDR_SIZE] = {0}; + code = tsdbWriteFile(writer->fd[ftype], 0, hdr, TSDB_FHDR_SIZE); + TSDB_CHECK_CODE(code, lino, _exit); + writer->files[ftype].size += TSDB_FHDR_SIZE; + + if (writer->ctx->reader) { + code = tsdbDataFileReadTombBlk(writer->ctx->reader, &writer->ctx->tombBlkArray); + TSDB_CHECK_CODE(code, lino, _exit); + + if (TARRAY2_SIZE(writer->ctx->tombBlkArray) > 0) { + writer->ctx->hasOldTomb = true; + } + + writer->ctx->tombBlkArrayIdx = 0; + tTombBlockClear(writer->ctx->tombBlock); + writer->ctx->tombBlockIdx = 0; + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record) { + int32_t code = 0; + int32_t lino = 0; + + if (!writer->ctx->opened) { + code = tsdbDataFileWriterDoOpen(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (writer->fd[TSDB_FTYPE_TOMB] == NULL) { + code = tsdbDataFileWriterOpenTombFD(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDataFileDoWriteTombRecord(writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h new file mode 100644 index 0000000000000000000000000000000000000000..827b58fb4a247a0d68c606c7c5a4eaa05b4db5c4 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDef.h" +#include "tsdbFSet2.h" +#include "tsdbSttFileRW.h" +#include "tsdbUtil2.h" + +#ifndef _TSDB_DATA_FILE_RW_H +#define _TSDB_DATA_FILE_RW_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef TARRAY2(SBlockIdx) TBlockIdxArray; +typedef TARRAY2(SDataBlk) TDataBlkArray; +typedef TARRAY2(SColumnDataAgg) TColumnDataAggArray; + +typedef struct { + SFDataPtr brinBlkPtr[1]; + SFDataPtr rsrvd[2]; +} SHeadFooter; + +typedef struct { + SFDataPtr tombBlkPtr[1]; + SFDataPtr rsrvd[2]; +} STombFooter; + +// SDataFileReader ============================================= +typedef struct SDataFileReader SDataFileReader; +typedef struct SDataFileReaderConfig { + STsdb *tsdb; + int32_t szPage; + struct { + bool exist; + STFile file; + } files[TSDB_FTYPE_MAX]; + uint8_t **bufArr; +} SDataFileReaderConfig; + +int32_t tsdbDataFileReaderOpen(const char *fname[/* TSDB_FTYPE_MAX */], const SDataFileReaderConfig *config, + SDataFileReader **reader); +int32_t tsdbDataFileReaderClose(SDataFileReader **reader); +// .head +int32_t tsdbDataFileReadBrinBlk(SDataFileReader *reader, const TBrinBlkArray **brinBlkArray); +int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinBlk, SBrinBlock *brinBlock); +// .data +int32_t tsdbDataFileReadBlockData(SDataFileReader *reader, const SBrinRecord *record, SBlockData *bData); +int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRecord *record, SBlockData *bData, + STSchema *pTSchema, int16_t cids[], int32_t ncid); +// .sma +int32_t tsdbDataFileReadBlockSma(SDataFileReader *reader, const SBrinRecord *record, + TColumnDataAggArray *columnDataAggArray); +// .tomb +int32_t tsdbDataFileReadTombBlk(SDataFileReader *reader, const TTombBlkArray **tombBlkArray); +int32_t tsdbDataFileReadTombBlock(SDataFileReader *reader, const STombBlk *tombBlk, STombBlock *tData); + +// SDataFileWriter ============================================= +typedef struct SDataFileWriter SDataFileWriter; +typedef struct SDataFileWriterConfig { + STsdb *tsdb; + int8_t cmprAlg; + int32_t maxRow; + int32_t szPage; + int32_t fid; + int64_t cid; + SDiskID did; + int64_t compactVersion; + struct { + bool exist; + STFile file; + } files[TSDB_FTYPE_MAX]; + SSkmInfo *skmTb; + SSkmInfo *skmRow; + uint8_t **bufArr; +} SDataFileWriterConfig; + +int32_t tsdbDataFileWriterOpen(const SDataFileWriterConfig *config, SDataFileWriter **writer); +int32_t tsdbDataFileWriterClose(SDataFileWriter **writer, bool abort, TFileOpArray *opArr); + +int32_t tsdbDataFileWriteRow(SDataFileWriter *writer, SRowInfo *row); +int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData); +int32_t tsdbDataFileFlush(SDataFileWriter *writer); + +int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_DATA_FILE_RW_H*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbDef.h b/source/dnode/vnode/src/tsdb/tsdbDef.h new file mode 100644 index 0000000000000000000000000000000000000000..e768f68b15654ab6cc9327795b80783557896c3e --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbDef.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tarray2.h" +#include "tsdb.h" + +#ifndef _TD_TSDB_DEF_H_ +#define _TD_TSDB_DEF_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define TSDB_ERROR_LOG(vid, lino, code) \ + tsdbError("vgId:%d %s failed at line %d since %s", vid, __func__, lino, tstrerror(code)) + +typedef struct SFDataPtr { + int64_t offset; + int64_t size; +} SFDataPtr; + +extern int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD); +extern void tsdbCloseFile(STsdbFD **ppFD); +extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size); +extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size); +extern int32_t tsdbFsyncFile(STsdbFD *pFD); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TSDB_DEF_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 5519d43012f46dc2e24ad56083ff421d5fcff1c5..ec116c717e05bc3a1be7379b18386c20eb8ef67d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -14,6 +14,7 @@ */ #include "tsdb.h" +#include "vnd.h" // ================================================================================================= static int32_t tsdbFSToBinary(uint8_t *p, STsdbFS *pFS) { @@ -180,10 +181,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { TSDB_CHECK_CODE(code, lino, _exit); } - if (size != tsdbLogicToFileSize(pTsdb->fs.pDelFile->size, pTsdb->pVnode->config.tsdbPageSize)) { - code = TSDB_CODE_FILE_CORRUPTED; - TSDB_CHECK_CODE(code, lino, _exit); - } + // if (size != tsdbLogicToFileSize(pTsdb->fs.pDelFile->size, pTsdb->pVnode->config.tsdbPageSize)) { + // code = TSDB_CODE_FILE_CORRUPTED; + // TSDB_CHECK_CODE(code, lino, _exit); + // } } // SArray @@ -198,10 +199,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); } - if (size != tsdbLogicToFileSize(pSet->pHeadF->size, pTsdb->pVnode->config.tsdbPageSize)) { - code = TSDB_CODE_FILE_CORRUPTED; - TSDB_CHECK_CODE(code, lino, _exit); - } + // if (size != tsdbLogicToFileSize(pSet->pHeadF->size, pTsdb->pVnode->config.tsdbPageSize)) { + // code = TSDB_CODE_FILE_CORRUPTED; + // TSDB_CHECK_CODE(code, lino, _exit); + // } // data ========= tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); @@ -209,10 +210,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); } - if (size < tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) { - code = TSDB_CODE_FILE_CORRUPTED; - TSDB_CHECK_CODE(code, lino, _exit); - } + // if (size < tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) { + // code = TSDB_CODE_FILE_CORRUPTED; + // TSDB_CHECK_CODE(code, lino, _exit); + // } // else if (size > tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) { // code = tsdbDFileRollback(pTsdb, pSet, TSDB_DATA_FILE); // TSDB_CHECK_CODE(code, lino, _exit); @@ -224,10 +225,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); } - if (size < tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) { - code = TSDB_CODE_FILE_CORRUPTED; - TSDB_CHECK_CODE(code, lino, _exit); - } + // if (size < tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) { + // code = TSDB_CODE_FILE_CORRUPTED; + // TSDB_CHECK_CODE(code, lino, _exit); + // } // else if (size > tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) { // code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE); // TSDB_CHECK_CODE(code, lino, _exit); @@ -240,10 +241,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); } - if (size != tsdbLogicToFileSize(pSet->aSttF[iStt]->size, pTsdb->pVnode->config.tsdbPageSize)) { - code = TSDB_CODE_FILE_CORRUPTED; - TSDB_CHECK_CODE(code, lino, _exit); - } + // if (size != tsdbLogicToFileSize(pSet->aSttF[iStt]->size, pTsdb->pVnode->config.tsdbPageSize)) { + // code = TSDB_CODE_FILE_CORRUPTED; + // TSDB_CHECK_CODE(code, lino, _exit); + // } } } @@ -269,28 +270,26 @@ int32_t tDFileSetCmprFn(const void *p1, const void *p2) { return 0; } -static void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t) { +void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t) { SVnode *pVnode = pTsdb->pVnode; - if (pVnode->pTfs) { - if (current) { - snprintf(current, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP, - pTsdb->path, TD_DIRSEP); - } - if (current_t) { - snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT.t", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP, - pTsdb->path, TD_DIRSEP); - } - } else { - if (current) { - snprintf(current, TSDB_FILENAME_LEN - 1, "%s%sCURRENT", pTsdb->path, TD_DIRSEP); - } - if (current_t) { - snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%sCURRENT.t", pTsdb->path, TD_DIRSEP); - } + int32_t offset = 0; + + // CURRENT + if (current) { + vnodeGetPrimaryDir(pTsdb->path, pVnode->diskPrimary, pVnode->pTfs, current, TSDB_FILENAME_LEN); + offset = strlen(current); + snprintf(current + offset, TSDB_FILENAME_LEN - offset - 1, "%sCURRENT", TD_DIRSEP); + } + + // CURRENT.t + if (current_t) { + vnodeGetPrimaryDir(pTsdb->path, pVnode->diskPrimary, pVnode->pTfs, current_t, TSDB_FILENAME_LEN); + offset = strlen(current_t); + snprintf(current_t + offset, TSDB_FILENAME_LEN - offset - 1, "%sCURRENT.t", TD_DIRSEP); } } -static int32_t tsdbLoadFSFromFile(const char *fname, STsdbFS *pFS) { +static int32_t load_fs(const char *fname, STsdbFS *pFS) { int32_t code = 0; int32_t lino = 0; uint8_t *pData = NULL; @@ -667,7 +666,7 @@ static int32_t tsdbFSApplyChange(STsdb *pTsdb, STsdbFS *pFS) { taosArrayRemove(pTsdb->fs.aDFileSet, iOld); } else { code = tsdbNewFileSet(pTsdb, &fSet, pSetNew); - TSDB_CHECK_CODE(code, lino, _exit) + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -683,7 +682,7 @@ static int32_t tsdbFSApplyChange(STsdb *pTsdb, STsdbFS *pFS) { taosArrayRemove(pTsdb->fs.aDFileSet, iOld); } else { code = tsdbNewFileSet(pTsdb, &fSet, pSetNew); - TSDB_CHECK_CODE(code, lino, _exit) + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -724,7 +723,7 @@ int32_t tsdbFSCommit(STsdb *pTsdb) { code = tsdbFSCreate(&fs); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbLoadFSFromFile(current, &fs); + code = load_fs(current, &fs); TSDB_CHECK_CODE(code, lino, _exit); // apply file change @@ -769,7 +768,7 @@ int32_t tsdbFSOpen(STsdb *pTsdb, int8_t rollback) { tsdbGetCurrentFName(pTsdb, current, current_t); if (taosCheckExistFile(current)) { - code = tsdbLoadFSFromFile(current, &pTsdb->fs); + code = load_fs(current, &pTsdb->fs); TSDB_CHECK_CODE(code, lino, _exit); if (taosCheckExistFile(current_t)) { @@ -1142,4 +1141,4 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { } taosArrayDestroy(pFS->aDFileSet); -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c new file mode 100644 index 0000000000000000000000000000000000000000..1a0104945fa52c33d06be198501b97510ca481c9 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -0,0 +1,885 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFS2.h" +#include "tsdbUpgrade.h" +#include "vnd.h" + +extern int vnodeScheduleTask(int (*execute)(void *), void *arg); +extern int vnodeScheduleTaskEx(int tpid, int (*execute)(void *), void *arg); + +#define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT +#define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1) + +enum { + TSDB_FS_STATE_NONE = 0, + TSDB_FS_STATE_OPEN, + TSDB_FS_STATE_EDIT, + TSDB_FS_STATE_CLOSE, +}; + +static const char *gCurrentFname[] = { + [TSDB_FCURRENT] = "current.json", + [TSDB_FCURRENT_C] = "current.c.json", + [TSDB_FCURRENT_M] = "current.m.json", +}; + +static int32_t create_fs(STsdb *pTsdb, STFileSystem **fs) { + fs[0] = taosMemoryCalloc(1, sizeof(*fs[0])); + if (fs[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + fs[0]->tsdb = pTsdb; + tsem_init(&fs[0]->canEdit, 0, 1); + fs[0]->state = TSDB_FS_STATE_NONE; + fs[0]->neid = 0; + TARRAY2_INIT(fs[0]->fSetArr); + TARRAY2_INIT(fs[0]->fSetArrTmp); + + // background task queue + taosThreadMutexInit(fs[0]->mutex, NULL); + fs[0]->bgTaskQueue->next = fs[0]->bgTaskQueue; + fs[0]->bgTaskQueue->prev = fs[0]->bgTaskQueue; + + return 0; +} + +static int32_t destroy_fs(STFileSystem **fs) { + if (fs[0] == NULL) return 0; + taosThreadMutexDestroy(fs[0]->mutex); + + ASSERT(fs[0]->bgTaskNum == 0); + + TARRAY2_DESTROY(fs[0]->fSetArr, NULL); + TARRAY2_DESTROY(fs[0]->fSetArrTmp, NULL); + tsem_destroy(&fs[0]->canEdit); + taosMemoryFree(fs[0]); + fs[0] = NULL; + return 0; +} + +int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype) { + int32_t offset = 0; + + vnodeGetPrimaryDir(pTsdb->path, pTsdb->pVnode->diskPrimary, pTsdb->pVnode->pTfs, fname, TSDB_FILENAME_LEN); + offset = strlen(fname); + snprintf(fname + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, gCurrentFname[ftype]); + + return 0; +} + +static int32_t save_json(const cJSON *json, const char *fname) { + int32_t code = 0; + + char *data = cJSON_PrintUnformatted(json); + if (data == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + TdFilePtr fp = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (fp == NULL) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + + if (taosWriteFile(fp, data, strlen(data)) < 0) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + + if (taosFsyncFile(fp) < 0) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + + taosCloseFile(&fp); + +_exit: + taosMemoryFree(data); + return code; +} + +static int32_t load_json(const char *fname, cJSON **json) { + int32_t code = 0; + char *data = NULL; + + TdFilePtr fp = taosOpenFile(fname, TD_FILE_READ); + if (fp == NULL) return TAOS_SYSTEM_ERROR(code); + + int64_t size; + if (taosFStatFile(fp, &size, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + + data = taosMemoryMalloc(size + 1); + if (data == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + if (taosReadFile(fp, data, size) < 0) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + data[size] = '\0'; + + json[0] = cJSON_Parse(data); + if (json[0] == NULL) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + +_exit: + taosCloseFile(&fp); + if (data) taosMemoryFree(data); + if (code) json[0] = NULL; + return code; +} + +int32_t save_fs(const TFileSetArray *arr, const char *fname) { + int32_t code = 0; + int32_t lino = 0; + + cJSON *json = cJSON_CreateObject(); + if (!json) return TSDB_CODE_OUT_OF_MEMORY; + + // fmtv + if (cJSON_AddNumberToObject(json, "fmtv", 1) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // fset + cJSON *ajson = cJSON_AddArrayToObject(json, "fset"); + if (!ajson) TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + const STFileSet *fset; + TARRAY2_FOREACH(arr, fset) { + cJSON *item = cJSON_CreateObject(); + if (!item) TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + cJSON_AddItemToArray(ajson, item); + + code = tsdbTFileSetToJson(fset, item); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = save_json(json, fname); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + cJSON_Delete(json); + return code; +} + +static int32_t load_fs(STsdb *pTsdb, const char *fname, TFileSetArray *arr) { + int32_t code = 0; + int32_t lino = 0; + + TARRAY2_CLEAR(arr, tsdbTFileSetClear); + + // load json + cJSON *json = NULL; + code = load_json(fname, &json); + TSDB_CHECK_CODE(code, lino, _exit); + + // parse json + const cJSON *item1; + + /* fmtv */ + item1 = cJSON_GetObjectItem(json, "fmtv"); + if (cJSON_IsNumber(item1)) { + ASSERT(item1->valuedouble == 1); + } else { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } + + /* fset */ + item1 = cJSON_GetObjectItem(json, "fset"); + if (cJSON_IsArray(item1)) { + const cJSON *item2; + cJSON_ArrayForEach(item2, item1) { + STFileSet *fset; + code = tsdbJsonToTFileSet(pTsdb, item2, &fset); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(arr, fset); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); + } + +_exit: + if (code) { + tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); + } + if (json) cJSON_Delete(json); + return code; +} + +static bool is_same_file(const STFile *f1, const STFile f2) { + if (f1->type != f2.type) return false; + if (f1->did.level != f2.did.level) return false; + if (f1->did.id != f2.did.id) return false; + if (f1->cid != f2.cid) return false; + return true; +} + +static int32_t apply_commit(STFileSystem *fs) { + int32_t code = 0; + TFileSetArray *fsetArray1 = fs->fSetArr; + TFileSetArray *fsetArray2 = fs->fSetArrTmp; + int32_t i1 = 0, i2 = 0; + + while (i1 < TARRAY2_SIZE(fsetArray1) || i2 < TARRAY2_SIZE(fsetArray2)) { + STFileSet *fset1 = i1 < TARRAY2_SIZE(fsetArray1) ? TARRAY2_GET(fsetArray1, i1) : NULL; + STFileSet *fset2 = i2 < TARRAY2_SIZE(fsetArray2) ? TARRAY2_GET(fsetArray2, i2) : NULL; + + if (fset1 && fset2) { + if (fset1->fid < fset2->fid) { + // delete fset1 + TARRAY2_REMOVE(fsetArray1, i1, tsdbTFileSetRemove); + } else if (fset1->fid > fset2->fid) { + // create new file set with fid of fset2->fid + code = tsdbTFileSetInitDup(fs->tsdb, fset2, &fset1); + if (code) return code; + code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn); + if (code) return code; + i1++; + i2++; + } else { + // edit + code = tsdbTFileSetApplyEdit(fs->tsdb, fset2, fset1); + if (code) return code; + i1++; + i2++; + } + } else if (fset1) { + // delete fset1 + TARRAY2_REMOVE(fsetArray1, i1, tsdbTFileSetRemove); + } else { + // create new file set with fid of fset2->fid + code = tsdbTFileSetInitDup(fs->tsdb, fset2, &fset1); + if (code) return code; + code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn); + if (code) return code; + i1++; + i2++; + } + } + + return 0; +} + +static int32_t commit_edit(STFileSystem *fs) { + char current[TSDB_FILENAME_LEN]; + char current_t[TSDB_FILENAME_LEN]; + + current_fname(fs->tsdb, current, TSDB_FCURRENT); + if (fs->etype == TSDB_FEDIT_COMMIT) { + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C); + } else if (fs->etype == TSDB_FEDIT_MERGE) { + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M); + } else { + ASSERT(0); + } + + int32_t code; + int32_t lino; + if ((code = taosRenameFile(current_t, current))) { + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); + } + + code = apply_commit(fs); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(fs->tsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); + } + return code; +} + +// static int32_t +static int32_t apply_abort(STFileSystem *fs) { + // TODO + return 0; +} + +static int32_t abort_edit(STFileSystem *fs) { + char fname[TSDB_FILENAME_LEN]; + + if (fs->etype == TSDB_FEDIT_COMMIT) { + current_fname(fs->tsdb, fname, TSDB_FCURRENT_C); + } else if (fs->etype == TSDB_FEDIT_MERGE) { + current_fname(fs->tsdb, fname, TSDB_FCURRENT_M); + } else { + ASSERT(0); + } + + int32_t code; + int32_t lino; + if ((code = taosRemoveFile(fname))) { + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); + } + + code = apply_abort(fs); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed since %s", TD_VID(fs->tsdb->pVnode), __func__, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); + } + return code; +} + +static int32_t tsdbFSScanAndFix(STFileSystem *fs) { + fs->neid = 0; + + // get max commit id + const STFileSet *fset; + TARRAY2_FOREACH(fs->fSetArr, fset) { fs->neid = TMAX(fs->neid, tsdbTFileSetMaxCid(fset)); } + + // TODO + return 0; +} + +static int32_t tsdbFSDupState(STFileSystem *fs) { + int32_t code; + + const TFileSetArray *src = fs->fSetArr; + TFileSetArray *dst = fs->fSetArrTmp; + + TARRAY2_CLEAR(dst, tsdbTFileSetClear); + + const STFileSet *fset1; + TARRAY2_FOREACH(src, fset1) { + STFileSet *fset2; + code = tsdbTFileSetInitDup(fs->tsdb, fset1, &fset2); + if (code) return code; + code = TARRAY2_APPEND(dst, fset2); + if (code) return code; + } + + return 0; +} + +static int32_t open_fs(STFileSystem *fs, int8_t rollback) { + int32_t code = 0; + int32_t lino = 0; + STsdb *pTsdb = fs->tsdb; + + char fCurrent[TSDB_FILENAME_LEN]; + char cCurrent[TSDB_FILENAME_LEN]; + char mCurrent[TSDB_FILENAME_LEN]; + + current_fname(pTsdb, fCurrent, TSDB_FCURRENT); + current_fname(pTsdb, cCurrent, TSDB_FCURRENT_C); + current_fname(pTsdb, mCurrent, TSDB_FCURRENT_M); + + if (taosCheckExistFile(fCurrent)) { // current.json exists + code = load_fs(pTsdb, fCurrent, fs->fSetArr); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosCheckExistFile(cCurrent)) { + // current.c.json exists + + fs->etype = TSDB_FEDIT_COMMIT; + if (rollback) { + code = abort_edit(fs); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = load_fs(pTsdb, cCurrent, fs->fSetArrTmp); + TSDB_CHECK_CODE(code, lino, _exit); + + code = commit_edit(fs); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else if (taosCheckExistFile(mCurrent)) { + // current.m.json exists + fs->etype = TSDB_FEDIT_MERGE; + code = abort_edit(fs); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbFSDupState(fs); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSScanAndFix(fs); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = save_fs(fs->fSetArr, fCurrent); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s success", TD_VID(pTsdb->pVnode), __func__); + } + return 0; +} + +static int32_t close_file_system(STFileSystem *fs) { + TARRAY2_CLEAR(fs->fSetArr, tsdbTFileSetClear); + TARRAY2_CLEAR(fs->fSetArrTmp, tsdbTFileSetClear); + // TODO + return 0; +} + +static int32_t apply_edit(STFileSystem *pFS) { + int32_t code = 0; + ASSERTS(0, "TODO: Not implemented yet"); + return code; +} + +static int32_t fset_cmpr_fn(const struct STFileSet *pSet1, const struct STFileSet *pSet2) { + if (pSet1->fid < pSet2->fid) { + return -1; + } else if (pSet1->fid > pSet2->fid) { + return 1; + } + return 0; +} + +static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFSDupState(fs); + if (code) return code; + + TFileSetArray *fsetArray = fs->fSetArrTmp; + STFileSet *fset = NULL; + const STFileOp *op; + TARRAY2_FOREACH_PTR(opArray, op) { + if (!fset || fset->fid != op->fid) { + STFileSet tfset = {.fid = op->fid}; + fset = &tfset; + STFileSet **fsetPtr = TARRAY2_SEARCH(fsetArray, &fset, tsdbTFileSetCmprFn, TD_EQ); + fset = (fsetPtr == NULL) ? NULL : *fsetPtr; + + if (!fset) { + code = tsdbTFileSetInit(op->fid, &fset); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_SORT_INSERT(fsetArray, fset, tsdbTFileSetCmprFn); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + code = tsdbTFileSetEdit(fs->tsdb, fset, op); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // remove empty file set + int32_t i = 0; + while (i < TARRAY2_SIZE(fsetArray)) { + fset = TARRAY2_GET(fsetArray, i); + if (tsdbTFileSetIsEmpty(fset)) { + TARRAY2_REMOVE(fsetArray, i, tsdbTFileSetClear); + } else { + i++; + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(fs->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbOpenFS(STsdb *pTsdb, STFileSystem **fs, int8_t rollback) { + int32_t code; + int32_t lino; + + code = tsdbCheckAndUpgradeFileSystem(pTsdb, rollback); + TSDB_CHECK_CODE(code, lino, _exit); + + code = create_fs(pTsdb, fs); + TSDB_CHECK_CODE(code, lino, _exit); + + code = open_fs(fs[0], rollback); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + destroy_fs(fs); + } else { + tsdbInfo("vgId:%d %s success", TD_VID(pTsdb->pVnode), __func__); + } + return 0; +} + +static void tsdbDoWaitBgTask(STFileSystem *fs, STFSBgTask *task) { + task->numWait++; + taosThreadCondWait(task->done, fs->mutex); + task->numWait--; + + if (task->numWait == 0) { + taosThreadCondDestroy(task->done); + taosMemoryFree(task); + } +} + +static void tsdbDoDoneBgTask(STFileSystem *fs, STFSBgTask *task) { + if (task->numWait > 0) { + taosThreadCondBroadcast(task->done); + } else { + taosThreadCondDestroy(task->done); + taosMemoryFree(task); + } +} + +int32_t tsdbCloseFS(STFileSystem **fs) { + if (fs[0] == NULL) return 0; + + taosThreadMutexLock(fs[0]->mutex); + fs[0]->stop = true; + + if (fs[0]->bgTaskRunning) { + tsdbDoWaitBgTask(fs[0], fs[0]->bgTaskRunning); + } + taosThreadMutexUnlock(fs[0]->mutex); + + close_file_system(fs[0]); + destroy_fs(fs); + return 0; +} + +int64_t tsdbFSAllocEid(STFileSystem *fs) { + taosThreadRwlockRdlock(&fs->tsdb->rwLock); + int64_t cid = ++fs->neid; + taosThreadRwlockUnlock(&fs->tsdb->rwLock); + return cid; +} + +int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype) { + int32_t code = 0; + int32_t lino; + char current_t[TSDB_FILENAME_LEN]; + + switch (etype) { + case TSDB_FEDIT_COMMIT: + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C); + break; + case TSDB_FEDIT_MERGE: + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M); + break; + default: + ASSERT(0); + } + + tsem_wait(&fs->canEdit); + fs->etype = etype; + + // edit + code = edit_fs(fs, opArray); + TSDB_CHECK_CODE(code, lino, _exit); + + // save fs + code = save_fs(fs->fSetArrTmp, current_t); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, lino, + tstrerror(code), etype); + } else { + tsdbInfo("vgId:%d %s done, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, etype); + } + return code; +} + +int32_t tsdbFSEditCommit(STFileSystem *fs) { + int32_t code = 0; + int32_t lino = 0; + + // commit + code = commit_edit(fs); + TSDB_CHECK_CODE(code, lino, _exit); + + // schedule merge + if (fs->tsdb->pVnode->config.sttTrigger != 1) { + STFileSet *fset; + TARRAY2_FOREACH_REVERSE(fs->fSetArr, fset) { + if (TARRAY2_SIZE(fset->lvlArr) == 0) continue; + + SSttLvl *lvl = TARRAY2_FIRST(fset->lvlArr); + if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < fs->tsdb->pVnode->config.sttTrigger) continue; + + code = tsdbFSScheduleBgTask(fs, TSDB_BG_TASK_MERGER, tsdbMerge, fs->tsdb, NULL); + TSDB_CHECK_CODE(code, lino, _exit); + + break; + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(fs->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); + tsem_post(&fs->canEdit); + } + return code; +} + +int32_t tsdbFSEditAbort(STFileSystem *fs) { + int32_t code = abort_edit(fs); + tsem_post(&fs->canEdit); + return code; +} + +int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset) { + STFileSet tfset = {.fid = fid}; + STFileSet *pset = &tfset; + STFileSet **fsetPtr = TARRAY2_SEARCH(fs->fSetArr, &pset, tsdbTFileSetCmprFn, TD_EQ); + fset[0] = (fsetPtr == NULL) ? NULL : fsetPtr[0]; + return 0; +} + +int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr) { + int32_t code = 0; + STFileSet *fset; + STFileSet *fset1; + + fsetArr[0] = taosMemoryMalloc(sizeof(TFileSetArray)); + if (fsetArr == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + TARRAY2_INIT(fsetArr[0]); + + taosThreadRwlockRdlock(&fs->tsdb->rwLock); + TARRAY2_FOREACH(fs->fSetArr, fset) { + code = tsdbTFileSetInitDup(fs->tsdb, fset, &fset1); + if (code) break; + + code = TARRAY2_APPEND(fsetArr[0], fset1); + if (code) break; + } + taosThreadRwlockUnlock(&fs->tsdb->rwLock); + + if (code) { + TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear); + taosMemoryFree(fsetArr[0]); + fsetArr[0] = NULL; + } + return code; +} + +int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr) { + if (fsetArr[0]) { + TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear); + taosMemoryFree(fsetArr[0]); + fsetArr[0] = NULL; + } + return 0; +} + +int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr) { + int32_t code = 0; + STFileSet *fset, *fset1; + + fsetArr[0] = taosMemoryCalloc(1, sizeof(*fsetArr[0])); + if (fsetArr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + taosThreadRwlockRdlock(&fs->tsdb->rwLock); + TARRAY2_FOREACH(fs->fSetArr, fset) { + code = tsdbTFileSetInitRef(fs->tsdb, fset, &fset1); + if (code) break; + + code = TARRAY2_APPEND(fsetArr[0], fset1); + if (code) break; + } + taosThreadRwlockUnlock(&fs->tsdb->rwLock); + + if (code) { + TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear); + fsetArr[0] = NULL; + } + return code; +} + +int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) { + if (fsetArr[0]) { + TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear); + taosMemoryFreeClear(fsetArr[0]); + fsetArr[0] = NULL; + } + return 0; +} + +const char *gFSBgTaskName[] = {NULL, "MERGE", "RETENTION", "COMPACT"}; + +static int32_t tsdbFSRunBgTask(void *arg) { + STFileSystem *fs = (STFileSystem *)arg; + + ASSERT(fs->bgTaskRunning != NULL); + + fs->bgTaskRunning->launchTime = taosGetTimestampMs(); + fs->bgTaskRunning->run(fs->bgTaskRunning->arg); + fs->bgTaskRunning->finishTime = taosGetTimestampMs(); + + tsdbDebug("vgId:%d bg task:%s task id:%" PRId64 " finished, schedule time:%" PRId64 " launch time:%" PRId64 + " finish time:%" PRId64, + TD_VID(fs->tsdb->pVnode), gFSBgTaskName[fs->bgTaskRunning->type], fs->bgTaskRunning->taskid, + fs->bgTaskRunning->scheduleTime, fs->bgTaskRunning->launchTime, fs->bgTaskRunning->finishTime); + + taosThreadMutexLock(fs->mutex); + + // free last + tsdbDoDoneBgTask(fs, fs->bgTaskRunning); + fs->bgTaskRunning = NULL; + + // schedule next + if (fs->bgTaskNum > 0) { + if (fs->stop) { + while (fs->bgTaskNum > 0) { + STFSBgTask *task = fs->bgTaskQueue->next; + task->prev->next = task->next; + task->next->prev = task->prev; + fs->bgTaskNum--; + tsdbDoDoneBgTask(fs, task); + } + } else { + // pop task from head + fs->bgTaskRunning = fs->bgTaskQueue->next; + fs->bgTaskRunning->prev->next = fs->bgTaskRunning->next; + fs->bgTaskRunning->next->prev = fs->bgTaskRunning->prev; + fs->bgTaskNum--; + vnodeScheduleTaskEx(1, tsdbFSRunBgTask, arg); + } + } + + taosThreadMutexUnlock(fs->mutex); + return 0; +} + +static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void *arg, + int64_t *taskid) { + if (fs->stop) { + return 0; // TODO: use a better error code + } + + // check if same task is on + // if (fs->bgTaskRunning && fs->bgTaskRunning->type == type) { + // return 0; + // } + + for (STFSBgTask *task = fs->bgTaskQueue->next; task != fs->bgTaskQueue; task = task->next) { + if (task->type == type) { + return 0; + } + } + + // do schedule task + STFSBgTask *task = taosMemoryCalloc(1, sizeof(STFSBgTask)); + if (task == NULL) return TSDB_CODE_OUT_OF_MEMORY; + taosThreadCondInit(task->done, NULL); + + task->type = type; + task->run = run; + task->arg = arg; + task->scheduleTime = taosGetTimestampMs(); + task->taskid = ++fs->taskid; + + if (fs->bgTaskRunning == NULL && fs->bgTaskNum == 0) { + // launch task directly + fs->bgTaskRunning = task; + vnodeScheduleTaskEx(1, tsdbFSRunBgTask, fs); + } else { + // add to the queue tail + fs->bgTaskNum++; + task->next = fs->bgTaskQueue; + task->prev = fs->bgTaskQueue->prev; + task->prev->next = task; + task->next->prev = task; + } + + if (taskid) *taskid = task->taskid; + return 0; +} + +int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void *arg, int64_t *taskid) { + taosThreadMutexLock(fs->mutex); + int32_t code = tsdbFSScheduleBgTaskImpl(fs, type, run, arg, taskid); + taosThreadMutexUnlock(fs->mutex); + return code; +} + +int32_t tsdbFSWaitBgTask(STFileSystem *fs, int64_t taskid) { + STFSBgTask *task = NULL; + + taosThreadMutexLock(fs->mutex); + + if (fs->bgTaskRunning && fs->bgTaskRunning->taskid == taskid) { + task = fs->bgTaskRunning; + } else { + for (STFSBgTask *taskt = fs->bgTaskQueue->next; taskt != fs->bgTaskQueue; taskt = taskt->next) { + if (taskt->taskid == taskid) { + task = taskt; + break; + } + } + } + + if (task) { + tsdbDoWaitBgTask(fs, task); + } + + taosThreadMutexUnlock(fs->mutex); + return 0; +} + +int32_t tsdbFSWaitAllBgTask(STFileSystem *fs) { + taosThreadMutexLock(fs->mutex); + + while (fs->bgTaskRunning) { + taosThreadCondWait(fs->bgTaskRunning->done, fs->mutex); + } + + taosThreadMutexUnlock(fs->mutex); + return 0; +} + +static int32_t tsdbFSDoDisableBgTask(STFileSystem *fs) { + fs->stop = true; + + if (fs->bgTaskRunning) { + tsdbDoWaitBgTask(fs, fs->bgTaskRunning); + } + return 0; +} + +int32_t tsdbFSDisableBgTask(STFileSystem *fs) { + taosThreadMutexLock(fs->mutex); + int32_t code = tsdbFSDoDisableBgTask(fs); + taosThreadMutexUnlock(fs->mutex); + return code; +} + +int32_t tsdbFSEnableBgTask(STFileSystem *fs) { + taosThreadMutexLock(fs->mutex); + fs->stop = false; + taosThreadMutexUnlock(fs->mutex); + return 0; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h new file mode 100644 index 0000000000000000000000000000000000000000..8270581e58e23468fb1f6e782fc1015c27b2e71c --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFSet2.h" + +#ifndef _TSDB_FILE_SYSTEM_H +#define _TSDB_FILE_SYSTEM_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Exposed Handle */ +typedef struct STFileSystem STFileSystem; +typedef struct STFSBgTask STFSBgTask; +// typedef TARRAY2(STFileSet *) TFileSetArray; + +typedef enum { + TSDB_FEDIT_COMMIT = 1, // + TSDB_FEDIT_MERGE +} EFEditT; + +typedef enum { + TSDB_BG_TASK_MERGER = 1, + TSDB_BG_TASK_RETENTION, + TSDB_BG_TASK_COMPACT, +} EFSBgTaskT; + +typedef enum { + TSDB_FCURRENT = 1, + TSDB_FCURRENT_C, // for commit + TSDB_FCURRENT_M, // for merge +} EFCurrentT; + +/* Exposed APIs */ +// open/close +int32_t tsdbOpenFS(STsdb *pTsdb, STFileSystem **fs, int8_t rollback); +int32_t tsdbCloseFS(STFileSystem **fs); +// snapshot +int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr); +int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr); +int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr); +int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr); +// txn +int64_t tsdbFSAllocEid(STFileSystem *fs); +int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype); +int32_t tsdbFSEditCommit(STFileSystem *fs); +int32_t tsdbFSEditAbort(STFileSystem *fs); +// background task +int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void *arg, int64_t *taskid); +int32_t tsdbFSWaitBgTask(STFileSystem *fs, int64_t taskid); +int32_t tsdbFSWaitAllBgTask(STFileSystem *fs); +int32_t tsdbFSDisableBgTask(STFileSystem *fs); +int32_t tsdbFSEnableBgTask(STFileSystem *fs); +// other +int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset); + +struct STFSBgTask { + EFSBgTaskT type; + int32_t (*run)(void *arg); + void *arg; + + TdThreadCond done[1]; + int32_t numWait; + + int64_t taskid; + int64_t scheduleTime; + int64_t launchTime; + int64_t finishTime; + + struct STFSBgTask *prev; + struct STFSBgTask *next; +}; + +/* Exposed Structs */ +struct STFileSystem { + STsdb *tsdb; + tsem_t canEdit; + int32_t state; + int64_t neid; + EFEditT etype; + TFileSetArray fSetArr[1]; + TFileSetArray fSetArrTmp[1]; + + // background task queue + TdThreadMutex mutex[1]; + bool stop; + int64_t taskid; + int32_t bgTaskNum; + STFSBgTask bgTaskQueue[1]; + STFSBgTask *bgTaskRunning; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_FILE_SYSTEM_H*/ diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c new file mode 100644 index 0000000000000000000000000000000000000000..7bc9743ecb726c9305572ba0ab0db184355bbc08 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -0,0 +1,542 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFSet2.h" + +int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl) { + if (!(lvl[0] = taosMemoryMalloc(sizeof(SSttLvl)))) return TSDB_CODE_OUT_OF_MEMORY; + lvl[0]->level = level; + TARRAY2_INIT(lvl[0]->fobjArr); + return 0; +} + +static void tsdbSttLvlClearFObj(void *data) { tsdbTFileObjUnref(*(STFileObj **)data); } + +int32_t tsdbSttLvlClear(SSttLvl **lvl) { + if (lvl[0] != NULL) { + TARRAY2_DESTROY(lvl[0]->fobjArr, tsdbSttLvlClearFObj); + taosMemoryFree(lvl[0]); + lvl[0] = NULL; + } + return 0; +} + +static int32_t tsdbSttLvlInitEx(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl **lvl) { + int32_t code = tsdbSttLvlInit(lvl1->level, lvl); + if (code) return code; + + const STFileObj *fobj1; + TARRAY2_FOREACH(lvl1->fobjArr, fobj1) { + STFileObj *fobj; + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj); + if (code) { + tsdbSttLvlClear(lvl); + return code; + } + + TARRAY2_APPEND(lvl[0]->fobjArr, fobj); + } + return 0; +} + +static int32_t tsdbSttLvlInitRef(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl **lvl) { + int32_t code = tsdbSttLvlInit(lvl1->level, lvl); + if (code) return code; + + STFileObj *fobj1; + TARRAY2_FOREACH(lvl1->fobjArr, fobj1) { + tsdbTFileObjRef(fobj1); + code = TARRAY2_APPEND(lvl[0]->fobjArr, fobj1); + if (code) return code; + } + return 0; +} + +static void tsdbSttLvlRemoveFObj(void *data) { tsdbTFileObjRemove(*(STFileObj **)data); } +static void tsdbSttLvlRemove(SSttLvl **lvl) { + TARRAY2_DESTROY(lvl[0]->fobjArr, tsdbSttLvlRemoveFObj); + taosMemoryFree(lvl[0]); + lvl[0] = NULL; +} + +static int32_t tsdbSttLvlApplyEdit(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl *lvl2) { + int32_t code = 0; + + ASSERT(lvl1->level == lvl2->level); + + int32_t i1 = 0, i2 = 0; + while (i1 < TARRAY2_SIZE(lvl1->fobjArr) || i2 < TARRAY2_SIZE(lvl2->fobjArr)) { + STFileObj *fobj1 = i1 < TARRAY2_SIZE(lvl1->fobjArr) ? TARRAY2_GET(lvl1->fobjArr, i1) : NULL; + STFileObj *fobj2 = i2 < TARRAY2_SIZE(lvl2->fobjArr) ? TARRAY2_GET(lvl2->fobjArr, i2) : NULL; + + if (fobj1 && fobj2) { + if (fobj1->f->cid < fobj2->f->cid) { + // create a file obj + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj2); + if (code) return code; + code = TARRAY2_APPEND(lvl2->fobjArr, fobj2); + if (code) return code; + i1++; + i2++; + } else if (fobj1->f->cid > fobj2->f->cid) { + // remove a file obj + TARRAY2_REMOVE(lvl2->fobjArr, i2, tsdbSttLvlRemoveFObj); + } else { + if (tsdbIsSameTFile(fobj1->f, fobj2->f)) { + if (tsdbIsTFileChanged(fobj1->f, fobj2->f)) { + fobj2->f[0] = fobj1->f[0]; + } + } else { + TARRAY2_REMOVE(lvl2->fobjArr, i2, tsdbSttLvlRemoveFObj); + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj2); + if (code) return code; + code = TARRAY2_SORT_INSERT(lvl2->fobjArr, fobj2, tsdbTFileObjCmpr); + if (code) return code; + } + i1++; + i2++; + } + } else if (fobj1) { + // create a file obj + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj2); + if (code) return code; + code = TARRAY2_APPEND(lvl2->fobjArr, fobj2); + if (code) return code; + i1++; + i2++; + } else { + // remove a file obj + TARRAY2_REMOVE(lvl2->fobjArr, i2, tsdbSttLvlRemoveFObj); + } + } + return 0; +} + +static int32_t tsdbSttLvlCmprFn(const SSttLvl **lvl1, const SSttLvl **lvl2) { + if (lvl1[0]->level < lvl2[0]->level) return -1; + if (lvl1[0]->level > lvl2[0]->level) return 1; + return 0; +} + +static int32_t tsdbSttLvlToJson(const SSttLvl *lvl, cJSON *json) { + if (cJSON_AddNumberToObject(json, "level", lvl->level) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + cJSON *ajson = cJSON_AddArrayToObject(json, "files"); + if (ajson == NULL) return TSDB_CODE_OUT_OF_MEMORY; + const STFileObj *fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + cJSON *item = cJSON_CreateObject(); + if (item == NULL) return TSDB_CODE_OUT_OF_MEMORY; + cJSON_AddItemToArray(ajson, item); + + int32_t code = tsdbTFileToJson(fobj->f, item); + if (code) return code; + } + + return 0; +} + +static int32_t tsdbJsonToSttLvl(STsdb *pTsdb, const cJSON *json, SSttLvl **lvl) { + const cJSON *item1, *item2; + int32_t level; + + item1 = cJSON_GetObjectItem(json, "level"); + if (cJSON_IsNumber(item1)) { + level = item1->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + int32_t code = tsdbSttLvlInit(level, lvl); + if (code) return code; + + item1 = cJSON_GetObjectItem(json, "files"); + if (!cJSON_IsArray(item1)) { + tsdbSttLvlClear(lvl); + return TSDB_CODE_FILE_CORRUPTED; + } + + cJSON_ArrayForEach(item2, item1) { + STFile tf; + code = tsdbJsonToTFile(item2, TSDB_FTYPE_STT, &tf); + if (code) { + tsdbSttLvlClear(lvl); + return code; + } + + STFileObj *fobj; + code = tsdbTFileObjInit(pTsdb, &tf, &fobj); + if (code) { + tsdbSttLvlClear(lvl); + return code; + } + + TARRAY2_APPEND(lvl[0]->fobjArr, fobj); + } + return 0; +} + +int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json) { + int32_t code = 0; + cJSON *item1, *item2; + + // fid + if (cJSON_AddNumberToObject(json, "fid", fset->fid) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset->farr[ftype] == NULL) continue; + + code = tsdbTFileToJson(fset->farr[ftype]->f, json); + if (code) return code; + } + + // each level + item1 = cJSON_AddArrayToObject(json, "stt lvl"); + if (item1 == NULL) return TSDB_CODE_OUT_OF_MEMORY; + const SSttLvl *lvl; + TARRAY2_FOREACH(fset->lvlArr, lvl) { + item2 = cJSON_CreateObject(); + if (!item2) return TSDB_CODE_OUT_OF_MEMORY; + cJSON_AddItemToArray(item1, item2); + + code = tsdbSttLvlToJson(lvl, item2); + if (code) return code; + } + + return 0; +} + +int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset) { + int32_t code; + const cJSON *item1, *item2; + int32_t fid; + STFile tf; + + // fid + item1 = cJSON_GetObjectItem(json, "fid"); + if (cJSON_IsNumber(item1)) { + fid = item1->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + code = tsdbTFileSetInit(fid, fset); + if (code) return code; + + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + code = tsdbJsonToTFile(json, ftype, &tf); + if (code == TSDB_CODE_NOT_FOUND) { + continue; + } else if (code) { + tsdbTFileSetClear(fset); + return code; + } else { + code = tsdbTFileObjInit(pTsdb, &tf, &(*fset)->farr[ftype]); + if (code) return code; + } + } + + // each level + item1 = cJSON_GetObjectItem(json, "stt lvl"); + if (cJSON_IsArray(item1)) { + cJSON_ArrayForEach(item2, item1) { + SSttLvl *lvl; + code = tsdbJsonToSttLvl(pTsdb, item2, &lvl); + if (code) { + tsdbTFileSetClear(fset); + return code; + } + + TARRAY2_APPEND((*fset)->lvlArr, lvl); + } + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + return 0; +} + +// NOTE: the api does not remove file, only do memory operation +int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) { + int32_t code = 0; + + if (op->optype == TSDB_FOP_CREATE) { + // create a new file + STFileObj *fobj; + code = tsdbTFileObjInit(pTsdb, &op->nf, &fobj); + if (code) return code; + + if (fobj->f->type == TSDB_FTYPE_STT) { + SSttLvl *lvl = tsdbTFileSetGetSttLvl(fset, fobj->f->stt->level); + if (!lvl) { + code = tsdbSttLvlInit(fobj->f->stt->level, &lvl); + if (code) return code; + + code = TARRAY2_SORT_INSERT(fset->lvlArr, lvl, tsdbSttLvlCmprFn); + if (code) return code; + } + + code = TARRAY2_SORT_INSERT(lvl->fobjArr, fobj, tsdbTFileObjCmpr); + if (code) return code; + } else { + ASSERT(fset->farr[fobj->f->type] == NULL); + fset->farr[fobj->f->type] = fobj; + } + } else if (op->optype == TSDB_FOP_REMOVE) { + // delete a file + if (op->of.type == TSDB_FTYPE_STT) { + SSttLvl *lvl = tsdbTFileSetGetSttLvl(fset, op->of.stt->level); + ASSERT(lvl); + + STFileObj tfobj = {.f[0] = {.cid = op->of.cid}}; + STFileObj *tfobjp = &tfobj; + int32_t idx = TARRAY2_SEARCH_IDX(lvl->fobjArr, &tfobjp, tsdbTFileObjCmpr, TD_EQ); + ASSERT(idx >= 0); + TARRAY2_REMOVE(lvl->fobjArr, idx, tsdbSttLvlClearFObj); + + if (TARRAY2_SIZE(lvl->fobjArr) == 0) { + // TODO: remove the stt level if no file exists anymore + // TARRAY2_REMOVE(&fset->lvlArr, lvl - fset->lvlArr.data, tsdbSttLvlClear); + } + } else { + ASSERT(tsdbIsSameTFile(&op->of, fset->farr[op->of.type]->f)); + tsdbTFileObjUnref(fset->farr[op->of.type]); + fset->farr[op->of.type] = NULL; + } + } else { + if (op->nf.type == TSDB_FTYPE_STT) { + SSttLvl *lvl = tsdbTFileSetGetSttLvl(fset, op->of.stt->level); + ASSERT(lvl); + + STFileObj tfobj = {.f[0] = {.cid = op->of.cid}}, *tfobjp = &tfobj; + STFileObj **fobjPtr = TARRAY2_SEARCH(lvl->fobjArr, &tfobjp, tsdbTFileObjCmpr, TD_EQ); + tfobjp = (fobjPtr ? *fobjPtr : NULL); + + ASSERT(tfobjp); + + tfobjp->f[0] = op->nf; + } else { + fset->farr[op->nf.type]->f[0] = op->nf; + } + } + + return 0; +} + +int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *fset2) { + int32_t code = 0; + + ASSERT(fset1->fid == fset2->fid); + + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (!fset1->farr[ftype] && !fset2->farr[ftype]) continue; + + STFileObj *fobj1 = fset1->farr[ftype]; + STFileObj *fobj2 = fset2->farr[ftype]; + + if (fobj1 && fobj2) { + if (tsdbIsSameTFile(fobj1->f, fobj2->f)) { + if (tsdbIsTFileChanged(fobj1->f, fobj2->f)) { + fobj2->f[0] = fobj1->f[0]; + } + } else { + tsdbTFileObjRemove(fobj2); + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fset2->farr[ftype]); + if (code) return code; + } + } else if (fobj1) { + // create a new file + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fset2->farr[ftype]); + if (code) return code; + } else { + // remove the file + tsdbTFileObjRemove(fobj2); + fset2->farr[ftype] = NULL; + } + } + + // stt part + int32_t i1 = 0, i2 = 0; + while (i1 < TARRAY2_SIZE(fset1->lvlArr) || i2 < TARRAY2_SIZE(fset2->lvlArr)) { + SSttLvl *lvl1 = i1 < TARRAY2_SIZE(fset1->lvlArr) ? TARRAY2_GET(fset1->lvlArr, i1) : NULL; + SSttLvl *lvl2 = i2 < TARRAY2_SIZE(fset2->lvlArr) ? TARRAY2_GET(fset2->lvlArr, i2) : NULL; + + if (lvl1 && lvl2) { + if (lvl1->level < lvl2->level) { + // add a new stt level + code = tsdbSttLvlInitEx(pTsdb, lvl1, &lvl2); + if (code) return code; + code = TARRAY2_SORT_INSERT(fset2->lvlArr, lvl2, tsdbSttLvlCmprFn); + if (code) return code; + i1++; + i2++; + } else if (lvl1->level > lvl2->level) { + // remove the stt level + TARRAY2_REMOVE(fset2->lvlArr, i2, tsdbSttLvlRemove); + } else { + // apply edit on stt level + code = tsdbSttLvlApplyEdit(pTsdb, lvl1, lvl2); + if (code) return code; + i1++; + i2++; + } + } else if (lvl1) { + // add a new stt level + code = tsdbSttLvlInitEx(pTsdb, lvl1, &lvl2); + if (code) return code; + code = TARRAY2_SORT_INSERT(fset2->lvlArr, lvl2, tsdbSttLvlCmprFn); + if (code) return code; + i1++; + i2++; + } else { + // remove the stt level + TARRAY2_REMOVE(fset2->lvlArr, i2, tsdbSttLvlRemove); + } + } + + return 0; +} + +int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) { + fset[0] = taosMemoryCalloc(1, sizeof(STFileSet)); + if (fset[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + fset[0]->fid = fid; + TARRAY2_INIT(fset[0]->lvlArr); + return 0; +} + +int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset) { + int32_t code = tsdbTFileSetInit(fset1->fid, fset); + if (code) return code; + + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset1->farr[ftype] == NULL) continue; + + code = tsdbTFileObjInit(pTsdb, fset1->farr[ftype]->f, &fset[0]->farr[ftype]); + if (code) { + tsdbTFileSetClear(fset); + return code; + } + } + + const SSttLvl *lvl1; + TARRAY2_FOREACH(fset1->lvlArr, lvl1) { + SSttLvl *lvl; + code = tsdbSttLvlInitEx(pTsdb, lvl1, &lvl); + if (code) { + tsdbTFileSetClear(fset); + return code; + } + + code = TARRAY2_APPEND(fset[0]->lvlArr, lvl); + if (code) return code; + } + + return 0; +} + +int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset) { + int32_t code = tsdbTFileSetInit(fset1->fid, fset); + if (code) return code; + + for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset1->farr[ftype] == NULL) continue; + + tsdbTFileObjRef(fset1->farr[ftype]); + fset[0]->farr[ftype] = fset1->farr[ftype]; + } + + const SSttLvl *lvl1; + TARRAY2_FOREACH(fset1->lvlArr, lvl1) { + SSttLvl *lvl; + code = tsdbSttLvlInitRef(pTsdb, lvl1, &lvl); + if (code) { + tsdbTFileSetClear(fset); + return code; + } + + code = TARRAY2_APPEND(fset[0]->lvlArr, lvl); + if (code) return code; + } + + return 0; +} + +int32_t tsdbTFileSetClear(STFileSet **fset) { + if (!fset[0]) return 0; + + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset[0]->farr[ftype] == NULL) continue; + tsdbTFileObjUnref(fset[0]->farr[ftype]); + } + + TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlClear); + + taosMemoryFree(fset[0]); + fset[0] = NULL; + + return 0; +} + +int32_t tsdbTFileSetRemove(STFileSet **fset) { + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset[0]->farr[ftype] == NULL) continue; + tsdbTFileObjRemove(fset[0]->farr[ftype]); + } + + TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlRemove); + taosMemoryFree(fset[0]); + fset[0] = NULL; + return 0; +} + +SSttLvl *tsdbTFileSetGetSttLvl(STFileSet *fset, int32_t level) { + SSttLvl sttLvl = {.level = level}; + SSttLvl *lvl = &sttLvl; + SSttLvl **lvlPtr = TARRAY2_SEARCH(fset->lvlArr, &lvl, tsdbSttLvlCmprFn, TD_EQ); + return lvlPtr ? lvlPtr[0] : NULL; +} + +int32_t tsdbTFileSetCmprFn(const STFileSet **fset1, const STFileSet **fset2) { + if (fset1[0]->fid < fset2[0]->fid) return -1; + if (fset1[0]->fid > fset2[0]->fid) return 1; + return 0; +} + +int64_t tsdbTFileSetMaxCid(const STFileSet *fset) { + int64_t maxCid = 0; + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset->farr[ftype] == NULL) continue; + maxCid = TMAX(maxCid, fset->farr[ftype]->f->cid); + } + const SSttLvl *lvl; + const STFileObj *fobj; + TARRAY2_FOREACH(fset->lvlArr, lvl) { + TARRAY2_FOREACH(lvl->fobjArr, fobj) { maxCid = TMAX(maxCid, fobj->f->cid); } + } + return maxCid; +} + +bool tsdbTFileSetIsEmpty(const STFileSet *fset) { + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (fset->farr[ftype] != NULL) return false; + } + return TARRAY2_SIZE(fset->lvlArr) == 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h new file mode 100644 index 0000000000000000000000000000000000000000..d7b3c1fc8cb3478b8ad6fe85d94ec7a992dad0c8 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFile2.h" + +#ifndef _TSDB_FILE_SET2_H +#define _TSDB_FILE_SET2_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct STFileSet STFileSet; +typedef struct STFileOp STFileOp; +typedef struct SSttLvl SSttLvl; +typedef TARRAY2(STFileObj *) TFileObjArray; +typedef TARRAY2(SSttLvl *) TSttLvlArray; +typedef TARRAY2(STFileOp) TFileOpArray; + +typedef enum { + TSDB_FOP_NONE = 0, + TSDB_FOP_CREATE, + TSDB_FOP_REMOVE, + TSDB_FOP_MODIFY, +} tsdb_fop_t; + +#define TFILE_SET(fid_) \ + (STFileSet) { .fid = (fid_) } + +// init/clear +int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset); +int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset); +int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset); +int32_t tsdbTFileSetClear(STFileSet **fset); +int32_t tsdbTFileSetRemove(STFileSet **fset); +// to/from json +int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json); +int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset); +// cmpr +int32_t tsdbTFileSetCmprFn(const STFileSet **fset1, const STFileSet **fset2); +// edit +int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op); +int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *fset); +// max commit id +int64_t tsdbTFileSetMaxCid(const STFileSet *fset); +// get +SSttLvl *tsdbTFileSetGetSttLvl(STFileSet *fset, int32_t level); +// is empty +bool tsdbTFileSetIsEmpty(const STFileSet *fset); + +struct STFileOp { + tsdb_fop_t optype; + int32_t fid; + STFile of; // old file state + STFile nf; // new file state +}; + +struct SSttLvl { + int32_t level; + TFileObjArray fobjArr[1]; +}; + +struct STFileSet { + int32_t fid; + STFileObj *farr[TSDB_FTYPE_MAX]; // file array + TSttLvlArray lvlArr[1]; // level array +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_FILE_SET2_H*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c new file mode 100644 index 0000000000000000000000000000000000000000..83ae8c24291542f179a53fe9d7e2215c8f4ca8ca --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFSetRW.h" + +// SFSetWriter ================================================== +struct SFSetWriter { + SFSetWriterConfig config[1]; + + SSkmInfo skmTb[1]; + SSkmInfo skmRow[1]; + uint8_t *bufArr[10]; + + struct { + TABLEID tbid[1]; + } ctx[1]; + + // writer + SBlockData blockData[2]; + int32_t blockDataIdx; + SDataFileWriter *dataWriter; + SSttFileWriter *sttWriter; +}; + +static int32_t tsdbFSetWriteTableDataBegin(SFSetWriter *writer, const TABLEID *tbid) { + int32_t code = 0; + int32_t lino = 0; + + writer->ctx->tbid->suid = tbid->suid; + writer->ctx->tbid->uid = tbid->uid; + + code = tsdbUpdateSkmTb(writer->config->tsdb, writer->ctx->tbid, writer->skmTb); + TSDB_CHECK_CODE(code, lino, _exit); + + writer->blockDataIdx = 0; + for (int32_t i = 0; i < ARRAY_SIZE(writer->blockData); i++) { + code = tBlockDataInit(&writer->blockData[i], writer->ctx->tbid, writer->skmTb->pTSchema, NULL, 0); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbFSetWriteTableDataEnd(SFSetWriter *writer) { + if (writer->ctx->tbid->uid == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + int32_t cidx = writer->blockDataIdx; + int32_t pidx = ((cidx + 1) & 1); + int32_t numRow = ((writer->blockData[pidx].nRow + writer->blockData[cidx].nRow) >> 1); + + if (writer->blockData[pidx].nRow > 0 && numRow >= writer->config->minRow) { + ASSERT(writer->blockData[pidx].nRow == writer->config->maxRow); + + SRowInfo row = { + .suid = writer->ctx->tbid->suid, + .uid = writer->ctx->tbid->uid, + .row = tsdbRowFromBlockData(writer->blockData + pidx, 0), + }; + + for (int32_t i = 0; i < numRow; i++) { + row.row.iRow = i; + + code = tsdbDataFileWriteRow(writer->dataWriter, &row); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDataFileFlush(writer->dataWriter); + TSDB_CHECK_CODE(code, lino, _exit); + + for (int32_t i = numRow; i < writer->blockData[pidx].nRow; i++) { + row.row.iRow = i; + code = tsdbDataFileWriteRow(writer->dataWriter, &row); + TSDB_CHECK_CODE(code, lino, _exit); + } + + row.row = tsdbRowFromBlockData(writer->blockData + cidx, 0); + for (int32_t i = 0; i < writer->blockData[cidx].nRow; i++) { + row.row.iRow = i; + code = tsdbDataFileWriteRow(writer->dataWriter, &row); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + // pidx + if (writer->blockData[pidx].nRow > 0) { + code = tsdbDataFileWriteBlockData(writer->dataWriter, &writer->blockData[pidx]); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // cidx + if (writer->blockData[cidx].nRow < writer->config->minRow) { + code = tsdbSttFileWriteBlockData(writer->sttWriter, &writer->blockData[cidx]); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDataFileWriteBlockData(writer->dataWriter, &writer->blockData[cidx]); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + for (int32_t i = 0; i < ARRAY_SIZE(writer->blockData); i++) { + tBlockDataReset(&writer->blockData[i]); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) { + int32_t code = 0; + int32_t lino = 0; + + writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); + if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->config[0] = config[0]; + + // data writer + if (!config->toSttOnly) { + SDataFileWriterConfig dataWriterConfig = { + .tsdb = config->tsdb, + .cmprAlg = config->cmprAlg, + .maxRow = config->maxRow, + .szPage = config->szPage, + .fid = config->fid, + .cid = config->cid, + .did = config->did, + .compactVersion = config->compactVersion, + .skmTb = writer[0]->skmTb, + .skmRow = writer[0]->skmRow, + .bufArr = writer[0]->bufArr, + }; + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) { + dataWriterConfig.files[ftype].exist = config->files[ftype].exist; + dataWriterConfig.files[ftype].file = config->files[ftype].file; + } + + code = tsdbDataFileWriterOpen(&dataWriterConfig, &writer[0]->dataWriter); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // stt writer + SSttFileWriterConfig sttWriterConfig = { + .tsdb = config->tsdb, + .maxRow = config->maxRow, + .szPage = config->szPage, + .cmprAlg = config->cmprAlg, + .compactVersion = config->compactVersion, + .did = config->did, + .fid = config->fid, + .cid = config->cid, + .level = config->level, + .skmTb = writer[0]->skmTb, + .skmRow = writer[0]->skmRow, + .bufArr = writer[0]->bufArr, + }; + code = tsdbSttFileWriterOpen(&sttWriterConfig, &writer[0]->sttWriter); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetWriterClose(SFSetWriter **writer, bool abort, TFileOpArray *fopArr) { + if (writer[0] == NULL) return 0; + + int32_t code = 0; + int32_t lino = 0; + + STsdb *tsdb = writer[0]->config->tsdb; + + // end + if (!writer[0]->config->toSttOnly) { + code = tsdbFSetWriteTableDataEnd(writer[0]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFileWriterClose(&writer[0]->dataWriter, abort, fopArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbSttFileWriterClose(&writer[0]->sttWriter, abort, fopArr); + TSDB_CHECK_CODE(code, lino, _exit); + + // free + for (int32_t i = 0; i < ARRAY_SIZE(writer[0]->blockData); i++) { + tBlockDataDestroy(&writer[0]->blockData[i]); + } + for (int32_t i = 0; i < ARRAY_SIZE(writer[0]->bufArr); i++) { + tFree(writer[0]->bufArr[i]); + } + tDestroyTSchema(writer[0]->skmRow->pTSchema); + tDestroyTSchema(writer[0]->skmTb->pTSchema); + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetWriteRow(SFSetWriter *writer, SRowInfo *row) { + int32_t code = 0; + int32_t lino = 0; + + if (writer->config->toSttOnly) { + code = tsdbSttFileWriteRow(writer->sttWriter, row); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + if (writer->ctx->tbid->uid != row->uid) { + code = tsdbFSetWriteTableDataEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSetWriteTableDataBegin(writer, (TABLEID *)row); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (row->row.type == TSDBROW_ROW_FMT) { + code = tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, TSDBROW_SVERSION(&row->row), writer->skmRow); + TSDB_CHECK_CODE(code, lino, _exit); + } + + TSDBKEY key = TSDBROW_KEY(&row->row); + if (key.version <= writer->config->compactVersion // + && writer->blockData[writer->blockDataIdx].nRow > 0 // + && key.ts == writer->blockData[writer->blockDataIdx].aTSKEY[writer->blockData[writer->blockDataIdx].nRow - 1]) { + code = tBlockDataUpdateRow(&writer->blockData[writer->blockDataIdx], &row->row, writer->skmRow->pTSchema); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + if (writer->blockData[writer->blockDataIdx].nRow >= writer->config->maxRow) { + int32_t idx = ((writer->blockDataIdx + 1) & 1); + if (writer->blockData[idx].nRow >= writer->config->maxRow) { + code = tsdbDataFileWriteBlockData(writer->dataWriter, &writer->blockData[idx]); + TSDB_CHECK_CODE(code, lino, _exit); + + tBlockDataClear(&writer->blockData[idx]); + } + writer->blockDataIdx = idx; + } + + code = + tBlockDataAppendRow(&writer->blockData[writer->blockDataIdx], &row->row, writer->skmRow->pTSchema, row->uid); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFSetWriteTombRecord(SFSetWriter *writer, const STombRecord *tombRecord) { + int32_t code = 0; + int32_t lino = 0; + + if (writer->config->toSttOnly || tsdbSttFileWriterIsOpened(writer->sttWriter)) { + code = tsdbSttFileWriteTombRecord(writer->sttWriter, tombRecord); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDataFileWriteTombRecord(writer->dataWriter, tombRecord); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.h b/source/dnode/vnode/src/tsdb/tsdbFSetRW.h new file mode 100644 index 0000000000000000000000000000000000000000..b5710407cfe40e28736c2949d3f3421e131c6624 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRW.h" +#include "tsdbSttFileRW.h" + +#ifndef _TSDB_FSET_RW_H +#define _TSDB_FSET_RW_H + +#ifdef __cplusplus +extern "C" { +#endif + +// +typedef struct SFSetWriter SFSetWriter; +typedef struct { + STsdb *tsdb; + bool toSttOnly; + int64_t compactVersion; + int32_t minRow; + int32_t maxRow; + int32_t szPage; + int8_t cmprAlg; + int32_t fid; + int64_t cid; + SDiskID did; + int32_t level; + struct { + bool exist; + STFile file; + } files[TSDB_FTYPE_MAX]; +} SFSetWriterConfig; + +int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer); +int32_t tsdbFSetWriterClose(SFSetWriter **writer, bool abort, TFileOpArray *fopArr); +int32_t tsdbFSetWriteRow(SFSetWriter *writer, SRowInfo *row); +int32_t tsdbFSetWriteTombRecord(SFSetWriter *writer, const STombRecord *tombRecord); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_FSET_RW_H*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index d91475376b240812b8942227fa85e37cb5671dec..62b37cd0a66a9a9d2f1515a17d1ce8f6f11cbdde 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -14,6 +14,7 @@ */ #include "tsdb.h" +#include "vnd.h" int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) { int32_t n = 0; @@ -111,7 +112,10 @@ static char* getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t p += titoa(TD_VID(pTsdb->pVnode), 10, p); *(p++) = 'f'; - p += titoa(fid, 10, p); + if (fid < 0) { + *(p++) = '-'; + } + p += titoa((fid < 0) ? -fid : fid, 10, p); memcpy(p, "ver", 3); p += 3; @@ -282,8 +286,13 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { // SDelFile =============================================== void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]) { - snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%dver%" PRId64 "%s", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), - TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), pFile->commitID, ".del"); + int32_t offset = 0; + SVnode *pVnode = pTsdb->pVnode; + + vnodeGetPrimaryDir(pTsdb->path, pVnode->diskPrimary, pVnode->pTfs, fname, TSDB_FILENAME_LEN); + offset = strlen(fname); + snprintf((char *)fname + offset, TSDB_FILENAME_LEN - offset - 1, "%sv%dver%" PRId64 ".del", TD_DIRSEP, + TD_VID(pTsdb->pVnode), pFile->commitID); } int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c new file mode 100644 index 0000000000000000000000000000000000000000..be021169cd890676e8a90bf0588d8617b97343f2 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFile2.h" + +// to_json +static int32_t head_to_json(const STFile *file, cJSON *json); +static int32_t data_to_json(const STFile *file, cJSON *json); +static int32_t sma_to_json(const STFile *file, cJSON *json); +static int32_t tomb_to_json(const STFile *file, cJSON *json); +static int32_t stt_to_json(const STFile *file, cJSON *json); + +// from_json +static int32_t head_from_json(const cJSON *json, STFile *file); +static int32_t data_from_json(const cJSON *json, STFile *file); +static int32_t sma_from_json(const cJSON *json, STFile *file); +static int32_t tomb_from_json(const cJSON *json, STFile *file); +static int32_t stt_from_json(const cJSON *json, STFile *file); + +static const struct { + const char *suffix; + int32_t (*to_json)(const STFile *file, cJSON *json); + int32_t (*from_json)(const cJSON *json, STFile *file); +} g_tfile_info[] = { + [TSDB_FTYPE_HEAD] = {"head", head_to_json, head_from_json}, + [TSDB_FTYPE_DATA] = {"data", data_to_json, data_from_json}, + [TSDB_FTYPE_SMA] = {"sma", sma_to_json, sma_from_json}, + [TSDB_FTYPE_TOMB] = {"tomb", tomb_to_json, tomb_from_json}, + [TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json}, +}; + +static void remove_file(const char *fname) { + taosRemoveFile(fname); + tsdbInfo("file:%s is removed", fname); +} + +static int32_t tfile_to_json(const STFile *file, cJSON *json) { + /* did.level */ + if (cJSON_AddNumberToObject(json, "did.level", file->did.level) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + /* did.id */ + if (cJSON_AddNumberToObject(json, "did.id", file->did.id) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + /* fid */ + if (cJSON_AddNumberToObject(json, "fid", file->fid) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + /* cid */ + if (cJSON_AddNumberToObject(json, "cid", file->cid) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + /* size */ + if (cJSON_AddNumberToObject(json, "size", file->size) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return 0; +} + +static int32_t tfile_from_json(const cJSON *json, STFile *file) { + const cJSON *item; + + /* did.level */ + item = cJSON_GetObjectItem(json, "did.level"); + if (cJSON_IsNumber(item)) { + file->did.level = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + /* did.id */ + item = cJSON_GetObjectItem(json, "did.id"); + if (cJSON_IsNumber(item)) { + file->did.id = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + /* fid */ + item = cJSON_GetObjectItem(json, "fid"); + if (cJSON_IsNumber(item)) { + file->fid = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + /* cid */ + item = cJSON_GetObjectItem(json, "cid"); + if (cJSON_IsNumber(item)) { + file->cid = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + /* size */ + item = cJSON_GetObjectItem(json, "size"); + if (cJSON_IsNumber(item)) { + file->size = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + return 0; +} + +static int32_t head_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } +static int32_t data_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } +static int32_t sma_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } +static int32_t tomb_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } +static int32_t stt_to_json(const STFile *file, cJSON *json) { + int32_t code = tfile_to_json(file, json); + if (code) return code; + + /* lvl */ + if (cJSON_AddNumberToObject(json, "level", file->stt->level) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return 0; +} + +static int32_t head_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } +static int32_t data_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } +static int32_t sma_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } +static int32_t tomb_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } +static int32_t stt_from_json(const cJSON *json, STFile *file) { + int32_t code = tfile_from_json(json, file); + if (code) return code; + + const cJSON *item; + + /* lvl */ + item = cJSON_GetObjectItem(json, "level"); + if (cJSON_IsNumber(item)) { + file->stt->level = item->valuedouble; + } else { + return TSDB_CODE_FILE_CORRUPTED; + } + + return 0; +} + +int32_t tsdbTFileToJson(const STFile *file, cJSON *json) { + if (file->type == TSDB_FTYPE_STT) { + return g_tfile_info[file->type].to_json(file, json); + } else { + cJSON *item = cJSON_AddObjectToObject(json, g_tfile_info[file->type].suffix); + if (item == NULL) return TSDB_CODE_OUT_OF_MEMORY; + return g_tfile_info[file->type].to_json(file, item); + } +} + +int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f) { + f[0] = (STFile){.type = ftype}; + + if (ftype == TSDB_FTYPE_STT) { + int32_t code = g_tfile_info[ftype].from_json(json, f); + if (code) return code; + } else { + const cJSON *item = cJSON_GetObjectItem(json, g_tfile_info[ftype].suffix); + if (cJSON_IsObject(item)) { + int32_t code = g_tfile_info[ftype].from_json(item, f); + if (code) return code; + } else { + return TSDB_CODE_NOT_FOUND; + } + } + + return 0; +} + +int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) { + fobj[0] = taosMemoryMalloc(sizeof(*fobj[0])); + if (!fobj[0]) return TSDB_CODE_OUT_OF_MEMORY; + + taosThreadMutexInit(&fobj[0]->mutex, NULL); + fobj[0]->f[0] = f[0]; + fobj[0]->state = TSDB_FSTATE_LIVE; + fobj[0]->ref = 1; + tsdbTFileName(pTsdb, f, fobj[0]->fname); + return 0; +} + +int32_t tsdbTFileObjRef(STFileObj *fobj) { + int32_t nRef; + taosThreadMutexLock(&fobj->mutex); + ASSERT(fobj->ref > 0 && fobj->state == TSDB_FSTATE_LIVE); + nRef = ++fobj->ref; + taosThreadMutexUnlock(&fobj->mutex); + tsdbTrace("ref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); + return 0; +} + +int32_t tsdbTFileObjUnref(STFileObj *fobj) { + taosThreadMutexLock(&fobj->mutex); + int32_t nRef = --fobj->ref; + taosThreadMutexUnlock(&fobj->mutex); + ASSERT(nRef >= 0); + tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); + if (nRef == 0) { + if (fobj->state == TSDB_FSTATE_DEAD) { + remove_file(fobj->fname); + } + taosMemoryFree(fobj); + } + + return 0; +} + +int32_t tsdbTFileObjRemove(STFileObj *fobj) { + taosThreadMutexLock(&fobj->mutex); + ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0); + fobj->state = TSDB_FSTATE_DEAD; + int32_t nRef = --fobj->ref; + taosThreadMutexUnlock(&fobj->mutex); + tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); + if (nRef == 0) { + remove_file(fobj->fname); + taosMemoryFree(fobj); + } + return 0; +} + +int32_t tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]) { + SVnode *pVnode = pTsdb->pVnode; + STfs *pTfs = pVnode->pTfs; + + if (pTfs) { + snprintf(fname, // + TSDB_FILENAME_LEN, // + "%s%s%s%sv%df%dver%" PRId64 ".%s", // + tfsGetDiskPath(pTfs, f->did), // + TD_DIRSEP, // + pTsdb->path, // + TD_DIRSEP, // + TD_VID(pVnode), // + f->fid, // + f->cid, // + g_tfile_info[f->type].suffix); + } else { + snprintf(fname, // + TSDB_FILENAME_LEN, // + "%s%sv%df%dver%" PRId64 ".%s", // + pTsdb->path, // + TD_DIRSEP, // + TD_VID(pVnode), // + f->fid, // + f->cid, // + g_tfile_info[f->type].suffix); + } + return 0; +} + +bool tsdbIsSameTFile(const STFile *f1, const STFile *f2) { + if (f1->type != f2->type) return false; + if (f1->did.level != f2->did.level) return false; + if (f1->did.id != f2->did.id) return false; + if (f1->fid != f2->fid) return false; + if (f1->cid != f2->cid) return false; + return true; +} + +bool tsdbIsTFileChanged(const STFile *f1, const STFile *f2) { + if (f1->size != f2->size) return true; + // if (f1->type == TSDB_FTYPE_STT && f1->stt->nseg != f2->stt->nseg) return true; + return false; +} + +int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2) { + if (fobj1[0]->f->cid < fobj2[0]->f->cid) { + return -1; + } else if (fobj1[0]->f->cid > fobj2[0]->f->cid) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.h b/source/dnode/vnode/src/tsdb/tsdbFile2.h new file mode 100644 index 0000000000000000000000000000000000000000..11d08e45e667ddea6f8150239b472c86426f841a --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDef.h" + +#ifndef _TSDB_FILE_H +#define _TSDB_FILE_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct STFile STFile; +typedef struct STFileObj STFileObj; + +typedef enum { + TSDB_FTYPE_HEAD = 0, // .head + TSDB_FTYPE_DATA, // .data + TSDB_FTYPE_SMA, // .sma + TSDB_FTYPE_TOMB, // .tomb + TSDB_FTYPE_STT = TSDB_FTYPE_TOMB + 2, // .stt +} tsdb_ftype_t; + +enum { + TSDB_FSTATE_LIVE = 1, + TSDB_FSTATE_DEAD, +}; + +#define TSDB_FTYPE_MIN TSDB_FTYPE_HEAD +#define TSDB_FTYPE_MAX (TSDB_FTYPE_TOMB + 1) + +// STFile +int32_t tsdbTFileToJson(const STFile *f, cJSON *json); +int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f); +int32_t tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]); +bool tsdbIsSameTFile(const STFile *f1, const STFile *f2); +bool tsdbIsTFileChanged(const STFile *f1, const STFile *f2); + +// STFileObj +int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj); +int32_t tsdbTFileObjRef(STFileObj *fobj); +int32_t tsdbTFileObjUnref(STFileObj *fobj); +int32_t tsdbTFileObjRemove(STFileObj *fobj); +int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2); + +struct STFile { + tsdb_ftype_t type; + SDiskID did; // disk id + int32_t fid; // file id + int64_t cid; // commit id + int64_t size; + union { + struct { + int32_t level; + } stt[1]; + }; +}; + +struct STFileObj { + TdThreadMutex mutex; + STFile f[1]; + int32_t state; + int32_t ref; + char fname[TSDB_FILENAME_LEN]; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_FILE_H*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbIter.c b/source/dnode/vnode/src/tsdb/tsdbIter.c new file mode 100644 index 0000000000000000000000000000000000000000..9780cc6be63661869576bdfe7f1af544c59f1d59 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbIter.c @@ -0,0 +1,780 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbIter.h" + +// STsdbIter ================ +struct STsdbIter { + EIterType type; + bool noMoreData; + bool filterByVersion; + int64_t range[2]; + union { + SRowInfo row[1]; + STombRecord record[1]; + }; + SRBTreeNode node[1]; + union { + struct { + SSttFileReader *reader; + const TSttBlkArray *sttBlkArray; + int32_t sttBlkArrayIdx; + SBlockData blockData[1]; + int32_t blockDataIdx; + } sttData[1]; + struct { + SDataFileReader *reader; + const TBrinBlkArray *brinBlkArray; + int32_t brinBlkArrayIdx; + SBrinBlock brinBlock[1]; + int32_t brinBlockIdx; + SBlockData blockData[1]; + int32_t blockDataIdx; + } dataData[1]; + struct { + SMemTable *memt; + TSDBKEY from[1]; + SRBTreeIter iter[1]; + STbData *tbData; + STbDataIter tbIter[1]; + } memtData[1]; + struct { + SSttFileReader *reader; + const TTombBlkArray *tombBlkArray; + int32_t tombBlkArrayIdx; + STombBlock tombBlock[1]; + int32_t tombBlockIdx; + } sttTomb[1]; + struct { + SDataFileReader *reader; + const TTombBlkArray *tombBlkArray; + int32_t tombBlkArrayIdx; + STombBlock tombBlock[1]; + int32_t tombBlockIdx; + } dataTomb[1]; + struct { + SMemTable *memt; + SRBTreeIter rbtIter[1]; + STbData *tbData; + SDelData *delData; + } memtTomb[1]; + }; +}; + +static int32_t tsdbSttIterNext(STsdbIter *iter, const TABLEID *tbid) { + while (!iter->noMoreData) { + for (; iter->sttData->blockDataIdx < iter->sttData->blockData->nRow; iter->sttData->blockDataIdx++) { + int64_t version = iter->sttData->blockData->aVersion[iter->sttData->blockDataIdx]; + + if (iter->filterByVersion && (version < iter->range[0] || version > iter->range[1])) { + continue; + } + + iter->row->suid = iter->sttData->blockData->suid; + iter->row->uid = iter->sttData->blockData->uid ? iter->sttData->blockData->uid + : iter->sttData->blockData->aUid[iter->sttData->blockDataIdx]; + + if (tbid && iter->row->suid == tbid->suid && iter->row->uid == tbid->uid) { + continue; + } + + iter->row->row = tsdbRowFromBlockData(iter->sttData->blockData, iter->sttData->blockDataIdx); + iter->sttData->blockDataIdx++; + goto _exit; + } + + if (iter->sttData->sttBlkArrayIdx >= TARRAY2_SIZE(iter->sttData->sttBlkArray)) { + iter->noMoreData = true; + break; + } + + for (; iter->sttData->sttBlkArrayIdx < TARRAY2_SIZE(iter->sttData->sttBlkArray); iter->sttData->sttBlkArrayIdx++) { + const SSttBlk *sttBlk = TARRAY2_GET_PTR(iter->sttData->sttBlkArray, iter->sttData->sttBlkArrayIdx); + + if (iter->filterByVersion && (sttBlk->maxVer < iter->range[0] || sttBlk->minVer > iter->range[1])) { + continue; + } + + if (tbid && tbid->suid == sttBlk->suid && tbid->uid == sttBlk->minUid && tbid->uid == sttBlk->maxUid) { + continue; + } + + int32_t code = tsdbSttFileReadBlockData(iter->sttData->reader, sttBlk, iter->sttData->blockData); + if (code) return code; + + iter->sttData->blockDataIdx = 0; + iter->sttData->sttBlkArrayIdx++; + break; + } + } + +_exit: + return 0; +} + +static int32_t tsdbDataIterNext(STsdbIter *iter, const TABLEID *tbid) { + int32_t code; + + while (!iter->noMoreData) { + for (;;) { + // SBlockData + for (; iter->dataData->blockDataIdx < iter->dataData->blockData->nRow; iter->dataData->blockDataIdx++) { + int64_t version = iter->dataData->blockData->aVersion[iter->dataData->blockDataIdx]; + if (iter->filterByVersion && (version < iter->range[0] || version > iter->range[1])) { + continue; + } + + if (tbid && tbid->suid == iter->dataData->blockData->suid && tbid->uid == iter->dataData->blockData->uid) { + iter->dataData->blockDataIdx = iter->dataData->blockData->nRow; + break; + } + + iter->row->row = tsdbRowFromBlockData(iter->dataData->blockData, iter->dataData->blockDataIdx); + iter->dataData->blockDataIdx++; + goto _exit; + } + + // SBrinBlock + if (iter->dataData->brinBlockIdx >= BRIN_BLOCK_SIZE(iter->dataData->brinBlock)) { + break; + } + + for (; iter->dataData->brinBlockIdx < BRIN_BLOCK_SIZE(iter->dataData->brinBlock); + iter->dataData->brinBlockIdx++) { + SBrinRecord record[1]; + tBrinBlockGet(iter->dataData->brinBlock, iter->dataData->brinBlockIdx, record); + + if (iter->filterByVersion && (record->maxVer < iter->range[0] || record->minVer > iter->range[1])) { + continue; + } + + if (tbid && tbid->suid == record->suid && tbid->uid == record->uid) { + continue; + } + + iter->row->suid = record->suid; + iter->row->uid = record->uid; + + code = tsdbDataFileReadBlockData(iter->dataData->reader, record, iter->dataData->blockData); + if (code) return code; + + iter->dataData->blockDataIdx = 0; + iter->dataData->brinBlockIdx++; + break; + } + } + + if (iter->dataData->brinBlkArrayIdx >= TARRAY2_SIZE(iter->dataData->brinBlkArray)) { + iter->noMoreData = true; + break; + } + + for (; iter->dataData->brinBlkArrayIdx < TARRAY2_SIZE(iter->dataData->brinBlkArray); + iter->dataData->brinBlkArrayIdx++) { + const SBrinBlk *brinBlk = TARRAY2_GET_PTR(iter->dataData->brinBlkArray, iter->dataData->brinBlkArrayIdx); + + if (iter->filterByVersion && (brinBlk->maxVer < iter->range[0] || brinBlk->minVer > iter->range[1])) { + continue; + } + + if (tbid && tbid->uid == brinBlk->minTbid.uid && tbid->uid == brinBlk->maxTbid.uid) { + continue; + } + + code = tsdbDataFileReadBrinBlock(iter->dataData->reader, brinBlk, iter->dataData->brinBlock); + if (code) return code; + + iter->dataData->brinBlockIdx = 0; + iter->dataData->brinBlkArrayIdx++; + break; + } + } + +_exit: + return 0; +} + +static int32_t tsdbMemTableIterNext(STsdbIter *iter, const TABLEID *tbid) { + SRBTreeNode *node; + + while (!iter->noMoreData) { + for (TSDBROW *row; iter->memtData->tbData && (row = tsdbTbDataIterGet(iter->memtData->tbIter));) { + if (tbid && tbid->suid == iter->memtData->tbData->suid && tbid->uid == iter->memtData->tbData->uid) { + iter->memtData->tbData = NULL; + break; + } + + if (iter->filterByVersion) { + int64_t version = TSDBROW_VERSION(row); + if (version < iter->range[0] || version > iter->range[1]) { + continue; + } + } + + iter->row->row = row[0]; + + tsdbTbDataIterNext(iter->memtData->tbIter); + goto _exit; + } + + for (;;) { + node = tRBTreeIterNext(iter->memtData->iter); + if (!node) { + iter->noMoreData = true; + goto _exit; + } + + iter->memtData->tbData = TCONTAINER_OF(node, STbData, rbtn); + if (tbid && tbid->suid == iter->memtData->tbData->suid && tbid->uid == iter->memtData->tbData->uid) { + continue; + } else { + iter->row->suid = iter->memtData->tbData->suid; + iter->row->uid = iter->memtData->tbData->uid; + tsdbTbDataIterOpen(iter->memtData->tbData, iter->memtData->from, 0, iter->memtData->tbIter); + break; + } + } + } + +_exit: + return 0; +} + +static int32_t tsdbDataTombIterNext(STsdbIter *iter, const TABLEID *tbid) { + while (!iter->noMoreData) { + for (; iter->dataTomb->tombBlockIdx < TOMB_BLOCK_SIZE(iter->dataTomb->tombBlock); iter->dataTomb->tombBlockIdx++) { + iter->record->suid = TARRAY2_GET(iter->dataTomb->tombBlock->suid, iter->dataTomb->tombBlockIdx); + iter->record->uid = TARRAY2_GET(iter->dataTomb->tombBlock->uid, iter->dataTomb->tombBlockIdx); + iter->record->version = TARRAY2_GET(iter->dataTomb->tombBlock->version, iter->dataTomb->tombBlockIdx); + + if (iter->filterByVersion && (iter->record->version < iter->range[0] || iter->record->version > iter->range[1])) { + continue; + } + + if (tbid && iter->record->suid == tbid->suid && iter->record->uid == tbid->uid) { + continue; + } + + iter->record->skey = TARRAY2_GET(iter->dataTomb->tombBlock->skey, iter->dataTomb->tombBlockIdx); + iter->record->ekey = TARRAY2_GET(iter->dataTomb->tombBlock->ekey, iter->dataTomb->tombBlockIdx); + iter->dataTomb->tombBlockIdx++; + goto _exit; + } + + if (iter->dataTomb->tombBlkArrayIdx >= TARRAY2_SIZE(iter->dataTomb->tombBlkArray)) { + iter->noMoreData = true; + goto _exit; + } + + for (; iter->dataTomb->tombBlkArrayIdx < TARRAY2_SIZE(iter->dataTomb->tombBlkArray); + iter->dataTomb->tombBlkArrayIdx++) { + const STombBlk *tombBlk = TARRAY2_GET_PTR(iter->dataTomb->tombBlkArray, iter->dataTomb->tombBlkArrayIdx); + + if (tbid && tbid->suid == tombBlk->minTbid.suid && tbid->uid == tombBlk->minTbid.uid && + tbid->suid == tombBlk->maxTbid.suid && tbid->uid == tombBlk->maxTbid.uid) { + continue; + } + + int32_t code = tsdbDataFileReadTombBlock(iter->dataTomb->reader, tombBlk, iter->dataTomb->tombBlock); + if (code) return code; + + iter->dataTomb->tombBlockIdx = 0; + iter->dataTomb->tombBlkArrayIdx++; + break; + } + } + +_exit: + return 0; +} + +static int32_t tsdbMemTombIterNext(STsdbIter *iter, const TABLEID *tbid) { + while (!iter->noMoreData) { + for (; iter->memtTomb->delData;) { + if (tbid && tbid->uid == iter->memtTomb->tbData->uid) { + iter->memtTomb->delData = NULL; + break; + } + + if (iter->filterByVersion && + (iter->memtTomb->delData->version < iter->range[0] || iter->memtTomb->delData->version > iter->range[1])) { + continue; + } + + iter->record->suid = iter->memtTomb->tbData->suid; + iter->record->uid = iter->memtTomb->tbData->uid; + iter->record->version = iter->memtTomb->delData->version; + iter->record->skey = iter->memtTomb->delData->sKey; + iter->record->ekey = iter->memtTomb->delData->eKey; + + iter->memtTomb->delData = iter->memtTomb->delData->pNext; + goto _exit; + } + + for (;;) { + SRBTreeNode *node = tRBTreeIterNext(iter->memtTomb->rbtIter); + if (node == NULL) { + iter->noMoreData = true; + goto _exit; + } + + iter->memtTomb->tbData = TCONTAINER_OF(node, STbData, rbtn); + if (tbid && tbid->uid == iter->memtTomb->tbData->uid) { + continue; + } else { + iter->memtTomb->delData = iter->memtTomb->tbData->pHead; + break; + } + } + } + +_exit: + return 0; +} + +static int32_t tsdbSttIterOpen(STsdbIter *iter) { + int32_t code; + + code = tsdbSttFileReadSttBlk(iter->sttData->reader, &iter->sttData->sttBlkArray); + if (code) return code; + + if (TARRAY2_SIZE(iter->sttData->sttBlkArray) == 0) { + iter->noMoreData = true; + return 0; + } + + iter->sttData->sttBlkArrayIdx = 0; + tBlockDataCreate(iter->sttData->blockData); + iter->sttData->blockDataIdx = 0; + + return tsdbSttIterNext(iter, NULL); +} + +static int32_t tsdbDataIterOpen(STsdbIter *iter) { + int32_t code; + + // SBrinBlk + code = tsdbDataFileReadBrinBlk(iter->dataData->reader, &iter->dataData->brinBlkArray); + if (code) return code; + + if (TARRAY2_SIZE(iter->dataData->brinBlkArray) == 0) { + iter->noMoreData = true; + return 0; + } + + iter->dataData->brinBlkArrayIdx = 0; + + // SBrinBlock + tBrinBlockInit(iter->dataData->brinBlock); + iter->dataData->brinBlockIdx = 0; + + // SBlockData + tBlockDataCreate(iter->dataData->blockData); + iter->dataData->blockDataIdx = 0; + + return tsdbDataIterNext(iter, NULL); +} + +static int32_t tsdbMemTableIterOpen(STsdbIter *iter) { + if (iter->memtData->memt->nRow == 0) { + iter->noMoreData = true; + return 0; + } + + iter->memtData->iter[0] = tRBTreeIterCreate(iter->memtData->memt->tbDataTree, 1); + return tsdbMemTableIterNext(iter, NULL); +} + +static int32_t tsdbSttIterClose(STsdbIter *iter) { + tBlockDataDestroy(iter->sttData->blockData); + return 0; +} + +static int32_t tsdbDataTombIterOpen(STsdbIter *iter) { + int32_t code; + + code = tsdbDataFileReadTombBlk(iter->dataTomb->reader, &iter->dataTomb->tombBlkArray); + if (code) return code; + + if (TARRAY2_SIZE(iter->dataTomb->tombBlkArray) == 0) { + iter->noMoreData = true; + return 0; + } + iter->dataTomb->tombBlkArrayIdx = 0; + + tTombBlockInit(iter->dataTomb->tombBlock); + iter->dataTomb->tombBlockIdx = 0; + + return tsdbDataTombIterNext(iter, NULL); +} + +static int32_t tsdbMemTombIterOpen(STsdbIter *iter) { + int32_t code; + + if (iter->memtTomb->memt->nDel == 0) { + iter->noMoreData = true; + return 0; + } + + iter->memtTomb->rbtIter[0] = tRBTreeIterCreate(iter->memtTomb->memt->tbDataTree, 1); + return tsdbMemTombIterNext(iter, NULL); +} + +static int32_t tsdbDataIterClose(STsdbIter *iter) { + tBrinBlockDestroy(iter->dataData->brinBlock); + tBlockDataDestroy(iter->dataData->blockData); + return 0; +} + +static int32_t tsdbMemTableIterClose(STsdbIter *iter) { return 0; } + +static int32_t tsdbSttTombIterNext(STsdbIter *iter, const TABLEID *tbid) { + while (!iter->noMoreData) { + for (; iter->sttTomb->tombBlockIdx < TOMB_BLOCK_SIZE(iter->sttTomb->tombBlock); iter->sttTomb->tombBlockIdx++) { + iter->record->suid = TARRAY2_GET(iter->sttTomb->tombBlock->suid, iter->sttTomb->tombBlockIdx); + iter->record->uid = TARRAY2_GET(iter->sttTomb->tombBlock->uid, iter->sttTomb->tombBlockIdx); + iter->record->version = TARRAY2_GET(iter->sttTomb->tombBlock->version, iter->sttTomb->tombBlockIdx); + + if (iter->filterByVersion && (iter->record->version < iter->range[0] || iter->record->version > iter->range[1])) { + continue; + } + + if (tbid && iter->record->suid == tbid->suid && iter->record->uid == tbid->uid) { + continue; + } + + iter->record->skey = TARRAY2_GET(iter->sttTomb->tombBlock->skey, iter->sttTomb->tombBlockIdx); + iter->record->ekey = TARRAY2_GET(iter->sttTomb->tombBlock->ekey, iter->sttTomb->tombBlockIdx); + iter->sttTomb->tombBlockIdx++; + goto _exit; + } + + if (iter->sttTomb->tombBlkArrayIdx >= TARRAY2_SIZE(iter->sttTomb->tombBlkArray)) { + iter->noMoreData = true; + goto _exit; + } + + for (; iter->sttTomb->tombBlkArrayIdx < TARRAY2_SIZE(iter->sttTomb->tombBlkArray); + iter->sttTomb->tombBlkArrayIdx++) { + const STombBlk *tombBlk = TARRAY2_GET_PTR(iter->sttTomb->tombBlkArray, iter->sttTomb->tombBlkArrayIdx); + + if (iter->filterByVersion && (tombBlk->maxVer < iter->range[0] || tombBlk->minVer > iter->range[1])) { + continue; + } + + if (tbid && tbid->suid == tombBlk->minTbid.suid && tbid->uid == tombBlk->minTbid.uid && + tbid->suid == tombBlk->maxTbid.suid && tbid->uid == tombBlk->maxTbid.uid) { + continue; + } + + int32_t code = tsdbSttFileReadTombBlock(iter->sttTomb->reader, tombBlk, iter->sttTomb->tombBlock); + if (code) return code; + + iter->sttTomb->tombBlockIdx = 0; + iter->sttTomb->tombBlkArrayIdx++; + break; + } + } + +_exit: + return 0; +} + +static int32_t tsdbSttTombIterOpen(STsdbIter *iter) { + int32_t code; + + code = tsdbSttFileReadTombBlk(iter->sttTomb->reader, &iter->sttTomb->tombBlkArray); + if (code) return code; + + if (TARRAY2_SIZE(iter->sttTomb->tombBlkArray) == 0) { + iter->noMoreData = true; + return 0; + } + + iter->sttTomb->tombBlkArrayIdx = 0; + tTombBlockInit(iter->sttTomb->tombBlock); + iter->sttTomb->tombBlockIdx = 0; + + return tsdbSttTombIterNext(iter, NULL); +} + +int32_t tsdbIterOpen(const STsdbIterConfig *config, STsdbIter **iter) { + int32_t code; + + iter[0] = taosMemoryCalloc(1, sizeof(*iter[0])); + if (iter[0] == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + iter[0]->type = config->type; + iter[0]->noMoreData = false; + iter[0]->filterByVersion = config->filterByVersion; + if (iter[0]->filterByVersion) { + iter[0]->range[0] = config->verRange[0]; + iter[0]->range[1] = config->verRange[1]; + } + + switch (config->type) { + case TSDB_ITER_TYPE_STT: + iter[0]->sttData->reader = config->sttReader; + code = tsdbSttIterOpen(iter[0]); + break; + case TSDB_ITER_TYPE_DATA: + iter[0]->dataData->reader = config->dataReader; + code = tsdbDataIterOpen(iter[0]); + break; + case TSDB_ITER_TYPE_MEMT: + iter[0]->memtData->memt = config->memt; + iter[0]->memtData->from[0] = config->from[0]; + code = tsdbMemTableIterOpen(iter[0]); + break; + case TSDB_ITER_TYPE_STT_TOMB: + iter[0]->sttTomb->reader = config->sttReader; + code = tsdbSttTombIterOpen(iter[0]); + break; + case TSDB_ITER_TYPE_DATA_TOMB: + iter[0]->dataTomb->reader = config->dataReader; + code = tsdbDataTombIterOpen(iter[0]); + break; + case TSDB_ITER_TYPE_MEMT_TOMB: + iter[0]->memtTomb->memt = config->memt; + code = tsdbMemTombIterOpen(iter[0]); + break; + default: + code = TSDB_CODE_INVALID_PARA; + ASSERTS(false, "Not implemented"); + } + + if (code) { + taosMemoryFree(iter[0]); + iter[0] = NULL; + } + return code; +} + +static int32_t tsdbSttTombIterClose(STsdbIter *iter) { + tTombBlockDestroy(iter->sttTomb->tombBlock); + return 0; +} + +static int32_t tsdbDataTombIterClose(STsdbIter *iter) { + tTombBlockDestroy(iter->dataTomb->tombBlock); + return 0; +} + +int32_t tsdbIterClose(STsdbIter **iter) { + switch (iter[0]->type) { + case TSDB_ITER_TYPE_STT: + tsdbSttIterClose(iter[0]); + break; + case TSDB_ITER_TYPE_DATA: + tsdbDataIterClose(iter[0]); + break; + case TSDB_ITER_TYPE_MEMT: + tsdbMemTableIterClose(iter[0]); + break; + case TSDB_ITER_TYPE_STT_TOMB: + tsdbSttTombIterClose(iter[0]); + break; + case TSDB_ITER_TYPE_DATA_TOMB: + tsdbDataTombIterClose(iter[0]); + break; + case TSDB_ITER_TYPE_MEMT_TOMB: + break; + default: + ASSERT(false); + } + taosMemoryFree(iter[0]); + iter[0] = NULL; + return 0; +} + +int32_t tsdbIterNext(STsdbIter *iter) { + switch (iter->type) { + case TSDB_ITER_TYPE_STT: + return tsdbSttIterNext(iter, NULL); + case TSDB_ITER_TYPE_DATA: + return tsdbDataIterNext(iter, NULL); + case TSDB_ITER_TYPE_MEMT: + return tsdbMemTableIterNext(iter, NULL); + case TSDB_ITER_TYPE_STT_TOMB: + return tsdbSttTombIterNext(iter, NULL); + case TSDB_ITER_TYPE_DATA_TOMB: + return tsdbDataTombIterNext(iter, NULL); + case TSDB_ITER_TYPE_MEMT_TOMB: + return tsdbMemTombIterNext(iter, NULL); + default: + ASSERT(false); + } + return 0; +} + +static int32_t tsdbIterSkipTableData(STsdbIter *iter, const TABLEID *tbid) { + switch (iter->type) { + case TSDB_ITER_TYPE_STT: + return tsdbSttIterNext(iter, tbid); + case TSDB_ITER_TYPE_DATA: + return tsdbDataIterNext(iter, tbid); + case TSDB_ITER_TYPE_MEMT: + return tsdbMemTableIterNext(iter, tbid); + case TSDB_ITER_TYPE_STT_TOMB: + return tsdbSttTombIterNext(iter, tbid); + case TSDB_ITER_TYPE_DATA_TOMB: + return tsdbDataTombIterNext(iter, tbid); + case TSDB_ITER_TYPE_MEMT_TOMB: + return tsdbMemTombIterNext(iter, tbid); + default: + ASSERT(false); + } + return 0; +} + +static int32_t tsdbIterCmprFn(const SRBTreeNode *n1, const SRBTreeNode *n2) { + STsdbIter *iter1 = TCONTAINER_OF(n1, STsdbIter, node); + STsdbIter *iter2 = TCONTAINER_OF(n2, STsdbIter, node); + return tRowInfoCmprFn(&iter1->row, &iter2->row); +} + +static int32_t tsdbTombIterCmprFn(const SRBTreeNode *n1, const SRBTreeNode *n2) { + STsdbIter *iter1 = TCONTAINER_OF(n1, STsdbIter, node); + STsdbIter *iter2 = TCONTAINER_OF(n2, STsdbIter, node); + + if (iter1->record->suid < iter2->record->suid) { + return -1; + } else if (iter1->record->suid > iter2->record->suid) { + return 1; + } + + if (iter1->record->uid < iter2->record->uid) { + return -1; + } else if (iter1->record->uid > iter2->record->uid) { + return 1; + } + + if (iter1->record->version < iter2->record->version) { + return -1; + } else if (iter1->record->version > iter2->record->version) { + return 1; + } + + return 0; +} + +// SIterMerger ================ +struct SIterMerger { + bool isTomb; + STsdbIter *iter; + SRBTree iterTree[1]; +}; + +int32_t tsdbIterMergerOpen(const TTsdbIterArray *iterArray, SIterMerger **merger, bool isTomb) { + STsdbIter *iter; + SRBTreeNode *node; + + merger[0] = taosMemoryCalloc(1, sizeof(*merger[0])); + if (merger[0] == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + merger[0]->isTomb = isTomb; + if (isTomb) { + tRBTreeCreate(merger[0]->iterTree, tsdbTombIterCmprFn); + } else { + tRBTreeCreate(merger[0]->iterTree, tsdbIterCmprFn); + } + TARRAY2_FOREACH(iterArray, iter) { + if (iter->noMoreData) continue; + node = tRBTreePut(merger[0]->iterTree, iter->node); + ASSERT(node); + } + + return tsdbIterMergerNext(merger[0]); +} + +int32_t tsdbIterMergerClose(SIterMerger **merger) { + if (merger[0]) { + taosMemoryFree(merger[0]); + merger[0] = NULL; + } + return 0; +} + +int32_t tsdbIterMergerNext(SIterMerger *merger) { + int32_t code; + int32_t c; + SRBTreeNode *node; + + if (merger->iter) { + code = tsdbIterNext(merger->iter); + if (code) return code; + + if (merger->iter->noMoreData) { + merger->iter = NULL; + } else if ((node = tRBTreeMin(merger->iterTree))) { + c = merger->iterTree->cmprFn(merger->iter->node, node); + ASSERT(c); + if (c > 0) { + node = tRBTreePut(merger->iterTree, merger->iter->node); + ASSERT(node); + merger->iter = NULL; + } + } + } + + if (merger->iter == NULL && (node = tRBTreeDropMin(merger->iterTree))) { + merger->iter = TCONTAINER_OF(node, STsdbIter, node); + } + + return 0; +} + +SRowInfo *tsdbIterMergerGetData(SIterMerger *merger) { + ASSERT(!merger->isTomb); + return merger->iter ? merger->iter->row : NULL; +} + +STombRecord *tsdbIterMergerGetTombRecord(SIterMerger *merger) { + ASSERT(merger->isTomb); + return merger->iter ? merger->iter->record : NULL; +} + +int32_t tsdbIterMergerSkipTableData(SIterMerger *merger, const TABLEID *tbid) { + int32_t code; + int32_t c; + SRBTreeNode *node; + + while (merger->iter && tbid->suid == merger->iter->row->suid && tbid->uid == merger->iter->row->uid) { + int32_t code = tsdbIterSkipTableData(merger->iter, tbid); + if (code) return code; + + if (merger->iter->noMoreData) { + merger->iter = NULL; + } else if ((node = tRBTreeMin(merger->iterTree))) { + c = merger->iterTree->cmprFn(merger->iter->node, node); + ASSERT(c); + if (c > 0) { + node = tRBTreePut(merger->iterTree, merger->iter->node); + ASSERT(node); + merger->iter = NULL; + } + } + + if (!merger->iter && (node = tRBTreeDropMin(merger->iterTree))) { + merger->iter = TCONTAINER_OF(node, STsdbIter, node); + } + } + + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbIter.h b/source/dnode/vnode/src/tsdb/tsdbIter.h new file mode 100644 index 0000000000000000000000000000000000000000..367901bd848df5b752a3439249dba95f76369b45 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbIter.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "trbtree.h" +#include "tsdbDataFileRW.h" +#include "tsdbDef.h" +#include "tsdbSttFileRW.h" + +#ifndef _TSDB_ITER_H_ +#define _TSDB_ITER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SIterMerger SIterMerger; +typedef struct STsdbIter STsdbIter; +typedef TARRAY2(STsdbIter *) TTsdbIterArray; + +typedef enum { + TSDB_ITER_TYPE_STT = 1, + TSDB_ITER_TYPE_DATA, + TSDB_ITER_TYPE_MEMT, + TSDB_ITER_TYPE_STT_TOMB, + TSDB_ITER_TYPE_DATA_TOMB, + TSDB_ITER_TYPE_MEMT_TOMB, +} EIterType; + +typedef struct { + EIterType type; + union { + SSttFileReader *sttReader; // TSDB_ITER_TYPE_STT || TSDB_ITER_TYPE_STT_TOMB + SDataFileReader *dataReader; // TSDB_ITER_TYPE_DATA || TSDB_ITER_TYPE_DATA_TOMB + struct { + SMemTable *memt; // TSDB_ITER_TYPE_MEMT_TOMB + TSDBKEY from[1]; + }; // TSDB_ITER_TYPE_MEMT + }; + bool filterByVersion; + int64_t verRange[2]; +} STsdbIterConfig; + +// STsdbIter =============== +int32_t tsdbIterOpen(const STsdbIterConfig *config, STsdbIter **iter); +int32_t tsdbIterClose(STsdbIter **iter); +int32_t tsdbIterNext(STsdbIter *iter); + +// SIterMerger =============== +int32_t tsdbIterMergerOpen(const TTsdbIterArray *iterArray, SIterMerger **merger, bool isTomb); +int32_t tsdbIterMergerClose(SIterMerger **merger); +int32_t tsdbIterMergerNext(SIterMerger *merger); +int32_t tsdbIterMergerSkipTableData(SIterMerger *merger, const TABLEID *tbid); + +SRowInfo *tsdbIterMergerGetData(SIterMerger *merger); +STombRecord *tsdbIterMergerGetTombRecord(SIterMerger *merger); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_ITER_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 6d223e00c54272a47377f3521a181d350ab2c4a2..ee3abf7559ecae536ab6dc522d541ee611477237 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -38,6 +38,16 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, SSubmitTbData *pSubmitTbData, int32_t *affectedRows); +static int32_t tTbDataCmprFn(const SRBTreeNode *n1, const SRBTreeNode *n2) { + STbData *tbData1 = TCONTAINER_OF(n1, STbData, rbtn); + STbData *tbData2 = TCONTAINER_OF(n2, STbData, rbtn); + if (tbData1->suid < tbData2->suid) return -1; + if (tbData1->suid > tbData2->suid) return 1; + if (tbData1->uid < tbData2->uid) return -1; + if (tbData1->uid > tbData2->uid) return 1; + return 0; +} + int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { int32_t code = 0; SMemTable *pMemTable = NULL; @@ -66,6 +76,7 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { goto _err; } vnodeBufPoolRef(pMemTable->pPool); + tRBTreeCreate(pMemTable->tbDataTree, tTbDataCmprFn); *ppMemTable = pMemTable; return code; @@ -406,6 +417,8 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid pMemTable->aBucket[idx] = pTbData; pMemTable->nTbData++; + tRBTreePut(pMemTable->tbDataTree, pTbData->rbtn); + taosWUnLockLatch(&pMemTable->latch); _exit: diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c new file mode 100644 index 0000000000000000000000000000000000000000..ec0ea3c60ffa1c8ef0a5564dcdb2e2f59bd6935a --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -0,0 +1,454 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbMerge.h" + +typedef struct { + STsdb *tsdb; + TFileSetArray *fsetArr; + + int32_t sttTrigger; + int32_t maxRow; + int32_t minRow; + int32_t szPage; + int8_t cmprAlg; + int64_t compactVersion; + int64_t cid; + + // context + struct { + bool opened; + int64_t now; + STFileSet *fset; + bool toData; + int32_t level; + SSttLvl *lvl; + TABLEID tbid[1]; + } ctx[1]; + + TFileOpArray fopArr[1]; + + // reader + TSttFileReaderArray sttReaderArr[1]; + // iter + TTsdbIterArray dataIterArr[1]; + SIterMerger *dataIterMerger; + TTsdbIterArray tombIterArr[1]; + SIterMerger *tombIterMerger; + // writer + SFSetWriter *writer; +} SMerger; + +static int32_t tsdbMergerOpen(SMerger *merger) { + merger->ctx->now = taosGetTimestampSec(); + merger->maxRow = merger->tsdb->pVnode->config.tsdbCfg.maxRows; + merger->minRow = merger->tsdb->pVnode->config.tsdbCfg.minRows; + merger->szPage = merger->tsdb->pVnode->config.tsdbPageSize; + merger->cmprAlg = merger->tsdb->pVnode->config.tsdbCfg.compression; + merger->compactVersion = INT64_MAX; + merger->cid = tsdbFSAllocEid(merger->tsdb->pFS); + merger->ctx->opened = true; + return 0; +} + +static int32_t tsdbMergerClose(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + SVnode *pVnode = merger->tsdb->pVnode; + + // edit file system + code = tsdbFSEditBegin(merger->tsdb->pFS, merger->fopArr, TSDB_FEDIT_MERGE); + TSDB_CHECK_CODE(code, lino, _exit); + + taosThreadRwlockWrlock(&merger->tsdb->rwLock); + code = tsdbFSEditCommit(merger->tsdb->pFS); + if (code) { + taosThreadRwlockUnlock(&merger->tsdb->rwLock); + TSDB_CHECK_CODE(code, lino, _exit); + } + taosThreadRwlockUnlock(&merger->tsdb->rwLock); + + ASSERT(merger->writer == NULL); + ASSERT(merger->dataIterMerger == NULL); + ASSERT(merger->tombIterMerger == NULL); + ASSERT(TARRAY2_SIZE(merger->dataIterArr) == 0); + ASSERT(TARRAY2_SIZE(merger->tombIterArr) == 0); + ASSERT(TARRAY2_SIZE(merger->sttReaderArr) == 0); + + // clear the merge + TARRAY2_DESTROY(merger->tombIterArr, NULL); + TARRAY2_DESTROY(merger->dataIterArr, NULL); + TARRAY2_DESTROY(merger->sttReaderArr, NULL); + TARRAY2_DESTROY(merger->fopArr, NULL); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pVnode), lino, code); + } + return code; +} + +static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + + merger->ctx->toData = true; + merger->ctx->level = 0; + + // TODO: optimize merge strategy + for (int32_t i = 0;; ++i) { + if (i >= TARRAY2_SIZE(merger->ctx->fset->lvlArr)) { + merger->ctx->lvl = NULL; + break; + } + + merger->ctx->lvl = TARRAY2_GET(merger->ctx->fset->lvlArr, i); + if (merger->ctx->lvl->level != merger->ctx->level || + TARRAY2_SIZE(merger->ctx->lvl->fobjArr) + 1 < merger->sttTrigger) { + merger->ctx->toData = false; + merger->ctx->lvl = NULL; + break; + } + + merger->ctx->level++; + + STFileObj *fobj; + int32_t numFile = 0; + TARRAY2_FOREACH(merger->ctx->lvl->fobjArr, fobj) { + if (numFile == merger->sttTrigger) { + break; + } + + STFileOp op = { + .optype = TSDB_FOP_REMOVE, + .fid = merger->ctx->fset->fid, + .of = fobj->f[0], + }; + code = TARRAY2_APPEND(merger->fopArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + SSttFileReader *reader; + SSttFileReaderConfig config = { + .tsdb = merger->tsdb, + .szPage = merger->szPage, + .file[0] = fobj->f[0], + }; + + code = tsdbSttFileReaderOpen(fobj->fname, &config, &reader); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(merger->sttReaderArr, reader); + TSDB_CHECK_CODE(code, lino, _exit); + + numFile++; + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbMergeFileSetBeginOpenIter(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + int32_t vid = TD_VID(merger->tsdb->pVnode); + + SSttFileReader *sttReader; + TARRAY2_FOREACH(merger->sttReaderArr, sttReader) { + STsdbIter *iter; + STsdbIterConfig config = {0}; + + // data iter + config.type = TSDB_ITER_TYPE_STT; + config.sttReader = sttReader; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(merger->dataIterArr, iter); + TSDB_CHECK_CODE(code, lino, _exit); + + // tomb iter + config.type = TSDB_ITER_TYPE_STT_TOMB; + config.sttReader = sttReader; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(merger->tombIterArr, iter); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbIterMergerOpen(merger->dataIterArr, &merger->dataIterMerger, false); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbIterMergerOpen(merger->tombIterArr, &merger->tombIterMerger, true); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(vid, lino, code); + } + return code; +} + +static int32_t tsdbMergeFileSetBeginOpenWriter(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + int32_t vid = TD_VID(merger->tsdb->pVnode); + + SDiskID did; + int32_t level = tsdbFidLevel(merger->ctx->fset->fid, &merger->tsdb->keepCfg, merger->ctx->now); + if (tfsAllocDisk(merger->tsdb->pVnode->pTfs, level, &did) < 0) { + code = TSDB_CODE_FS_NO_VALID_DISK; + TSDB_CHECK_CODE(code, lino, _exit); + } + tfsMkdirRecurAt(merger->tsdb->pVnode->pTfs, merger->tsdb->path, did); + SFSetWriterConfig config = { + .tsdb = merger->tsdb, + .toSttOnly = true, + .compactVersion = merger->compactVersion, + .minRow = merger->minRow, + .maxRow = merger->maxRow, + .szPage = merger->szPage, + .cmprAlg = merger->cmprAlg, + .fid = merger->ctx->fset->fid, + .cid = merger->cid, + .did = did, + .level = merger->ctx->level, + }; + + if (merger->ctx->toData) { + config.toSttOnly = false; + + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (merger->ctx->fset->farr[ftype]) { + config.files[ftype].exist = true; + config.files[ftype].file = merger->ctx->fset->farr[ftype]->f[0]; + } else { + config.files[ftype].exist = false; + } + } + } + + code = tsdbFSetWriterOpen(&config, &merger->writer); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(vid, lino, code); + } + return code; +} + +static int32_t tsdbMergeFileSetBegin(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(TARRAY2_SIZE(merger->sttReaderArr) == 0); + ASSERT(TARRAY2_SIZE(merger->dataIterArr) == 0); + ASSERT(merger->dataIterMerger == NULL); + ASSERT(merger->writer == NULL); + + merger->ctx->tbid->suid = 0; + merger->ctx->tbid->uid = 0; + + // open reader + code = tsdbMergeFileSetBeginOpenReader(merger); + TSDB_CHECK_CODE(code, lino, _exit); + + // open iterator + code = tsdbMergeFileSetBeginOpenIter(merger); + TSDB_CHECK_CODE(code, lino, _exit); + + // open writer + code = tsdbMergeFileSetBeginOpenWriter(merger); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbMergeFileSetEndCloseWriter(SMerger *merger) { + return tsdbFSetWriterClose(&merger->writer, 0, merger->fopArr); +} + +static int32_t tsdbMergeFileSetEndCloseIter(SMerger *merger) { + tsdbIterMergerClose(&merger->tombIterMerger); + TARRAY2_CLEAR(merger->tombIterArr, tsdbIterClose); + tsdbIterMergerClose(&merger->dataIterMerger); + TARRAY2_CLEAR(merger->dataIterArr, tsdbIterClose); + return 0; +} + +static int32_t tsdbMergeFileSetEndCloseReader(SMerger *merger) { + TARRAY2_CLEAR(merger->sttReaderArr, tsdbSttFileReaderClose); + return 0; +} + +static int32_t tsdbMergeFileSetEnd(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbMergeFileSetEndCloseWriter(merger); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbMergeFileSetEndCloseIter(merger); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbMergeFileSetEndCloseReader(merger); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbMergeFileSet(SMerger *merger, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + + merger->ctx->fset = fset; + code = tsdbMergeFileSetBegin(merger); + TSDB_CHECK_CODE(code, lino, _exit); + + // data + SMetaInfo info; + SRowInfo *row; + merger->ctx->tbid->suid = 0; + merger->ctx->tbid->uid = 0; + while ((row = tsdbIterMergerGetData(merger->dataIterMerger)) != NULL) { + if (row->uid != merger->ctx->tbid->uid) { + merger->ctx->tbid->uid = row->uid; + merger->ctx->tbid->suid = row->suid; + + if (metaGetInfo(merger->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) { + code = tsdbIterMergerSkipTableData(merger->dataIterMerger, merger->ctx->tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } + } + + code = tsdbFSetWriteRow(merger->writer, row); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbIterMergerNext(merger->dataIterMerger); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // tomb + merger->ctx->tbid->suid = 0; + merger->ctx->tbid->uid = 0; + for (STombRecord *record; (record = tsdbIterMergerGetTombRecord(merger->tombIterMerger)) != NULL;) { + if (record->uid != merger->ctx->tbid->uid) { + merger->ctx->tbid->uid = record->uid; + merger->ctx->tbid->suid = record->suid; + + if (metaGetInfo(merger->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) { + code = tsdbIterMergerSkipTableData(merger->tombIterMerger, merger->ctx->tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } + } + code = tsdbFSetWriteTombRecord(merger->writer, record); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbIterMergerNext(merger->tombIterMerger); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbMergeFileSetEnd(merger); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(merger->tsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(merger->tsdb->pVnode), __func__, fset->fid); + } + return code; +} + +static int32_t tsdbDoMerge(SMerger *merger) { + int32_t code = 0; + int32_t lino = 0; + + STFileSet *fset; + TARRAY2_FOREACH(merger->fsetArr, fset) { + if (TARRAY2_SIZE(fset->lvlArr) == 0) continue; + + SSttLvl *lvl = TARRAY2_FIRST(fset->lvlArr); + + if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < merger->sttTrigger) continue; + + if (!merger->ctx->opened) { + code = tsdbMergerOpen(merger); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbMergeFileSet(merger, fset); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (merger->ctx->opened) { + code = tsdbMergerClose(merger); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done", TD_VID(merger->tsdb->pVnode), __func__); + } + return code; +} + +int32_t tsdbMerge(void *arg) { + int32_t code = 0; + int32_t lino = 0; + STsdb *tsdb = (STsdb *)arg; + + SMerger merger[1] = {{ + .tsdb = tsdb, + .sttTrigger = tsdb->pVnode->config.sttTrigger, + }}; + + ASSERT(merger->sttTrigger > 1); + + code = tsdbFSCreateCopySnapshot(tsdb->pFS, &merger->fsetArr); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDoMerge(merger); + TSDB_CHECK_CODE(code, lino, _exit); + + tsdbFSDestroyCopySnapshot(&merger->fsetArr); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } else if (merger->ctx->opened) { + tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); + } + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.h b/source/dnode/vnode/src/tsdb/tsdbMerge.h new file mode 100644 index 0000000000000000000000000000000000000000..69d802fd2776eddba8d65090dfe5717ba4bb76bc --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDataFileRW.h" +#include "tsdbFS2.h" +#include "tsdbFSetRW.h" +#include "tsdbIter.h" +#include "tsdbSttFileRW.h" +#include "tsdbUtil2.h" + +#ifndef _TD_TSDB_MERGE_H_ +#define _TD_TSDB_MERGE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Exposed Handle */ + +/* Exposed APIs */ + +/* Exposed Structs */ + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TSDB_MERGE_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 79f4a17f65430c3dc0efc8177c7f131db60934e3..ea5b574ceda600b7322cc97f75b3947f25dc3fe5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -14,6 +14,11 @@ */ #include "tsdb.h" +#include "tsdbFSet2.h" +#include "tsdbReadUtil.h" +#include "tsdbSttFileRW.h" + +static void tLDataIterClose2(SLDataIter *pIter); // SLDataIter ================================================= SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, @@ -24,8 +29,6 @@ SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, return NULL; } - pLoadInfo->numOfStt = numOfSttTrigger; - for (int32_t i = 0; i < numOfSttTrigger; ++i) { pLoadInfo[i].blockIndex[0] = -1; pLoadInfo[i].blockIndex[1] = -1; @@ -50,8 +53,37 @@ SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, return pLoadInfo; } +SSttBlockLoadInfo *tCreateOneLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols) { + SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo)); + if (pLoadInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pLoadInfo->blockIndex[0] = -1; + pLoadInfo->blockIndex[1] = -1; + pLoadInfo->currentLoadBlockIndex = 1; + + int32_t code = tBlockDataCreate(&pLoadInfo->blockData[0]); + if (code) { + terrno = code; + } + + code = tBlockDataCreate(&pLoadInfo->blockData[1]); + if (code) { + terrno = code; + } + + pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk)); + pLoadInfo->pSchema = pSchema; + pLoadInfo->colIds = colList; + pLoadInfo->numOfCols = numOfCols; + + return pLoadInfo; +} + void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { - for (int32_t i = 0; i < pLoadInfo->numOfStt; ++i) { + for (int32_t i = 0; i < 1; ++i) { pLoadInfo[i].currentLoadBlockIndex = 1; pLoadInfo[i].blockIndex[0] = -1; pLoadInfo[i].blockIndex[1] = -1; @@ -65,18 +97,24 @@ void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { } void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el) { - for (int32_t i = 0; i < pLoadInfo->numOfStt; ++i) { + for (int32_t i = 0; i < 1; ++i) { *el += pLoadInfo[i].elapsedTime; *blocks += pLoadInfo[i].loadBlocks; } } +static void freeTombBlock(void *param) { + STombBlock **pTombBlock = (STombBlock **)param; + tTombBlockDestroy(*pTombBlock); + taosMemoryFree(*pTombBlock); +} + void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { if (pLoadInfo == NULL) { return NULL; } - for (int32_t i = 0; i < pLoadInfo->numOfStt; ++i) { + for (int32_t i = 0; i < 1; ++i) { pLoadInfo[i].currentLoadBlockIndex = 1; pLoadInfo[i].blockIndex[0] = -1; pLoadInfo[i].blockIndex[1] = -1; @@ -91,6 +129,33 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { return NULL; } +static void destroyLDataIter(SLDataIter *pIter) { + tLDataIterClose2(pIter); + destroyLastBlockLoadInfo(pIter->pBlockLoadInfo); + taosMemoryFree(pIter); +} + +void *destroySttBlockReader(SArray *pLDataIterArray, int64_t *blocks, double *el) { + if (pLDataIterArray == NULL) { + return NULL; + } + + int32_t numOfLevel = taosArrayGetSize(pLDataIterArray); + for (int32_t i = 0; i < numOfLevel; ++i) { + SArray *pList = taosArrayGetP(pLDataIterArray, i); + for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) { + SLDataIter *pIter = taosArrayGetP(pList, j); + *el += pIter->pBlockLoadInfo->elapsedTime; + *blocks += pIter->pBlockLoadInfo->loadBlocks; + destroyLDataIter(pIter); + } + taosArrayDestroy(pList); + } + + taosArrayDestroy(pLDataIterArray); + return NULL; +} + static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { int32_t code = 0; @@ -122,20 +187,8 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { int64_t st = taosGetTimestampUs(); SBlockData *pBlock = &pInfo->blockData[pInfo->currentLoadBlockIndex]; - - TABLEID id = {0}; - if (pIter->pSttBlk->suid != 0) { - id.suid = pIter->pSttBlk->suid; - } else { - id.uid = pIter->uid; - } - - code = tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } - - code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock); + code = tsdbSttFileReadBlockDataByColumn(pIter->pReader, pIter->pSttBlk, pBlock, pInfo->pSchema, &pInfo->colIds[1], + pInfo->numOfCols - 1); if (code != TSDB_CODE_SUCCESS) { goto _exit; } @@ -255,74 +308,153 @@ static int32_t binarySearchForStartRowIndex(uint64_t *uidList, int32_t num, uint int32_t tLDataIterOpen(struct SLDataIter *pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo *pBlockLoadInfo, const char *idStr, bool strictTimeRange) { + return 0; +} + +static int32_t extractSttBlockInfo(SLDataIter *pIter, const TSttBlkArray *pArray, SSttBlockLoadInfo *pBlockLoadInfo, + uint64_t suid) { + if (TARRAY2_SIZE(pArray) <= 0) { + return TSDB_CODE_SUCCESS; + } + + SSttBlk *pStart = &pArray->data[0]; + SSttBlk *pEnd = &pArray->data[TARRAY2_SIZE(pArray) - 1]; + + // all identical + if (pStart->suid == pEnd->suid) { + if (pStart->suid != suid) { // no qualified stt block existed + taosArrayClear(pBlockLoadInfo->aSttBlk); + pIter->iSttBlk = -1; + return TSDB_CODE_SUCCESS; + } else { // all blocks are qualified + taosArrayClear(pBlockLoadInfo->aSttBlk); + taosArrayAddBatch(pBlockLoadInfo->aSttBlk, pArray->data, pArray->size); + } + } else { + SArray *pTmp = taosArrayInit(TARRAY2_SIZE(pArray), sizeof(SSttBlk)); + for (int32_t i = 0; i < TARRAY2_SIZE(pArray); ++i) { + SSttBlk *p = &pArray->data[i]; + if (p->suid < suid) { + continue; + } + + if (p->suid == suid) { + taosArrayPush(pTmp, p); + } else if (p->suid > suid) { + break; + } + } + + taosArrayDestroy(pBlockLoadInfo->aSttBlk); + pBlockLoadInfo->aSttBlk = pTmp; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t uidComparFn(const void *p1, const void *p2) { + const uint64_t *uid1 = p1; + const uint64_t *uid2 = p2; + return (*uid1) - (*uid2); +} + +static bool existsFromSttBlkStatis(const TStatisBlkArray *pStatisBlkArray, uint64_t suid, uint64_t uid, + SSttFileReader *pReader) { + if (TARRAY2_SIZE(pStatisBlkArray) <= 0) { + return true; + } + + int32_t i = 0; + for (i = 0; i < TARRAY2_SIZE(pStatisBlkArray); ++i) { + SStatisBlk *p = &pStatisBlkArray->data[i]; + if (p->minTbid.suid <= suid && p->maxTbid.suid >= suid) { + break; + } + } + + // for (; i < TARRAY2_SIZE(pStatisBlkArray); ++i) { + // SStatisBlk *p = &pStatisBlkArray->data[i]; + // if (p->minTbid.uid <= uid && p->maxTbid.uid >= uid) { + // break; + // } + // + // if (p->maxTbid.uid < uid) { + // break; + // } + // } + + if (i >= TARRAY2_SIZE(pStatisBlkArray)) { + return false; + } + + SStatisBlk *p = &pStatisBlkArray->data[i]; + STbStatisBlock block = {0}; + tsdbSttFileReadStatisBlock(pReader, p, &block); + + int32_t index = tarray2SearchIdx(block.uid, &uid, sizeof(int64_t), uidComparFn, TD_EQ); + tStatisBlockDestroy(&block); + + return (index != -1); +} + +int32_t tLDataIterOpen2(struct SLDataIter *pIter, SSttFileReader *pSttFileReader, int32_t iStt, int8_t backward, + uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, + SSttBlockLoadInfo *pBlockLoadInfo, const char *idStr, bool strictTimeRange, + _load_tomb_fn loadTombFn, void *pReader1) { int32_t code = TSDB_CODE_SUCCESS; pIter->uid = uid; - pIter->pReader = pReader; pIter->iStt = iStt; pIter->backward = backward; pIter->verRange.minVer = pRange->minVer; pIter->verRange.maxVer = pRange->maxVer; pIter->timeWindow.skey = pTimeWindow->skey; pIter->timeWindow.ekey = pTimeWindow->ekey; - + pIter->pReader = pSttFileReader; pIter->pBlockLoadInfo = pBlockLoadInfo; if (!pBlockLoadInfo->sttBlockLoaded) { int64_t st = taosGetTimestampUs(); + + const TSttBlkArray *pSttBlkArray = NULL; pBlockLoadInfo->sttBlockLoaded = true; - code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk); - if (code) { + // load the stt block info for each stt-block + code = tsdbSttFileReadSttBlk(pIter->pReader, &pSttBlkArray); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("load stt blk failed, code:%s, %s", tstrerror(code), idStr); return code; } - // only apply to the child tables, ordinary tables will not incur this filter procedure. - size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); - - if (size >= 1) { - SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); - SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); - - // all identical - if (pStart->suid == pEnd->suid) { - if (pStart->suid != suid) { - // no qualified stt block existed - taosArrayClear(pBlockLoadInfo->aSttBlk); - - pIter->iSttBlk = -1; - double el = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr); - return code; - } - } else { - SArray *pTmp = taosArrayInit(size, sizeof(SSttBlk)); - for (int32_t i = 0; i < size; ++i) { - SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); - uint64_t s = p->suid; - if (s < suid) { - continue; - } - - if (s == suid) { - taosArrayPush(pTmp, p); - } else if (s > suid) { - break; - } - } + code = extractSttBlockInfo(pIter, pSttBlkArray, pBlockLoadInfo, suid); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("load stt block info failed, code:%s, %s", tstrerror(code), idStr); + return code; + } - taosArrayDestroy(pBlockLoadInfo->aSttBlk); - pBlockLoadInfo->aSttBlk = pTmp; - } + // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file + code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray **)&pBlockLoadInfo->pSttStatisBlkArray); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), idStr); + return code; } + code = loadTombFn(pReader1, pIter->pReader, pIter->pBlockLoadInfo); + double el = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr); + tsdbDebug("load the stt file info completed, elapsed time:%.2fms, %s", el, idStr); } - size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + // bool exists = existsFromSttBlkStatis(pBlockLoadInfo->pSttStatisBlkArray, suid, uid, pIter->pReader); + // if (!exists) { + // pIter->iSttBlk = -1; + // pIter->pSttBlk = NULL; + // return TSDB_CODE_SUCCESS; + // } - // find the start block + // find the start block, actually we could load the position to avoid repeatly searching for the start position when + // the skey is updated. + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); pIter->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward); if (pIter->iSttBlk != -1) { pIter->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, pIter->iSttBlk); @@ -343,7 +475,10 @@ int32_t tLDataIterOpen(struct SLDataIter *pIter, SDataFReader *pReader, int32_t return code; } -void tLDataIterClose(SLDataIter *pIter) { /*taosMemoryFree(pIter); */} +void tLDataIterClose2(SLDataIter *pIter) { + tsdbSttFileReaderClose(&pIter->pReader); + pIter->pReader = NULL; +} void tLDataIterNextBlock(SLDataIter *pIter, const char *idStr) { int32_t step = pIter->backward ? -1 : 1; @@ -395,25 +530,23 @@ void tLDataIterNextBlock(SLDataIter *pIter, const char *idStr) { if (index != -1) { pIter->iSttBlk = index; pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk); - tsdbDebug("try next last file block:%d from %d, trigger by uid:%" PRIu64 ", file index:%d, %s", pIter->iSttBlk, - oldIndex, pIter->uid, pIter->iStt, idStr); + tsdbDebug("try next last file block:%d from stt fileIdx:%d, trigger by uid:%" PRIu64 ", file index:%d, %s", + pIter->iSttBlk, oldIndex, pIter->uid, pIter->iStt, idStr); } else { tsdbDebug("no more last block qualified, uid:%" PRIu64 ", file index:%d, %s", pIter->uid, oldIndex, idStr); } } static void findNextValidRow(SLDataIter *pIter, const char *idStr) { - int32_t step = pIter->backward ? -1 : 1; - bool hasVal = false; + int32_t step = pIter->backward ? -1 : 1; int32_t i = pIter->iRow; - SBlockData *pBlockData = loadLastBlock(pIter, idStr); + SBlockData *pData = loadLastBlock(pIter, idStr); // mostly we only need to find the start position for a given table - if ((((i == 0) && (!pIter->backward)) || (i == pBlockData->nRow - 1 && pIter->backward)) && - pBlockData->aUid != NULL) { - i = binarySearchForStartRowIndex((uint64_t *)pBlockData->aUid, pBlockData->nRow, pIter->uid, pIter->backward); + if ((((i == 0) && (!pIter->backward)) || (i == pData->nRow - 1 && pIter->backward)) && pData->aUid != NULL) { + i = binarySearchForStartRowIndex((uint64_t *)pData->aUid, pData->nRow, pIter->uid, pIter->backward); if (i == -1) { tsdbDebug("failed to find the data in pBlockData, uid:%" PRIu64 " , %s", pIter->uid, idStr); pIter->iRow = -1; @@ -421,20 +554,20 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) { } } - for (; i < pBlockData->nRow && i >= 0; i += step) { - if (pBlockData->aUid != NULL) { + for (; i < pData->nRow && i >= 0; i += step) { + if (pData->aUid != NULL) { if (!pIter->backward) { - if (pBlockData->aUid[i] > pIter->uid) { + if (pData->aUid[i] > pIter->uid) { break; } } else { - if (pBlockData->aUid[i] < pIter->uid) { + if (pData->aUid[i] < pIter->uid) { break; } } } - int64_t ts = pBlockData->aTSKEY[i]; + int64_t ts = pData->aTSKEY[i]; if (!pIter->backward) { // asc if (ts > pIter->timeWindow.ekey) { // no more data break; @@ -449,7 +582,7 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) { } } - int64_t ver = pBlockData->aVersion[i]; + int64_t ver = pData->aVersion[i]; if (ver < pIter->verRange.minVer) { continue; } @@ -485,7 +618,6 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { while (1) { bool skipBlock = false; - findNextValidRow(pIter, idStr); if (pIter->pBlockLoadInfo->checkRemainingRow) { @@ -570,7 +702,7 @@ static FORCE_INLINE int32_t tLDataIterDescCmprFn(const SRBTreeNode *p1, const SR int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, - bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter) { + bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter *pLDataIter) { int32_t code = TSDB_CODE_SUCCESS; pMTree->backward = backward; @@ -612,6 +744,95 @@ _end: return code; } +int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) { + int32_t code = TSDB_CODE_SUCCESS; + + pMTree->pIter = NULL; + pMTree->backward = pConf->backward; + pMTree->idStr = pConf->idstr; + + if (!pMTree->backward) { // asc + tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); + } else { // desc + tRBTreeCreate(&pMTree->rbt, tLDataIterDescCmprFn); + } + + pMTree->ignoreEarlierTs = false; + + // todo handle other level of stt files, here only deal with the first level stt + int32_t size = ((STFileSet *)pConf->pCurrentFileset)->lvlArr->size; + if (size == 0) { + goto _end; + } + + // add the list/iter placeholder + while (taosArrayGetSize(pConf->pSttFileBlockIterArray) < size) { + SArray *pList = taosArrayInit(4, POINTER_BYTES); + taosArrayPush(pConf->pSttFileBlockIterArray, &pList); + } + + for (int32_t j = 0; j < size; ++j) { + SSttLvl *pSttLevel = ((STFileSet *)pConf->pCurrentFileset)->lvlArr->data[j]; + ASSERT(pSttLevel->level == j); + + SArray *pList = taosArrayGetP(pConf->pSttFileBlockIterArray, j); + int32_t numOfIter = taosArrayGetSize(pList); + + if (numOfIter < TARRAY2_SIZE(pSttLevel->fobjArr)) { + int32_t inc = TARRAY2_SIZE(pSttLevel->fobjArr) - numOfIter; + for (int32_t k = 0; k < inc; ++k) { + SLDataIter *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); + taosArrayPush(pList, &pIter); + } + } + + for (int32_t i = 0; i < TARRAY2_SIZE(pSttLevel->fobjArr); ++i) { // open all last file + SLDataIter *pIter = taosArrayGetP(pList, i); + + SSttFileReader *pSttFileReader = pIter->pReader; + SSttBlockLoadInfo *pLoadInfo = pIter->pBlockLoadInfo; + + // open stt file reader if not + if (pSttFileReader == NULL) { + SSttFileReaderConfig conf = {.tsdb = pConf->pTsdb, .szPage = pConf->pTsdb->pVnode->config.tsdbPageSize}; + conf.file[0] = *pSttLevel->fobjArr->data[i]->f; + + code = tsdbSttFileReaderOpen(pSttLevel->fobjArr->data[i]->fname, &conf, &pSttFileReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + if (pLoadInfo == NULL) { + pLoadInfo = tCreateOneLastBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); + } + + memset(pIter, 0, sizeof(SLDataIter)); + code = tLDataIterOpen2(pIter, pSttFileReader, i, pMTree->backward, pConf->suid, pConf->uid, &pConf->timewindow, + &pConf->verRange, pLoadInfo, pMTree->idStr, pConf->strictTimeRange, pConf->loadTombFn, + pConf->pReader); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + + bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr); + if (hasVal) { + tMergeTreeAddIter(pMTree, pIter); + } else { + if (!pMTree->ignoreEarlierTs) { + pMTree->ignoreEarlierTs = pIter->ignoreEarlierTs; + } + } + } + } + + return code; + +_end: + tMergeTreeClose(pMTree); + return code; +} + void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); } bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree) { return pMTree->ignoreEarlierTs; } diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 8901f644598ec4ca5343f4a35a7b063bf39096fd..c684ad51848d1a6af49ff2881f05b96eca6e51dc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -14,6 +14,7 @@ */ #include "tsdb.h" +#include "tsdbFS2.h" int32_t tsdbSetKeepCfg(STsdb *pTsdb, STsdbCfg *pCfg) { STsdbKeepCfg *pKeepCfg = &pTsdb->keepCfg; @@ -66,7 +67,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee } // open tsdb - if (tsdbFSOpen(pTsdb, rollback) < 0) { + if (tsdbOpenFS(pTsdb, &pTsdb->pFS, rollback) < 0) { goto _err; } @@ -94,7 +95,7 @@ int tsdbClose(STsdb **pTsdb) { taosThreadRwlockDestroy(&(*pTsdb)->rwLock); - tsdbFSClose(*pTsdb); + tsdbCloseFS(&(*pTsdb)->pFS); tsdbCloseCache(*pTsdb); taosMemoryFreeClear(*pTsdb); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 88027e2891dd9daa1150177a6c8b53de40874fe4..2aa21bd86f31e8167d0f5f0fd9406dad9a50d513 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -17,7 +17,7 @@ #include "tsdb.h" #include "tsimplehash.h" -#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define getCurrentKeyInLastBlock(_r) ((_r)->currentKey) typedef enum { @@ -30,12 +30,12 @@ typedef enum { EXTERNAL_ROWS_MAIN = 0x2, EXTERNAL_ROWS_NEXT = 0x3, } EContentData; - +/* typedef enum { READ_MODE_COUNT_ONLY = 0x1, READ_MODE_ALL, } EReadMode; - +*/ typedef struct { STbDataIter* iter; int32_t index; @@ -166,7 +166,7 @@ typedef struct SReaderStatus { SDataBlockIter blockIter; SLDataIter* pLDataIter; SRowMerger merger; - SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data + SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data } SReaderStatus; typedef struct SBlockInfoBuf { @@ -248,7 +248,7 @@ static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdb STbData* piMemTbData); static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr, int8_t* pLevel); -static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); +static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, const char* id); static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static int32_t doBuildDataBlock(STsdbReader* pReader); static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); @@ -292,7 +292,7 @@ static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInf if (j < pSupInfo->numOfCols && PRIMARYKEY_TIMESTAMP_COL_ID == pSupInfo->colId[j]) { j += 1; } - + while (i < pSchema->numOfCols && j < pSupInfo->numOfCols) { STColumn* pTCol = &pSchema->columns[i]; if (pTCol->colId == pSupInfo->colId[j]) { @@ -410,7 +410,7 @@ static int32_t uidComparFunc(const void* p1, const void* p2) { // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model static SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, - STableUidList* pUidList, int32_t numOfTables) { + STableUidList* pUidList, int32_t numOfTables) { // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); @@ -461,7 +461,7 @@ static SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBu } static void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { - void *p = NULL; + void* p = NULL; int32_t iter = 0; while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { @@ -505,7 +505,7 @@ static void clearBlockScanInfo(STableBlockScanInfo* p) { } static void destroyAllBlockScanInfo(SSHashObj* pTableMap) { - void* p = NULL; + void* p = NULL; int32_t iter = 0; while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { @@ -743,7 +743,8 @@ void tsdbReleaseDataBlock(STsdbReader* pReader) { } } -static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacity, SSDataBlock* pResBlock, SQueryTableDataCond* pCond) { +static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacity, SSDataBlock* pResBlock, + SQueryTableDataCond* pCond) { pResBlockInfo->capacity = capacity; pResBlockInfo->pResBlock = pResBlock; terrno = 0; @@ -779,7 +780,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void pReader->order = pCond->order; pReader->idStr = (idstr != NULL) ? taosStrdup(idstr) : NULL; - pReader->verRange = getQueryVerRange(pVnode, pCond, level); + pReader->verRange = getQueryVerRange(pVnode, pCond, idstr); pReader->type = pCond->type; pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket @@ -921,9 +922,9 @@ static void cleanupTableScanInfo(SReaderStatus* pStatus) { return; } - SSHashObj* pTableMap = pStatus->pTableMap; + SSHashObj* pTableMap = pStatus->pTableMap; STableBlockScanInfo** px = NULL; - int32_t iter = 0; + int32_t iter = 0; while (1) { px = tSimpleHashIterate(pTableMap, px, &iter); @@ -937,9 +938,10 @@ static void cleanupTableScanInfo(SReaderStatus* pStatus) { pStatus->mapDataCleaned = true; } -static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, SArray* pTableScanInfoList) { - size_t sizeInDisk = 0; - size_t numOfTables = taosArrayGetSize(pIndexList); +static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, + SArray* pTableScanInfoList) { + size_t sizeInDisk = 0; + size_t numOfTables = taosArrayGetSize(pIndexList); int64_t st = taosGetTimestampUs(); cleanupTableScanInfo(&pReader->status); @@ -1125,18 +1127,18 @@ static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData endPos = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, key, pReader->order); } - if ((pReader->verRange.maxVer >= pBlock->minVer && pReader->verRange.maxVer < pBlock->maxVer)|| + if ((pReader->verRange.maxVer >= pBlock->minVer && pReader->verRange.maxVer < pBlock->maxVer) || (pReader->verRange.minVer <= pBlock->maxVer && pReader->verRange.minVer > pBlock->minVer)) { int32_t i = endPos; if (asc) { - for(; i >= 0; --i) { + for (; i >= 0; --i) { if (pBlockData->aVersion[i] <= pReader->verRange.maxVer) { break; } } } else { - for(; i < pBlock->nRow; ++i) { + for (; i < pBlock->nRow; ++i) { if (pBlockData->aVersion[i] >= pReader->verRange.minVer) { break; } @@ -1309,17 +1311,17 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { ASSERT(pReader->verRange.minVer <= pBlock->maxVer && pReader->verRange.maxVer >= pBlock->minVer); // find the appropriate start position that satisfies the version requirement. - if ((pReader->verRange.maxVer >= pBlock->minVer && pReader->verRange.maxVer < pBlock->maxVer)|| + if ((pReader->verRange.maxVer >= pBlock->minVer && pReader->verRange.maxVer < pBlock->maxVer) || (pReader->verRange.minVer <= pBlock->maxVer && pReader->verRange.minVer > pBlock->minVer)) { int32_t i = pDumpInfo->rowIndex; if (asc) { - for(; i < pBlock->nRow; ++i) { + for (; i < pBlock->nRow; ++i) { if (pBlockData->aVersion[i] >= pReader->verRange.minVer) { break; } } } else { - for(; i >= 0; --i) { + for (; i >= 0; --i) { if (pBlockData->aVersion[i] <= pReader->verRange.maxVer) { break; } @@ -1562,7 +1564,8 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter, const char* idStr) return TSDB_CODE_SUCCESS; } -static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) { +static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, + SArray* pTableList) { bool asc = ASCENDING_TRAVERSE(pReader->order); SBlockOrderSupporter sup = {0}; @@ -1967,13 +1970,14 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc } TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - int64_t key = pRow->pBlockData->aTSKEY[pRow->iRow]; - int64_t ver = pRow->pBlockData->aVersion[pRow->iRow]; + int64_t key = pRow->pBlockData->aTSKEY[pRow->iRow]; + int64_t ver = pRow->pBlockData->aVersion[pRow->iRow]; pLastBlockReader->currentKey = key; pScanInfo->lastKeyInStt = key; - if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order, pVerRange)) { + if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order, + pVerRange)) { return true; } } @@ -2030,7 +2034,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* } STSchema* ptr = NULL; - int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; @@ -2153,7 +2157,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return terrno; } - int32_t code = tsdbRowMergerAdd(pMerger, pRow, pSchema); + int32_t code = tsdbRowMergerAdd(pMerger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2208,7 +2212,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, bool mergeBlockData) { - SRowMerger* pMerger = &pReader->status.merger; + SRowMerger* pMerger = &pReader->status.merger; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); @@ -2218,9 +2222,10 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); // create local variable to hold the row value - TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; + TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; - tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid, pReader->idStr); + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid, + pReader->idStr); // only last block exists if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { @@ -2240,7 +2245,8 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tsdbRowMergerAdd(pMerger, pRow1, NULL); - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->verRange, + pReader->idStr); code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { @@ -2290,7 +2296,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader* pLastBlockReader, int64_t key, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SRowMerger* pMerger = &pReader->status.merger; + SRowMerger* pMerger = &pReader->status.merger; // merge is not initialized yet, due to the fact that the pReader->pSchema is not initialized if (pMerger->pArray == NULL) { @@ -2316,7 +2322,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader if (key < ts) { // imem, mem are all empty, file blocks (data blocks and last block) exist return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key == ts) { - SRow* pTSRow = NULL; + SRow* pTSRow = NULL; int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2723,7 +2729,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc } else { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - SRow* pTSRow = NULL; + SRow* pTSRow = NULL; code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2837,11 +2843,11 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - bool asc = ASCENDING_TRAVERSE(pReader->order); - int64_t st = taosGetTimestampUs(); - int32_t step = asc ? 1 : -1; - double el = 0; - SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); + bool asc = ASCENDING_TRAVERSE(pReader->order); + int64_t st = taosGetTimestampUs(); + int32_t step = asc ? 1 : -1; + double el = 0; + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; STableBlockScanInfo* pBlockScanInfo = NULL; @@ -2874,7 +2880,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } } else { // file blocks not exist pBlockScanInfo = *pReader->status.pTableIter; - if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockScanInfo->uid, sizeof(pBlockScanInfo->uid))) { + if (pReader->pIgnoreTables && + taosHashGet(*pReader->pIgnoreTables, &pBlockScanInfo->uid, sizeof(pBlockScanInfo->uid))) { setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); return code; } @@ -3172,6 +3179,16 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { // load the last data block of current table STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; + if (pScanInfo == NULL) { + tsdbError("table Iter is null, invalid pScanInfo, try next table %s", pReader->idStr); + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + + continue; + } + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pScanInfo->uid, sizeof(pScanInfo->uid))) { // reset the index in last block when handing a new file doCleanupTableScanInfo(pScanInfo); @@ -3238,7 +3255,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { } static int32_t doBuildDataBlock(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; @@ -3261,7 +3278,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return terrno; } - initLastBlockReader(pLastBlockReader, pScanInfo, pReader); TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); @@ -3338,7 +3354,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } } - return (pReader->code != TSDB_CODE_SUCCESS)? pReader->code:code; + return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code; } static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { @@ -3493,14 +3509,15 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { } STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; - if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) { + if (pReader->pIgnoreTables && + taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) { bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { return TSDB_CODE_SUCCESS; } pBlockScanInfo = pStatus->pTableIter; } - + initMemDataIterator(*pBlockScanInfo, pReader); int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? INT64_MAX : INT64_MIN; @@ -3544,7 +3561,7 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { SBlockNumber num = {0}; - SArray* pTableList = taosArrayInit(40, POINTER_BYTES); + SArray* pTableList = taosArrayInit(40, POINTER_BYTES); int32_t code = moveToNextFile(pReader, &num, pTableList); if (code != TSDB_CODE_SUCCESS) { @@ -3589,7 +3606,7 @@ static ERetrieveType doReadDataFromLastFiles(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; SDataBlockIter* pBlockIter = &pReader->status.blockIter; - while(1) { + while (1) { terrno = 0; code = doLoadLastBlockSequentially(pReader); @@ -3612,7 +3629,7 @@ static ERetrieveType doReadDataFromLastFiles(STsdbReader* pReader) { return TSDB_READ_RETURN; } - if (pBlockIter->numOfBlocks > 0) { // there are data blocks existed. + if (pBlockIter->numOfBlocks > 0) { // there are data blocks existed. return TSDB_READ_CONTINUE; } else { // all blocks in data file are checked, let's check the data in last files resetTableListIndex(&pReader->status); @@ -3625,7 +3642,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { bool asc = ASCENDING_TRAVERSE(pReader->order); SDataBlockIter* pBlockIter = &pReader->status.blockIter; - SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; if (pBlockIter->numOfBlocks == 0) { // let's try to extract data from stt files. @@ -3726,7 +3743,7 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret return VND_TSDB(pVnode); } -SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) { +SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, const char* id) { int64_t startVer = (pCond->startVersion == -1) ? 0 : pCond->startVersion; int64_t endVer = 0; @@ -3737,10 +3754,14 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_ endVer = (pCond->endVersion > pVnode->state.applied) ? pVnode->state.applied : pCond->endVersion; } + tsdbDebug("queried verRange:%" PRId64 "-%" PRId64 ", revised query verRange:%" PRId64 "-%" PRId64 ", %s", + pCond->startVersion, pCond->endVersion, startVer, endVer, id); + return (SVersionRange){.minVer = startVer, .maxVer = endVer}; } -bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, SVersionRange* pVerRange) { +bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, + SVersionRange* pVerRange) { if (pDelList == NULL) { return false; } @@ -3758,8 +3779,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t return false; } else if (key == last->ts) { TSDBKEY* prev = taosArrayGet(pDelList, num - 2); - return (prev->version >= ver && prev->version <= pVerRange->maxVer && - prev->version >= pVerRange->minVer); + return (prev->version >= ver && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer); } } else { TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); @@ -3968,9 +3988,9 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SRowMerger* pMerger = &pReader->status.merger; - bool asc = ASCENDING_TRAVERSE(pReader->order); - int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; - int32_t step = asc ? 1 : -1; + bool asc = ASCENDING_TRAVERSE(pReader->order); + int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + int32_t step = asc ? 1 : -1; pDumpInfo->rowIndex += step; if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) { @@ -4067,14 +4087,14 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, return terrno; } - tsdbRowMergerAdd(&pReader->status.merger,pNextRow, pTSchema1); + tsdbRowMergerAdd(&pReader->status.merger, pNextRow, pTSchema1); } else { // let's merge rows in file block code = tsdbRowMergerAdd(&pReader->status.merger, ¤t, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - tsdbRowMergerAdd(&pReader->status.merger,pNextRow, NULL); + tsdbRowMergerAdd(&pReader->status.merger, pNextRow, NULL); } code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, pReader); @@ -4121,9 +4141,8 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tsdbRowMergerAdd(&pReader->status.merger,pRow, pSchema); - code = - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4362,7 +4381,7 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n int32_t size = tSimpleHashGetSize(pReader->status.pTableMap); STableBlockScanInfo** p = NULL; - int32_t iter = 0; + int32_t iter = 0; while ((p = tSimpleHashIterate(pReader->status.pTableMap, p, &iter)) != NULL) { clearBlockScanInfo(*p); @@ -4449,15 +4468,16 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { } static void freeSchemaFunc(void* param) { - void **p = (void **)param; + void** p = (void**)param; taosMemoryFreeClear(*p); } // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, - SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, SHashObj** pIgnoreTables) { + SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, + SHashObj** pIgnoreTables) { STimeWindow window = pCond->twindows; - SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); + SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); int32_t capacity = pConf->tsdbCfg.maxRows; if (pResBlock != NULL) { @@ -4726,7 +4746,7 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; - int32_t iter = 0; + int32_t iter = 0; while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; @@ -4748,7 +4768,7 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { } else { // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; - int32_t iter = 0; + int32_t iter = 0; while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; @@ -4947,8 +4967,9 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { *hasNext = false; - if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT || pReader->code != TSDB_CODE_SUCCESS) { - return (pReader->code != TSDB_CODE_SUCCESS)? pReader->code:code; + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT || + pReader->code != TSDB_CODE_SUCCESS) { + return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code; } SReaderStatus* pStatus = &pReader->status; @@ -5084,7 +5105,7 @@ static bool doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ return hasNullSMA; } -int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool *hasNullSMA) { +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) { SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg; int32_t code = 0; @@ -5193,9 +5214,9 @@ STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, c } static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; - int32_t code = TSDB_CODE_SUCCESS; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); + SReaderStatus* pStatus = &pReader->status; + int32_t code = TSDB_CODE_SUCCESS; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); if (pReader->code != TSDB_CODE_SUCCESS) { return NULL; @@ -5457,7 +5478,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { int32_t tsdbGetTableSchema(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { SMetaReader mr = {0}; - metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0); + metaReaderDoInit(&mr, ((SVnode*)pVnode)->pMeta, 0); int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c new file mode 100644 index 0000000000000000000000000000000000000000..d5d8ba130c632b0b50caa607ffcd7084b37cde28 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -0,0 +1,4957 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "osDef.h" +#include "tsdb.h" +#include "tsdbDataFileRW.h" +#include "tsdbFS2.h" +#include "tsdbMerge.h" +#include "tsdbReadUtil.h" +#include "tsdbUtil2.h" +#include "tsimplehash.h" + +#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define getCurrentKeyInLastBlock(_r) ((_r)->currentKey) + +static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); +static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, + STsdbReader* pReader); +static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); +static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader); +static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, + SRowMerger* pMerger, SVersionRange* pVerRange, const char* id); +static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, STsdbReader* pReader); +static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, + STableBlockScanInfo* pScanInfo); +static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, + int32_t rowIndex); +static void setComposedBlockFlag(STsdbReader* pReader, bool composed); +static bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, + SVersionRange* pVerRange); + +static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, + TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow); +static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader, SRow** pTSRow); +static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, + STsdbReader* pReader); + +static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost); +static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr, + int8_t* pLevel); +static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); +static int32_t doBuildDataBlock(STsdbReader* pReader); +static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); +static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo); +static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); +static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order); + +static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } + +static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, + int32_t numOfCols) { + pSupInfo->smaValid = true; + pSupInfo->numOfCols = numOfCols; + pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t) * 2 + POINTER_BYTES)); + if (pSupInfo->colId == NULL) { + taosMemoryFree(pSupInfo->colId); + return TSDB_CODE_OUT_OF_MEMORY; + } + + pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols)); + pSupInfo->buildBuf = (char**)((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols)); + for (int32_t i = 0; i < numOfCols; ++i) { + pSupInfo->colId[i] = pCols[i].colId; + pSupInfo->slotId[i] = pSlotIdList[i]; + + if (IS_VAR_DATA_TYPE(pCols[i].type)) { + pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); + } else { + pSupInfo->buildBuf[i] = NULL; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) { + int32_t i = 0, j = 0; + + while (i < pSchema->numOfCols && j < pSupInfo->numOfCols) { + STColumn* pTCol = &pSchema->columns[i]; + if (pTCol->colId == pSupInfo->colId[j]) { + if (!IS_BSMA_ON(pTCol)) { + pSupInfo->smaValid = false; + return TSDB_CODE_SUCCESS; + } + + i += 1; + j += 1; + } else if (pTCol->colId < pSupInfo->colId[j]) { // do nothing + i += 1; + } else { + return TSDB_CODE_INVALID_PARA; + } + } + + return TSDB_CODE_SUCCESS; +} + +static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) { return pWindow->skey > pWindow->ekey; } + +// Update the query time window according to the data time to live(TTL) information, in order to avoid to return +// the expired data to client, even it is queried already. +static STimeWindow updateQueryTimeWindow(STsdb* pTsdb, STimeWindow* pWindow) { + STsdbKeepCfg* pCfg = &pTsdb->keepCfg; + + int64_t now = taosGetTimestamp(pCfg->precision); + int64_t earilyTs = now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick + + STimeWindow win = *pWindow; + if (win.skey < earilyTs) { + win.skey = earilyTs; + } + + return win; +} + +// init file iterator +static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetArray, STsdbReader* pReader) { + size_t numOfFileset = TARRAY2_SIZE(pFileSetArray); + + pIter->index = ASCENDING_TRAVERSE(pReader->info.order) ? -1 : numOfFileset; + pIter->order = pReader->info.order; + pIter->pFilesetList = pFileSetArray; + pIter->numOfFiles = numOfFileset; + + if (pIter->pLastBlockReader == NULL) { + pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader)); + if (pIter->pLastBlockReader == NULL) { + int32_t code = TSDB_CODE_OUT_OF_MEMORY; + tsdbError("failed to prepare the last block iterator, since:%s %s", tstrerror(code), pReader->idStr); + return code; + } + } + + SLastBlockReader* pLReader = pIter->pLastBlockReader; + pLReader->order = pReader->info.order; + pLReader->window = pReader->info.window; + pLReader->verRange = pReader->info.verRange; + + pLReader->uid = 0; + tMergeTreeClose(&pLReader->mergeTree); + tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); + return TSDB_CODE_SUCCESS; +} + +static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bool* hasNext) { + bool asc = ASCENDING_TRAVERSE(pIter->order); + int32_t step = asc ? 1 : -1; + int32_t code = 0; + + pIter->index += step; + if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) { + *hasNext = false; + return TSDB_CODE_SUCCESS; + } + + SCostSummary* pSum = &pReader->cost; + + pIter->pLastBlockReader->uid = 0; + tMergeTreeClose(&pIter->pLastBlockReader->mergeTree); + + pReader->status.pLDataIterArray = + destroySttBlockReader(pReader->status.pLDataIterArray, &pSum->lastBlockLoad, &pSum->lastBlockLoadTime); + pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + + // check file the time range of coverage + STimeWindow win = {0}; + + while (1) { + if (pReader->pFileReader != NULL) { + tsdbDataFileReaderClose(&pReader->pFileReader); + } + + pReader->status.pCurrentFileset = pIter->pFilesetList->data[pIter->index]; + + STFileObj** pFileObj = pReader->status.pCurrentFileset->farr; + if (pFileObj[0] != NULL || pFileObj[3] != NULL) { + SDataFileReaderConfig conf = {.tsdb = pReader->pTsdb, .szPage = pReader->pTsdb->pVnode->config.tsdbPageSize}; + + const char* filesName[4] = {0}; + + if (pFileObj[0] != NULL) { + conf.files[0].file = *pFileObj[0]->f; + conf.files[0].exist = true; + filesName[0] = pFileObj[0]->fname; + + conf.files[1].file = *pFileObj[1]->f; + conf.files[1].exist = true; + filesName[1] = pFileObj[1]->fname; + + conf.files[2].file = *pFileObj[2]->f; + conf.files[2].exist = true; + filesName[2] = pFileObj[2]->fname; + } + + if (pFileObj[3] != NULL) { + conf.files[3].exist = true; + conf.files[3].file = *pFileObj[3]->f; + filesName[3] = pFileObj[3]->fname; + } + + code = tsdbDataFileReaderOpen(filesName, &conf, &pReader->pFileReader); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + pReader->cost.headFileLoad += 1; + } + + int32_t fid = pReader->status.pCurrentFileset->fid; + tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey); + + // current file are no longer overlapped with query time window, ignore remain files + if ((asc && win.skey > pReader->info.window.ekey) || (!asc && win.ekey < pReader->info.window.skey)) { + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pReader, + pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); + *hasNext = false; + return TSDB_CODE_SUCCESS; + } + + if ((asc && (win.ekey < pReader->info.window.skey)) || ((!asc) && (win.skey > pReader->info.window.ekey))) { + pIter->index += step; + if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) { + *hasNext = false; + return TSDB_CODE_SUCCESS; + } + continue; + } + + tsdbDebug("%p file found fid:%d for qrange:%" PRId64 "-%" PRId64 ", %s", pReader, fid, pReader->info.window.skey, + pReader->info.window.ekey, pReader->idStr); + *hasNext = true; + return TSDB_CODE_SUCCESS; + } + +_err: + *hasNext = false; + return code; +} + +static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order) { + pIter->order = order; + pIter->index = -1; + pIter->numOfBlocks = 0; + if (pIter->blockList == NULL) { + pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); + } else { + taosArrayClear(pIter->blockList); + } +} + +static void cleanupDataBlockIterator(SDataBlockIter* pIter) { taosArrayDestroy(pIter->blockList); } + +static void initReaderStatus(SReaderStatus* pStatus) { + pStatus->pTableIter = NULL; + pStatus->loadFromFile = true; +} + +static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) { + SSDataBlock* pResBlock = createDataBlock(); + if (pResBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for (int32_t i = 0; i < pCond->numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + colInfo.info = pCond->colList[i]; + blockDataAppendColInfo(pResBlock, &colInfo); + } + + int32_t code = blockDataEnsureCapacity(pResBlock, capacity); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + taosMemoryFree(pResBlock); + return NULL; + } + return pResBlock; +} + +static int32_t tsdbInitReaderLock(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexInit(&pReader->readerMutex, NULL); + + qTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +static int32_t tsdbUninitReaderLock(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexDestroy(&pReader->readerMutex); + + qTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +static int32_t tsdbAcquireReader(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexLock(&pReader->readerMutex); + + qTrace("tsdb/read: %p, post-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +static int32_t tsdbTryAcquireReader(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexTryLock(&pReader->readerMutex); + + qTrace("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +static int32_t tsdbReleaseReader(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-untake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexUnlock(&pReader->readerMutex); + + qTrace("tsdb/read: %p, post-untake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +void tsdbReleaseDataBlock2(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + if (!pStatus->composedDataBlock) { + tsdbReleaseReader(pReader); + } +} + +static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacity, SSDataBlock* pResBlock, + SQueryTableDataCond* pCond) { + pResBlockInfo->capacity = capacity; + pResBlockInfo->pResBlock = pResBlock; + terrno = 0; + + if (pResBlockInfo->pResBlock == NULL) { + pResBlockInfo->freeBlock = true; + pResBlockInfo->pResBlock = createResBlock(pCond, pResBlockInfo->capacity); + } else { + pResBlockInfo->freeBlock = false; + } + + return terrno; +} + +static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void** ppReader, int32_t capacity, + SSDataBlock* pResBlock, const char* idstr) { + int32_t code = 0; + int8_t level = 0; + STsdbReader* pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader)); + if (pReader == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (VND_IS_TSMA(pVnode)) { + tsdbDebug("vgId:%d, tsma is selected to query, %s", TD_VID(pVnode), idstr); + } + + initReaderStatus(&pReader->status); + + pReader->pTsdb = getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level); + pReader->info.suid = pCond->suid; + pReader->info.order = pCond->order; + + pReader->idStr = (idstr != NULL) ? taosStrdup(idstr) : NULL; + pReader->info.verRange = getQueryVerRange(pVnode, pCond, level); + pReader->type = pCond->type; + pReader->info.window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); + pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket + + code = initResBlockInfo(&pReader->resBlockInfo, capacity, pResBlock, pCond); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + + if (pCond->numOfCols <= 0) { + tsdbError("vgId:%d, invalid column number %d in query cond, %s", TD_VID(pVnode), pCond->numOfCols, idstr); + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + + // allocate buffer in order to load data blocks from file + SBlockLoadSuppInfo* pSup = &pReader->suppInfo; + pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; + setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); + + code = tBlockDataCreate(&pReader->status.fileBlockData); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + goto _end; + } + + if (pReader->suppInfo.colId[0] != PRIMARYKEY_TIMESTAMP_COL_ID) { + tsdbError("the first column isn't primary timestamp, %d, %s", pReader->suppInfo.colId[0], pReader->idStr); + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + + pReader->status.pPrimaryTsCol = taosArrayGet(pReader->resBlockInfo.pResBlock->pDataBlock, pSup->slotId[0]); + int32_t type = pReader->status.pPrimaryTsCol->info.type; + if (type != TSDB_DATA_TYPE_TIMESTAMP) { + tsdbError("the first column isn't primary timestamp in result block, actual: %s, %s", tDataTypes[type].name, + pReader->idStr); + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + + tsdbInitReaderLock(pReader); + + *ppReader = pReader; + return code; + +_end: + tsdbReaderClose(pReader); + *ppReader = NULL; + return code; +} + +static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileReader, SArray* pIndexList) { + int64_t st = taosGetTimestampUs(); + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + if (pFileReader == NULL) { + return TSDB_CODE_SUCCESS; + } + + const TBrinBlkArray* pBlkArray = NULL; + + int32_t code = tsdbDataFileReadBrinBlk(pFileReader, &pBlkArray); + +#if 0 + LRUHandle* handle = NULL; + + int32_t code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle); + if (code != TSDB_CODE_SUCCESS || handle == NULL) { + goto _end; + } + + + SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle); + size_t num = taosArrayGetSize(aBlockIdx); + if (num == 0) { + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return TSDB_CODE_SUCCESS; + } +#endif + + // todo binary search to the start position + int64_t et1 = taosGetTimestampUs(); + + SBrinBlk* pBrinBlk = NULL; + STableUidList* pList = &pReader->status.uidList; + + int32_t i = 0; + + while (i < TARRAY2_SIZE(pBlkArray)) { + pBrinBlk = &pBlkArray->data[i]; + if (pBrinBlk->maxTbid.suid < pReader->info.suid) { + i += 1; + continue; + } + + if (pBrinBlk->minTbid.suid > pReader->info.suid) { // not include the queried table/super table, quit the loop + break; + } + + ASSERT(pBrinBlk->minTbid.suid <= pReader->info.suid && pBrinBlk->maxTbid.suid >= pReader->info.suid); + if (pBrinBlk->maxTbid.suid == pReader->info.suid && pBrinBlk->maxTbid.uid < pList->tableUidList[0]) { + i += 1; + continue; + } + + if (pBrinBlk->minTbid.suid == pReader->info.suid && pBrinBlk->minTbid.uid > pList->tableUidList[numOfTables - 1]) { + break; + } + + taosArrayPush(pIndexList, pBrinBlk); + i += 1; + } + + int64_t et2 = taosGetTimestampUs(); + tsdbDebug("load block index for %d/%d tables completed, elapsed time:%.2f ms, set BrinBlk:%.2f ms, size:%.2f Kb %s", + numOfTables, (int32_t)pBlkArray->size, (et1 - st) / 1000.0, (et2 - et1) / 1000.0, + pBlkArray->size * sizeof(SBrinBlk) / 1024.0, pReader->idStr); + + pReader->cost.headFileLoadTime += (et1 - st) / 1000.0; + +_end: + // tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return code; +} + +static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, + SArray* pTableScanInfoList) { + size_t sizeInDisk = 0; + int64_t st = taosGetTimestampUs(); + + // clear info for the new file + cleanupInfoFoxNextFileset(pReader->status.pTableMap); + + int32_t k = 0; + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; + STimeWindow w = pReader->info.window; + SBrinRecord* pRecord = NULL; + + SBrinRecordIter iter = {0}; + initBrinRecordIter(&iter, pReader->pFileReader, pIndexList); + + while (((pRecord = getNextBrinRecord(&iter)) != NULL)) { + if (pRecord->suid > pReader->info.suid) { + break; + } + + uint64_t uid = pReader->status.uidList.tableUidList[k]; + if (pRecord->suid < pReader->info.suid) { + continue; + } + + if (uid < pRecord->uid) { // forward the table uid index + while (k < numOfTables && pReader->status.uidList.tableUidList[k] < pRecord->uid) { + k += 1; + } + + if (k >= numOfTables) { + break; + } + + uid = pReader->status.uidList.tableUidList[k]; + } + + if (pRecord->uid < uid) { + continue; + } + + ASSERT(pRecord->suid == pReader->info.suid && uid == pRecord->uid); + + STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + if (ASCENDING_TRAVERSE(pReader->info.order)) { + w.skey = pScanInfo->lastKey + step; + } else { + w.ekey = pScanInfo->lastKey + step; + } + + if (isEmptyQueryTimeWindow(&w)) { + k += 1; + continue; + } + + // 1. time range check + if (pRecord->firstKey > w.ekey || pRecord->lastKey < w.skey) { + continue; + } + + // 2. version range check + if (pRecord->minVer > pReader->info.verRange.maxVer || pRecord->maxVer < pReader->info.verRange.minVer) { + continue; + } + + if (pScanInfo->pBlockList == NULL) { + pScanInfo->pBlockList = taosArrayInit(4, sizeof(SBrinRecord)); + } + + void* p1 = taosArrayPush(pScanInfo->pBlockList, pRecord); + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pBlockNum->numOfBlocks += 1; + if (taosArrayGetSize(pTableScanInfoList) == 0) { + taosArrayPush(pTableScanInfoList, &pScanInfo); + } else { + STableBlockScanInfo** p = taosArrayGetLast(pTableScanInfoList); + if ((*p)->uid != uid) { + taosArrayPush(pTableScanInfoList, &pScanInfo); + } + } + } + + clearBrinBlockIter(&iter); + + pBlockNum->numOfLastFiles = pReader->status.pCurrentFileset->lvlArr->size; + int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks; + + double el = (taosGetTimestampUs() - st) / 1000.0; + tsdbDebug( + "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " + "time:%.2f ms %s", + numOfTables, pBlockNum->numOfBlocks, (int32_t)taosArrayGetSize(pTableScanInfoList), pBlockNum->numOfLastFiles, + sizeInDisk / 1000.0, el, pReader->idStr); + + pReader->cost.numOfBlocks += total; + pReader->cost.headFileLoadTime += el; + + return TSDB_CODE_SUCCESS; +} + +static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, int64_t maxKey, int32_t order) { + int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1; + pDumpInfo->allDumped = true; + pDumpInfo->lastKey = maxKey + step; +} + +static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, + SBlockLoadSuppInfo* pSup) { + if (IS_VAR_DATA_TYPE(pColVal->type)) { + if (!COL_VAL_IS_VALUE(pColVal)) { + colDataSetNULL(pColInfoData, rowIndex); + } else { + varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData); + if (pColVal->value.nData > pColInfoData->info.bytes) { + tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, pColVal->value.nData, + pColInfoData->info.bytes); + return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + } + if (pColVal->value.nData > 0) { // pData may be null, if nData is 0 + memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); + } + + colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); + } + } else { + colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value, !COL_VAL_IS_VALUE(pColVal)); + } + + return TSDB_CODE_SUCCESS; +} + +static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { + size_t num = TARRAY_SIZE(pBlockIter->blockList); + if (num == 0) { + ASSERT(pBlockIter->numOfBlocks == num); + return NULL; + } + + SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); + return pBlockInfo; +} + +static int doBinarySearchKey(TSKEY* keyList, int num, int pos, TSKEY key, int order) { + // start end position + int s, e; + s = pos; + + // check + ASSERT(pos >= 0 && pos < num && num > 0); + if (order == TSDB_ORDER_ASC) { + // find the first position which is smaller than the key + e = num - 1; + if (key < keyList[pos]) return -1; + while (1) { + // check can return + if (key >= keyList[e]) return e; + if (key <= keyList[s]) return s; + if (e - s <= 1) return s; + + // change start or end position + int mid = s + (e - s + 1) / 2; + if (keyList[mid] > key) + e = mid; + else if (keyList[mid] < key) + s = mid; + else + return mid; + } + } else { // DESC + // find the first position which is bigger than the key + e = 0; + if (key > keyList[pos]) return -1; + while (1) { + // check can return + if (key <= keyList[e]) return e; + if (key >= keyList[s]) return s; + if (s - e <= 1) return s; + + // change start or end position + int mid = s - (s - e + 1) / 2; + if (keyList[mid] < key) + e = mid; + else if (keyList[mid] > key) + s = mid; + else + return mid; + } + } +} + +static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SBrinRecord* pRecord, int32_t pos) { + // NOTE: reverse the order to find the end position in data block + int32_t endPos = -1; + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + + if (asc && pReader->info.window.ekey >= pRecord->lastKey) { + endPos = pRecord->numRow - 1; + } else if (!asc && pReader->info.window.skey <= pRecord->firstKey) { + endPos = 0; + } else { + int64_t key = asc ? pReader->info.window.ekey : pReader->info.window.skey; + endPos = doBinarySearchKey(pBlockData->aTSKEY, pRecord->numRow, pos, key, pReader->info.order); + } + + if ((pReader->info.verRange.maxVer >= pRecord->minVer && pReader->info.verRange.maxVer < pRecord->maxVer) || + (pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.minVer > pRecord->minVer)) { + int32_t i = endPos; + + if (asc) { + for (; i >= 0; --i) { + if (pBlockData->aVersion[i] <= pReader->info.verRange.maxVer) { + break; + } + } + } else { + for (; i < pRecord->numRow; ++i) { + if (pBlockData->aVersion[i] >= pReader->info.verRange.minVer) { + break; + } + } + } + + endPos = i; + } + + return endPos; +} + +static void copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, + int32_t dumpedRows, bool asc) { + if (asc) { + memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t)); + } else { + int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; + memcpy(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t)); + + // todo: opt perf by extract the loop + // reverse the array list + int32_t mid = dumpedRows >> 1u; + int64_t* pts = (int64_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + } +} + +// a faster version of copy procedure. +static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, + int32_t dumpedRows, bool asc) { + uint8_t* p = NULL; + if (asc) { + p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; + } else { + int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; + p = pData->pData + tDataTypes[pData->type].bytes * startIndex; + } + + int32_t step = asc ? 1 : -1; + + // make sure it is aligned to 8bit, the allocated memory address is aligned to 256bit + // ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); + + // 1. copy data in a batch model + memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); + + // 2. reverse the array list in case of descending order scan data block + if (!asc) { + switch (pColData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: { + int32_t mid = dumpedRows >> 1u; + int64_t* pts = (int64_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: { + int32_t mid = dumpedRows >> 1u; + int8_t* pts = (int8_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int8_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: { + int32_t mid = dumpedRows >> 1u; + int16_t* pts = (int16_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: { + int32_t mid = dumpedRows >> 1u; + int32_t* pts = (int32_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int32_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + } + } + + // 3. if the null value exists, check items one-by-one + if (pData->flag != HAS_VALUE) { + int32_t rowIndex = 0; + + for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step, rowIndex++) { + uint8_t v = tColDataGetBitValue(pData, j); + if (v == 0 || v == 1) { + colDataSetNull_f(pColData->nullbitmap, rowIndex); + pColData->hasNull = true; + } + } + } +} + +static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + SBlockData* pBlockData = &pStatus->fileBlockData; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + int32_t numOfOutputCols = pSupInfo->numOfCols; + int32_t code = TSDB_CODE_SUCCESS; + + SColVal cv = {0}; + int64_t st = taosGetTimestampUs(); + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + int32_t step = asc ? 1 : -1; + + SBrinRecord* pRecord = &pBlockInfo->record; + + // no data exists, return directly. + if (pBlockData->nRow == 0 || pBlockData->aTSKEY == 0) { + tsdbWarn("%p no need to copy since no data in blockData, table uid:%" PRIu64 " has been dropped, %s", pReader, + pBlockInfo->uid, pReader->idStr); + pResBlock->info.rows = 0; + return 0; + } + + // row index of dump info remain the initial position, let's find the appropriate start position. + if ((pDumpInfo->rowIndex == 0 && asc) || (pDumpInfo->rowIndex == pRecord->numRow - 1 && (!asc))) { + if (asc && pReader->info.window.skey <= pRecord->firstKey && pReader->info.verRange.minVer <= pRecord->minVer) { + // pDumpInfo->rowIndex = 0; + } else if (!asc && pReader->info.window.ekey >= pRecord->lastKey && + pReader->info.verRange.maxVer >= pRecord->maxVer) { + // pDumpInfo->rowIndex = pRecord->numRow - 1; + } else { // find the appropriate the start position in current block, and set it to be the current rowIndex + int32_t pos = asc ? pRecord->numRow - 1 : 0; + int32_t order = asc ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; + int64_t key = asc ? pReader->info.window.skey : pReader->info.window.ekey; + pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pRecord->numRow, pos, key, order); + + if (pDumpInfo->rowIndex < 0) { + tsdbError( + "%p failed to locate the start position in current block, global index:%d, table index:%d, brange:%" PRId64 + "-%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 " %s", + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey, pRecord->lastKey, pRecord->minVer, + pRecord->maxVer, pReader->idStr); + return TSDB_CODE_INVALID_PARA; + } + + ASSERT(pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.maxVer >= pRecord->minVer); + + // find the appropriate start position that satisfies the version requirement. + if ((pReader->info.verRange.maxVer >= pRecord->minVer && pReader->info.verRange.maxVer < pRecord->maxVer) || + (pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.minVer > pRecord->minVer)) { + int32_t i = pDumpInfo->rowIndex; + if (asc) { + for (; i < pRecord->numRow; ++i) { + if (pBlockData->aVersion[i] >= pReader->info.verRange.minVer) { + break; + } + } + } else { + for (; i >= 0; --i) { + if (pBlockData->aVersion[i] <= pReader->info.verRange.maxVer) { + break; + } + } + } + + pDumpInfo->rowIndex = i; + } + } + } + + // time window check + int32_t endIndex = getEndPosInDataBlock(pReader, pBlockData, pRecord, pDumpInfo->rowIndex); + if (endIndex == -1) { + setBlockAllDumped(pDumpInfo, pReader->info.window.ekey, pReader->info.order); + return TSDB_CODE_SUCCESS; + } + + endIndex += step; + int32_t dumpedRows = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex); + if (dumpedRows > pReader->resBlockInfo.capacity) { // output buffer check + dumpedRows = pReader->resBlockInfo.capacity; + } else if (dumpedRows <= 0) { // no qualified rows in current data block, abort directly. + setBlockAllDumped(pDumpInfo, pReader->info.window.ekey, pReader->info.order); + return TSDB_CODE_SUCCESS; + } + + int32_t i = 0; + int32_t rowIndex = 0; + + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { + copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); + i += 1; + } + + int32_t colIndex = 0; + int32_t num = pBlockData->nColData; + while (i < numOfOutputCols && colIndex < num) { + rowIndex = 0; + + SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex); + if (pData->cid < pSupInfo->colId[i]) { + colIndex += 1; + } else if (pData->cid == pSupInfo->colId[i]) { + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + + if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { + colDataSetNNULL(pColData, 0, dumpedRows); + } else { + if (IS_MATHABLE_TYPE(pColData->info.type)) { + copyNumericCols(pData, pDumpInfo, pColData, dumpedRows, asc); + } else { // varchar/nchar type + for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) { + tColDataGetValue(pData, j, &cv); + code = doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); + if (code) { + return code; + } + } + } + } + + colIndex += 1; + i += 1; + } else { // the specified column does not exist in file block, fill with null data + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + colDataSetNNULL(pColData, 0, dumpedRows); + i += 1; + } + } + + // fill the mis-matched columns with null value + while (i < numOfOutputCols) { + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + colDataSetNNULL(pColData, 0, dumpedRows); + i += 1; + } + + pResBlock->info.dataLoad = 1; + pResBlock->info.rows = dumpedRows; + pDumpInfo->rowIndex += step * dumpedRows; + + // check if current block are all handled + if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pRecord->numRow) { + int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + if (outOfTimeWindow(ts, + &pReader->info.window)) { // the remain data has out of query time window, ignore current block + setBlockAllDumped(pDumpInfo, ts, pReader->info.order); + } + } else { + int64_t ts = asc ? pRecord->lastKey : pRecord->firstKey; + setBlockAllDumped(pDumpInfo, ts, pReader->info.order); + } + + double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; + pReader->cost.blockLoadTime += elapsedTime; + + int32_t unDumpedRows = asc ? pRecord->numRow - pDumpInfo->rowIndex : pDumpInfo->rowIndex + 1; + tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 + ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", uid:%" PRIu64 " elapsed time:%.2f ms, %s", + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey, pRecord->lastKey, dumpedRows, + unDumpedRows, pRecord->minVer, pRecord->maxVer, pBlockInfo->uid, elapsedTime, pReader->idStr); + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE STSchema* getTableSchemaImpl(STsdbReader* pReader, uint64_t uid) { + ASSERT(pReader->info.pSchema == NULL); + + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, -1, &pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS || pReader->info.pSchema == NULL) { + terrno = code; + tsdbError("failed to get table schema, uid:%" PRIu64 ", it may have been dropped, ver:-1, %s", uid, pReader->idStr); + return NULL; + } + + code = tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + tsdbError("failed to init merger, code:%s, %s", tstrerror(code), pReader->idStr); + return NULL; + } + + return pReader->info.pSchema; +} + +static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData, + uint64_t uid) { + int32_t code = 0; + STSchema* pSchema = pReader->info.pSchema; + int64_t st = taosGetTimestampUs(); + + tBlockDataReset(pBlockData); + + if (pReader->info.pSchema == NULL) { + pSchema = getTableSchemaImpl(pReader, uid); + if (pSchema == NULL) { + tsdbDebug("%p table uid:%" PRIu64 " has been dropped, no data existed, %s", pReader, uid, pReader->idStr); + return code; + } + } + + SBlockLoadSuppInfo* pSup = &pReader->suppInfo; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + SBrinRecord* pRecord = &pBlockInfo->record; + code = tsdbDataFileReadBlockDataByColumn(pReader->pFileReader, pRecord, pBlockData, pSchema, &pSup->colId[1], + pSup->numOfCols - 1); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 + ", rows:%d, code:%s %s", + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.firstKey, + pBlockInfo->record.lastKey, pBlockInfo->record.numRow, tstrerror(code), pReader->idStr); + return code; + } + + double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; + + tsdbDebug("%p load file block into buffer, global index:%d, index in table block list:%d, brange:%" PRId64 "-%" PRId64 + ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey, pRecord->lastKey, pRecord->numRow, + pRecord->minVer, pRecord->maxVer, elapsedTime, pReader->idStr); + + pReader->cost.blockLoadTime += elapsedTime; + pDumpInfo->allDumped = false; + + return TSDB_CODE_SUCCESS; +} + +/** + * This is an two rectangles overlap cases. + */ +static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SFileDataBlockInfo* pBlock) { + return (pWindow->ekey < pBlock->record.lastKey && pWindow->ekey >= pBlock->record.firstKey) || + (pWindow->skey > pBlock->record.firstKey && pWindow->skey <= pBlock->record.lastKey) || + (pVerRange->minVer > pBlock->record.minVer && pVerRange->minVer <= pBlock->record.maxVer) || + (pVerRange->maxVer < pBlock->record.maxVer && pVerRange->maxVer >= pBlock->record.minVer); +} + +static bool getNeighborBlockOfSameTable(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, + int32_t* nextIndex, int32_t order, SBrinRecord* pRecord) { + bool asc = ASCENDING_TRAVERSE(order); + if (asc && pBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockList) - 1) { + return false; + } + + if (!asc && pBlockInfo->tbBlockIdx == 0) { + return false; + } + + int32_t step = asc ? 1 : -1; + // *nextIndex = pBlockInfo->tbBlockIdx + step; + // *pBlockIndex = *(SBlockIndex*)taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); + SBrinRecord* p = taosArrayGet(pTableBlockScanInfo->pBlockList, pBlockInfo->tbBlockIdx + step); + memcpy(pRecord, p, sizeof(SBrinRecord)); + + // tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, pIndex->ordinalIndex, pBlock, tGetDataBlk); + return true; +} + +static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pFBlockInfo) { + int32_t step = ASCENDING_TRAVERSE(pBlockIter->order) ? 1 : -1; + int32_t index = pBlockIter->index; + + while (index < pBlockIter->numOfBlocks && index >= 0) { + SFileDataBlockInfo* pFBlock = taosArrayGet(pBlockIter->blockList, index); + if (pFBlock->uid == pFBlockInfo->uid && pFBlock->tbBlockIdx == pFBlockInfo->tbBlockIdx) { + return index; + } + + index += step; + } + + return -1; +} + +static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t index, int32_t step) { + if (index < 0 || index >= pBlockIter->numOfBlocks) { + return -1; + } + + SFileDataBlockInfo fblock = *(SFileDataBlockInfo*)taosArrayGet(pBlockIter->blockList, index); + pBlockIter->index += step; + + if (index != pBlockIter->index) { + taosArrayRemove(pBlockIter->blockList, index); + taosArrayInsert(pBlockIter->blockList, pBlockIter->index, &fblock); + + SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); + ASSERT(pBlockInfo->uid == fblock.uid && pBlockInfo->tbBlockIdx == fblock.tbBlockIdx); + } + + return TSDB_CODE_SUCCESS; +} + +// todo: this attribute could be acquired during extractin the global ordered block list. +static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order) { + // it is the last block in current file, no chance to overlap with neighbor blocks. + if (ASCENDING_TRAVERSE(order)) { + return pBlock->record.lastKey == pRec->firstKey; + } else { + return pBlock->record.firstKey == pRec->lastKey; + } +} + +static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SFileDataBlockInfo* pBlock) { + bool ascScan = ASCENDING_TRAVERSE(order); + + return (ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts <= pBlock->record.firstKey)) || + (!ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts >= pBlock->record.lastKey)); +} + +static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersionRange* pVerRange) { + return (key.ts >= pBlock->record.firstKey && key.ts <= pBlock->record.lastKey) && + (pBlock->record.maxVer >= pVerRange->minVer) && (pBlock->record.minVer <= pVerRange->maxVer); +} + +static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, + int32_t startIndex) { + size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); + + for (int32_t i = startIndex; i < num; i += 1) { + TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i); + if (p->ts >= pRecord->firstKey && p->ts <= pRecord->lastKey) { + if (p->version >= pRecord->minVer) { + return true; + } + } else if (p->ts < pRecord->firstKey) { // p->ts < pBlock->minKey.ts + if (p->version >= pRecord->minVer) { + if (i < num - 1) { + TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1); + if (pnext->ts >= pRecord->firstKey) { + return true; + } + } else { // it must be the last point + ASSERT(p->version == 0); + } + } + } else { // (p->ts > pBlock->maxKey.ts) { + return false; + } + } + + return false; +} + +static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order) { + if (pBlockScanInfo->delSkyline == NULL || (taosArrayGetSize(pBlockScanInfo->delSkyline) == 0)) { + return false; + } + + // ts is not overlap + TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0); + TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline); + if (pRecord->firstKey > pLast->ts || pRecord->lastKey < pFirst->ts) { + return false; + } + + // version is not overlap + if (ASCENDING_TRAVERSE(order)) { + return doCheckforDatablockOverlap(pBlockScanInfo, pRecord, pBlockScanInfo->fileDelIndex); + } else { + int32_t index = pBlockScanInfo->fileDelIndex; + while (1) { + TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index); + if (p->ts > pRecord->firstKey && index > 0) { + index -= 1; + } else { // find the first point that is smaller than the minKey.ts of dataBlock. + if (p->ts == pRecord->firstKey && p->version < pRecord->maxVer && index > 0) { + index -= 1; + } + break; + } + } + + return doCheckforDatablockOverlap(pBlockScanInfo, pRecord, index); + } +} + +typedef struct { + bool overlapWithNeighborBlock; + bool hasDupTs; + bool overlapWithDelInfo; + bool overlapWithLastBlock; + bool overlapWithKeyInBuf; + bool partiallyRequired; + bool moreThanCapcity; +} SDataBlockToLoadInfo; + +static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, + STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader, + STsdbReader* pReader) { + int32_t neighborIndex = 0; + SBrinRecord rec = {0}; + + bool hasNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pScanInfo, &neighborIndex, pReader->info.order, &rec); + + // overlap with neighbor + if (hasNeighbor) { + pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock2(pBlockInfo, &rec, pReader->info.order); + } + + // has duplicated ts of different version in this block + pInfo->hasDupTs = (pBlockInfo->record.numRow > pBlockInfo->record.count); + pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pBlockInfo->record, pReader->info.order); + + if (hasDataInLastBlock(pLastBlockReader)) { + int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + pInfo->overlapWithLastBlock = !(pBlockInfo->record.lastKey < tsLast || pBlockInfo->record.firstKey > tsLast); + } + + pInfo->moreThanCapcity = pBlockInfo->record.numRow > pReader->resBlockInfo.capacity; + pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->info.window, &pReader->info.verRange, pBlockInfo); + pInfo->overlapWithKeyInBuf = keyOverlapFileBlock(keyInBuf, pBlockInfo, &pReader->info.verRange); +} + +// 1. the version of all rows should be less than the endVersion +// 2. current block should not overlap with next neighbor block +// 3. current timestamp should not be overlap with each other +// 4. output buffer should be large enough to hold all rows in current block +// 5. delete info should not overlap with current block data +// 6. current block should not contain the duplicated ts +static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, + TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) { + SDataBlockToLoadInfo info = {0}; + getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pLastBlockReader, pReader); + + bool loadDataBlock = + (info.overlapWithNeighborBlock || info.hasDupTs || info.partiallyRequired || info.overlapWithKeyInBuf || + info.moreThanCapcity || info.overlapWithDelInfo || info.overlapWithLastBlock); + + // log the reason why load the datablock for profile + if (loadDataBlock) { + tsdbDebug("%p uid:%" PRIu64 + " need to load the datablock, overlapneighbor:%d, hasDup:%d, partiallyRequired:%d, " + "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s", + pReader, pBlockInfo->uid, info.overlapWithNeighborBlock, info.hasDupTs, info.partiallyRequired, + info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithLastBlock, + pReader->idStr); + } + + return loadDataBlock; +} + +static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, + TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) { + SDataBlockToLoadInfo info = {0}; + getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pLastBlockReader, pReader); + bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || + info.overlapWithDelInfo || info.overlapWithLastBlock); + return isCleanFileBlock; +} + +static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, int64_t endKey) { + if (!(pBlockScanInfo->iiter.hasVal || pBlockScanInfo->iter.hasVal)) { + return TSDB_CODE_SUCCESS; + } + + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + + int64_t st = taosGetTimestampUs(); + int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->resBlockInfo.capacity, pReader); + + blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotId[0]); + pBlock->info.id.uid = pBlockScanInfo->uid; + + setComposedBlockFlag(pReader, true); + + double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; + tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%" PRId64 ", brange:%" PRId64 + " - %" PRId64 ", uid:%" PRIu64 ", %s", + pReader, elapsedTime, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, + pBlockScanInfo->uid, pReader->idStr); + + pReader->cost.buildmemBlock += elapsedTime; + return code; +} + +static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, int64_t key, + SFileBlockDumpInfo* pDumpInfo, bool* copied) { + // opt version + // 1. it is not a border point + // 2. the direct next point is not an duplicated timestamp + int32_t code = TSDB_CODE_SUCCESS; + + *copied = false; + bool asc = (pReader->info.order == TSDB_ORDER_ASC); + if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) { + int32_t step = pReader->info.order == TSDB_ORDER_ASC ? 1 : -1; + + int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; + if (nextKey != key) { // merge is not needed + code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); + if (code) { + return code; + } + pDumpInfo->rowIndex += step; + *copied = true; + } + } + + return code; +} + +static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, + SVersionRange* pVerRange) { + int32_t step = ASCENDING_TRAVERSE(pLastBlockReader->order) ? 1 : -1; + + while (1) { + bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); + if (!hasVal) { // the next value will be the accessed key in stt + pScanInfo->lastKeyInStt += step; + return false; + } + + TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + int64_t key = pRow->pBlockData->aTSKEY[pRow->iRow]; + int64_t ver = pRow->pBlockData->aVersion[pRow->iRow]; + + pLastBlockReader->currentKey = key; + pScanInfo->lastKeyInStt = key; + + if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order, + pVerRange)) { + return true; + } + } +} + +static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader, + STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader, + bool* copied) { + int32_t code = TSDB_CODE_SUCCESS; + + *copied = false; + + bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo, &pReader->info.verRange); + if (hasVal) { + int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); + if (next1 != ts) { + code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow); + if (code) { + return code; + } + + *copied = true; + return code; + } + } else { + code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow); + if (code) { + return code; + } + + *copied = true; + return code; + } + + return code; +} + +static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { + // always set the newest schema version in pReader->info.pSchema + if (pReader->info.pSchema == NULL) { + STSchema* ps = getTableSchemaImpl(pReader, uid); + if (ps == NULL) { + return NULL; + } + } + + if (pReader->info.pSchema && sversion == pReader->info.pSchema->version) { + return pReader->info.pSchema; + } + + void** p = tSimpleHashGet(pReader->pSchemaMap, &sversion, sizeof(sversion)); + if (p != NULL) { + return *(STSchema**)p; + } + + STSchema* ptr = NULL; + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, sversion, &ptr); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } else { + code = tSimpleHashPut(pReader->pSchemaMap, &sversion, sizeof(sversion), &ptr, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + return ptr; + } +} + +static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, + SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { + SRowMerger* pMerger = &pReader->status.merger; + SRow* pTSRow = NULL; + SBlockData* pBlockData = &pReader->status.fileBlockData; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + int64_t tsLast = INT64_MIN; + if (hasDataInLastBlock(pLastBlockReader)) { + tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + } + + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + + // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized + if (pMerger->pArray == NULL) { + ASSERT(pReader->info.pSchema == NULL); + STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); + if (ps == NULL) { + return terrno; + } + } + + int64_t minKey = 0; + if (pReader->info.order == TSDB_ORDER_ASC) { + minKey = INT64_MAX; // chosen the minimum value + if (minKey > tsLast && hasDataInLastBlock(pLastBlockReader)) { + minKey = tsLast; + } + + if (minKey > k.ts) { + minKey = k.ts; + } + + if (minKey > key && hasDataInFileBlock(pBlockData, pDumpInfo)) { + minKey = key; + } + } else { + minKey = INT64_MIN; + if (minKey < tsLast && hasDataInLastBlock(pLastBlockReader)) { + minKey = tsLast; + } + + if (minKey < k.ts) { + minKey = k.ts; + } + + if (minKey < key && hasDataInFileBlock(pBlockData, pDumpInfo)) { + minKey = key; + } + } + + // todo remove init + bool init = false; + + // ASC: file block ---> last block -----> imem -----> mem + // DESC: mem -----> imem -----> last block -----> file block + if (pReader->info.order == TSDB_ORDER_ASC) { + if (minKey == key) { + init = true; + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + } + + if (minKey == tsLast) { + TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + if (init) { + tsdbRowMergerAdd(pMerger, fRow1, NULL); + } else { + init = true; + int32_t code = tsdbRowMergerAdd(pMerger, fRow1, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, + pReader->idStr); + } + + if (minKey == k.ts) { + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return terrno; + } + if (init) { + tsdbRowMergerAdd(pMerger, pRow, pSchema); + } else { + init = true; + int32_t code = tsdbRowMergerAdd(pMerger, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + int32_t code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } else { + if (minKey == k.ts) { + init = true; + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return terrno; + } + + int32_t code = tsdbRowMergerAdd(pMerger, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS || pMerger->pTSchema == NULL) { + return code; + } + } + + if (minKey == tsLast) { + TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + if (init) { + tsdbRowMergerAdd(pMerger, fRow1, NULL); + } else { + init = true; + int32_t code = tsdbRowMergerAdd(pMerger, fRow1, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, + pReader->idStr); + } + + if (minKey == key) { + if (init) { + tsdbRowMergerAdd(pMerger, &fRow, NULL); + } else { + init = true; + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + } + } + + int32_t code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + + return code; +} + +static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader, + STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, + bool mergeBlockData) { + SRowMerger* pMerger = &pReader->status.merger; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + bool copied = false; + int32_t code = TSDB_CODE_SUCCESS; + SRow* pTSRow = NULL; + TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + + // create local variable to hold the row value + TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; + + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid, + pReader->idStr); + + // only last block exists + if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { + code = tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); + if (code) { + return code; + } + + if (copied) { + pBlockScanInfo->lastKey = tsLastBlock; + return TSDB_CODE_SUCCESS; + } else { + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, + pReader->idStr); + + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } else { // not merge block data + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, + pReader->idStr); + + // merge with block data if ts == key + if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + } + + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader* pLastBlockReader, int64_t key, + STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + SRowMerger* pMerger = &pReader->status.merger; + + // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized + if (pMerger->pArray == NULL) { + ASSERT(pReader->info.pSchema == NULL); + STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); + if (ps == NULL) { + return terrno; + } + } + + if (hasDataInFileBlock(pBlockData, pDumpInfo)) { + // no last block available, only data block exists + if (!hasDataInLastBlock(pLastBlockReader)) { + return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); + } + + // row in last file block + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + if (ASCENDING_TRAVERSE(pReader->info.order)) { + if (key < tsLast) { + return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); + } else if (key > tsLast) { + return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, NULL, false); + } + } else { + if (key > tsLast) { + return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); + } else if (key < tsLast) { + return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, NULL, false); + } + } + // the following for key == tsLast + SRow* pTSRow = NULL; + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); + + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); + + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + return code; + + } else { // only last block exists + return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, NULL, false); + } +} + +static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, + SLastBlockReader* pLastBlockReader) { + SRowMerger* pMerger = &pReader->status.merger; + SRow* pTSRow = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + SArray* pDelList = pBlockScanInfo->delSkyline; + + TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader); + TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); + + int64_t tsLast = INT64_MIN; + if (hasDataInLastBlock(pLastBlockReader)) { + tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + } + + int64_t key = hasDataInFileBlock(pBlockData, pDumpInfo) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; + + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return code; + } + + STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + if (piSchema == NULL) { + return code; + } + + // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized + if (pMerger->pArray == NULL) { + ASSERT(pReader->info.pSchema == NULL); + STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); + if (ps == NULL) { + return terrno; + } + } + + int64_t minKey = 0; + if (ASCENDING_TRAVERSE(pReader->info.order)) { + minKey = INT64_MAX; // let's find the minimum + if (minKey > k.ts) { + minKey = k.ts; + } + + if (minKey > ik.ts) { + minKey = ik.ts; + } + + if (minKey > key && hasDataInFileBlock(pBlockData, pDumpInfo)) { + minKey = key; + } + + if (minKey > tsLast && hasDataInLastBlock(pLastBlockReader)) { + minKey = tsLast; + } + } else { + minKey = INT64_MIN; // let find the maximum ts value + if (minKey < k.ts) { + minKey = k.ts; + } + + if (minKey < ik.ts) { + minKey = ik.ts; + } + + if (minKey < key && hasDataInFileBlock(pBlockData, pDumpInfo)) { + minKey = key; + } + + if (minKey < tsLast && hasDataInLastBlock(pLastBlockReader)) { + minKey = tsLast; + } + } + + bool init = false; + + // ASC: file block -----> last block -----> imem -----> mem + // DESC: mem -----> imem -----> last block -----> file block + if (ASCENDING_TRAVERSE(pReader->info.order)) { + if (minKey == key) { + init = true; + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + } + + if (minKey == tsLast) { + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + if (init) { + tsdbRowMergerAdd(pMerger, pRow1, NULL); + } else { + init = true; + code = tsdbRowMergerAdd(pMerger, pRow1, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, + pReader->idStr); + } + + if (minKey == ik.ts) { + if (init) { + tsdbRowMergerAdd(pMerger, piRow, piSchema); + } else { + init = true; + code = tsdbRowMergerAdd(pMerger, piRow, piSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + if (minKey == k.ts) { + if (init) { + tsdbRowMergerAdd(pMerger, pRow, pSchema); + } else { + // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + code = tsdbRowMergerAdd(pMerger, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } else { + if (minKey == k.ts) { + init = true; + code = tsdbRowMergerAdd(pMerger, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + if (minKey == ik.ts) { + if (init) { + tsdbRowMergerAdd(pMerger, piRow, piSchema); + } else { + init = true; + code = tsdbRowMergerAdd(pMerger, piRow, piSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + if (minKey == tsLast) { + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + if (init) { + tsdbRowMergerAdd(pMerger, pRow1, NULL); + } else { + init = true; + code = tsdbRowMergerAdd(pMerger, pRow1, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, + pReader->idStr); + } + + if (minKey == key) { + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + if (!init) { + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + tsdbRowMergerAdd(pMerger, &fRow, NULL); + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + } + } + + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + return code; +} + +int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScanInfo* pBlockScanInfo, TSDBKEY* pKey, + SMemTable* pMem, SIterInfo* pIter, const char* type) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t backward = (!ASCENDING_TRAVERSE(pReader->info.order)); + pIter->hasVal = false; + + if (pMem != NULL) { + *pData = tsdbGetTbDataFromMemTable(pMem, pReader->info.suid, pBlockScanInfo->uid); + + if ((*pData) != NULL) { + code = tsdbTbDataIterCreate((*pData), pKey, backward, &pIter->iter); + if (code == TSDB_CODE_SUCCESS) { + pIter->hasVal = (tsdbTbDataIterGet(pIter->iter) != NULL); + + tsdbDebug("%p uid:%" PRIu64 ", check data in %s from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + "-%" PRId64 " %s", + pReader, pBlockScanInfo->uid, type, pKey->ts, pReader->info.order, (*pData)->minKey, (*pData)->maxKey, + pReader->idStr); + } else { + tsdbError("%p uid:%" PRIu64 ", failed to create iterator for %s, code:%s, %s", pReader, pBlockScanInfo->uid, + type, tstrerror(code), pReader->idStr); + return code; + } + } + } else { + tsdbDebug("%p uid:%" PRIu64 ", no data in %s, %s", pReader, pBlockScanInfo->uid, type, pReader->idStr); + } + + return code; +} + +static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { + if (pBlockScanInfo->iterInit) { + return TSDB_CODE_SUCCESS; + } + + STbData* d = NULL; + TSDBKEY startKey = {0}; + if (ASCENDING_TRAVERSE(pReader->info.order)) { + startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey + 1, .version = pReader->info.verRange.minVer}; + } else { + startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey - 1, .version = pReader->info.verRange.maxVer}; + } + + int32_t code = + doInitMemDataIter(pReader, &d, pBlockScanInfo, &startKey, pReader->pReadSnap->pMem, &pBlockScanInfo->iter, "mem"); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + STbData* di = NULL; + code = doInitMemDataIter(pReader, &di, pBlockScanInfo, &startKey, pReader->pReadSnap->pIMem, &pBlockScanInfo->iiter, + "imem"); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + loadMemTombData(&pBlockScanInfo->pMemDelData, d, di, pReader->info.verRange.maxVer); + + pBlockScanInfo->iterInit = true; + return TSDB_CODE_SUCCESS; +} + +static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, + STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { + // it is an multi-table data block + if (pBlockData->aUid != NULL) { + uint64_t uid = pBlockData->aUid[pDumpInfo->rowIndex]; + if (uid != pBlockScanInfo->uid) { // move to next row + return false; + } + } + + // check for version and time range + int64_t ver = pBlockData->aVersion[pDumpInfo->rowIndex]; + if (ver > pReader->info.verRange.maxVer || ver < pReader->info.verRange.minVer) { + return false; + } + + int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + if (ts > pReader->info.window.ekey || ts < pReader->info.window.skey) { + return false; + } + + if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, pReader->info.order, + &pReader->info.verRange)) { + return false; + } + + return true; +} + +static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { + // the last block reader has been initialized for this table. + if (pLBlockReader->uid == pScanInfo->uid) { + return hasDataInLastBlock(pLBlockReader); + } + + if (pLBlockReader->uid != 0) { + tMergeTreeClose(&pLBlockReader->mergeTree); + } + + pLBlockReader->uid = pScanInfo->uid; + + STimeWindow w = pLBlockReader->window; + if (ASCENDING_TRAVERSE(pLBlockReader->order)) { + w.skey = pScanInfo->lastKeyInStt; + } else { + w.ekey = pScanInfo->lastKeyInStt; + } + + int64_t st = taosGetTimestampUs(); + tsdbDebug("init last block reader, window:%" PRId64 "-%" PRId64 ", uid:%" PRIu64 ", %s", w.skey, w.ekey, + pScanInfo->uid, pReader->idStr); + + SMergeTreeConf conf = { + .uid = pScanInfo->uid, + .suid = pReader->info.suid, + .pTsdb = pReader->pTsdb, + .timewindow = w, + .verRange = pLBlockReader->verRange, + .strictTimeRange = false, + .pSchema = pReader->info.pSchema, + .pCurrentFileset = pReader->status.pCurrentFileset, + .backward = (pLBlockReader->order == TSDB_ORDER_DESC), + .pSttFileBlockIterArray = pReader->status.pLDataIterArray, + .pCols = pReader->suppInfo.colId, + .numOfCols = pReader->suppInfo.numOfCols, + .loadTombFn = loadSttTombDataForAll, + .pReader = pReader, + .idstr = pReader->idStr, + }; + + int32_t code = tMergeTreeOpen2(&pLBlockReader->mergeTree, &conf); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + initMemDataIterator(pScanInfo, pReader); + initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost); + + code = nextRowFromLastBlocks(pLBlockReader, pScanInfo, &pReader->info.verRange); + + int64_t el = taosGetTimestampUs() - st; + pReader->cost.initLastBlockReader += (el / 1000.0); + + tsdbDebug("init last block reader completed, elapsed time:%" PRId64 "us %s", el, pReader->idStr); + return code; +} + +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } + +bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) { + if ((pBlockData->nRow > 0) && (pBlockData->nRow != pDumpInfo->totalRows)) { + return false; // this is an invalid result. + } + return pBlockData->nRow > 0 && (!pDumpInfo->allDumped); +} + +int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, + STsdbReader* pReader) { + SRowMerger* pMerger = &pReader->status.merger; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + bool copied = false; + + int32_t code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo, &copied); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized + if (pMerger->pArray == NULL) { + ASSERT(pReader->info.pSchema == NULL); + STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); + if (ps == NULL) { + return terrno; + } + } + + if (copied) { + pBlockScanInfo->lastKey = key; + return TSDB_CODE_SUCCESS; + } else { + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + + SRow* pTSRow = NULL; + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + code = tsdbRowMergerGetRow(pMerger, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); + + taosMemoryFree(pTSRow); + tsdbRowMergerClear(pMerger); + return code; + } +} + +static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, + SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + TSDBROW *pRow = NULL, *piRow = NULL; + int64_t key = (pBlockData->nRow > 0 && (!pDumpInfo->allDumped)) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : + (ASCENDING_TRAVERSE(pReader->info.order) ? INT64_MAX : INT64_MIN); + if (pBlockScanInfo->iter.hasVal) { + pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); + } + + if (pBlockScanInfo->iiter.hasVal) { + piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); + } + + // two levels of mem-table does contain the valid rows + if (pRow != NULL && piRow != NULL) { + return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); + } + + // imem + file + last block + if (pBlockScanInfo->iiter.hasVal) { + return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader); + } + + // mem + file + last block + if (pBlockScanInfo->iter.hasVal) { + return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pLastBlockReader); + } + + // files data blocks + last block + return mergeFileBlockAndLastBlock(pReader, pLastBlockReader, key, pBlockScanInfo, pBlockData); +} + +static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader, bool* loadNeighbor) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; + int32_t nextIndex = -1; + + *loadNeighbor = false; + + SBrinRecord rec = {0}; + bool hasNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pBlockScanInfo, &nextIndex, pReader->info.order, &rec); + if (!hasNeighbor) { // do nothing + return code; + } + + if (overlapWithNeighborBlock2(pBlockInfo, &rec, pReader->info.order)) { // load next block + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + + // 1. find the next neighbor block in the scan block list + SFileDataBlockInfo fb = {.uid = pBlockInfo->uid, .tbBlockIdx = nextIndex}; + int32_t neighborIndex = findFileBlockInfoIndex(pBlockIter, &fb); + + // 2. remove it from the scan block list + setFileBlockActiveInBlockIter(pBlockIter, neighborIndex, step); + + // 3. load the neighbor block, and set it to be the currently accessed file data block + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pBlockInfo->uid); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // 4. check the data values + initBlockDumpInfo(pReader, pBlockIter); + *loadNeighbor = true; + } + + return code; +} + +static void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo) { + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + + pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; + pResBlock->info.dataLoad = 1; + blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); + + setComposedBlockFlag(pReader, true); + + pReader->cost.composedBlocks += 1; + pReader->cost.buildComposedBlockTime += el; +} + +static int32_t buildComposedDataBlock(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + int64_t st = taosGetTimestampUs(); + int32_t step = asc ? 1 : -1; + double el = 0; + SBrinRecord* pRecord = &pBlockInfo->record; + + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + STableBlockScanInfo* pBlockScanInfo = NULL; + if (pBlockInfo != NULL) { + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { + setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + return code; + } + + pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + if (pBlockScanInfo == NULL) { + goto _end; + } + + pRecord = &pBlockInfo->record; + TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); + + // it is a clean block, load it directly + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf, pLastBlockReader) && + (pRecord->numRow <= pReader->resBlockInfo.capacity)) { + if (asc || (!hasDataInLastBlock(pLastBlockReader))) { + code = copyBlockDataToSDataBlock(pReader); + if (code) { + goto _end; + } + + // record the last key value + pBlockScanInfo->lastKey = asc ? pRecord->lastKey : pRecord->firstKey; + goto _end; + } + } + } else { // file blocks not exist + ASSERT(0); + pBlockScanInfo = *pReader->status.pTableIter; + if (pReader->pIgnoreTables && + taosHashGet(*pReader->pIgnoreTables, &pBlockScanInfo->uid, sizeof(pBlockScanInfo->uid))) { + // setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->info.order); + return code; + } + } + + SBlockData* pBlockData = &pReader->status.fileBlockData; + + while (1) { + bool hasBlockData = false; + { + while (pBlockData->nRow > 0 && pBlockData->uid == pBlockScanInfo->uid) { + // find the first qualified row in data block + if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { + hasBlockData = true; + break; + } + + pDumpInfo->rowIndex += step; + + if (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0) { + pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); // NOTE: get the new block info + + // continue check for the next file block if the last ts in the current block + // is overlapped with the next neighbor block + bool loadNeighbor = false; + code = loadNeighborIfOverlap(pBlockInfo, pBlockScanInfo, pReader, &loadNeighbor); + if ((!loadNeighbor) || (code != 0)) { + setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + break; + } + } + } + } + + // no data in last block and block, no need to proceed. + if (hasBlockData == false) { + break; + } + + code = buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); + if (code) { + goto _end; + } + + // currently loaded file data block is consumed + if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { + setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order); + break; + } + + if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) { + break; + } + } + +_end: + el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pBlockScanInfo); + + if (pResBlock->info.rows > 0) { + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", + pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pResBlock->info.rows, el, pReader->idStr); + } + + return code; +} + +void setComposedBlockFlag(STsdbReader* pReader, bool composed) { pReader->status.composedDataBlock = composed; } + +int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order) { + if (pDelSkyline == NULL) { + return 0; + } + + return ASCENDING_TRAVERSE(order) ? 0 : taosArrayGetSize(pDelSkyline) - 1; +} + +int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost) { + int32_t code = 0; + int32_t newDelDataInFile = taosArrayGetSize(pBlockScanInfo->pfileDelData); + if (newDelDataInFile == 0 && + ((pBlockScanInfo->delSkyline != NULL) || (TARRAY_SIZE(pBlockScanInfo->pMemDelData) == 0))) { + return code; + } + + int64_t st = taosGetTimestampUs(); + + if (pBlockScanInfo->delSkyline != NULL) { + taosArrayClear(pBlockScanInfo->delSkyline); + } else { + pBlockScanInfo->delSkyline = taosArrayInit(4, sizeof(TSDBKEY)); + } + + SArray* pSource = pBlockScanInfo->pfileDelData; + if (pSource == NULL) { + pSource = pBlockScanInfo->pMemDelData; + } else { + taosArrayAddAll(pSource, pBlockScanInfo->pMemDelData); + } + + code = tsdbBuildDeleteSkyline(pSource, 0, taosArrayGetSize(pSource) - 1, pBlockScanInfo->delSkyline); + + taosArrayClear(pBlockScanInfo->pfileDelData); + int32_t index = getInitialDelIndex(pBlockScanInfo->delSkyline, order); + + pBlockScanInfo->iter.index = index; + pBlockScanInfo->iiter.index = index; + pBlockScanInfo->fileDelIndex = index; + pBlockScanInfo->lastBlockDelIndex = index; + + double el = taosGetTimestampUs() - st; + pCost->createSkylineIterTime = el / 1000.0; + + return code; +} + +TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}, ikey = {.ts = TSKEY_INITIAL_VAL}; + + bool hasKey = false, hasIKey = false; + TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); + if (pRow != NULL) { + hasKey = true; + key = TSDBROW_KEY(pRow); + } + + TSDBROW* pIRow = getValidMemRow(&pScanInfo->iiter, pScanInfo->delSkyline, pReader); + if (pIRow != NULL) { + hasIKey = true; + ikey = TSDBROW_KEY(pIRow); + } + + if (hasKey) { + if (hasIKey) { // has data in mem & imem + if (asc) { + return key.ts <= ikey.ts ? key : ikey; + } else { + return key.ts <= ikey.ts ? ikey : key; + } + } else { // no data in imem + return key; + } + } else { + // no data in mem & imem, return the initial value + // only imem has data, return ikey + return ikey; + } +} + +static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pTableList) { + SReaderStatus* pStatus = &pReader->status; + pBlockNum->numOfBlocks = 0; + pBlockNum->numOfLastFiles = 0; + + size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBrinBlk)); + + while (1) { + // only check here, since the iterate data in memory is very fast. + if (pReader->code != TSDB_CODE_SUCCESS) { + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + return pReader->code; + } + + bool hasNext = false; + int32_t code = filesetIteratorNext(&pStatus->fileIter, pReader, &hasNext); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pIndexList); + return code; + } + + if (!hasNext) { // no data files on disk + break; + } + + taosArrayClear(pIndexList); + code = doLoadBlockIndex(pReader, pReader->pFileReader, pIndexList); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pIndexList); + return code; + } + + if (taosArrayGetSize(pIndexList) > 0 || pReader->status.pCurrentFileset->lvlArr->size > 0) { + code = doLoadFileBlock(pReader, pIndexList, pBlockNum, pTableList); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pIndexList); + return code; + } + + if (pBlockNum->numOfBlocks + pBlockNum->numOfLastFiles > 0) { + break; + } + } + + // no blocks in current file, try next files + } + + taosArrayDestroy(pIndexList); + return loadDataFileTombDataForAll(pReader); +} + +static void resetTableListIndex(SReaderStatus* pStatus) { + STableUidList* pList = &pStatus->uidList; + + pList->currentIndex = 0; + uint64_t uid = pList->tableUidList[0]; + pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid)); +} + +static bool moveToNextTable(STableUidList* pOrderedCheckInfo, SReaderStatus* pStatus) { + pOrderedCheckInfo->currentIndex += 1; + if (pOrderedCheckInfo->currentIndex >= tSimpleHashGetSize(pStatus->pTableMap)) { + pStatus->pTableIter = NULL; + return false; + } + + uint64_t uid = pOrderedCheckInfo->tableUidList[pOrderedCheckInfo->currentIndex]; + pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid)); + return (pStatus->pTableIter != NULL); +} + +static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader; + STableUidList* pUidList = &pStatus->uidList; + int32_t code = TSDB_CODE_SUCCESS; + + if (tSimpleHashGetSize(pStatus->pTableMap) == 0) { + return TSDB_CODE_SUCCESS; + } + + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + + while (1) { + if (pReader->code != TSDB_CODE_SUCCESS) { + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + return pReader->code; + } + + // load the last data block of current table + STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; + if (pScanInfo == NULL) { + tsdbError("table Iter is null, invalid pScanInfo, try next table %s", pReader->idStr); + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + + continue; + } + + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pScanInfo->uid, sizeof(pScanInfo->uid))) { + // reset the index in last block when handing a new file + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + + continue; + } + + bool hasDataInLastFile = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + if (!hasDataInLastFile) { + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + + continue; + } + + int64_t st = taosGetTimestampUs(); + while (1) { + bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); + + // no data in last block and block, no need to proceed. + if (hasBlockLData == false) { + break; + } + + code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + if (code) { + return code; + } + + if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) { + break; + } + } + + double el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pScanInfo); + + if (pResBlock->info.rows > 0) { + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", + pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pResBlock->info.rows, el, pReader->idStr); + return TSDB_CODE_SUCCESS; + } + + // current table is exhausted, let's try next table + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + } +} + +static int32_t doBuildDataBlock(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + STableBlockScanInfo* pScanInfo = NULL; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) { + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order); + return code; + } + + if (pReader->code != TSDB_CODE_SUCCESS) { + return pReader->code; + } + + pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + if (pScanInfo == NULL) { + return terrno; + } + + initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); + + if (fileBlockShouldLoad(pReader, pBlockInfo, pScanInfo, keyInBuf, pLastBlockReader)) { + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // build composed data block + code = buildComposedDataBlock(pReader); + } else if (bufferDataInFileBlockGap(pReader->info.order, keyInBuf, pBlockInfo)) { + // data in memory that are earlier than current file block + // rows in buffer should be less than the file block in asc, greater than file block in desc + int64_t endKey = + (ASCENDING_TRAVERSE(pReader->info.order)) ? pBlockInfo->record.firstKey : pBlockInfo->record.lastKey; + code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); + } else { + bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader); + int64_t tsLast = bHasDataInLastBlock ? getCurrentKeyInLastBlock(pLastBlockReader) : INT64_MIN; + if (!bHasDataInLastBlock || ((ASCENDING_TRAVERSE(pReader->info.order) && pBlockInfo->record.lastKey < tsLast) || + (!ASCENDING_TRAVERSE(pReader->info.order) && pBlockInfo->record.firstKey > tsLast))) { + // whole block is required, return it directly + SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info; + pInfo->rows = pBlockInfo->record.numRow; + pInfo->id.uid = pScanInfo->uid; + pInfo->dataLoad = 0; + pInfo->window = (STimeWindow){.skey = pBlockInfo->record.firstKey, .ekey = pBlockInfo->record.lastKey}; + setComposedBlockFlag(pReader, false); + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order); + + // update the last key for the corresponding table + pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? pInfo->window.ekey : pInfo->window.skey; + tsdbDebug("%p uid:%" PRIu64 + " clean file block retrieved from file, global index:%d, " + "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s", + pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.numRow, + pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr); + } else { + SBlockData* pBData = &pReader->status.fileBlockData; + tBlockDataReset(pBData); + + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + tsdbDebug("load data in last block firstly %s", pReader->idStr); + + int64_t st = taosGetTimestampUs(); + + while (1) { + bool hasBlockLData = hasDataInLastBlock(pLastBlockReader); + + // no data in last block and block, no need to proceed. + if (hasBlockLData == false) { + break; + } + + code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + if (code) { + return code; + } + + if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) { + break; + } + } + + double el = (taosGetTimestampUs() - st) / 1000.0; + updateComposedBlockInfo(pReader, el, pScanInfo); + + if (pResBlock->info.rows > 0) { + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", + pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pResBlock->info.rows, el, pReader->idStr); + } + } + + } + + return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code; +} + +static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { + int64_t st = taosGetTimestampUs(); + LRUHandle* handle = NULL; + int32_t code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle); + if (code != TSDB_CODE_SUCCESS || handle == NULL) { + goto _end; + } + + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + + SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle); + size_t num = taosArrayGetSize(aBlockIdx); + if (num == 0) { + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return TSDB_CODE_SUCCESS; + } + + SBlockIdx* pBlockIdx = NULL; + for (int32_t i = 0; i < num; ++i) { + pBlockIdx = (SBlockIdx*)taosArrayGet(aBlockIdx, i); + if (pBlockIdx->suid != pReader->info.suid) { + continue; + } + + STableBlockScanInfo** p = tSimpleHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(pBlockIdx->uid)); + if (p == NULL) { + continue; + } + + STableBlockScanInfo* pScanInfo = *p; + SDataBlk block = {0}; + // for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { + // tGetDataBlk(pScanInfo->mapData.pData + pScanInfo->mapData.aOffset[j], &block); + // pReader->rowsNum += block.nRow; + // } + } + +_end: + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return code; +} + +static int32_t doSumSttBlockRows(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + SSttBlockLoadInfo* pBlockLoadInfo = NULL; +#if 0 + for (int32_t i = 0; i < pReader->pFileReader->pSet->nSttF; ++i) { // open all last file + pBlockLoadInfo = &pLastBlockReader->pInfo[i]; + + code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk); + if (code) { + return code; + } + + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + if (size >= 1) { + SSttBlk* pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); + SSttBlk* pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); + + // all identical + if (pStart->suid == pEnd->suid) { + if (pStart->suid != pReader->info.suid) { + // no qualified stt block existed + taosArrayClear(pBlockLoadInfo->aSttBlk); + continue; + } + for (int32_t j = 0; j < size; ++j) { + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, j); + pReader->rowsNum += p->nRow; + } + } else { + for (int32_t j = 0; j < size; ++j) { + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, j); + uint64_t s = p->suid; + if (s < pReader->info.suid) { + continue; + } + + if (s == pReader->info.suid) { + pReader->rowsNum += p->nRow; + } else if (s > pReader->info.suid) { + break; + } + } + } + } + } +#endif + + return code; +} + +static int32_t readRowsCountFromFiles(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + + while (1) { + bool hasNext = false; + code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext); + if (code) { + return code; + } + + if (!hasNext) { // no data files on disk + break; + } + + // code = doSumFileBlockRows(pReader, pReader->pFileReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doSumSttBlockRows(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + pReader->status.loadFromFile = false; + + return code; +} + +static int32_t readRowsCountFromMem(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t memNum = 0, imemNum = 0; + if (pReader->pReadSnap->pMem != NULL) { + tsdbMemTableCountRows(pReader->pReadSnap->pMem, pReader->status.pTableMap, &memNum); + } + + if (pReader->pReadSnap->pIMem != NULL) { + tsdbMemTableCountRows(pReader->pReadSnap->pIMem, pReader->status.pTableMap, &imemNum); + } + + pReader->rowsNum += memNum + imemNum; + + return code; +} + +static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + STableUidList* pUidList = &pStatus->uidList; + + while (1) { + if (pReader->code != TSDB_CODE_SUCCESS) { + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + return pReader->code; + } + + STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; + if (pReader->pIgnoreTables && + taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) { + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + pBlockScanInfo = pStatus->pTableIter; + } + + initMemDataIterator(*pBlockScanInfo, pReader); + initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost); + + int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN; + int32_t code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pReader->resBlockInfo.pResBlock->info.rows > 0) { + return TSDB_CODE_SUCCESS; + } + + // current table is exhausted, let's try next table + bool hasNexTable = moveToNextTable(pUidList, pStatus); + if (!hasNexTable) { + return TSDB_CODE_SUCCESS; + } + } +} + +// set the correct start position in case of the first/last file block, according to the query time window +static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { + int64_t lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? INT64_MIN : INT64_MAX; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SReaderStatus* pStatus = &pReader->status; + SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; + + if (pBlockInfo) { + STableBlockScanInfo* pScanInfo = tSimpleHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + if (pScanInfo) { + lastKey = pScanInfo->lastKey; + } + + pDumpInfo->totalRows = pBlockInfo->record.numRow; + pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->record.numRow - 1; + } else { + pDumpInfo->totalRows = 0; + pDumpInfo->rowIndex = 0; + } + + pDumpInfo->allDumped = false; + pDumpInfo->lastKey = lastKey; +} + +static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { + SBlockNumber num = {0}; + SArray* pTableList = taosArrayInit(40, POINTER_BYTES); + + int32_t code = moveToNextFile(pReader, &num, pTableList); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pTableList); + return code; + } + + // all data files are consumed, try data in buffer + if (num.numOfBlocks + num.numOfLastFiles == 0) { + pReader->status.loadFromFile = false; + taosArrayDestroy(pTableList); + return code; + } + + // initialize the block iterator for a new fileset + if (num.numOfBlocks > 0) { + code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks, pTableList); + } else { // no block data, only last block exists + tBlockDataReset(&pReader->status.fileBlockData); + resetDataBlockIterator(pBlockIter, pReader->info.order); + resetTableListIndex(&pReader->status); + } + + // set the correct start position according to the query time window + initBlockDumpInfo(pReader, pBlockIter); + taosArrayDestroy(pTableList); + return code; +} + +static bool fileBlockPartiallyRead(SFileBlockDumpInfo* pDumpInfo, bool asc) { + return (!pDumpInfo->allDumped) && + ((pDumpInfo->rowIndex > 0 && asc) || (pDumpInfo->rowIndex < (pDumpInfo->totalRows - 1) && (!asc))); +} + +typedef enum { + TSDB_READ_RETURN = 0x1, + TSDB_READ_CONTINUE = 0x2, +} ERetrieveType; + +static ERetrieveType doReadDataFromLastFiles(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + SDataBlockIter* pBlockIter = &pReader->status.blockIter; + + while (1) { + terrno = 0; + + code = doLoadLastBlockSequentially(pReader); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return TSDB_READ_RETURN; + } + + if (pResBlock->info.rows > 0) { + return TSDB_READ_RETURN; + } + + // all data blocks are checked in this last block file, now let's try the next file + ASSERT(pReader->status.pTableIter == NULL); + code = initForFirstBlockInFile(pReader, pBlockIter); + + // error happens or all the data files are completely checked + if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { + terrno = code; + return TSDB_READ_RETURN; + } + + if (pBlockIter->numOfBlocks > 0) { // there are data blocks existed. + return TSDB_READ_CONTINUE; + } else { // all blocks in data file are checked, let's check the data in last files + resetTableListIndex(&pReader->status); + } + } +} + +static int32_t buildBlockFromFiles(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + + SDataBlockIter* pBlockIter = &pReader->status.blockIter; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + + if (pBlockIter->numOfBlocks == 0) { + // let's try to extract data from stt files. + ERetrieveType type = doReadDataFromLastFiles(pReader); + if (type == TSDB_READ_RETURN) { + return terrno; + } + + code = doBuildDataBlock(pReader); + if (code != TSDB_CODE_SUCCESS || pResBlock->info.rows > 0) { + return code; + } + } + + while (1) { + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + if (fileBlockPartiallyRead(pDumpInfo, asc)) { // file data block is partially loaded + code = buildComposedDataBlock(pReader); + } else { + // current block are exhausted, try the next file block + if (pDumpInfo->allDumped) { + // try next data block in current file + bool hasNext = blockIteratorNext(&pReader->status.blockIter, pReader->idStr); + if (hasNext) { // check for the next block in the block accessed order list + initBlockDumpInfo(pReader, pBlockIter); + } else { + // all data blocks in files are checked, let's check the data in last files. + // data blocks in current file are exhausted, let's try the next file now + SBlockData* pBlockData = &pReader->status.fileBlockData; + if (pBlockData->uid != 0) { + tBlockDataClear(pBlockData); + } + + tBlockDataReset(pBlockData); + resetDataBlockIterator(pBlockIter, pReader->info.order); + resetTableListIndex(&pReader->status); + + ERetrieveType type = doReadDataFromLastFiles(pReader); + if (type == TSDB_READ_RETURN) { + return terrno; + } + } + } + + code = doBuildDataBlock(pReader); + } + + if (code != TSDB_CODE_SUCCESS || pResBlock->info.rows > 0) { + return code; + } + } +} + +static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idStr, + int8_t* pLevel) { + if (VND_IS_RSMA(pVnode)) { + int8_t level = 0; + int8_t precision = pVnode->config.tsdbCfg.precision; + int64_t now = taosGetTimestamp(precision); + int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1L + : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L + : 1000000L); + + for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) { + SRetention* pRetention = retentions + level; + if (pRetention->keep <= 0) { + if (level > 0) { + --level; + } + break; + } + if ((now - pRetention->keep) <= (winSKey + offset)) { + break; + } + ++level; + } + + const char* str = (idStr != NULL) ? idStr : ""; + + if (level == TSDB_RETENTION_L0) { + *pLevel = TSDB_RETENTION_L0; + tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L0, str); + return VND_RSMA0(pVnode); + } else if (level == TSDB_RETENTION_L1) { + *pLevel = TSDB_RETENTION_L1; + tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L1, str); + return VND_RSMA1(pVnode); + } else { + *pLevel = TSDB_RETENTION_L2; + tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L2, str); + return VND_RSMA2(pVnode); + } + } + + return VND_TSDB(pVnode); +} + +SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) { + int64_t startVer = (pCond->startVersion == -1) ? 0 : pCond->startVersion; + + int64_t endVer = 0; + if (pCond->endVersion == -1) { + // user not specified end version, set current maximum version of vnode as the endVersion + endVer = pVnode->state.applied; + } else { + endVer = (pCond->endVersion > pVnode->state.applied) ? pVnode->state.applied : pCond->endVersion; + } + + return (SVersionRange){.minVer = startVer, .maxVer = endVer}; +} + +bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, + SVersionRange* pVerRange) { + if (pDelList == NULL || (taosArrayGetSize(pDelList) == 0)) { + return false; + } + + size_t num = taosArrayGetSize(pDelList); + bool asc = ASCENDING_TRAVERSE(order); + int32_t step = asc ? 1 : -1; + + if (asc) { + if (*index >= num - 1) { + TSDBKEY* last = taosArrayGetLast(pDelList); + ASSERT(key >= last->ts); + + if (key > last->ts) { + return false; + } else if (key == last->ts) { + TSDBKEY* prev = taosArrayGet(pDelList, num - 2); + return (prev->version >= ver && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer); + } + } else { + TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); + TSDBKEY* pNext = taosArrayGet(pDelList, (*index) + 1); + + if (key < pCurrent->ts) { + return false; + } + + if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver && + pVerRange->maxVer >= pCurrent->version) { + return true; + } + + while (pNext->ts <= key && (*index) < num - 1) { + (*index) += 1; + + if ((*index) < num - 1) { + pCurrent = taosArrayGet(pDelList, *index); + pNext = taosArrayGet(pDelList, (*index) + 1); + + // it is not a consecutive deletion range, ignore it + if (pCurrent->version == 0 && pNext->version > 0) { + continue; + } + + if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver && + pVerRange->maxVer >= pCurrent->version) { + return true; + } + } + } + + return false; + } + } else { + if (*index <= 0) { + TSDBKEY* pFirst = taosArrayGet(pDelList, 0); + + if (key < pFirst->ts) { + return false; + } else if (key == pFirst->ts) { + return pFirst->version >= ver; + } else { + ASSERT(0); + } + } else { + TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); + TSDBKEY* pPrev = taosArrayGet(pDelList, (*index) - 1); + + if (key > pCurrent->ts) { + return false; + } + + if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) { + return true; + } + + while (pPrev->ts >= key && (*index) > 1) { + (*index) += step; + + if ((*index) >= 1) { + pCurrent = taosArrayGet(pDelList, *index); + pPrev = taosArrayGet(pDelList, (*index) - 1); + + // it is not a consecutive deletion range, ignore it + if (pCurrent->version > 0 && pPrev->version == 0) { + continue; + } + + if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) { + return true; + } + } + } + + return false; + } + } + + return false; +} + +TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader) { + if (!pIter->hasVal) { + return NULL; + } + + TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); + TSDBKEY key = TSDBROW_KEY(pRow); + + if (outOfTimeWindow(key.ts, &pReader->info.window)) { + pIter->hasVal = false; + return NULL; + } + + // it is a valid data version + if ((key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) && + (!hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, pReader->info.order, &pReader->info.verRange))) { + return pRow; + } + + while (1) { + pIter->hasVal = tsdbTbDataIterNext(pIter->iter); + if (!pIter->hasVal) { + return NULL; + } + + pRow = tsdbTbDataIterGet(pIter->iter); + + key = TSDBROW_KEY(pRow); + if (outOfTimeWindow(key.ts, &pReader->info.window)) { + pIter->hasVal = false; + return NULL; + } + + if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer && + (!hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, pReader->info.order, &pReader->info.verRange))) { + return pRow; + } + } +} + +int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, STsdbReader* pReader) { + SRowMerger* pMerger = &pReader->status.merger; + + while (1) { + pIter->hasVal = tsdbTbDataIterNext(pIter->iter); + if (!pIter->hasVal) { + break; + } + + // data exists but not valid + TSDBROW* pRow = getValidMemRow(pIter, pDelList, pReader); + if (pRow == NULL) { + break; + } + + // ts is not identical, quit + TSDBKEY k = TSDBROW_KEY(pRow); + if (k.ts != ts) { + break; + } + + if (pRow->type == TSDBROW_ROW_FMT) { + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } + + tsdbRowMergerAdd(pMerger, pRow, pTSchema); + } else { // column format + tsdbRowMergerAdd(pMerger, pRow, NULL); + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowIndex, int64_t key, SRowMerger* pMerger, + SVersionRange* pVerRange, int32_t step) { + while (rowIndex < pBlockData->nRow && rowIndex >= 0 && pBlockData->aTSKEY[rowIndex] == key) { + if (pBlockData->aVersion[rowIndex] > pVerRange->maxVer || pBlockData->aVersion[rowIndex] < pVerRange->minVer) { + rowIndex += step; + continue; + } + + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); + tsdbRowMergerAdd(pMerger, &fRow, NULL); + rowIndex += step; + } + + return rowIndex; +} + +typedef enum { + CHECK_FILEBLOCK_CONT = 0x1, + CHECK_FILEBLOCK_QUIT = 0x2, +} CHECK_FILEBLOCK_STATE; + +static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, + SFileDataBlockInfo* pFBlock, SRowMerger* pMerger, int64_t key, + CHECK_FILEBLOCK_STATE* state) { + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + SBlockData* pBlockData = &pReader->status.fileBlockData; + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + + *state = CHECK_FILEBLOCK_QUIT; + int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; + + bool loadNeighbor = true; + int32_t code = loadNeighborIfOverlap(pFBlock, pScanInfo, pReader, &loadNeighbor); + + if (loadNeighbor && (code == TSDB_CODE_SUCCESS)) { + pDumpInfo->rowIndex = + doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, key, pMerger, &pReader->info.verRange, step); + if ((pDumpInfo->rowIndex >= pDumpInfo->totalRows && asc) || (pDumpInfo->rowIndex < 0 && !asc)) { + *state = CHECK_FILEBLOCK_CONT; + } + } + + return code; +} + +int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + + SRowMerger* pMerger = &pReader->status.merger; + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + int32_t step = asc ? 1 : -1; + + pDumpInfo->rowIndex += step; + if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) { + pDumpInfo->rowIndex = + doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, key, pMerger, &pReader->info.verRange, step); + } + + // all rows are consumed, let's try next file block + if ((pDumpInfo->rowIndex >= pBlockData->nRow && asc) || (pDumpInfo->rowIndex < 0 && !asc)) { + while (1) { + CHECK_FILEBLOCK_STATE st; + + SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + if (pFileBlockInfo == NULL) { + st = CHECK_FILEBLOCK_QUIT; + break; + } + + checkForNeighborFileBlock(pReader, pScanInfo, pFileBlockInfo, pMerger, key, &st); + if (st == CHECK_FILEBLOCK_QUIT) { + break; + } + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, + SRowMerger* pMerger, SVersionRange* pVerRange, const char* idStr) { + while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo, pVerRange)) { + int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); + if (next1 == ts) { + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); + } else { + tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, + pScanInfo->lastBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt, + idStr); + break; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, TSDBROW* pResRow, + STsdbReader* pReader, bool* freeTSRow) { + TSDBROW* pNextRow = NULL; + TSDBROW current = *pRow; + + { // if the timestamp of the next valid row has a different ts, return current row directly + pIter->hasVal = tsdbTbDataIterNext(pIter->iter); + + if (!pIter->hasVal) { + *pResRow = *pRow; + *freeTSRow = false; + return TSDB_CODE_SUCCESS; + } else { // has next point in mem/imem + pNextRow = getValidMemRow(pIter, pDelList, pReader); + if (pNextRow == NULL) { + *pResRow = current; + *freeTSRow = false; + return TSDB_CODE_SUCCESS; + } + + if (TSDBROW_TS(¤t) != TSDBROW_TS(pNextRow)) { + *pResRow = current; + *freeTSRow = false; + return TSDB_CODE_SUCCESS; + } + } + } + + terrno = 0; + int32_t code = 0; + + // start to merge duplicated rows + if (current.type == TSDBROW_ROW_FMT) { + // get the correct schema for data in memory + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } + + code = tsdbRowMergerAdd(&pReader->status.merger, ¤t, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + if (pTSchema1 == NULL) { + return terrno; + } + + tsdbRowMergerAdd(&pReader->status.merger, pNextRow, pTSchema1); + } else { // let's merge rows in file block + code = tsdbRowMergerAdd(&pReader->status.merger, ¤t, pReader->info.pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + tsdbRowMergerAdd(&pReader->status.merger, pNextRow, NULL); + } + + code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = tsdbRowMergerGetRow(&pReader->status.merger, &pResRow->pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + pResRow->type = TSDBROW_ROW_FMT; + tsdbRowMergerClear(&pReader->status.merger); + *freeTSRow = true; + + return TSDB_CODE_SUCCESS; +} + +int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, + SRow** pTSRow) { + SRowMerger* pMerger = &pReader->status.merger; + + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return terrno; + } + + STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + if (piSchema == NULL) { + return terrno; + } + + if (ASCENDING_TRAVERSE(pReader->info.order)) { // ascending order imem --> mem + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + } else { + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS || pMerger->pTSchema == NULL) { + return code; + } + + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + int32_t code = tsdbRowMergerGetRow(pMerger, pTSRow); + tsdbRowMergerClear(pMerger); + return code; +} + +static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow, + int64_t endKey, bool* freeTSRow) { + TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); + TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); + SArray* pDelList = pBlockScanInfo->delSkyline; + uint64_t uid = pBlockScanInfo->uid; + + // todo refactor + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + if (pBlockScanInfo->iter.hasVal) { + TSDBKEY k = TSDBROW_KEY(pRow); + if ((k.ts >= endKey && asc) || (k.ts <= endKey && !asc)) { + pRow = NULL; + } + } + + if (pBlockScanInfo->iiter.hasVal) { + TSDBKEY k = TSDBROW_KEY(piRow); + if ((k.ts >= endKey && asc) || (k.ts <= endKey && !asc)) { + piRow = NULL; + } + } + + if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal && pRow != NULL && piRow != NULL) { + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + + int32_t code = TSDB_CODE_SUCCESS; + if (ik.ts != k.ts) { + if (((ik.ts < k.ts) && asc) || ((ik.ts > k.ts) && (!asc))) { // ik.ts < k.ts + code = doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pResRow, pReader, freeTSRow); + } else if (((k.ts < ik.ts) && asc) || ((k.ts > ik.ts) && (!asc))) { + code = doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, pResRow, pReader, freeTSRow); + } + } else { // ik.ts == k.ts + *freeTSRow = true; + pResRow->type = TSDBROW_ROW_FMT; + code = doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pResRow->pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return code; + } + + if (pBlockScanInfo->iter.hasVal && pRow != NULL) { + return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pResRow, pReader, + freeTSRow); + } + + if (pBlockScanInfo->iiter.hasVal && piRow != NULL) { + return doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pResRow, pReader, freeTSRow); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) { + int32_t outputRowIndex = pBlock->info.rows; + int64_t uid = pScanInfo->uid; + int32_t code = TSDB_CODE_SUCCESS; + + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); + if (pSchema == NULL) { + return terrno; + } + + SColVal colVal = {0}; + int32_t i = 0, j = 0; + + if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; + i += 1; + } + + while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) { + col_id_t colId = pSupInfo->colId[i]; + + if (colId == pSchema->columns[j].colId) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + + tRowGet(pTSRow, pSchema, j, &colVal); + code = doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); + if (code) { + return code; + } + i += 1; + j += 1; + } else if (colId < pSchema->columns[j].colId) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + + colDataSetNULL(pColInfoData, outputRowIndex); + i += 1; + } else if (colId > pSchema->columns[j].colId) { + j += 1; + } + } + + // set null value since current column does not exist in the "pSchema" + while (i < pSupInfo->numOfCols) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + colDataSetNULL(pColInfoData, outputRowIndex); + i += 1; + } + + pBlock->info.dataLoad = 1; + pBlock->info.rows += 1; + pScanInfo->lastKey = pTSRow->ts; + return TSDB_CODE_SUCCESS; +} + +int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, + int32_t rowIndex) { + int32_t i = 0, j = 0; + int32_t outputRowIndex = pResBlock->info.rows; + int32_t code = TSDB_CODE_SUCCESS; + + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + ((int64_t*)pReader->status.pPrimaryTsCol->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; + i += 1; + + SColVal cv = {0}; + int32_t numOfInputCols = pBlockData->nColData; + int32_t numOfOutputCols = pSupInfo->numOfCols; + + while (i < numOfOutputCols && j < numOfInputCols) { + SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); + if (pData->cid < pSupInfo->colId[i]) { + j += 1; + continue; + } + + SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pData->cid == pSupInfo->colId[i]) { + tColDataGetValue(pData, rowIndex, &cv); + code = doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); + if (code) { + return code; + } + j += 1; + } else if (pData->cid > pCol->info.colId) { + // the specified column does not exist in file block, fill with null data + colDataSetNULL(pCol, outputRowIndex); + } + + i += 1; + } + + while (i < numOfOutputCols) { + SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + colDataSetNULL(pCol, outputRowIndex); + i += 1; + } + + pResBlock->info.dataLoad = 1; + pResBlock->info.rows += 1; + return TSDB_CODE_SUCCESS; +} + +int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, + STsdbReader* pReader) { + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + int32_t code = TSDB_CODE_SUCCESS; + + do { + // SRow* pTSRow = NULL; + TSDBROW row = {.type = -1}; + bool freeTSRow = false; + tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); + if (row.type == -1) { + break; + } + + if (row.type == TSDBROW_ROW_FMT) { + code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo); + + if (freeTSRow) { + taosMemoryFree(row.pTSRow); + } + + if (code) { + return code; + } + } else { + code = doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); + if (code) { + break; + } + } + + // no data in buffer, return immediately + if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) { + break; + } + + if (pBlock->info.rows >= capacity) { + break; + } + } while (1); + + return code; +} + +// TODO refactor: with createDataBlockScanInfo +int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t num) { + int32_t size = tSimpleHashGetSize(pReader->status.pTableMap); + + STableBlockScanInfo** p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pReader->status.pTableMap, p, &iter)) != NULL) { + clearBlockScanInfo(*p); + } + + if (size < num) { + int32_t code = ensureBlockScanInfoBuf(&pReader->blockInfoBuf, num); + if (code) { + return code; + } + + char* p1 = taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num); + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pReader->status.uidList.tableUidList = (uint64_t*)p1; + } + + tSimpleHashClear(pReader->status.pTableMap); + STableUidList* pUidList = &pReader->status.uidList; + pUidList->currentIndex = 0; + + STableKeyInfo* pList = (STableKeyInfo*)pTableList; + for (int32_t i = 0; i < num; ++i) { + STableBlockScanInfo* pInfo = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i); + pInfo->uid = pList[i].uid; + pUidList->tableUidList[i] = pList[i].uid; + + // todo extract method + if (ASCENDING_TRAVERSE(pReader->info.order)) { + int64_t skey = pReader->info.window.skey; + pInfo->lastKey = (skey > INT64_MIN) ? (skey - 1) : skey; + pInfo->lastKeyInStt = skey; + } else { + int64_t ekey = pReader->info.window.ekey; + pInfo->lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey; + pInfo->lastKeyInStt = ekey; + } + + tSimpleHashPut(pReader->status.pTableMap, &pInfo->uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); + } + + return TDB_CODE_SUCCESS; +} + +void* tsdbGetIdx2(SMeta* pMeta) { + if (pMeta == NULL) { + return NULL; + } + return metaGetIdx(pMeta); +} + +void* tsdbGetIvtIdx2(SMeta* pMeta) { + if (pMeta == NULL) { + return NULL; + } + return metaGetIvtIdx(pMeta); +} + +uint64_t tsdbGetReaderMaxVersion2(STsdbReader* pReader) { return pReader->info.verRange.maxVer; } + +static int32_t doOpenReaderImpl(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + + initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + resetDataBlockIterator(&pStatus->blockIter, pReader->info.order); + + int32_t code = TSDB_CODE_SUCCESS; + if (pStatus->fileIter.numOfFiles == 0) { + pStatus->loadFromFile = false; + } else if (READ_MODE_COUNT_ONLY == pReader->info.readMode) { + // DO NOTHING + } else { + code = initForFirstBlockInFile(pReader, pBlockIter); + } + + if (!pStatus->loadFromFile) { + resetTableListIndex(pStatus); + } + + return code; +} + +static void freeSchemaFunc(void* param) { + void** p = (void**)param; + taosMemoryFreeClear(*p); +} + +static void clearSharedPtr(STsdbReader* p) { + p->status.pTableMap = NULL; + p->status.uidList.tableUidList = NULL; + p->info.pSchema = NULL; + p->pReadSnap = NULL; + p->pSchemaMap = NULL; +} + +static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { + pDst->status.pTableMap = pSrc->status.pTableMap; + pDst->status.uidList = pSrc->status.uidList; + pDst->info.pSchema = pSrc->info.pSchema; + pDst->pSchemaMap = pSrc->pSchemaMap; + pDst->pReadSnap = pSrc->pReadSnap; + pDst->pReadSnap->pfSetArray = pSrc->pReadSnap->pfSetArray; + + if (pDst->info.pSchema) { + tsdbRowMergerInit(&pDst->status.merger, pDst->info.pSchema); + } +} + +// ====================================== EXPOSED APIs ====================================== +int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, + SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, + SHashObj** pIgnoreTables) { + STimeWindow window = pCond->twindows; + SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); + + int32_t capacity = pConf->tsdbCfg.maxRows; + if (pResBlock != NULL) { + blockDataEnsureCapacity(pResBlock, capacity); + } + + int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + // check for query time window + STsdbReader* pReader = *ppReader; + if (isEmptyQueryTimeWindow(&pReader->info.window) && pCond->type == TIMEWINDOW_RANGE_CONTAINED) { + tsdbDebug("%p query window not overlaps with the data set, no result returned, %s", pReader, pReader->idStr); + return TSDB_CODE_SUCCESS; + } + + if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { + // update the SQueryTableDataCond to create inner reader + int32_t order = pCond->order; + if (order == TSDB_ORDER_ASC) { + pCond->twindows.ekey = window.skey - 1; + pCond->twindows.skey = INT64_MIN; + pCond->order = TSDB_ORDER_DESC; + } else { + pCond->twindows.skey = window.ekey + 1; + pCond->twindows.ekey = INT64_MAX; + pCond->order = TSDB_ORDER_ASC; + } + + // here we only need one more row, so the capacity is set to be ONE. + code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[0], 1, pResBlock, idstr); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + if (order == TSDB_ORDER_ASC) { + pCond->twindows.skey = window.ekey + 1; + pCond->twindows.ekey = INT64_MAX; + } else { + pCond->twindows.skey = INT64_MIN; + pCond->twindows.ekey = window.ekey - 1; + } + pCond->order = order; + + code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[1], 1, pResBlock, idstr); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + } + + // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here. + // no valid error code set in metaGetTbTSchema, so let's set the error code here. + // we should proceed in case of tmq processing. + if (pCond->suid != 0) { + pReader->info.pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, -1, 1); + if (pReader->info.pSchema == NULL) { + tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->info.suid, pReader->idStr); + } + } else if (numOfTables > 0) { + STableKeyInfo* pKey = pTableList; + pReader->info.pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1); + if (pReader->info.pSchema == NULL) { + tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr); + } + } + + if (pReader->info.pSchema != NULL) { + tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema); + } + + pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash); + if (pReader->pSchemaMap == NULL) { + tsdbError("failed init schema hash for reader %s", pReader->idStr); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + tSimpleHashSetFreeFp(pReader->pSchemaMap, freeSchemaFunc); + if (pReader->info.pSchema != NULL) { + code = updateBlockSMAInfo(pReader->info.pSchema, &pReader->suppInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + } + + STsdbReader* p = (pReader->innerReader[0] != NULL) ? pReader->innerReader[0] : pReader; + pReader->status.pTableMap = + createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables); + if (pReader->status.pTableMap == NULL) { + *ppReader = NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); + if (pReader->status.pLDataIterArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + pReader->flag = READER_STATUS_SUSPEND; + + if (countOnly) { + pReader->info.readMode = READ_MODE_COUNT_ONLY; + } + + pReader->pIgnoreTables = pIgnoreTables; + + tsdbDebug("%p total numOfTable:%d, window:%" PRId64 " - %" PRId64 ", verRange:%" PRId64 " - %" PRId64 + " in this query %s", + pReader, numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->info.verRange.minVer, + pReader->info.verRange.maxVer, pReader->idStr); + + return code; + +_err: + tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); + tsdbReaderClose2(*ppReader); + *ppReader = NULL; // reset the pointer value. + return code; +} + +void tsdbReaderClose2(STsdbReader* pReader) { + if (pReader == NULL) { + return; + } + + tsdbAcquireReader(pReader); + + { + if (pReader->innerReader[0] != NULL || pReader->innerReader[1] != NULL) { + STsdbReader* p = pReader->innerReader[0]; + clearSharedPtr(p); + + p = pReader->innerReader[1]; + clearSharedPtr(p); + + tsdbReaderClose2(pReader->innerReader[0]); + tsdbReaderClose2(pReader->innerReader[1]); + } + } + + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + TARRAY2_DESTROY(&pSupInfo->colAggArray, NULL); + for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) { + if (pSupInfo->buildBuf[i] != NULL) { + taosMemoryFreeClear(pSupInfo->buildBuf[i]); + } + } + + if (pReader->resBlockInfo.freeBlock) { + pReader->resBlockInfo.pResBlock = blockDataDestroy(pReader->resBlockInfo.pResBlock); + } + + taosMemoryFree(pSupInfo->colId); + tBlockDataDestroy(&pReader->status.fileBlockData); + cleanupDataBlockIterator(&pReader->status.blockIter); + + size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + if (pReader->status.pTableMap != NULL) { + destroyAllBlockScanInfo(pReader->status.pTableMap); + clearBlockScanInfoBuf(&pReader->blockInfoBuf); + pReader->status.pTableMap = NULL; + } + + if (pReader->pFileReader != NULL) { + tsdbDataFileReaderClose(&pReader->pFileReader); + } + + qTrace("tsdb/reader-close: %p, untake snapshot", pReader); + tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, true); + pReader->pReadSnap = NULL; + + tsdbReleaseReader(pReader); + tsdbUninitReaderLock(pReader); + + SCostSummary* pCost = &pReader->cost; + SFilesetIter* pFilesetIter = &pReader->status.fileIter; + if (pFilesetIter->pLastBlockReader != NULL) { + SLastBlockReader* pLReader = pFilesetIter->pLastBlockReader; + tMergeTreeClose(&pLReader->mergeTree); + taosMemoryFree(pLReader); + } + + destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->lastBlockLoad, &pCost->lastBlockLoadTime); + taosMemoryFreeClear(pReader->status.uidList.tableUidList); + + tsdbDebug( + "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 + " SMA-time:%.2f ms, fileBlocks:%" PRId64 + ", fileBlocks-load-time:%.2f ms, " + "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 + ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, createTime:%.2f ms,createSkylineIterTime:%.2f " + "ms, initLastBlockReader:%.2fms, %s", + pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks, + pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, pCost->lastBlockLoadTime, pCost->composedBlocks, + pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pCost->createScanInfoList, + pCost->createSkylineIterTime, pCost->initLastBlockReader, pReader->idStr); + + taosMemoryFree(pReader->idStr); + + tsdbRowMergerCleanup(&pReader->status.merger); + taosMemoryFree(pReader->info.pSchema); + + tSimpleHashCleanup(pReader->pSchemaMap); + taosMemoryFreeClear(pReader); +} + +int32_t tsdbReaderSuspend2(STsdbReader* pReader) { + int32_t code = 0; + + // save reader's base state & reset top state to be reconstructed from base state + SReaderStatus* pStatus = &pReader->status; + STableBlockScanInfo* pBlockScanInfo = NULL; + + if (pStatus->loadFromFile) { + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + if (pBlockInfo != NULL) { + pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); + if (pBlockScanInfo == NULL) { + goto _err; + } + } else { + pBlockScanInfo = *pStatus->pTableIter; + } + + tsdbDataFileReaderClose(&pReader->pFileReader); + + // resetDataBlockScanInfo excluding lastKey + STableBlockScanInfo** p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { + STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; + + pInfo->iterInit = false; + pInfo->iter.hasVal = false; + pInfo->iiter.hasVal = false; + + if (pInfo->iter.iter != NULL) { + pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); + } + + if (pInfo->iiter.iter != NULL) { + pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); + } + + pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + pInfo->pfileDelData = taosArrayDestroy(pInfo->pfileDelData); + } + } else { + // resetDataBlockScanInfo excluding lastKey + STableBlockScanInfo** p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { + STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; + + pInfo->iterInit = false; + pInfo->iter.hasVal = false; + pInfo->iiter.hasVal = false; + + if (pInfo->iter.iter != NULL) { + pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); + } + + if (pInfo->iiter.iter != NULL) { + pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); + } + + pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + } + + pBlockScanInfo = pStatus->pTableIter == NULL ? NULL : *pStatus->pTableIter; + if (pBlockScanInfo) { + // save lastKey to restore memory iterator + STimeWindow w = pReader->resBlockInfo.pResBlock->info.window; + pBlockScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? w.ekey : w.skey; + + // reset current current table's data block scan info, + pBlockScanInfo->iterInit = false; + + pBlockScanInfo->iter.hasVal = false; + pBlockScanInfo->iiter.hasVal = false; + if (pBlockScanInfo->iter.iter != NULL) { + pBlockScanInfo->iter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iter.iter); + } + + if (pBlockScanInfo->iiter.iter != NULL) { + pBlockScanInfo->iiter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iiter.iter); + } + + pBlockScanInfo->pBlockList = taosArrayDestroy(pBlockScanInfo->pBlockList); + // TODO: keep skyline for reuse + pBlockScanInfo->delSkyline = taosArrayDestroy(pBlockScanInfo->delSkyline); + } + } + + tsdbUntakeReadSnap(pReader, pReader->pReadSnap, false); + pReader->pReadSnap = NULL; + pReader->flag = READER_STATUS_SUSPEND; + + tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, + pReader->idStr); + return code; + +_err: + tsdbError("failed to suspend data reader, code:%s %s", tstrerror(code), pReader->idStr); + return code; +} + +static int32_t tsdbSetQueryReseek(void* pQHandle) { + int32_t code = 0; + STsdbReader* pReader = pQHandle; + + code = tsdbTryAcquireReader(pReader); + if (code == 0) { + if (pReader->flag == READER_STATUS_SUSPEND) { + tsdbReleaseReader(pReader); + return code; + } + + tsdbReaderSuspend2(pReader); + + tsdbReleaseReader(pReader); + + return code; + } else if (code == EBUSY) { + return TSDB_CODE_VND_QUERY_BUSY; + } else { + terrno = TAOS_SYSTEM_ERROR(code); + return TSDB_CODE_FAILED; + } +} + +int32_t tsdbReaderResume2(STsdbReader* pReader) { + int32_t code = 0; + + STableBlockScanInfo** pBlockScanInfo = pReader->status.pTableIter; + + // restore reader's state + // task snapshot + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + if (numOfTables > 0) { + qTrace("tsdb/reader: %p, take snapshot", pReader); + code = tsdbTakeReadSnap2(pReader, tsdbSetQueryReseek, &pReader->pReadSnap); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + STsdbReader* pPrevReader = pReader->innerReader[0]; + STsdbReader* pNextReader = pReader->innerReader[1]; + + // we need only one row + pPrevReader->resBlockInfo.capacity = 1; + setSharedPtr(pPrevReader, pReader); + + pNextReader->resBlockInfo.capacity = 1; + setSharedPtr(pNextReader, pReader); + + code = doOpenReaderImpl(pPrevReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } + + pReader->flag = READER_STATUS_NORMAL; + tsdbDebug("reader: %p resumed uid %" PRIu64 ", numOfTable:%" PRId32 ", in this query %s", pReader, + pBlockScanInfo ? (*pBlockScanInfo)->uid : 0, numOfTables, pReader->idStr); + return code; + +_err: + tsdbError("failed to resume data reader, code:%s %s", tstrerror(code), pReader->idStr); + return code; +} + +static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + + if (pReader->status.loadFromFile == false) { + return false; + } + + code = readRowsCountFromFiles(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + code = readRowsCountFromMem(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + pBlock->info.rows = pReader->rowsNum; + pBlock->info.id.uid = 0; + pBlock->info.dataLoad = 0; + + pReader->rowsNum = 0; + + return pBlock->info.rows > 0; +} + +static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { + int32_t code = TSDB_CODE_SUCCESS; + + // cleanup the data that belongs to the previous data block + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + blockDataCleanup(pBlock); + + *hasNext = false; + + SReaderStatus* pStatus = &pReader->status; + if (tSimpleHashGetSize(pStatus->pTableMap) == 0) { + return code; + } + + if (READ_MODE_COUNT_ONLY == pReader->info.readMode) { + return tsdbReadRowsCountOnly(pReader); + } + + if (pStatus->loadFromFile) { + code = buildBlockFromFiles(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pBlock->info.rows <= 0) { + resetTableListIndex(&pReader->status); + code = buildBlockFromBufferSequentially(pReader); + } + } else { // no data in files, let's try the buffer + code = buildBlockFromBufferSequentially(pReader); + } + + *hasNext = pBlock->info.rows > 0; + + return code; +} + +int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { + int32_t code = TSDB_CODE_SUCCESS; + + *hasNext = false; + + if (isEmptyQueryTimeWindow(&pReader->info.window) || pReader->step == EXTERNAL_ROWS_NEXT || + pReader->code != TSDB_CODE_SUCCESS) { + return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code; + } + + SReaderStatus* pStatus = &pReader->status; + + code = tsdbAcquireReader(pReader); + qTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); + + if (pReader->flag == READER_STATUS_SUSPEND) { + code = tsdbReaderResume2(pReader); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + } + + if (pReader->innerReader[0] != NULL && pReader->step == 0) { + code = doTsdbNextDataBlock2(pReader->innerReader[0], hasNext); + if (code) { + tsdbReleaseReader(pReader); + return code; + } + + pReader->step = EXTERNAL_ROWS_PREV; + if (*hasNext) { + pStatus = &pReader->innerReader[0]->status; + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + } + + return code; + } + } + + if (pReader->step == EXTERNAL_ROWS_PREV) { + // prepare for the main scan + code = doOpenReaderImpl(pReader); + int32_t step = 1; + resetAllDataBlockScanInfo(pReader->status.pTableMap, pReader->innerReader[0]->info.window.ekey, step); + + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + pReader->step = EXTERNAL_ROWS_MAIN; + } + + code = doTsdbNextDataBlock2(pReader, hasNext); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + + if (*hasNext) { + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + } + + return code; + } + + if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) { + // prepare for the next row scan + int32_t step = -1; + code = doOpenReaderImpl(pReader->innerReader[1]); + resetAllDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->info.window.ekey, step); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doTsdbNextDataBlock2(pReader->innerReader[1], hasNext); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + + pReader->step = EXTERNAL_ROWS_NEXT; + if (*hasNext) { + pStatus = &pReader->innerReader[1]->status; + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + } + + return code; + } + } + + qTrace("tsdb/read: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + + return code; +} + +static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { + // do fill all null column value SMA info + int32_t i = 0, j = 0; + int32_t size = (int32_t)TARRAY2_SIZE(&pSup->colAggArray); + TARRAY2_INSERT_PTR(&pSup->colAggArray, 0, pTsAgg); + size++; + + while (j < numOfCols && i < size) { + SColumnDataAgg* pAgg = &pSup->colAggArray.data[i]; + if (pAgg->colId == pSup->colId[j]) { + i += 1; + j += 1; + } else if (pAgg->colId < pSup->colId[j]) { + i += 1; + } else if (pSup->colId[j] < pAgg->colId) { + if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; + TARRAY2_INSERT_PTR(&pSup->colAggArray, i, &nullColAgg); + i += 1; + size++; + } + j += 1; + } + } + + while (j < numOfCols) { + if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; + TARRAY2_INSERT_PTR(&pSup->colAggArray, i, &nullColAgg); + i += 1; + } + j++; + } +} + +int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) { + SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg; + + int32_t code = 0; + *allHave = false; + *pBlockSMA = NULL; + + if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { + return TSDB_CODE_SUCCESS; + } + + // there is no statistics data for composed block + if (pReader->status.composedDataBlock || (!pReader->suppInfo.smaValid)) { + return TSDB_CODE_SUCCESS; + } + + SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); + SBlockLoadSuppInfo* pSup = &pReader->suppInfo; + + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + if (pResBlock->info.id.uid != pFBlock->uid) { + return TSDB_CODE_SUCCESS; + } + + // int64_t st = taosGetTimestampUs(); + TARRAY2_CLEAR(&pSup->colAggArray, 0); + + code = tsdbDataFileReadBlockSma(pReader->pFileReader, &pFBlock->record, &pSup->colAggArray); + if (code != TSDB_CODE_SUCCESS) { + tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code), + pReader->idStr); + return code; + } + + if (pSup->colAggArray.size > 0) { + *allHave = true; + } else { + *pBlockSMA = NULL; + return TSDB_CODE_SUCCESS; + } + + // always load the first primary timestamp column data + SColumnDataAgg* pTsAgg = &pSup->tsColAgg; + + pTsAgg->numOfNull = 0; + pTsAgg->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + pTsAgg->min = pResBlock->info.window.skey; + pTsAgg->max = pResBlock->info.window.ekey; + + // update the number of NULL data rows + size_t numOfCols = pSup->numOfCols; + + if (pResBlock->pBlockAgg == NULL) { + size_t num = taosArrayGetSize(pResBlock->pDataBlock); + pResBlock->pBlockAgg = taosMemoryCalloc(num, POINTER_BYTES); + } + + // do fill all null column value SMA info + doFillNullColSMA(pSup, pFBlock->record.numRow, numOfCols, pTsAgg); + + size_t size = pSup->colAggArray.size; + + int32_t i = 0, j = 0; + while (j < numOfCols && i < size) { + SColumnDataAgg* pAgg = &pSup->colAggArray.data[i]; + if (pAgg->colId == pSup->colId[j]) { + pResBlock->pBlockAgg[pSup->slotId[j]] = pAgg; + i += 1; + j += 1; + } else if (pAgg->colId < pSup->colId[j]) { + i += 1; + } else if (pSup->colId[j] < pAgg->colId) { + pResBlock->pBlockAgg[pSup->slotId[j]] = NULL; + *allHave = false; + j += 1; + } + } + + *pBlockSMA = pResBlock->pBlockAgg; + pReader->cost.smaDataLoad += 1; + + // double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; + pReader->cost.smaLoadTime += 0; // elapsedTime; + + tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr); + return code; +} + +static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + int32_t code = TSDB_CODE_SUCCESS; + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); + + if (pReader->code != TSDB_CODE_SUCCESS) { + return NULL; + } + + STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); + if (pBlockScanInfo == NULL) { + return NULL; + } + + code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); + if (code != TSDB_CODE_SUCCESS) { + tBlockDataDestroy(&pStatus->fileBlockData); + terrno = code; + return NULL; + } + + code = copyBlockDataToSDataBlock(pReader); + if (code != TSDB_CODE_SUCCESS) { + tBlockDataDestroy(&pStatus->fileBlockData); + terrno = code; + return NULL; + } + + return pReader->resBlockInfo.pResBlock; +} + +SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { + STsdbReader* pTReader = pReader; + if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { + if (pReader->step == EXTERNAL_ROWS_PREV) { + pTReader = pReader->innerReader[0]; + } else if (pReader->step == EXTERNAL_ROWS_NEXT) { + pTReader = pReader->innerReader[1]; + } + } + + SReaderStatus* pStatus = &pTReader->status; + if (pStatus->composedDataBlock) { + return pTReader->resBlockInfo.pResBlock; + } + + SSDataBlock* ret = doRetrieveDataBlock(pTReader); + + qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + + return ret; +} + +int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { + int32_t code = TSDB_CODE_SUCCESS; + + qTrace("tsdb/reader-reset: %p, take read mutex", pReader); + tsdbAcquireReader(pReader); + + if (pReader->flag == READER_STATUS_SUSPEND) { + code = tsdbReaderResume2(pReader); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + } + + if (isEmptyQueryTimeWindow(&pReader->info.window) || pReader->pReadSnap == NULL) { + tsdbDebug("tsdb reader reset return %p, %s", pReader->pReadSnap, pReader->idStr); + tsdbReleaseReader(pReader); + return TSDB_CODE_SUCCESS; + } + + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + + pReader->info.order = pCond->order; + pReader->type = TIMEWINDOW_RANGE_CONTAINED; + pStatus->loadFromFile = true; + pStatus->pTableIter = NULL; + pReader->info.window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); + + // allocate buffer in order to load data blocks from file + memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg)); + + pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; + tsdbDataFileReaderClose(&pReader->pFileReader); + + int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap); + + initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); + resetDataBlockIterator(pBlockIter, pReader->info.order); + resetTableListIndex(&pReader->status); + + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + int32_t step = asc ? 1 : -1; + int64_t ts = asc ? pReader->info.window.skey - 1 : pReader->info.window.ekey + 1; + resetAllDataBlockScanInfo(pStatus->pTableMap, ts, step); + + // no data in files, let's try buffer in memory + if (pStatus->fileIter.numOfFiles == 0) { + pStatus->loadFromFile = false; + resetTableListIndex(pStatus); + } else { + code = initForFirstBlockInFile(pReader, pBlockIter); + if (code != TSDB_CODE_SUCCESS) { + tsdbError("%p reset reader failed, numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", pReader, + numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr); + + tsdbReleaseReader(pReader); + return code; + } + } + + tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, skey:%" PRId64 ", query range:%" PRId64 " - %" PRId64 + " in query %s", + pReader, pReader->info.suid, numOfTables, pCond->twindows.skey, pReader->info.window.skey, + pReader->info.window.ekey, pReader->idStr); + + tsdbReleaseReader(pReader); + + return code; +} + +static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t numOfRows, int32_t numOfBucket) { + if (numOfRows < startRow) { + return 0; + } + int32_t bucketIndex = ((numOfRows - startRow) / bucketRange); + if (bucketIndex == numOfBucket) { + bucketIndex -= 1; + } + return bucketIndex; +} + +int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pTableBlockInfo) { + int32_t code = TSDB_CODE_SUCCESS; + pTableBlockInfo->totalSize = 0; + pTableBlockInfo->totalRows = 0; + pTableBlockInfo->numOfVgroups = 1; + + const int32_t numOfBuckets = 20.0; + const int32_t defaultRows = 4096; + + // find the start data block in file + tsdbAcquireReader(pReader); + if (pReader->flag == READER_STATUS_SUSPEND) { + code = tsdbReaderResume2(pReader); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + } + SReaderStatus* pStatus = &pReader->status; + + STsdbCfg* pc = &pReader->pTsdb->pVnode->config.tsdbCfg; + pTableBlockInfo->defMinRows = pc->minRows; + pTableBlockInfo->defMaxRows = pc->maxRows; + + int32_t bucketRange = ceil(((double)(pc->maxRows - pc->minRows)) / numOfBuckets); + + pTableBlockInfo->numOfFiles += 1; + + int32_t numOfTables = (int32_t)tSimpleHashGetSize(pStatus->pTableMap); + + SDataBlockIter* pBlockIter = &pStatus->blockIter; + pTableBlockInfo->numOfFiles += pStatus->fileIter.numOfFiles; + + if (pBlockIter->numOfBlocks > 0) { + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; + } + + pTableBlockInfo->numOfTables = numOfTables; + bool hasNext = (pBlockIter->numOfBlocks > 0); + + while (true) { + if (hasNext) { + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + int32_t numOfRows = pBlockInfo->record.numRow; + + pTableBlockInfo->totalRows += numOfRows; + + if (numOfRows > pTableBlockInfo->maxRows) { + pTableBlockInfo->maxRows = numOfRows; + } + + if (numOfRows < pTableBlockInfo->minRows) { + pTableBlockInfo->minRows = numOfRows; + } + + if (numOfRows < defaultRows) { + pTableBlockInfo->numOfSmallBlocks += 1; + } + + pTableBlockInfo->totalSize += pBlockInfo->record.blockSize; + + int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows, numOfBuckets); + pTableBlockInfo->blockRowsHisto[bucketIndex]++; + + hasNext = blockIteratorNext(&pStatus->blockIter, pReader->idStr); + } else { + code = initForFirstBlockInFile(pReader, pBlockIter); + if ((code != TSDB_CODE_SUCCESS) || (pStatus->loadFromFile == false)) { + break; + } + + pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks; + hasNext = (pBlockIter->numOfBlocks > 0); + } + + // tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, + // pReader->pFileGroup->fid, pReader->idStr); + } + tsdbReleaseReader(pReader); + return code; +} + +int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t rows = 0; + + SReaderStatus* pStatus = &pReader->status; + tsdbAcquireReader(pReader); + if (pReader->flag == READER_STATUS_SUSPEND) { + code = tsdbReaderResume2(pReader); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + } + + int32_t iter = 0; + pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter); + + while (pStatus->pTableIter != NULL) { + STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; + + STbData* d = NULL; + if (pReader->pReadSnap->pMem != NULL) { + d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->info.suid, pBlockScanInfo->uid); + if (d != NULL) { + rows += tsdbGetNRowsInTbData(d); + } + } + + STbData* di = NULL; + if (pReader->pReadSnap->pIMem != NULL) { + di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->info.suid, pBlockScanInfo->uid); + if (di != NULL) { + rows += tsdbGetNRowsInTbData(di); + } + } + + // current table is exhausted, let's try the next table + pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, pStatus->pTableIter, &iter); + } + + tsdbReleaseReader(pReader); + + return rows; +} + +int32_t tsdbGetTableSchema2(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { + SMetaReader mr = {0}; + metaReaderDoInit(&mr, ((SVnode*)pVnode)->pMeta, 0); + int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid); + if (code != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; + metaReaderClear(&mr); + return terrno; + } + + *suid = 0; + + // only child table and ordinary table is allowed, super table is not allowed. + if (mr.me.type == TSDB_CHILD_TABLE) { + tDecoderClear(&mr.coder); + *suid = mr.me.ctbEntry.suid; + code = metaReaderGetTableEntryByUidCache(&mr, *suid); + if (code != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; + metaReaderClear(&mr); + return terrno; + } + } else if (mr.me.type == TSDB_NORMAL_TABLE) { // do nothing + } else { + terrno = TSDB_CODE_INVALID_PARA; + metaReaderClear(&mr); + return terrno; + } + + metaReaderClear(&mr); + + // get the newest table schema version + code = metaGetTbTSchemaEx(((SVnode*)pVnode)->pMeta, *suid, uid, -1, pSchema); + return code; +} + +int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap) { + int32_t code = 0; + STsdb* pTsdb = pReader->pTsdb; + SVersionRange* pRange = &pReader->info.verRange; + + // lock + taosThreadRwlockRdlock(&pTsdb->rwLock); + + // alloc + STsdbReadSnap* pSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap)); + if (pSnap == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + // take snapshot + if (pTsdb->mem && (pRange->minVer <= pTsdb->mem->maxVer && pRange->maxVer >= pTsdb->mem->minVer)) { + pSnap->pMem = pTsdb->mem; + pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode)); + if (pSnap->pNode == NULL) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pSnap->pNode->pQHandle = pReader; + pSnap->pNode->reseek = reseek; + + tsdbRefMemTable(pTsdb->mem, pSnap->pNode); + } + + if (pTsdb->imem && (pRange->minVer <= pTsdb->imem->maxVer && pRange->maxVer >= pTsdb->imem->minVer)) { + pSnap->pIMem = pTsdb->imem; + pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode)); + if (pSnap->pINode == NULL) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pSnap->pINode->pQHandle = pReader; + pSnap->pINode->reseek = reseek; + + tsdbRefMemTable(pTsdb->imem, pSnap->pINode); + } + + // fs + code = tsdbFSCreateRefSnapshot(pTsdb->pFS, &pSnap->pfSetArray); + if (code) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _exit; + } + + // unlock + taosThreadRwlockUnlock(&pTsdb->rwLock); + + tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); + +_exit: + if (code) { + *ppSnap = NULL; + if (pSnap) { + if (pSnap->pNode) taosMemoryFree(pSnap->pNode); + if (pSnap->pINode) taosMemoryFree(pSnap->pINode); + taosMemoryFree(pSnap); + } + } else { + *ppSnap = pSnap; + } + + return code; +} + +void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proactive) { + STsdb* pTsdb = pReader->pTsdb; + + if (pSnap) { + if (pSnap->pMem) { + tsdbUnrefMemTable(pSnap->pMem, pSnap->pNode, proactive); + } + + if (pSnap->pIMem) { + tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive); + } + + tsdbFSUnref(pTsdb, &pSnap->fs); + if (pSnap->pNode) taosMemoryFree(pSnap->pNode); + if (pSnap->pINode) taosMemoryFree(pSnap->pINode); + + tsdbFSDestroyRefSnapshot(&pSnap->pfSetArray); + + taosMemoryFree(pSnap); + } + tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode)); +} + +// if failed, do nothing +void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { + taosMemoryFreeClear(pReader->idStr); + pReader->idStr = taosStrdup(idstr); + pReader->status.fileIter.pLastBlockReader->mergeTree.idStr = pReader->idStr; +} + +void tsdbReaderSetCloseFlag2(STsdbReader* pReader) { pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..635a74d8dd77f4a0e90bee9a5953c1538e8e292c --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -0,0 +1,630 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbReadUtil.h" +#include "osDef.h" +#include "tsdb.h" +#include "tsdbDataFileRW.h" +#include "tsdbFS2.h" +#include "tsdbMerge.h" +#include "tsdbUtil2.h" +#include "tsimplehash.h" + +static int32_t uidComparFunc(const void* p1, const void* p2) { + uint64_t pu1 = *(uint64_t*)p1; + uint64_t pu2 = *(uint64_t*)p2; + if (pu1 == pu2) { + return 0; + } else { + return (pu1 < pu2) ? -1 : 1; + } +} + +static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { + int32_t num = numOfTables / pBuf->numPerBucket; + int32_t remainder = numOfTables % pBuf->numPerBucket; + if (pBuf->pData == NULL) { + pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES); + } + + for (int32_t i = 0; i < num; ++i) { + char* p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush(pBuf->pData, &p); + } + + if (remainder > 0) { + char* p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosArrayPush(pBuf->pData, &p); + } + + pBuf->numOfTables = numOfTables; + + return TSDB_CODE_SUCCESS; +} + +int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { + if (numOfTables <= pBuf->numOfTables) { + return TSDB_CODE_SUCCESS; + } + + if (pBuf->numOfTables > 0) { + STableBlockScanInfo** p = (STableBlockScanInfo**)taosArrayPop(pBuf->pData); + taosMemoryFree(*p); + pBuf->numOfTables /= pBuf->numPerBucket; + } + + int32_t num = (numOfTables - pBuf->numOfTables) / pBuf->numPerBucket; + int32_t remainder = (numOfTables - pBuf->numOfTables) % pBuf->numPerBucket; + if (pBuf->pData == NULL) { + pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES); + } + + for (int32_t i = 0; i < num; ++i) { + char* p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush(pBuf->pData, &p); + } + + if (remainder > 0) { + char* p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosArrayPush(pBuf->pData, &p); + } + + pBuf->numOfTables = numOfTables; + + return TSDB_CODE_SUCCESS; +} + +void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) { + size_t num = taosArrayGetSize(pBuf->pData); + for (int32_t i = 0; i < num; ++i) { + char** p = taosArrayGet(pBuf->pData, i); + taosMemoryFree(*p); + } + + taosArrayDestroy(pBuf->pData); +} + +void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) { + int32_t bucketIndex = index / pBuf->numPerBucket; + char** pBucket = taosArrayGet(pBuf->pData, bucketIndex); + return (*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo); +} + +STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id) { + STableBlockScanInfo** p = tSimpleHashGet(pTableMap, &uid, sizeof(uid)); + if (p == NULL || *p == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + int32_t size = tSimpleHashGetSize(pTableMap); + tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); + return NULL; + } + + return *p; +} + +// NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model +SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, + STableUidList* pUidList, int32_t numOfTables) { + // allocate buffer in order to load data blocks from file + // todo use simple hash instead, optimize the memory consumption + SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + if (pTableMap == NULL) { + return NULL; + } + + int64_t st = taosGetTimestampUs(); + initBlockScanInfoBuf(pBuf, numOfTables); + + pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); + if (pUidList->tableUidList == NULL) { + tSimpleHashCleanup(pTableMap); + return NULL; + } + + pUidList->currentIndex = 0; + + for (int32_t j = 0; j < numOfTables; ++j) { + STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(pBuf, j); + + pScanInfo->uid = idList[j].uid; + pUidList->tableUidList[j] = idList[j].uid; + + if (ASCENDING_TRAVERSE(pTsdbReader->info.order)) { + int64_t skey = pTsdbReader->info.window.skey; + pScanInfo->lastKey = (skey > INT64_MIN) ? (skey - 1) : skey; + pScanInfo->lastKeyInStt = skey; + } else { + int64_t ekey = pTsdbReader->info.window.ekey; + pScanInfo->lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey; + pScanInfo->lastKeyInStt = ekey; + } + + tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); + tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReader, pScanInfo->uid, + pScanInfo->lastKey, pTsdbReader->idStr); + } + + taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc); + + pTsdbReader->cost.createScanInfoList = (taosGetTimestampUs() - st) / 1000.0; + tsdbDebug("%p create %d tables scan-info, size:%.2f Kb, elapsed time:%.2f ms, %s", pTsdbReader, numOfTables, + (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList, + pTsdbReader->idStr); + + return pTableMap; +} + +void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { + void* p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { + STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; + + pInfo->iterInit = false; + pInfo->iter.hasVal = false; + pInfo->iiter.hasVal = false; + + if (pInfo->iter.iter != NULL) { + pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); + } + + if (pInfo->iiter.iter != NULL) { + pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); + } + + pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + pInfo->lastKey = ts; + pInfo->lastKeyInStt = ts + step; + } +} + +void clearBlockScanInfo(STableBlockScanInfo* p) { + p->iterInit = false; + p->iter.hasVal = false; + p->iiter.hasVal = false; + + if (p->iter.iter != NULL) { + p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter); + } + + if (p->iiter.iter != NULL) { + p->iiter.iter = tsdbTbDataIterDestroy(p->iiter.iter); + } + + p->delSkyline = taosArrayDestroy(p->delSkyline); + p->pBlockList = taosArrayDestroy(p->pBlockList); + p->pMemDelData = taosArrayDestroy(p->pMemDelData); + p->pfileDelData = taosArrayDestroy(p->pfileDelData); +} + +void destroyAllBlockScanInfo(SSHashObj* pTableMap) { + void* p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { + clearBlockScanInfo(*(STableBlockScanInfo**)p); + } + + tSimpleHashCleanup(pTableMap); +} + +static void doCleanupInfoForNextFileset(STableBlockScanInfo* pScanInfo) { + // reset the index in last block when handing a new file + taosArrayClear(pScanInfo->pBlockList); + taosArrayClear(pScanInfo->pfileDelData); // del data from each file set +} + +void cleanupInfoFoxNextFileset(SSHashObj* pTableMap) { + STableBlockScanInfo** p = NULL; + + int32_t iter = 0; + while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { + doCleanupInfoForNextFileset(*p); + } +} + +// brin records iterator +void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList) { + memset(&pIter->block, 0, sizeof(SBrinBlock)); + memset(&pIter->record, 0, sizeof(SBrinRecord)); + pIter->blockIndex = -1; + pIter->recordIndex = -1; + + pIter->pReader = pReader; + pIter->pBrinBlockList = pList; +} + +SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { + if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= TARRAY2_SIZE(pIter->block.numRow)) { + pIter->blockIndex += 1; + if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) { + return NULL; + } + + pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex); + + tBrinBlockClear(&pIter->block); + tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); + pIter->recordIndex = -1; + } + + pIter->recordIndex += 1; + tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record); + return &pIter->record; +} + +void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); } + +// initialize the file block access order +// sort the file blocks according to the offset of each data block in the files +static void cleanupBlockOrderSupporter(SBlockOrderSupporter* pSup) { + taosMemoryFreeClear(pSup->numOfBlocksPerTable); + taosMemoryFreeClear(pSup->indexPerTable); + + for (int32_t i = 0; i < pSup->numOfTables; ++i) { + SBlockOrderWrapper* pBlockInfo = pSup->pDataBlockInfo[i]; + taosMemoryFreeClear(pBlockInfo); + } + + taosMemoryFreeClear(pSup->pDataBlockInfo); +} + +static int32_t initBlockOrderSupporter(SBlockOrderSupporter* pSup, int32_t numOfTables) { + pSup->numOfBlocksPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); + pSup->indexPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); + pSup->pDataBlockInfo = taosMemoryCalloc(1, POINTER_BYTES * numOfTables); + + if (pSup->numOfBlocksPerTable == NULL || pSup->indexPerTable == NULL || pSup->pDataBlockInfo == NULL) { + cleanupBlockOrderSupporter(pSup); + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) { + int32_t leftIndex = *(int32_t*)pLeft; + int32_t rightIndex = *(int32_t*)pRight; + + SBlockOrderSupporter* pSupporter = (SBlockOrderSupporter*)param; + + int32_t leftTableBlockIndex = pSupporter->indexPerTable[leftIndex]; + int32_t rightTableBlockIndex = pSupporter->indexPerTable[rightIndex]; + + if (leftTableBlockIndex > pSupporter->numOfBlocksPerTable[leftIndex]) { + /* left block is empty */ + return 1; + } else if (rightTableBlockIndex > pSupporter->numOfBlocksPerTable[rightIndex]) { + /* right block is empty */ + return -1; + } + + SBlockOrderWrapper* pLeftBlock = &pSupporter->pDataBlockInfo[leftIndex][leftTableBlockIndex]; + SBlockOrderWrapper* pRightBlock = &pSupporter->pDataBlockInfo[rightIndex][rightTableBlockIndex]; + + return pLeftBlock->offset > pRightBlock->offset ? 1 : -1; +} + +int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) { + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + + SBlockOrderSupporter sup = {0}; + pBlockIter->numOfBlocks = numOfBlocks; + taosArrayClear(pBlockIter->blockList); + + pBlockIter->pTableMap = pReader->status.pTableMap; + + // access data blocks according to the offset of each block in asc/desc order. + int32_t numOfTables = taosArrayGetSize(pTableList); + + int64_t st = taosGetTimestampUs(); + int32_t code = initBlockOrderSupporter(&sup, numOfTables); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int32_t cnt = 0; + + for (int32_t i = 0; i < numOfTables; ++i) { + STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, i); + // ASSERT(pTableScanInfo->pBlockList != NULL && taosArrayGetSize(pTableScanInfo->pBlockList) > 0); + + size_t num = taosArrayGetSize(pTableScanInfo->pBlockList); + sup.numOfBlocksPerTable[sup.numOfTables] = num; + + char* buf = taosMemoryMalloc(sizeof(SBlockOrderWrapper) * num); + if (buf == NULL) { + cleanupBlockOrderSupporter(&sup); + return TSDB_CODE_OUT_OF_MEMORY; + } + + sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf; + + for (int32_t k = 0; k < num; ++k) { + SBrinRecord* pRecord = taosArrayGet(pTableScanInfo->pBlockList, k); + sup.pDataBlockInfo[sup.numOfTables][k] = + (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pRecord->blockOffset, .pInfo = pTableScanInfo}; + cnt++; + } + + sup.numOfTables += 1; + } + + if (numOfBlocks != cnt && sup.numOfTables != numOfTables) { + cleanupBlockOrderSupporter(&sup); + return TSDB_CODE_INVALID_PARA; + } + + // since there is only one table qualified, blocks are not sorted + if (sup.numOfTables == 1) { + for (int32_t i = 0; i < numOfBlocks; ++i) { + SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[0][i].uid, .tbBlockIdx = i}; + blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[0][i].pInfo->pBlockList, i); + + taosArrayPush(pBlockIter->blockList, &blockInfo); + } + + int64_t et = taosGetTimestampUs(); + tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s", + pReader, numOfBlocks, (et - st) / 1000.0, pReader->idStr); + + pBlockIter->index = asc ? 0 : (numOfBlocks - 1); + cleanupBlockOrderSupporter(&sup); + return TSDB_CODE_SUCCESS; + } + + tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables, + pReader->idStr); + + SMultiwayMergeTreeInfo* pTree = NULL; + + uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); + if (ret != TSDB_CODE_SUCCESS) { + cleanupBlockOrderSupporter(&sup); + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t numOfTotal = 0; + while (numOfTotal < cnt) { + int32_t pos = tMergeTreeGetChosenIndex(pTree); + int32_t index = sup.indexPerTable[pos]++; + + SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[pos][index].uid, .tbBlockIdx = index}; + blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); + + taosArrayPush(pBlockIter->blockList, &blockInfo); + + // set data block index overflow, in order to disable the offset comparator + if (sup.indexPerTable[pos] >= sup.numOfBlocksPerTable[pos]) { + sup.indexPerTable[pos] = sup.numOfBlocksPerTable[pos] + 1; + } + + numOfTotal += 1; + tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree)); + } + + int64_t et = taosGetTimestampUs(); + tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks, + (et - st) / 1000.0, pReader->idStr); + cleanupBlockOrderSupporter(&sup); + taosMemoryFree(pTree); + + pBlockIter->index = asc ? 0 : (numOfBlocks - 1); + return TSDB_CODE_SUCCESS; +} + +bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr) { + bool asc = ASCENDING_TRAVERSE(pBlockIter->order); + + int32_t step = asc ? 1 : -1; + if ((pBlockIter->index >= pBlockIter->numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) { + return false; + } + + pBlockIter->index += step; + return true; +} + +typedef enum { + BLK_CHECK_CONTINUE = 0x1, + BLK_CHECK_QUIT = 0x2, +} ETombBlkCheckEnum; + +static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_t numOfTables, int32_t* j, + STableBlockScanInfo** pScanInfo, ETombBlkCheckEnum* pRet) { + int32_t code = 0; + STombRecord record = {0}; + uint64_t uid = pReader->status.uidList.tableUidList[*j]; + + for (int32_t k = 0; k < TARRAY2_SIZE(pBlock->suid); ++k) { + code = tTombBlockGet(pBlock, k, &record); + if (code != TSDB_CODE_SUCCESS) { + *pRet = BLK_CHECK_QUIT; + return code; + } + + if (record.suid < pReader->info.suid) { + continue; + } + + if (record.suid > pReader->info.suid) { + *pRet = BLK_CHECK_QUIT; + return TSDB_CODE_SUCCESS; + } + + bool newTable = false; + if (uid < record.uid) { + while ((*j) < numOfTables && pReader->status.uidList.tableUidList[*j] < record.uid) { + (*j) += 1; + newTable = true; + } + + if ((*j) >= numOfTables) { + *pRet = BLK_CHECK_QUIT; + return TSDB_CODE_SUCCESS; + } + + uid = pReader->status.uidList.tableUidList[*j]; + } + + if (record.uid < uid) { + continue; + } + + ASSERT(record.suid == pReader->info.suid && uid == record.uid); + + if (newTable) { + (*pScanInfo) = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + if ((*pScanInfo)->pfileDelData == NULL) { + (*pScanInfo)->pfileDelData = taosArrayInit(4, sizeof(SDelData)); + } + } + + if (record.version <= pReader->info.verRange.maxVer) { + SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; + taosArrayPush((*pScanInfo)->pfileDelData, &delData); + } + } + + *pRet = BLK_CHECK_CONTINUE; + return TSDB_CODE_SUCCESS; +} + +// load tomb data API +static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STsdbReader* pReader, void* pFileReader, + bool isFile) { + int32_t code = 0; + STableUidList* pList = &pReader->status.uidList; + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + + int32_t i = 0, j = 0; + while (i < pTombBlkArray->size && j < numOfTables) { + STombBlk* pTombBlk = &pTombBlkArray->data[i]; + if (pTombBlk->maxTbid.suid < pReader->info.suid) { + i += 1; + continue; + } + + if (pTombBlk->minTbid.suid > pReader->info.suid) { + break; + } + + ASSERT(pTombBlk->minTbid.suid <= pReader->info.suid && pTombBlk->maxTbid.suid >= pReader->info.suid); + if (pTombBlk->maxTbid.suid == pReader->info.suid && pTombBlk->maxTbid.uid < pList->tableUidList[0]) { + i += 1; + continue; + } + + if (pTombBlk->minTbid.suid == pReader->info.suid && pTombBlk->minTbid.uid > pList->tableUidList[numOfTables - 1]) { + break; + } + + STombBlock block = {0}; + code = isFile ? tsdbDataFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block) + : tsdbSttFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + uint64_t uid = pReader->status.uidList.tableUidList[j]; + + STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); + if (pScanInfo->pfileDelData == NULL) { + pScanInfo->pfileDelData = taosArrayInit(4, sizeof(SDelData)); + } + + ETombBlkCheckEnum ret = 0; + code = doCheckTombBlock(&block, pReader, numOfTables, &j, &pScanInfo, &ret); + + tTombBlockDestroy(&block); + if (code != TSDB_CODE_SUCCESS || ret == BLK_CHECK_QUIT) { + return code; + } + + i += 1; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t loadDataFileTombDataForAll(STsdbReader* pReader) { + if (pReader->status.pCurrentFileset == NULL || pReader->status.pCurrentFileset->farr[3] == NULL) { + return TSDB_CODE_SUCCESS; + } + + const TTombBlkArray* pBlkArray = NULL; + + int32_t code = tsdbDataFileReadTombBlk(pReader->pFileReader, &pBlkArray); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + return doLoadTombDataFromTombBlk(pBlkArray, pReader, pReader->pFileReader, true); +} + +int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo) { + const TTombBlkArray* pBlkArray = NULL; + int32_t code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + return doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false); +} + +void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) { + if (*ppMemDelData == NULL) { + *ppMemDelData = taosArrayInit(4, sizeof(SDelData)); + } + + SArray* pMemDelData = *ppMemDelData; + + SDelData* p = NULL; + if (pMemTbData != NULL) { + p = pMemTbData->pHead; + while (p) { + if (p->version <= ver) { + taosArrayPush(pMemDelData, p); + } + + p = p->pNext; + } + } + + if (piMemTbData != NULL) { + p = piMemTbData->pHead; + while (p) { + if (p->version <= ver) { + taosArrayPush(pMemDelData, p); + } + p = p->pNext; + } + } +} diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h new file mode 100644 index 0000000000000000000000000000000000000000..5c4737440d78afe8750cb2d2d7398bafc68b5c5e --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TSDBREADUTIL_H +#define TDENGINE_TSDBREADUTIL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tsdbDataFileRW.h" +#include "tsdbUtil2.h" + +#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) + +typedef enum { + READER_STATUS_SUSPEND = 0x1, + READER_STATUS_NORMAL = 0x2, +} EReaderStatus; + +typedef enum { + EXTERNAL_ROWS_PREV = 0x1, + EXTERNAL_ROWS_MAIN = 0x2, + EXTERNAL_ROWS_NEXT = 0x3, +} EContentData; + +typedef struct SBlockInfoBuf { + int32_t currentIndex; + SArray* pData; + int32_t numPerBucket; + int32_t numOfTables; +} SBlockInfoBuf; + +typedef struct { + STbDataIter* iter; + int32_t index; + bool hasVal; +} SIterInfo; + +typedef struct STableBlockScanInfo { + uint64_t uid; + TSKEY lastKey; + TSKEY lastKeyInStt; // last accessed key in stt + SArray* pBlockList; // block data index list, SArray + SArray* pMemDelData; // SArray + SArray* pfileDelData; // SArray from each file set + SIterInfo iter; // mem buffer skip list iterator + SIterInfo iiter; // imem buffer skip list iterator + SArray* delSkyline; // delete info for this table + int32_t fileDelIndex; // file block delete index + int32_t lastBlockDelIndex; // delete index for last block + bool iterInit; // whether to initialize the in-memory skip list iterator or not +} STableBlockScanInfo; + +typedef struct SResultBlockInfo { + SSDataBlock* pResBlock; + bool freeBlock; + int64_t capacity; +} SResultBlockInfo; + +typedef struct SCostSummary { + int64_t numOfBlocks; + double blockLoadTime; + double buildmemBlock; + int64_t headFileLoad; + double headFileLoadTime; + int64_t smaDataLoad; + double smaLoadTime; + int64_t lastBlockLoad; + double lastBlockLoadTime; + int64_t composedBlocks; + double buildComposedBlockTime; + double createScanInfoList; + double createSkylineIterTime; + double initLastBlockReader; +} SCostSummary; + +typedef struct STableUidList { + uint64_t* tableUidList; // access table uid list in uid ascending order list + int32_t currentIndex; // index in table uid list +} STableUidList; + +typedef struct { + int32_t numOfBlocks; + int32_t numOfLastFiles; +} SBlockNumber; + +typedef struct SBlockIndex { + int32_t ordinalIndex; + int64_t inFileOffset; + STimeWindow window; // todo replace it with overlap flag. +} SBlockIndex; + +typedef struct SBlockOrderWrapper { + int64_t uid; + int64_t offset; + STableBlockScanInfo* pInfo; +} SBlockOrderWrapper; + +typedef struct SBlockOrderSupporter { + SBlockOrderWrapper** pDataBlockInfo; + int32_t* indexPerTable; + int32_t* numOfBlocksPerTable; + int32_t numOfTables; +} SBlockOrderSupporter; + +typedef struct SBlockLoadSuppInfo { + TColumnDataAggArray colAggArray; + SColumnDataAgg tsColAgg; + int16_t* colId; + int16_t* slotId; + int32_t numOfCols; + char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. + bool smaValid; // the sma on all queried columns are activated +} SBlockLoadSuppInfo; + +typedef struct SLastBlockReader { + STimeWindow window; + SVersionRange verRange; + int32_t order; + uint64_t uid; + SMergeTree mergeTree; + SSttBlockLoadInfo* pInfo; + int64_t currentKey; +} SLastBlockReader; + +typedef struct SFilesetIter { + int32_t numOfFiles; // number of total files + int32_t index; // current accessed index in the list + TFileSetArray* pFilesetList; // data file set list + int32_t order; + SLastBlockReader* pLastBlockReader; // last file block reader +} SFilesetIter; + +typedef struct SFileDataBlockInfo { + // index position in STableBlockScanInfo in order to check whether neighbor block overlaps with it + uint64_t uid; + int32_t tbBlockIdx; + SBrinRecord record; +} SFileDataBlockInfo; + +typedef struct SDataBlockIter { + int32_t numOfBlocks; + int32_t index; + SArray* blockList; // SArray + int32_t order; + SDataBlk block; // current SDataBlk data + SSHashObj* pTableMap; +} SDataBlockIter; + +typedef struct SFileBlockDumpInfo { + int32_t totalRows; + int32_t rowIndex; + int64_t lastKey; + bool allDumped; +} SFileBlockDumpInfo; + +typedef struct SReaderStatus { + bool loadFromFile; // check file stage + bool composedDataBlock; // the returned data block is a composed block or not + SSHashObj* pTableMap; // SHash + STableBlockScanInfo** pTableIter; // table iterator used in building in-memory buffer data blocks. + STableUidList uidList; // check tables in uid order, to avoid the repeatly load of blocks in STT. + SFileBlockDumpInfo fBlockDumpInfo; + STFileSet* pCurrentFileset; // current opened file set + SBlockData fileBlockData; + SFilesetIter fileIter; + SDataBlockIter blockIter; + SArray* pLDataIterArray; + SRowMerger merger; + SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data +} SReaderStatus; + +struct STsdbReader { + STsdb* pTsdb; + STsdbReaderInfo info; + TdThreadMutex readerMutex; + EReaderStatus flag; + int32_t code; + uint64_t rowsNum; + SResultBlockInfo resBlockInfo; + SReaderStatus status; + char* idStr; // query info handle, for debug purpose + int32_t type; // query type: 1. retrieve all data blocks, 2. retrieve direct prev|next rows + SBlockLoadSuppInfo suppInfo; + STsdbReadSnap* pReadSnap; + SCostSummary cost; + SHashObj** pIgnoreTables; + SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema + SDataFileReader* pFileReader; // the file reader + SBlockInfoBuf blockInfoBuf; + EContentData step; + STsdbReader* innerReader[2]; +}; + +typedef struct SBrinRecordIter { + SArray* pBrinBlockList; + SBrinBlk* pCurrentBlk; + int32_t blockIndex; + int32_t recordIndex; + SDataFileReader* pReader; + SBrinBlock block; + SBrinRecord record; +} SBrinRecordIter; + +STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id); + +SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, + STableUidList* pUidList, int32_t numOfTables); +void clearBlockScanInfo(STableBlockScanInfo* p); +void destroyAllBlockScanInfo(SSHashObj* pTableMap); +void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step); +void cleanupInfoFoxNextFileset(SSHashObj* pTableMap); +int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables); +void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf); +void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index); + +// brin records iterator +void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList); +SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter); +void clearBrinBlockIter(SBrinRecordIter* pIter); + +// initialize block iterator API +int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList); +bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr); + +// load tomb data API (stt/mem only for one table each, tomb data from data files are load for all tables at one time) +void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver); +int32_t loadDataFileTombDataForAll(STsdbReader* pReader); +int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TSDBREADUTIL_H diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 4b677533e73983d4da4ef44c230d59f2a8e4d847..89b7d019ae194f11c33151555f1a52c872f7fe5e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -16,7 +16,7 @@ #include "tsdb.h" // =============== PAGE-WISE FILE =============== -static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) { +int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) { int32_t code = 0; STsdbFD *pFD = NULL; @@ -68,7 +68,7 @@ _exit: return code; } -static void tsdbCloseFile(STsdbFD **ppFD) { +void tsdbCloseFile(STsdbFD **ppFD) { STsdbFD *pFD = *ppFD; if (pFD) { taosMemoryFree(pFD->pBuf); @@ -141,7 +141,7 @@ _exit: return code; } -static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) { +int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) { int32_t code = 0; int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); @@ -173,7 +173,7 @@ _exit: return code; } -static int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { +int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { int32_t code = 0; int64_t n = 0; int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); @@ -202,7 +202,7 @@ _exit: return code; } -static int32_t tsdbFsyncFile(STsdbFD *pFD) { +int32_t tsdbFsyncFile(STsdbFD *pFD) { int32_t code = 0; code = tsdbWriteFilePage(pFD); @@ -749,7 +749,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { int64_t size; TdFilePtr pOutFD = NULL; TdFilePtr PInFD = NULL; - int32_t szPage = pTsdb->pVnode->config.szPage; + int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; char fNameFrom[TSDB_FILENAME_LEN]; char fNameTo[TSDB_FILENAME_LEN]; @@ -1489,7 +1489,7 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { } int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) { - return tsdbReadDelDatav1(pReader, pDelIdx, aDelData, INT64_MAX); + return tsdbReadDelDatav1(pReader, pDelIdx, aDelData, INT64_MAX); } int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) { @@ -1517,10 +1517,10 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa if (delData.version > maxVer) { continue; } - if (taosArrayPush(aDelData, &delData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + if (taosArrayPush(aDelData, &delData) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } } ASSERT(n == size); diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 7c7e1bd0f79254197e1c99dd5ca0544770656cea..ae5ac4ae3647012c39a20d9bee4cade563899b7a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -14,101 +14,265 @@ */ #include "tsdb.h" +#include "tsdbFS2.h" -static bool tsdbShouldDoRetentionImpl(STsdb *pTsdb, int64_t now) { - for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; +typedef struct { + STsdb *tsdb; + int32_t szPage; + int64_t now; + int64_t cid; - if (expLevel == pSet->diskId.level) continue; + TFileSetArray *fsetArr; + TFileOpArray fopArr[1]; - if (expLevel < 0) { - return true; - } else { - if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { - return false; - } + struct { + int32_t fsetArrIdx; + STFileSet *fset; + } ctx[1]; +} SRTNer; - if (did.level == pSet->diskId.level) continue; +static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) { + STFileOp op = { + .optype = TSDB_FOP_REMOVE, + .fid = fobj->f->fid, + .of = fobj->f[0], + }; - return true; - } + return TARRAY2_APPEND(rtner->fopArr, op); +} + +static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile *to) { + int32_t code = 0; + int32_t lino = 0; + + char fname[TSDB_FILENAME_LEN]; + TdFilePtr fdFrom = NULL; + TdFilePtr fdTo = NULL; + + tsdbTFileName(rtner->tsdb, to, fname); + + fdFrom = taosOpenFile(from->fname, TD_FILE_READ); + if (fdFrom == NULL) code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + + fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (fdTo == NULL) code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + + int64_t n = taosFSendFile(fdTo, fdFrom, 0, tsdbLogicToFileSize(from->f->size, rtner->szPage)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + taosCloseFile(&fdFrom); + taosCloseFile(&fdTo); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + taosCloseFile(&fdFrom); + taosCloseFile(&fdTo); } + return code; +} + +static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) { + int32_t code = 0; + int32_t lino = 0; + STFileOp op = {0}; + + // remove old + op = (STFileOp){ + .optype = TSDB_FOP_REMOVE, + .fid = fobj->f->fid, + .of = fobj->f[0], + }; + + code = TARRAY2_APPEND(rtner->fopArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + // create new + op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = fobj->f->fid, + .nf = + { + .type = fobj->f->type, + .did = did[0], + .fid = fobj->f->fid, + .cid = rtner->cid, + .size = fobj->f->size, + .stt[0] = + { + .level = fobj->f->stt[0].level, + }, + }, + }; + + code = TARRAY2_APPEND(rtner->fopArr, op); + TSDB_CHECK_CODE(code, lino, _exit); + + // do copy the file + code = tsdbDoCopyFile(rtner, fobj, &op.nf); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + } + return code; +} + +typedef struct { + STsdb *tsdb; + int64_t now; +} SRtnArg; + +static int32_t tsdbDoRetentionBegin(SRtnArg *arg, SRTNer *rtner) { + int32_t code = 0; + int32_t lino = 0; + + STsdb *tsdb = arg->tsdb; + + rtner->tsdb = tsdb; + rtner->szPage = tsdb->pVnode->config.tsdbPageSize; + rtner->now = arg->now; + rtner->cid = tsdbFSAllocEid(tsdb->pFS); - return false; + code = tsdbFSCreateCopySnapshot(tsdb->pFS, &rtner->fsetArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + } else { + tsdbInfo("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); + } + return code; } -bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { - bool should; - taosThreadRwlockRdlock(&pTsdb->rwLock); - should = tsdbShouldDoRetentionImpl(pTsdb, now); - taosThreadRwlockUnlock(&pTsdb->rwLock); - return should; + +static int32_t tsdbDoRetentionEnd(SRTNer *rtner) { + int32_t code = 0; + int32_t lino = 0; + + if (TARRAY2_SIZE(rtner->fopArr) == 0) goto _exit; + + code = tsdbFSEditBegin(rtner->tsdb->pFS, rtner->fopArr, TSDB_FEDIT_MERGE); + TSDB_CHECK_CODE(code, lino, _exit); + + taosThreadRwlockWrlock(&rtner->tsdb->rwLock); + + code = tsdbFSEditCommit(rtner->tsdb->pFS); + if (code) { + taosThreadRwlockUnlock(&rtner->tsdb->rwLock); + TSDB_CHECK_CODE(code, lino, _exit); + } + + taosThreadRwlockUnlock(&rtner->tsdb->rwLock); + + TARRAY2_DESTROY(rtner->fopArr, NULL); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + } else { + tsdbInfo("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); + } + tsdbFSDestroyCopySnapshot(&rtner->fsetArr); + return code; } -int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { +static int32_t tsdbDoRetention2(void *arg) { int32_t code = 0; int32_t lino = 0; - STsdbFS fs = {0}; + SRTNer rtner[1] = {0}; - code = tsdbFSCopy(pTsdb, &fs); + code = tsdbDoRetentionBegin(arg, rtner); TSDB_CHECK_CODE(code, lino, _exit); - for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); - SDiskID did; - - if (expLevel < 0) { - taosMemoryFree(pSet->pHeadF); - taosMemoryFree(pSet->pDataF); - taosMemoryFree(pSet->pSmaF); - for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { - taosMemoryFree(pSet->aSttF[iStt]); + for (rtner->ctx->fsetArrIdx = 0; rtner->ctx->fsetArrIdx < TARRAY2_SIZE(rtner->fsetArr); rtner->ctx->fsetArrIdx++) { + rtner->ctx->fset = TARRAY2_GET(rtner->fsetArr, rtner->ctx->fsetArrIdx); + + STFileObj *fobj; + int32_t expLevel = tsdbFidLevel(rtner->ctx->fset->fid, &rtner->tsdb->keepCfg, rtner->now); + + if (expLevel < 0) { // remove the file set + for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = rtner->ctx->fset->farr[ftype], 1); ++ftype) { + if (fobj == NULL) continue; + + code = tsdbDoRemoveFileObject(rtner, fobj); + TSDB_CHECK_CODE(code, lino, _exit); } - taosArrayRemove(fs.aDFileSet, iSet); - iSet--; + + SSttLvl *lvl; + TARRAY2_FOREACH(rtner->ctx->fset->lvlArr, lvl) { + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + code = tsdbDoRemoveFileObject(rtner, fobj); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } else if (expLevel == 0) { + continue; } else { - if (expLevel == 0) continue; - if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { + SDiskID did; + + if (tfsAllocDisk(rtner->tsdb->pVnode->pTfs, expLevel, &did) < 0) { code = terrno; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } + tfsMkdirRecurAt(rtner->tsdb->pVnode->pTfs, rtner->tsdb->path, did); - if (did.level == pSet->diskId.level) continue; + // data + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX && (fobj = rtner->ctx->fset->farr[ftype], 1); ++ftype) { + if (fobj == NULL) continue; - // copy file to new disk (todo) - SDFileSet fSet = *pSet; - fSet.diskId = did; + if (fobj->f->did.level == did.level) continue; + code = tsdbDoMigrateFileObj(rtner, fobj, &did); + TSDB_CHECK_CODE(code, lino, _exit); + } - code = tsdbDFileSetCopy(pTsdb, pSet, &fSet); - TSDB_CHECK_CODE(code, lino, _exit); + // stt + SSttLvl *lvl; + TARRAY2_FOREACH(rtner->ctx->fset->lvlArr, lvl) { + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + if (fobj->f->did.level == did.level) continue; - code = tsdbFSUpsertFSet(&fs, &fSet); - TSDB_CHECK_CODE(code, lino, _exit); + code = tsdbDoMigrateFileObj(rtner, fobj, &did); + TSDB_CHECK_CODE(code, lino, _exit); + } + } } } - // do change fs - code = tsdbFSPrepareCommit(pTsdb, &fs); + code = tsdbDoRetentionEnd(rtner); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); } - tsdbFSDestroy(&fs); + taosMemoryFree(arg); + return code; +} + +int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now, int64_t *taskid) { + SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); + if (arg == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + arg->tsdb = tsdb; + arg->now = now; + + int32_t code = tsdbFSScheduleBgTask(tsdb->pFS, TSDB_BG_TASK_RETENTION, tsdbDoRetention2, arg, taskid); + if (code) taosMemoryFree(arg); + return code; } -static int32_t tsdbCommitRetentionImpl(STsdb *pTsdb) { return tsdbFSCommit(pTsdb); } +int32_t tsdbSyncRetention(STsdb *tsdb, int64_t now) { + int64_t taskid; + + int32_t code = tsdbAsyncRetention(tsdb, now, &taskid); + if (code) return code; -int32_t tsdbCommitRetention(STsdb *pTsdb) { - taosThreadRwlockWrlock(&pTsdb->rwLock); - tsdbCommitRetentionImpl(pTsdb); - taosThreadRwlockUnlock(&pTsdb->rwLock); - tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); - return 0; + return tsdbFSWaitBgTask(tsdb->pFS, taskid); } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index b5ca716701b2c479fbc3e04ace61829d91184c69..011b9bd5a4c78e3d5392a3ed379b906f695bbab3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -14,553 +14,519 @@ */ #include "tsdb.h" +#include "tsdbDataFileRW.h" +#include "tsdbFS2.h" +#include "tsdbFSetRW.h" +#include "tsdbIter.h" +#include "tsdbSttFileRW.h" extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo); -extern int32_t tsdbWriteDataBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SMapData* mDataBlk, int8_t cmprAlg); -extern int32_t tsdbWriteSttBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SArray* aSttBlk, int8_t cmprAlg); // STsdbSnapReader ======================================== struct STsdbSnapReader { - STsdb* pTsdb; - int64_t sver; - int64_t ever; - int8_t type; + STsdb* tsdb; + int64_t sver; + int64_t ever; + int8_t type; + uint8_t* aBuf[5]; + SSkmInfo skmTb[1]; + + TFileSetArray* fsetArr; - STsdbFS fs; - TABLEID tbid; - SSkmInfo skmTable; - - // timeseries data - int8_t dataDone; - int32_t fid; - - SDataFReader* pDataFReader; - STsdbDataIter2* iterList; - STsdbDataIter2* pIter; - SRBTree rbt; - SBlockData bData; - - // tombstone data - int8_t delDone; - SDelFReader* pDelFReader; - STsdbDataIter2* pTIter; - SArray* aDelData; + // context + struct { + int32_t fsetArrIdx; + STFileSet* fset; + bool isDataDone; + bool isTombDone; + } ctx[1]; + + // reader + SDataFileReader* dataReader; + TSttFileReaderArray sttReaderArr[1]; + + // iter + TTsdbIterArray dataIterArr[1]; + SIterMerger* dataIterMerger; + TTsdbIterArray tombIterArr[1]; + SIterMerger* tombIterMerger; + + // data + SBlockData blockData[1]; + STombBlock tombBlock[1]; }; -static int32_t tsdbSnapReadFileDataStart(STsdbSnapReader* pReader) { +static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) { int32_t code = 0; int32_t lino = 0; - SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &(SDFileSet){.fid = pReader->fid}, tDFileSetCmprFn, TD_GT); - if (pSet == NULL) { - pReader->fid = INT32_MAX; - goto _exit; + ASSERT(reader->dataReader == NULL); + ASSERT(TARRAY2_SIZE(reader->sttReaderArr) == 0); + + // data + SDataFileReaderConfig config = { + .tsdb = reader->tsdb, + .szPage = reader->tsdb->pVnode->config.tsdbPageSize, + .bufArr = reader->aBuf, + }; + bool hasDataFile = false; + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ftype++) { + if (reader->ctx->fset->farr[ftype] != NULL) { + hasDataFile = true; + config.files[ftype].exist = true; + config.files[ftype].file = reader->ctx->fset->farr[ftype]->f[0]; + } } - pReader->fid = pSet->fid; + if (hasDataFile) { + code = tsdbDataFileReaderOpen(NULL, &config, &reader->dataReader); + TSDB_CHECK_CODE(code, lino, _exit); + } - tRBTreeCreate(&pReader->rbt, tsdbDataIterCmprFn); + // stt + SSttLvl* lvl; + TARRAY2_FOREACH(reader->ctx->fset->lvlArr, lvl) { + STFileObj* fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + SSttFileReader* sttReader; + SSttFileReaderConfig config = { + .tsdb = reader->tsdb, + .szPage = reader->tsdb->pVnode->config.tsdbPageSize, + .file = fobj->f[0], + .bufArr = reader->aBuf, + }; + + code = tsdbSttFileReaderOpen(fobj->fname, &config, &sttReader); + TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet); - TSDB_CHECK_CODE(code, lino, _exit); + code = TARRAY2_APPEND(reader->sttReaderArr, sttReader); + TSDB_CHECK_CODE(code, lino, _exit); + } + } - code = tsdbOpenDataFileDataIter(pReader->pDataFReader, &pReader->pIter); - TSDB_CHECK_CODE(code, lino, _exit); +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapReadFileSetCloseReader(STsdbSnapReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + TARRAY2_CLEAR(reader->sttReaderArr, tsdbSttFileReaderClose); + tsdbDataFileReaderClose(&reader->dataReader); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + +static int32_t tsdbSnapReadFileSetOpenIter(STsdbSnapReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(reader->dataIterMerger == NULL); + ASSERT(reader->tombIterMerger == NULL); + ASSERT(TARRAY2_SIZE(reader->dataIterArr) == 0); + ASSERT(TARRAY2_SIZE(reader->tombIterArr) == 0); + + STsdbIter* iter; + STsdbIterConfig config = { + .filterByVersion = true, + .verRange[0] = reader->sver, + .verRange[1] = reader->ever, + }; + + // data file + if (reader->dataReader) { + // data + config.type = TSDB_ITER_TYPE_DATA; + config.dataReader = reader->dataReader; + + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); - if (pReader->pIter) { - // iter to next with filter info (sver, ever) - code = tsdbDataIterNext2( - pReader->pIter, - &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag - .sver = pReader->sver, - .ever = pReader->ever}); + code = TARRAY2_APPEND(reader->dataIterArr, iter); TSDB_CHECK_CODE(code, lino, _exit); - if (pReader->pIter->rowInfo.suid || pReader->pIter->rowInfo.uid) { - // add to rbtree - tRBTreePut(&pReader->rbt, &pReader->pIter->rbtn); + // tomb + config.type = TSDB_ITER_TYPE_DATA_TOMB; + config.dataReader = reader->dataReader; - // add to iterList - pReader->pIter->next = pReader->iterList; - pReader->iterList = pReader->pIter; - } else { - tsdbCloseDataIter2(pReader->pIter); - } + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(reader->tombIterArr, iter); + TSDB_CHECK_CODE(code, lino, _exit); } - for (int32_t iStt = 0; iStt < pSet->nSttF; ++iStt) { - code = tsdbOpenSttFileDataIter(pReader->pDataFReader, iStt, &pReader->pIter); + // stt file + SSttFileReader* sttReader; + TARRAY2_FOREACH(reader->sttReaderArr, sttReader) { + // data + config.type = TSDB_ITER_TYPE_STT; + config.sttReader = sttReader; + + code = tsdbIterOpen(&config, &iter); TSDB_CHECK_CODE(code, lino, _exit); - if (pReader->pIter) { - // iter to valid row - code = tsdbDataIterNext2( - pReader->pIter, - &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag - .sver = pReader->sver, - .ever = pReader->ever}); - TSDB_CHECK_CODE(code, lino, _exit); + code = TARRAY2_APPEND(reader->dataIterArr, iter); + TSDB_CHECK_CODE(code, lino, _exit); - if (pReader->pIter->rowInfo.suid || pReader->pIter->rowInfo.uid) { - // add to rbtree - tRBTreePut(&pReader->rbt, &pReader->pIter->rbtn); + // tomb + config.type = TSDB_ITER_TYPE_STT_TOMB; + config.sttReader = sttReader; - // add to iterList - pReader->pIter->next = pReader->iterList; - pReader->iterList = pReader->pIter; - } else { - tsdbCloseDataIter2(pReader->pIter); - } - } + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(reader->tombIterArr, iter); + TSDB_CHECK_CODE(code, lino, _exit); } - pReader->pIter = NULL; + // merger + code = tsdbIterMergerOpen(reader->dataIterArr, &reader->dataIterMerger, false); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbIterMergerOpen(reader->tombIterArr, &reader->tombIterMerger, true); + TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pReader->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbInfo("vgId:%d %s done, fid:%d", TD_VID(pReader->pTsdb->pVnode), __func__, pReader->fid); + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } return code; } -static void tsdbSnapReadFileDataEnd(STsdbSnapReader* pReader) { - while (pReader->iterList) { - STsdbDataIter2* pIter = pReader->iterList; - pReader->iterList = pIter->next; - tsdbCloseDataIter2(pIter); - } - - tsdbDataFReaderClose(&pReader->pDataFReader); +static int32_t tsdbSnapReadFileSetCloseIter(STsdbSnapReader* reader) { + tsdbIterMergerClose(&reader->dataIterMerger); + tsdbIterMergerClose(&reader->tombIterMerger); + TARRAY2_CLEAR(reader->dataIterArr, tsdbIterClose); + TARRAY2_CLEAR(reader->tombIterArr, tsdbIterClose); + return 0; } -static int32_t tsdbSnapReadNextRow(STsdbSnapReader* pReader, SRowInfo** ppRowInfo) { +static int32_t tsdbSnapReadFileSetBegin(STsdbSnapReader* reader) { int32_t code = 0; int32_t lino = 0; - if (pReader->pIter) { - code = tsdbDataIterNext2(pReader->pIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | - TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag - .sver = pReader->sver, - .ever = pReader->ever}); - TSDB_CHECK_CODE(code, lino, _exit); + ASSERT(reader->ctx->fset == NULL); - if (pReader->pIter->rowInfo.suid == 0 && pReader->pIter->rowInfo.uid == 0) { - pReader->pIter = NULL; - } else { - SRBTreeNode* pNode = tRBTreeMin(&pReader->rbt); - if (pNode) { - int32_t c = tsdbDataIterCmprFn(&pReader->pIter->rbtn, pNode); - if (c > 0) { - tRBTreePut(&pReader->rbt, &pReader->pIter->rbtn); - pReader->pIter = NULL; - } else if (c == 0) { - ASSERT(0); - } - } - } - } + if (reader->ctx->fsetArrIdx < TARRAY2_SIZE(reader->fsetArr)) { + reader->ctx->fset = TARRAY2_GET(reader->fsetArr, reader->ctx->fsetArrIdx++); + reader->ctx->isDataDone = false; + reader->ctx->isTombDone = false; - if (pReader->pIter == NULL) { - SRBTreeNode* pNode = tRBTreeMin(&pReader->rbt); - if (pNode) { - tRBTreeDrop(&pReader->rbt, pNode); - pReader->pIter = TSDB_RBTN_TO_DATA_ITER(pNode); - } - } + code = tsdbSnapReadFileSetOpenReader(reader); + TSDB_CHECK_CODE(code, lino, _exit); - if (ppRowInfo) { - if (pReader->pIter) { - *ppRowInfo = &pReader->pIter->rowInfo; - } else { - *ppRowInfo = NULL; - } + code = tsdbSnapReadFileSetOpenIter(reader); + TSDB_CHECK_CODE(code, lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pReader->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } return code; } -static int32_t tsdbSnapReadGetRow(STsdbSnapReader* pReader, SRowInfo** ppRowInfo) { - if (pReader->pIter) { - *ppRowInfo = &pReader->pIter->rowInfo; - return 0; - } - - return tsdbSnapReadNextRow(pReader, ppRowInfo); +static int32_t tsdbSnapReadFileSetEnd(STsdbSnapReader* reader) { + tsdbSnapReadFileSetCloseIter(reader); + tsdbSnapReadFileSetCloseReader(reader); + reader->ctx->fset = NULL; + return 0; } -static int32_t tsdbSnapCmprData(STsdbSnapReader* pReader, uint8_t** ppData) { +static int32_t tsdbSnapCmprData(STsdbSnapReader* reader, uint8_t** data) { int32_t code = 0; - - ASSERT(pReader->bData.nRow); + int32_t lino = 0; int32_t aBufN[5] = {0}; - code = tCmprBlockData(&pReader->bData, NO_COMPRESSION, NULL, NULL, pReader->aBuf, aBufN); - if (code) goto _exit; + code = tCmprBlockData(reader->blockData, NO_COMPRESSION, NULL, NULL, reader->aBuf, aBufN); + TSDB_CHECK_CODE(code, lino, _exit); int32_t size = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; - *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); - if (*ppData == NULL) { + *data = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); + if (*data == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } - SSnapDataHdr* pHdr = (SSnapDataHdr*)*ppData; - pHdr->type = pReader->type; + SSnapDataHdr* pHdr = (SSnapDataHdr*)*data; + pHdr->type = reader->type; pHdr->size = size; - memcpy(pHdr->data, pReader->aBuf[3], aBufN[3]); - memcpy(pHdr->data + aBufN[3], pReader->aBuf[2], aBufN[2]); + memcpy(pHdr->data, reader->aBuf[3], aBufN[3]); + memcpy(pHdr->data + aBufN[3], reader->aBuf[2], aBufN[2]); if (aBufN[1]) { - memcpy(pHdr->data + aBufN[3] + aBufN[2], pReader->aBuf[1], aBufN[1]); + memcpy(pHdr->data + aBufN[3] + aBufN[2], reader->aBuf[1], aBufN[1]); } if (aBufN[0]) { - memcpy(pHdr->data + aBufN[3] + aBufN[2] + aBufN[1], pReader->aBuf[0], aBufN[0]); + memcpy(pHdr->data + aBufN[3] + aBufN[2] + aBufN[1], reader->aBuf[0], aBufN[0]); } _exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), lino, code); + } return code; } -static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* pReader, uint8_t** ppData) { - int32_t code = 0; - int32_t lino = 0; - - STsdb* pTsdb = pReader->pTsdb; +static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** data) { + int32_t code = 0; + int32_t lino = 0; + SMetaInfo info; + + tBlockDataReset(reader->blockData); + + TABLEID tbid[1] = {0}; + for (SRowInfo* row; (row = tsdbIterMergerGetData(reader->dataIterMerger));) { + // skip dropped table + if (row->uid != tbid->uid) { + tbid->suid = row->suid; + tbid->uid = row->uid; + if (metaGetInfo(reader->tsdb->pVnode->pMeta, tbid->uid, &info, NULL) != 0) { + code = tsdbIterMergerSkipTableData(reader->dataIterMerger, tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } + } - tBlockDataReset(&pReader->bData); + if (reader->blockData->suid == 0 && reader->blockData->uid == 0) { + code = tsdbUpdateSkmTb(reader->tsdb, (TABLEID*)row, reader->skmTb); + TSDB_CHECK_CODE(code, lino, _exit); - for (;;) { - // start a new file read if need - if (pReader->pDataFReader == NULL) { - code = tsdbSnapReadFileDataStart(pReader); + TABLEID tbid1 = { + .suid = row->suid, + .uid = row->suid ? 0 : row->uid, + }; + code = tBlockDataInit(reader->blockData, &tbid1, reader->skmTb->pTSchema, NULL, 0); TSDB_CHECK_CODE(code, lino, _exit); } - if (pReader->pDataFReader == NULL) break; - - SRowInfo* pRowInfo; - code = tsdbSnapReadGetRow(pReader, &pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pRowInfo == NULL) { - tsdbSnapReadFileDataEnd(pReader); - continue; + if (!TABLE_SAME_SCHEMA(reader->blockData->suid, reader->blockData->uid, row->suid, row->uid)) { + break; } - code = tsdbUpdateTableSchema(pTsdb->pVnode->pMeta, pRowInfo->suid, pRowInfo->uid, &pReader->skmTable); + code = tBlockDataAppendRow(reader->blockData, &row->row, NULL, row->uid); TSDB_CHECK_CODE(code, lino, _exit); - code = tBlockDataInit(&pReader->bData, (TABLEID*)pRowInfo, pReader->skmTable.pTSchema, NULL, 0); + code = tsdbIterMergerNext(reader->dataIterMerger); TSDB_CHECK_CODE(code, lino, _exit); - do { - if (!TABLE_SAME_SCHEMA(pReader->bData.suid, pReader->bData.uid, pRowInfo->suid, pRowInfo->uid)) break; - - if (pReader->bData.uid && pReader->bData.uid != pRowInfo->uid) { - code = tRealloc((uint8_t**)&pReader->bData.aUid, sizeof(int64_t) * (pReader->bData.nRow + 1)); - TSDB_CHECK_CODE(code, lino, _exit); - - for (int32_t iRow = 0; iRow < pReader->bData.nRow; ++iRow) { - pReader->bData.aUid[iRow] = pReader->bData.uid; - } - pReader->bData.uid = 0; - } - - code = tBlockDataAppendRow(&pReader->bData, &pRowInfo->row, NULL, pRowInfo->uid); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbSnapReadNextRow(pReader, &pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pReader->bData.nRow >= 81920) break; - } while (pRowInfo); - - ASSERT(pReader->bData.nRow > 0); - - break; + if (reader->blockData->nRow >= 81920) { + break; + } } - if (pReader->bData.nRow > 0) { - ASSERT(pReader->bData.suid || pReader->bData.uid); - - code = tsdbSnapCmprData(pReader, ppData); + if (reader->blockData->nRow > 0) { + ASSERT(reader->blockData->suid || reader->blockData->uid); + code = tsdbSnapCmprData(reader, data); TSDB_CHECK_CODE(code, lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } return code; } -static int32_t tsdbSnapCmprTombData(STsdbSnapReader* pReader, uint8_t** ppData) { +static int32_t tsdbSnapCmprTombData(STsdbSnapReader* reader, uint8_t** data) { int32_t code = 0; int32_t lino = 0; - int64_t size = sizeof(TABLEID); - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); ++iDelData) { - size += tPutDelData(NULL, taosArrayGet(pReader->aDelData, iDelData)); + int64_t size = sizeof(SSnapDataHdr); + for (int32_t i = 0; i < ARRAY_SIZE(reader->tombBlock->dataArr); i++) { + size += TARRAY2_DATA_LEN(reader->tombBlock->dataArr + i); } - uint8_t* pData = (uint8_t*)taosMemoryMalloc(sizeof(SSnapDataHdr) + size); - if (pData == NULL) { + data[0] = taosMemoryMalloc(size); + if (data[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } - SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; - pHdr->type = SNAP_DATA_DEL; - pHdr->size = size; + SSnapDataHdr* hdr = (SSnapDataHdr*)data[0]; + hdr->type = SNAP_DATA_DEL; + hdr->size = size; - TABLEID* pId = (TABLEID*)(pData + sizeof(SSnapDataHdr)); - *pId = pReader->tbid; - - size = sizeof(SSnapDataHdr) + sizeof(TABLEID); - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); ++iDelData) { - size += tPutDelData(pData + size, taosArrayGet(pReader->aDelData, iDelData)); + uint8_t* tdata = hdr->data; + for (int32_t i = 0; i < ARRAY_SIZE(reader->tombBlock->dataArr); i++) { + memcpy(tdata, TARRAY2_DATA(reader->tombBlock->dataArr + i), TARRAY2_DATA_LEN(reader->tombBlock->dataArr + i)); + tdata += TARRAY2_DATA_LEN(reader->tombBlock->dataArr + i); } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pReader->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } - *ppData = pData; return code; } -static void tsdbSnapReadGetTombData(STsdbSnapReader* pReader, SDelInfo** ppDelInfo) { - if (pReader->pTIter == NULL || (pReader->pTIter->delInfo.suid == 0 && pReader->pTIter->delInfo.uid == 0)) { - *ppDelInfo = NULL; - } else { - *ppDelInfo = &pReader->pTIter->delInfo; - } -} - -static int32_t tsdbSnapReadNextTombData(STsdbSnapReader* pReader, SDelInfo** ppDelInfo) { - int32_t code = 0; - int32_t lino = 0; +static int32_t tsdbSnapReadTombData(STsdbSnapReader* reader, uint8_t** data) { + int32_t code = 0; + int32_t lino = 0; + SMetaInfo info; - code = tsdbDataIterNext2( - pReader->pTIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, - .sver = pReader->sver, - .ever = pReader->ever}); - TSDB_CHECK_CODE(code, lino, _exit); + tTombBlockClear(reader->tombBlock); - if (ppDelInfo) { - tsdbSnapReadGetTombData(pReader, ppDelInfo); - } - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pReader->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbSnapReadTombData(STsdbSnapReader* pReader, uint8_t** ppData) { - int32_t code = 0; - int32_t lino = 0; - - STsdb* pTsdb = pReader->pTsdb; - - // open tombstone data iter if need - if (pReader->pDelFReader == NULL) { - if (pReader->fs.pDelFile == NULL) goto _exit; - - // open - code = tsdbDelFReaderOpen(&pReader->pDelFReader, pReader->fs.pDelFile, pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbOpenTombFileDataIter(pReader->pDelFReader, &pReader->pTIter); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pReader->pTIter) { - code = tsdbSnapReadNextTombData(pReader, NULL); - TSDB_CHECK_CODE(code, lino, _exit); + TABLEID tbid[1] = {0}; + for (STombRecord* record; (record = tsdbIterMergerGetTombRecord(reader->tombIterMerger)) != NULL;) { + if (record->uid != tbid->uid) { + tbid->suid = record->suid; + tbid->uid = record->uid; + if (metaGetInfo(reader->tsdb->pVnode->pMeta, tbid->uid, &info, NULL) != 0) { + code = tsdbIterMergerSkipTableData(reader->tombIterMerger, tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; + } } - } - - // loop to get tombstone data - SDelInfo* pDelInfo; - tsdbSnapReadGetTombData(pReader, &pDelInfo); - if (pDelInfo == NULL) goto _exit; - - pReader->tbid = *(TABLEID*)pDelInfo; - - if (pReader->aDelData) { - taosArrayClear(pReader->aDelData); - } else if ((pReader->aDelData = taosArrayInit(16, sizeof(SDelData))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = tTombBlockPut(reader->tombBlock, record); TSDB_CHECK_CODE(code, lino, _exit); - } - while (pDelInfo && pDelInfo->suid == pReader->tbid.suid && pDelInfo->uid == pReader->tbid.uid) { - if (taosArrayPush(pReader->aDelData, &pDelInfo->delData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + if (TOMB_BLOCK_SIZE(reader->tombBlock) >= 81920) { + break; } - - code = tsdbSnapReadNextTombData(pReader, &pDelInfo); - TSDB_CHECK_CODE(code, lino, _exit); } - // encode tombstone data - if (taosArrayGetSize(pReader->aDelData) > 0) { - code = tsdbSnapCmprTombData(pReader, ppData); + if (TOMB_BLOCK_SIZE(reader->tombBlock) > 0) { + code = tsdbSnapCmprTombData(reader, data); TSDB_CHECK_CODE(code, lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbDebug("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } return code; } -int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader) { +int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** reader) { int32_t code = 0; int32_t lino = 0; - // alloc - STsdbSnapReader* pReader = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); - if (pReader == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - pReader->pTsdb = pTsdb; - pReader->sver = sver; - pReader->ever = ever; - pReader->type = type; - - taosThreadRwlockRdlock(&pTsdb->rwLock); - code = tsdbFSRef(pTsdb, &pReader->fs); - if (code) { - taosThreadRwlockUnlock(&pTsdb->rwLock); - TSDB_CHECK_CODE(code, lino, _exit); - } - taosThreadRwlockUnlock(&pTsdb->rwLock); + reader[0] = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*reader[0])); + if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; - // init - pReader->fid = INT32_MIN; + reader[0]->tsdb = tsdb; + reader[0]->sver = sver; + reader[0]->ever = ever; + reader[0]->type = type; - code = tBlockDataCreate(&pReader->bData); + code = tsdbFSCreateRefSnapshot(tsdb->pFS, &reader[0]->fsetArr); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(pTsdb->pVnode), + tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code), sver, ever, type); - if (pReader) { - tBlockDataDestroy(&pReader->bData); - tsdbFSUnref(pTsdb, &pReader->fs); - taosMemoryFree(pReader); - pReader = NULL; - } + tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); + taosMemoryFree(reader[0]); + reader[0] = NULL; } else { - tsdbInfo("vgId:%d %s done, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(pTsdb->pVnode), __func__, sver, ever, + tsdbInfo("vgId:%d %s done, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, sver, ever, type); } - *ppReader = pReader; return code; } -int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { +int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { + if (reader[0] == NULL) return 0; + int32_t code = 0; int32_t lino = 0; - STsdbSnapReader* pReader = *ppReader; - STsdb* pTsdb = pReader->pTsdb; + STsdb* tsdb = reader[0]->tsdb; - // tombstone - if (pReader->pTIter) { - tsdbCloseDataIter2(pReader->pTIter); - pReader->pTIter = NULL; - } - if (pReader->pDelFReader) { - tsdbDelFReaderClose(&pReader->pDelFReader); - } - taosArrayDestroy(pReader->aDelData); + tTombBlockDestroy(reader[0]->tombBlock); + tBlockDataDestroy(reader[0]->blockData); - // timeseries - while (pReader->iterList) { - STsdbDataIter2* pIter = pReader->iterList; - pReader->iterList = pIter->next; - tsdbCloseDataIter2(pIter); - } - if (pReader->pDataFReader) { - tsdbDataFReaderClose(&pReader->pDataFReader); - } - tBlockDataDestroy(&pReader->bData); + tsdbIterMergerClose(&reader[0]->dataIterMerger); + tsdbIterMergerClose(&reader[0]->tombIterMerger); + TARRAY2_DESTROY(reader[0]->dataIterArr, tsdbIterClose); + TARRAY2_DESTROY(reader[0]->tombIterArr, tsdbIterClose); + TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose); + tsdbDataFileReaderClose(&reader[0]->dataReader); + + tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr); + tDestroyTSchema(reader[0]->skmTb->pTSchema); - // other - tDestroyTSchema(pReader->skmTable.pTSchema); - tsdbFSUnref(pReader->pTsdb, &pReader->fs); - for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(pReader->aBuf[0]); iBuf++) { - tFree(pReader->aBuf[iBuf]); + for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) { + tFree(reader[0]->aBuf[i]); } - taosMemoryFree(pReader); + + taosMemoryFree(reader[0]); + reader[0] = NULL; _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); } else { - tsdbDebug("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); } - *ppReader = NULL; return code; } -int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { +int32_t tsdbSnapRead(STsdbSnapReader* reader, uint8_t** data) { int32_t code = 0; int32_t lino = 0; - *ppData = NULL; + data[0] = NULL; - // read data file - if (!pReader->dataDone) { - code = tsdbSnapReadTimeSeriesData(pReader, ppData); - TSDB_CHECK_CODE(code, lino, _exit); - if (*ppData) { - goto _exit; - } else { - pReader->dataDone = 1; + for (;;) { + if (reader->ctx->fset == NULL) { + code = tsdbSnapReadFileSetBegin(reader); + TSDB_CHECK_CODE(code, lino, _exit); + + if (reader->ctx->fset == NULL) { + break; + } } - } - // read del file - if (!pReader->delDone) { - code = tsdbSnapReadTombData(pReader, ppData); - TSDB_CHECK_CODE(code, lino, _exit); - if (*ppData) { - goto _exit; - } else { - pReader->delDone = 1; + if (!reader->ctx->isDataDone) { + code = tsdbSnapReadTimeSeriesData(reader, data); + TSDB_CHECK_CODE(code, lino, _exit); + if (data[0]) { + goto _exit; + } else { + reader->ctx->isDataDone = true; + } + } + + if (!reader->ctx->isTombDone) { + code = tsdbSnapReadTombData(reader, data); + TSDB_CHECK_CODE(code, lino, _exit); + if (data[0]) { + goto _exit; + } else { + reader->ctx->isTombDone = true; + } } + + code = tsdbSnapReadFileSetEnd(reader); + TSDB_CHECK_CODE(code, lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pReader->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } else { - tsdbDebug("vgId:%d %s done", TD_VID(pReader->pTsdb->pVnode), __func__); + tsdbDebug("vgId:%d %s done", TD_VID(reader->tsdb->pVnode), __func__); } return code; } // STsdbSnapWriter ======================================== struct STsdbSnapWriter { - STsdb* pTsdb; + STsdb* tsdb; int64_t sver; int64_t ever; int32_t minutes; @@ -569,971 +535,595 @@ struct STsdbSnapWriter { int32_t maxRow; int8_t cmprAlg; int64_t commitID; + int32_t szPage; + int64_t compactVersion; + int64_t now; uint8_t* aBuf[5]; - STsdbFS fs; - TABLEID tbid; - - // time-series data - SBlockData inData; - - int32_t fid; - SSkmInfo skmTable; - - /* reader */ - SDataFReader* pDataFReader; - STsdbDataIter2* iterList; - STsdbDataIter2* pDIter; - STsdbDataIter2* pSIter; - SRBTree rbt; // SRBTree - - /* writer */ - SDataFWriter* pDataFWriter; - SArray* aBlockIdx; - SMapData mDataBlk; // SMapData - SArray* aSttBlk; // SArray - SBlockData bData; - SBlockData sData; - - // tombstone data - /* reader */ - SDelFReader* pDelFReader; - STsdbDataIter2* pTIter; - - /* writer */ - SDelFWriter* pDelFWriter; - SArray* aDelIdx; - SArray* aDelData; + TFileSetArray* fsetArr; + TFileOpArray fopArr[1]; + + struct { + bool fsetWriteBegin; + int32_t fid; + STFileSet* fset; + SDiskID did; + bool hasData; + bool hasTomb; + + // reader + SDataFileReader* dataReader; + TSttFileReaderArray sttReaderArr[1]; + + // iter/merger + TTsdbIterArray dataIterArr[1]; + SIterMerger* dataIterMerger; + TTsdbIterArray tombIterArr[1]; + SIterMerger* tombIterMerger; + + // writer + SFSetWriter* fsetWriter; + } ctx[1]; }; -// SNAP_DATA_TSDB -static int32_t tsdbSnapWriteTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pId) { +// APIs +static int32_t tsdbSnapWriteTimeSeriesRow(STsdbSnapWriter* writer, SRowInfo* row) { int32_t code = 0; int32_t lino = 0; - if (pId) { - pWriter->tbid = *pId; - } else { - pWriter->tbid = (TABLEID){INT64_MAX, INT64_MAX}; - } - - if (pWriter->pDIter) { - STsdbDataIter2* pIter = pWriter->pDIter; - - // assert last table data end - ASSERT(pIter->dIter.iRow >= pIter->dIter.bData.nRow); - ASSERT(pIter->dIter.iDataBlk >= pIter->dIter.mDataBlk.nItem); - - for (;;) { - if (pIter->dIter.iBlockIdx >= taosArrayGetSize(pIter->dIter.aBlockIdx)) { - pWriter->pDIter = NULL; - break; - } - - SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->dIter.aBlockIdx, pIter->dIter.iBlockIdx); - - int32_t c = tTABLEIDCmprFn(pBlockIdx, &pWriter->tbid); - if (c < 0) { - code = tsdbReadDataBlk(pIter->dIter.pReader, pBlockIdx, &pIter->dIter.mDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - SBlockIdx* pNewBlockIdx = taosArrayReserve(pWriter->aBlockIdx, 1); - if (pNewBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - pNewBlockIdx->suid = pBlockIdx->suid; - pNewBlockIdx->uid = pBlockIdx->uid; - - code = tsdbWriteDataBlk(pWriter->pDataFWriter, &pIter->dIter.mDataBlk, pNewBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - pIter->dIter.iBlockIdx++; - } else if (c == 0) { - code = tsdbReadDataBlk(pIter->dIter.pReader, pBlockIdx, &pIter->dIter.mDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - pIter->dIter.iDataBlk = 0; - pIter->dIter.iBlockIdx++; - - break; - } else { - pIter->dIter.iDataBlk = pIter->dIter.mDataBlk.nItem; - break; - } + while (writer->ctx->hasData) { + SRowInfo* row1 = tsdbIterMergerGetData(writer->ctx->dataIterMerger); + if (row1 == NULL) { + writer->ctx->hasData = false; + break; } - } - if (pId) { - code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pId->suid, pId->uid, &pWriter->skmTable); - TSDB_CHECK_CODE(code, lino, _exit); - - tMapDataReset(&pWriter->mDataBlk); - - code = tBlockDataInit(&pWriter->bData, pId, pWriter->skmTable.pTSchema, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (!TABLE_SAME_SCHEMA(pWriter->tbid.suid, pWriter->tbid.uid, pWriter->sData.suid, pWriter->sData.uid)) { - if ((pWriter->sData.nRow > 0)) { - code = tsdbWriteSttBlock(pWriter->pDataFWriter, &pWriter->sData, pWriter->aSttBlk, pWriter->cmprAlg); + int32_t c = tRowInfoCmprFn(row1, row); + if (c <= 0) { + code = tsdbFSetWriteRow(writer->ctx->fsetWriter, row1); TSDB_CHECK_CODE(code, lino, _exit); - } - if (pId) { - TABLEID id = {.suid = pWriter->tbid.suid, .uid = pWriter->tbid.suid ? 0 : pWriter->tbid.uid}; - code = tBlockDataInit(&pWriter->sData, &id, pWriter->skmTable.pTSchema, NULL, 0); + code = tsdbIterMergerNext(writer->ctx->dataIterMerger); TSDB_CHECK_CODE(code, lino, _exit); + } else { + break; } } -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbTrace("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), __func__, - pWriter->tbid.suid, pWriter->tbid.uid); + if (row->suid == INT64_MAX) { + ASSERT(writer->ctx->hasData == false); + goto _exit; } - return code; -} -static int32_t tsdbSnapWriteTableRowImpl(STsdbSnapWriter* pWriter, TSDBROW* pRow) { - int32_t code = 0; - int32_t lino = 0; - - code = tBlockDataAppendRow(&pWriter->bData, pRow, pWriter->skmTable.pTSchema, pWriter->tbid.uid); + code = tsdbFSetWriteRow(writer->ctx->fsetWriter, row); TSDB_CHECK_CODE(code, lino, _exit); - if (pWriter->bData.nRow >= pWriter->maxRow) { - code = tsdbWriteDataBlock(pWriter->pDataFWriter, &pWriter->bData, &pWriter->mDataBlk, pWriter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbSnapWriteTableRow(STsdbSnapWriter* pWriter, TSDBROW* pRow) { +static int32_t tsdbSnapWriteFileSetOpenReader(STsdbSnapWriter* writer) { int32_t code = 0; int32_t lino = 0; - TSDBKEY inKey = pRow ? TSDBROW_KEY(pRow) : TSDBKEY_MAX; + ASSERT(writer->ctx->dataReader == NULL); + ASSERT(TARRAY2_SIZE(writer->ctx->sttReaderArr) == 0); - if (pWriter->pDIter == NULL || (pWriter->pDIter->dIter.iRow >= pWriter->pDIter->dIter.bData.nRow && - pWriter->pDIter->dIter.iDataBlk >= pWriter->pDIter->dIter.mDataBlk.nItem)) { - goto _write_row; - } else { - for (;;) { - while (pWriter->pDIter->dIter.iRow < pWriter->pDIter->dIter.bData.nRow) { - TSDBROW row = tsdbRowFromBlockData(&pWriter->pDIter->dIter.bData, pWriter->pDIter->dIter.iRow); - - int32_t c = tsdbKeyCmprFn(&inKey, &TSDBROW_KEY(&row)); - if (c < 0) { - goto _write_row; - } else if (c > 0) { - code = tsdbSnapWriteTableRowImpl(pWriter, &row); - TSDB_CHECK_CODE(code, lino, _exit); - - pWriter->pDIter->dIter.iRow++; - } else { - ASSERT(0); - } - } + if (writer->ctx->fset) { + // open data reader + SDataFileReaderConfig dataFileReaderConfig = { + .tsdb = writer->tsdb, + .bufArr = writer->aBuf, + .szPage = writer->szPage, + }; - for (;;) { - if (pWriter->pDIter->dIter.iDataBlk >= pWriter->pDIter->dIter.mDataBlk.nItem) goto _write_row; - - // FIXME: Here can be slow, use array instead - SDataBlk dataBlk; - tMapDataGetItemByIdx(&pWriter->pDIter->dIter.mDataBlk, pWriter->pDIter->dIter.iDataBlk, &dataBlk, tGetDataBlk); - - int32_t c = tDataBlkCmprFn(&dataBlk, &(SDataBlk){.minKey = inKey, .maxKey = inKey}); - if (c > 0) { - goto _write_row; - } else if (c < 0) { - if (pWriter->bData.nRow > 0) { - code = tsdbWriteDataBlock(pWriter->pDataFWriter, &pWriter->bData, &pWriter->mDataBlk, pWriter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - - tMapDataPutItem(&pWriter->mDataBlk, &dataBlk, tPutDataBlk); - pWriter->pDIter->dIter.iDataBlk++; - } else { - code = tsdbReadDataBlockEx(pWriter->pDataFReader, &dataBlk, &pWriter->pDIter->dIter.bData); - TSDB_CHECK_CODE(code, lino, _exit); - - pWriter->pDIter->dIter.iRow = 0; - pWriter->pDIter->dIter.iDataBlk++; - break; - } + for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) { + if (writer->ctx->fset->farr[ftype] == NULL) { + continue; } + + dataFileReaderConfig.files[ftype].exist = true; + dataFileReaderConfig.files[ftype].file = writer->ctx->fset->farr[ftype]->f[0]; } - } -_write_row: - if (pRow) { - code = tsdbSnapWriteTableRowImpl(pWriter, pRow); + code = tsdbDataFileReaderOpen(NULL, &dataFileReaderConfig, &writer->ctx->dataReader); TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { - int32_t code = 0; - int32_t lino = 0; - // write a NULL row to end current table data write - code = tsdbSnapWriteTableRow(pWriter, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pWriter->bData.nRow > 0) { - if (pWriter->bData.nRow < pWriter->minRow) { - ASSERT(TABLE_SAME_SCHEMA(pWriter->sData.suid, pWriter->sData.uid, pWriter->tbid.suid, pWriter->tbid.uid)); - for (int32_t iRow = 0; iRow < pWriter->bData.nRow; iRow++) { - code = - tBlockDataAppendRow(&pWriter->sData, &tsdbRowFromBlockData(&pWriter->bData, iRow), NULL, pWriter->tbid.uid); + // open stt reader array + SSttLvl* lvl; + TARRAY2_FOREACH(writer->ctx->fset->lvlArr, lvl) { + STFileObj* fobj; + TARRAY2_FOREACH(lvl->fobjArr, fobj) { + SSttFileReader* reader; + SSttFileReaderConfig sttFileReaderConfig = { + .tsdb = writer->tsdb, + .szPage = writer->szPage, + .bufArr = writer->aBuf, + .file = fobj->f[0], + }; + + code = tsdbSttFileReaderOpen(fobj->fname, &sttFileReaderConfig, &reader); TSDB_CHECK_CODE(code, lino, _exit); - if (pWriter->sData.nRow >= pWriter->maxRow) { - code = tsdbWriteSttBlock(pWriter->pDataFWriter, &pWriter->sData, pWriter->aSttBlk, pWriter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } + code = TARRAY2_APPEND(writer->ctx->sttReaderArr, reader); + TSDB_CHECK_CODE(code, lino, _exit); } - - tBlockDataClear(&pWriter->bData); - } else { - code = tsdbWriteDataBlock(pWriter->pDataFWriter, &pWriter->bData, &pWriter->mDataBlk, pWriter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - if (pWriter->mDataBlk.nItem) { - SBlockIdx* pBlockIdx = taosArrayReserve(pWriter->aBlockIdx, 1); - if (pBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); } - - pBlockIdx->suid = pWriter->tbid.suid; - pBlockIdx->uid = pWriter->tbid.uid; - - code = tsdbWriteDataBlk(pWriter->pDataFWriter, &pWriter->mDataBlk, pBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbSnapWriteFileDataStart(STsdbSnapWriter* pWriter, int32_t fid) { +static int32_t tsdbSnapWriteFileSetCloseReader(STsdbSnapWriter* writer) { + TARRAY2_CLEAR(writer->ctx->sttReaderArr, tsdbSttFileReaderClose); + tsdbDataFileReaderClose(&writer->ctx->dataReader); + return 0; +} + +static int32_t tsdbSnapWriteFileSetOpenIter(STsdbSnapWriter* writer) { int32_t code = 0; int32_t lino = 0; - ASSERT(pWriter->pDataFWriter == NULL && pWriter->fid < fid); - - STsdb* pTsdb = pWriter->pTsdb; + // data ieter + if (writer->ctx->dataReader) { + STsdbIter* iter; + STsdbIterConfig config = {0}; - pWriter->fid = fid; - pWriter->tbid = (TABLEID){0}; - SDFileSet* pSet = taosArraySearch(pWriter->fs.aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ); + // data + config.type = TSDB_ITER_TYPE_DATA; + config.dataReader = writer->ctx->dataReader; - // open reader - pWriter->pDataFReader = NULL; - pWriter->iterList = NULL; - pWriter->pDIter = NULL; - pWriter->pSIter = NULL; - tRBTreeCreate(&pWriter->rbt, tsdbDataIterCmprFn); - if (pSet) { - code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet); + code = tsdbIterOpen(&config, &iter); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbOpenDataFileDataIter(pWriter->pDataFReader, &pWriter->pDIter); + code = TARRAY2_APPEND(writer->ctx->dataIterArr, iter); TSDB_CHECK_CODE(code, lino, _exit); - if (pWriter->pDIter) { - pWriter->pDIter->next = pWriter->iterList; - pWriter->iterList = pWriter->pDIter; - } - - for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { - code = tsdbOpenSttFileDataIter(pWriter->pDataFReader, iStt, &pWriter->pSIter); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pWriter->pSIter) { - code = tsdbDataIterNext2(pWriter->pSIter, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - // add to tree - tRBTreePut(&pWriter->rbt, &pWriter->pSIter->rbtn); + // tome + config.type = TSDB_ITER_TYPE_DATA_TOMB; + config.dataReader = writer->ctx->dataReader; - // add to list - pWriter->pSIter->next = pWriter->iterList; - pWriter->iterList = pWriter->pSIter; - } - } - - pWriter->pSIter = NULL; - } - - // open writer - SDiskID diskId; - if (pSet) { - diskId = pSet->diskId; - } else { - tfsAllocDisk(pTsdb->pVnode->pTfs, 0 /*TODO*/, &diskId); - tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, diskId); - } - SDFileSet wSet = {.diskId = diskId, - .fid = fid, - .pHeadF = &(SHeadFile){.commitID = pWriter->commitID}, - .pDataF = (pSet) ? pSet->pDataF : &(SDataFile){.commitID = pWriter->commitID}, - .pSmaF = (pSet) ? pSet->pSmaF : &(SSmaFile){.commitID = pWriter->commitID}, - .nSttF = 1, - .aSttF = {&(SSttFile){.commitID = pWriter->commitID}}}; - code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet); - TSDB_CHECK_CODE(code, lino, _exit); + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); - if (pWriter->aBlockIdx) { - taosArrayClear(pWriter->aBlockIdx); - } else if ((pWriter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = TARRAY2_APPEND(writer->ctx->tombIterArr, iter); TSDB_CHECK_CODE(code, lino, _exit); } - tMapDataReset(&pWriter->mDataBlk); + // stt iter + SSttFileReader* sttFileReader; + TARRAY2_FOREACH(writer->ctx->sttReaderArr, sttFileReader) { + STsdbIter* iter; + STsdbIterConfig config = {0}; - if (pWriter->aSttBlk) { - taosArrayClear(pWriter->aSttBlk); - } else if ((pWriter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + // data + config.type = TSDB_ITER_TYPE_STT; + config.sttReader = sttFileReader; + + code = tsdbIterOpen(&config, &iter); TSDB_CHECK_CODE(code, lino, _exit); - } - tBlockDataReset(&pWriter->bData); - tBlockDataReset(&pWriter->sData); + code = TARRAY2_APPEND(writer->ctx->dataIterArr, iter); + TSDB_CHECK_CODE(code, lino, _exit); -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code), - fid); - } else { - tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(pTsdb->pVnode), __func__, fid); - } - return code; -} + // tomb + config.type = TSDB_ITER_TYPE_STT_TOMB; + config.sttReader = sttFileReader; -static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, SRowInfo* pRowInfo) { - int32_t code = 0; - int32_t lino = 0; - - // switch to new table if need - if (pRowInfo == NULL || pRowInfo->uid != pWriter->tbid.uid) { - if (pWriter->tbid.uid) { - code = tsdbSnapWriteTableDataEnd(pWriter); - TSDB_CHECK_CODE(code, lino, _exit); - } + code = tsdbIterOpen(&config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbSnapWriteTableDataStart(pWriter, (TABLEID*)pRowInfo); + code = TARRAY2_APPEND(writer->ctx->tombIterArr, iter); TSDB_CHECK_CODE(code, lino, _exit); } - if (pRowInfo == NULL) goto _exit; + // open merger + code = tsdbIterMergerOpen(writer->ctx->dataIterArr, &writer->ctx->dataIterMerger, false); + TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbSnapWriteTableRow(pWriter, &pRowInfo->row); + code = tsdbIterMergerOpen(writer->ctx->tombIterArr, &writer->ctx->tombIterMerger, true); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbSnapWriteNextRow(STsdbSnapWriter* pWriter, SRowInfo** ppRowInfo) { - int32_t code = 0; - int32_t lino = 0; - - if (pWriter->pSIter) { - code = tsdbDataIterNext2(pWriter->pSIter, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pWriter->pSIter->rowInfo.suid == 0 && pWriter->pSIter->rowInfo.uid == 0) { - pWriter->pSIter = NULL; - } else { - SRBTreeNode* pNode = tRBTreeMin(&pWriter->rbt); - if (pNode) { - int32_t c = tsdbDataIterCmprFn(&pWriter->pSIter->rbtn, pNode); - if (c > 0) { - tRBTreePut(&pWriter->rbt, &pWriter->pSIter->rbtn); - pWriter->pSIter = NULL; - } else if (c == 0) { - ASSERT(0); - } - } - } - } - - if (pWriter->pSIter == NULL) { - SRBTreeNode* pNode = tRBTreeMin(&pWriter->rbt); - if (pNode) { - tRBTreeDrop(&pWriter->rbt, pNode); - pWriter->pSIter = TSDB_RBTN_TO_DATA_ITER(pNode); - } - } - - if (ppRowInfo) { - if (pWriter->pSIter) { - *ppRowInfo = &pWriter->pSIter->rowInfo; - } else { - *ppRowInfo = NULL; - } - } - -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; +static int32_t tsdbSnapWriteFileSetCloseIter(STsdbSnapWriter* writer) { + tsdbIterMergerClose(&writer->ctx->dataIterMerger); + tsdbIterMergerClose(&writer->ctx->tombIterMerger); + TARRAY2_CLEAR(writer->ctx->dataIterArr, tsdbIterClose); + TARRAY2_CLEAR(writer->ctx->tombIterArr, tsdbIterClose); + return 0; } -static int32_t tsdbSnapWriteGetRow(STsdbSnapWriter* pWriter, SRowInfo** ppRowInfo) { +static int32_t tsdbSnapWriteFileSetOpenWriter(STsdbSnapWriter* writer) { int32_t code = 0; int32_t lino = 0; - if (pWriter->pSIter) { - *ppRowInfo = &pWriter->pSIter->rowInfo; - goto _exit; - } - - code = tsdbSnapWriteNextRow(pWriter, ppRowInfo); + SFSetWriterConfig config = { + .tsdb = writer->tsdb, + .toSttOnly = false, + .compactVersion = writer->compactVersion, + .minRow = writer->minRow, + .maxRow = writer->maxRow, + .szPage = writer->szPage, + .cmprAlg = writer->cmprAlg, + .fid = writer->ctx->fid, + .cid = writer->commitID, + .did = writer->ctx->did, + .level = 0, + }; + + code = tsdbFSetWriterOpen(&config, &writer->ctx->fsetWriter); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbSnapWriteFileDataEnd(STsdbSnapWriter* pWriter) { +static int32_t tsdbSnapWriteFileSetCloseWriter(STsdbSnapWriter* writer) { + return tsdbFSetWriterClose(&writer->ctx->fsetWriter, 0, writer->fopArr); +} + +static int32_t tsdbSnapWriteFileSetBegin(STsdbSnapWriter* writer, int32_t fid) { int32_t code = 0; int32_t lino = 0; - ASSERT(pWriter->pDataFWriter); + ASSERT(writer->ctx->fsetWriteBegin == false); - // consume remain data and end with a NULL table row - SRowInfo* pRowInfo; - code = tsdbSnapWriteGetRow(pWriter, &pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - for (;;) { - code = tsdbSnapWriteTableData(pWriter, pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); + STFileSet* fset = &(STFileSet){.fid = fid}; - if (pRowInfo == NULL) break; + writer->ctx->fid = fid; + STFileSet** fsetPtr = TARRAY2_SEARCH(writer->fsetArr, &fset, tsdbTFileSetCmprFn, TD_EQ); + writer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr; - code = tsdbSnapWriteNextRow(pWriter, &pRowInfo); + int32_t level = tsdbFidLevel(fid, &writer->tsdb->keepCfg, taosGetTimestampSec()); + if (tfsAllocDisk(writer->tsdb->pVnode->pTfs, level, &writer->ctx->did)) { + code = TSDB_CODE_NO_AVAIL_DISK; TSDB_CHECK_CODE(code, lino, _exit); } + tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did); - // do file-level updates - code = tsdbWriteSttBlk(pWriter->pDataFWriter, pWriter->aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); + writer->ctx->hasData = true; + writer->ctx->hasTomb = true; - code = tsdbUpdateDFileSetHeader(pWriter->pDataFWriter); + code = tsdbSnapWriteFileSetOpenReader(writer); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->pDataFWriter->wSet); + code = tsdbSnapWriteFileSetOpenIter(writer); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1); + code = tsdbSnapWriteFileSetOpenWriter(writer); TSDB_CHECK_CODE(code, lino, _exit); - if (pWriter->pDataFReader) { - code = tsdbDataFReaderClose(&pWriter->pDataFReader); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // clear sources - while (pWriter->iterList) { - STsdbDataIter2* pIter = pWriter->iterList; - pWriter->iterList = pIter->next; - tsdbCloseDataIter2(pIter); - } + writer->ctx->fsetWriteBegin = true; _exit: if (code) { - tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code)); - } else { - tsdbDebug("vgId:%d %s is done", TD_VID(pWriter->pTsdb->pVnode), __func__); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbSnapWriteTimeSeriesData(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr) { +static int32_t tsdbSnapWriteTombRecord(STsdbSnapWriter* writer, const STombRecord* record) { int32_t code = 0; int32_t lino = 0; - code = tDecmprBlockData(pHdr->data, pHdr->size, &pWriter->inData, pWriter->aBuf); - TSDB_CHECK_CODE(code, lino, _exit); - - ASSERT(pWriter->inData.nRow > 0); + while (writer->ctx->hasTomb) { + STombRecord* record1 = tsdbIterMergerGetTombRecord(writer->ctx->tombIterMerger); + if (record1 == NULL) { + writer->ctx->hasTomb = false; + break; + } - // switch to new data file if need - int32_t fid = tsdbKeyFid(pWriter->inData.aTSKEY[0], pWriter->minutes, pWriter->precision); - if (pWriter->fid != fid) { - if (pWriter->pDataFWriter) { - code = tsdbSnapWriteFileDataEnd(pWriter); + int32_t c = tTombRecordCompare(record1, record); + if (c <= 0) { + code = tsdbFSetWriteTombRecord(writer->ctx->fsetWriter, record1); TSDB_CHECK_CODE(code, lino, _exit); + } else { + break; } + } - code = tsdbSnapWriteFileDataStart(pWriter, fid); - TSDB_CHECK_CODE(code, lino, _exit); + if (record->suid == INT64_MAX) { + ASSERT(writer->ctx->hasTomb == false); + goto _exit; } - // loop write each row - SRowInfo* pRowInfo; - code = tsdbSnapWriteGetRow(pWriter, &pRowInfo); + code = tsdbFSetWriteTombRecord(writer->ctx->fsetWriter, record); TSDB_CHECK_CODE(code, lino, _exit); - for (int32_t iRow = 0; iRow < pWriter->inData.nRow; ++iRow) { - SRowInfo rInfo = {.suid = pWriter->inData.suid, - .uid = pWriter->inData.uid ? pWriter->inData.uid : pWriter->inData.aUid[iRow], - .row = tsdbRowFromBlockData(&pWriter->inData, iRow)}; - - for (;;) { - if (pRowInfo == NULL) { - code = tsdbSnapWriteTableData(pWriter, &rInfo); - TSDB_CHECK_CODE(code, lino, _exit); - break; - } else { - int32_t c = tRowInfoCmprFn(&rInfo, pRowInfo); - if (c < 0) { - code = tsdbSnapWriteTableData(pWriter, &rInfo); - TSDB_CHECK_CODE(code, lino, _exit); - break; - } else if (c > 0) { - code = tsdbSnapWriteTableData(pWriter, pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbSnapWriteNextRow(pWriter, &pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - ASSERT(0); - } - } - } - } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbDebug("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64 " nRow:%d", TD_VID(pWriter->pTsdb->pVnode), __func__, - pWriter->inData.suid, pWriter->inData.uid, pWriter->inData.nRow); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -// SNAP_DATA_DEL -static int32_t tsdbSnapWriteDelTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pId) { +static int32_t tsdbSnapWriteFileSetEnd(STsdbSnapWriter* writer) { + if (!writer->ctx->fsetWriteBegin) return 0; + int32_t code = 0; int32_t lino = 0; - if (pId) { - pWriter->tbid = *pId; - } else { - pWriter->tbid = (TABLEID){.suid = INT64_MAX, .uid = INT64_MAX}; - } - - taosArrayClear(pWriter->aDelData); + SRowInfo row = { + .suid = INT64_MAX, + .uid = INT64_MAX, + }; - if (pWriter->pTIter) { - while (pWriter->pTIter->tIter.iDelIdx < taosArrayGetSize(pWriter->pTIter->tIter.aDelIdx)) { - SDelIdx* pDelIdx = taosArrayGet(pWriter->pTIter->tIter.aDelIdx, pWriter->pTIter->tIter.iDelIdx); + code = tsdbSnapWriteTimeSeriesRow(writer, &row); + TSDB_CHECK_CODE(code, lino, _exit); - int32_t c = tTABLEIDCmprFn(pDelIdx, &pWriter->tbid); - if (c < 0) { - code = tsdbReadDelDatav1(pWriter->pDelFReader, pDelIdx, pWriter->pTIter->tIter.aDelData, INT64_MAX); - TSDB_CHECK_CODE(code, lino, _exit); + STombRecord record = { + .suid = INT64_MAX, + .uid = INT64_MAX, + }; - SDelIdx* pDelIdxNew = taosArrayReserve(pWriter->aDelIdx, 1); - if (pDelIdxNew == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } + code = tsdbSnapWriteTombRecord(writer, &record); + TSDB_CHECK_CODE(code, lino, _exit); - pDelIdxNew->suid = pDelIdx->suid; - pDelIdxNew->uid = pDelIdx->uid; + // close write + code = tsdbSnapWriteFileSetCloseWriter(writer); + TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->pTIter->tIter.aDelData, pDelIdxNew); - TSDB_CHECK_CODE(code, lino, _exit); + code = tsdbSnapWriteFileSetCloseIter(writer); + TSDB_CHECK_CODE(code, lino, _exit); - pWriter->pTIter->tIter.iDelIdx++; - } else if (c == 0) { - code = tsdbReadDelDatav1(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, INT64_MAX); - TSDB_CHECK_CODE(code, lino, _exit); + code = tsdbSnapWriteFileSetCloseReader(writer); + TSDB_CHECK_CODE(code, lino, _exit); - pWriter->pTIter->tIter.iDelIdx++; - break; - } else { - break; - } - } - } + writer->ctx->fsetWriteBegin = false; _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbTrace("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), __func__, - pWriter->tbid.suid, pWriter->tbid.uid); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbSnapWriteDelTableDataEnd(STsdbSnapWriter* pWriter) { +static int32_t tsdbSnapWriteTimeSeriesData(STsdbSnapWriter* writer, SSnapDataHdr* hdr) { int32_t code = 0; int32_t lino = 0; - if (taosArrayGetSize(pWriter->aDelData) > 0) { - SDelIdx* pDelIdx = taosArrayReserve(pWriter->aDelIdx, 1); - if (pDelIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } + SBlockData blockData[1] = {0}; - pDelIdx->suid = pWriter->tbid.suid; - pDelIdx->uid = pWriter->tbid.uid; + code = tDecmprBlockData(hdr->data, hdr->size - sizeof(*hdr), blockData, writer->aBuf); + TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, pDelIdx); + int32_t fid = tsdbKeyFid(blockData->aTSKEY[0], writer->minutes, writer->precision); + if (!writer->ctx->fsetWriteBegin || fid != writer->ctx->fid) { + code = tsdbSnapWriteFileSetEnd(writer); TSDB_CHECK_CODE(code, lino, _exit); - } -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbTrace("vgId:%d %s done", TD_VID(pWriter->pTsdb->pVnode), __func__); - } - return code; -} - -static int32_t tsdbSnapWriteDelTableData(STsdbSnapWriter* pWriter, TABLEID* pId, uint8_t* pData, int64_t size) { - int32_t code = 0; - int32_t lino = 0; - - if (pId == NULL || pId->uid != pWriter->tbid.uid) { - if (pWriter->tbid.uid) { - code = tsdbSnapWriteDelTableDataEnd(pWriter); - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tsdbSnapWriteDelTableDataStart(pWriter, pId); + code = tsdbSnapWriteFileSetBegin(writer, fid); TSDB_CHECK_CODE(code, lino, _exit); } - if (pId == NULL) goto _exit; - - int64_t n = 0; - while (n < size) { - SDelData delData; - n += tGetDelData(pData + n, &delData); + for (int32_t i = 0; i < blockData->nRow; ++i) { + SRowInfo rowInfo = { + .suid = blockData->suid, + .uid = blockData->uid ? blockData->uid : blockData->aUid[i], + .row = tsdbRowFromBlockData(blockData, i), + }; - if (taosArrayPush(pWriter->aDelData, &delData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } + code = tsdbSnapWriteTimeSeriesRow(writer, &rowInfo); + TSDB_CHECK_CODE(code, lino, _exit); } - ASSERT(n == size); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); + } else { + tsdbDebug("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64 " nRow:%d", TD_VID(writer->tsdb->pVnode), __func__, + blockData->suid, blockData->uid, blockData->nRow); } + tBlockDataDestroy(blockData); return code; } -static int32_t tsdbSnapWriteDelDataStart(STsdbSnapWriter* pWriter) { +static int32_t tsdbSnapWriteDecmprTombBlock(SSnapDataHdr* hdr, STombBlock* tombBlock) { int32_t code = 0; int32_t lino = 0; - STsdb* pTsdb = pWriter->pTsdb; - SDelFile* pDelFile = pWriter->fs.pDelFile; - - pWriter->tbid = (TABLEID){0}; + int64_t size = hdr->size - sizeof(*hdr); + ASSERT(size % TOMB_RECORD_ELEM_NUM == 0); + size = size / TOMB_RECORD_ELEM_NUM; + ASSERT(size % sizeof(int64_t) == 0); - // reader - if (pDelFile) { - code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbOpenTombFileDataIter(pWriter->pDelFReader, &pWriter->pTIter); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // writer - code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &(SDelFile){.commitID = pWriter->commitID}, pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - - if ((pWriter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - if ((pWriter->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + int64_t* data = (int64_t*)hdr->data; + for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { + code = TARRAY2_APPEND_BATCH(&tombBlock->dataArr[i], hdr->data + i * size, size / sizeof(int64_t)); TSDB_CHECK_CODE(code, lino, _exit); } _exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbDebug("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); - } return code; } -static int32_t tsdbSnapWriteDelDataEnd(STsdbSnapWriter* pWriter) { +static int32_t tsdbSnapWriteTombData(STsdbSnapWriter* writer, SSnapDataHdr* hdr) { int32_t code = 0; int32_t lino = 0; - STsdb* pTsdb = pWriter->pTsdb; - - // end remaining table with NULL data - code = tsdbSnapWriteDelTableData(pWriter, NULL, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); - - // update file-level info - code = tsdbWriteDelIdx(pWriter->pDelFWriter, pWriter->aDelIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter); - TSDB_CHECK_CODE(code, lino, _exit); + STombRecord record; + STombBlock tombBlock[1] = {0}; - code = tsdbFSUpsertDelFile(&pWriter->fs, &pWriter->pDelFWriter->fDel); + code = tsdbSnapWriteDecmprTombBlock(hdr, tombBlock); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbDelFWriterClose(&pWriter->pDelFWriter, 1); - TSDB_CHECK_CODE(code, lino, _exit); + tTombBlockGet(tombBlock, 0, &record); + int32_t fid = tsdbKeyFid(record.skey, writer->minutes, writer->precision); + if (!writer->ctx->fsetWriteBegin || fid != writer->ctx->fid) { + code = tsdbSnapWriteFileSetEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); - if (pWriter->pDelFReader) { - code = tsdbDelFReaderClose(&pWriter->pDelFReader); + code = tsdbSnapWriteFileSetBegin(writer, fid); TSDB_CHECK_CODE(code, lino, _exit); } - if (pWriter->pTIter) { - tsdbCloseDataIter2(pWriter->pTIter); - pWriter->pTIter = NULL; - } + if (writer->ctx->hasData) { + SRowInfo row = { + .suid = INT64_MAX, + .uid = INT64_MAX, + }; -_exit: - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + code = tsdbSnapWriteTimeSeriesRow(writer, &row); + TSDB_CHECK_CODE(code, lino, _exit); } - return code; -} -static int32_t tsdbSnapWriteDelData(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr) { - int32_t code = 0; - int32_t lino = 0; + ASSERT(writer->ctx->hasData == false); - STsdb* pTsdb = pWriter->pTsdb; + for (int32_t i = 0; i < TOMB_BLOCK_SIZE(tombBlock); ++i) { + tTombBlockGet(tombBlock, i, &record); - // start to write del data if need - if (pWriter->pDelFWriter == NULL) { - code = tsdbSnapWriteDelDataStart(pWriter); + code = tsdbSnapWriteTombRecord(writer, &record); TSDB_CHECK_CODE(code, lino, _exit); } - // do write del data - code = tsdbSnapWriteDelTableData(pWriter, (TABLEID*)pHdr->data, pHdr->data + sizeof(TABLEID), - pHdr->size - sizeof(TABLEID)); - TSDB_CHECK_CODE(code, lino, _exit); + tTombBlockDestroy(tombBlock); _exit: if (code) { - tsdbError("vgId:%d %s failed since %s", TD_VID(pTsdb->pVnode), __func__, tstrerror(code)); - } else { - tsdbTrace("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } return code; } -// APIs -int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) { +int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** writer) { int32_t code = 0; int32_t lino = 0; - // alloc - STsdbSnapWriter* pWriter = (STsdbSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); - if (pWriter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - pWriter->pTsdb = pTsdb; - pWriter->sver = sver; - pWriter->ever = ever; - pWriter->minutes = pTsdb->keepCfg.days; - pWriter->precision = pTsdb->keepCfg.precision; - pWriter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; - pWriter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; - pWriter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; - pWriter->commitID = pTsdb->pVnode->state.commitID; - - code = tsdbFSCopy(pTsdb, &pWriter->fs); - TSDB_CHECK_CODE(code, lino, _exit); - - // SNAP_DATA_TSDB - code = tBlockDataCreate(&pWriter->inData); - TSDB_CHECK_CODE(code, lino, _exit); - - pWriter->fid = INT32_MIN; - - code = tBlockDataCreate(&pWriter->bData); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tBlockDataCreate(&pWriter->sData); + writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); + if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->tsdb = pTsdb; + writer[0]->sver = sver; + writer[0]->ever = ever; + writer[0]->minutes = pTsdb->keepCfg.days; + writer[0]->precision = pTsdb->keepCfg.precision; + writer[0]->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; + writer[0]->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; + writer[0]->commitID = tsdbFSAllocEid(pTsdb->pFS); + writer[0]->szPage = pTsdb->pVnode->config.tsdbPageSize; + writer[0]->compactVersion = INT64_MAX; + writer[0]->now = taosGetTimestampMs(); + + code = tsdbFSCreateCopySnapshot(pTsdb->pFS, &writer[0]->fsetArr); TSDB_CHECK_CODE(code, lino, _exit); - // SNAP_DATA_DEL + tsdbFSDisableBgTask(pTsdb->pFS); _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - if (pWriter) { - tBlockDataDestroy(&pWriter->sData); - tBlockDataDestroy(&pWriter->bData); - tBlockDataDestroy(&pWriter->inData); - tsdbFSDestroy(&pWriter->fs); - taosMemoryFree(pWriter); - pWriter = NULL; - } } else { tsdbInfo("vgId:%d %s done, sver:%" PRId64 " ever:%" PRId64, TD_VID(pTsdb->pVnode), __func__, sver, ever); } - *ppWriter = pWriter; return code; } -int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter) { +int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* writer) { int32_t code = 0; int32_t lino = 0; - if (pWriter->pDataFWriter) { - code = tsdbSnapWriteFileDataEnd(pWriter); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (pWriter->pDelFWriter) { - code = tsdbSnapWriteDelDataEnd(pWriter); - TSDB_CHECK_CODE(code, lino, _exit); - } + code = tsdbSnapWriteFileSetEnd(writer); + TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbFSPrepareCommit(pWriter->pTsdb, &pWriter->fs); + code = tsdbFSEditBegin(writer->tsdb->pFS, writer->fopArr, TSDB_FEDIT_COMMIT); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code); } else { - tsdbDebug("vgId:%d %s done", TD_VID(pWriter->pTsdb->pVnode), __func__); + tsdbDebug("vgId:%d %s done", TD_VID(writer->tsdb->pVnode), __func__); } return code; } -int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { +int32_t tsdbSnapWriterClose(STsdbSnapWriter** writer, int8_t rollback) { + if (writer[0] == NULL) return 0; + int32_t code = 0; int32_t lino = 0; - STsdbSnapWriter* pWriter = *ppWriter; - STsdb* pTsdb = pWriter->pTsdb; + STsdb* tsdb = writer[0]->tsdb; if (rollback) { - tsdbRollbackCommit(pWriter->pTsdb); + code = tsdbFSEditAbort(writer[0]->tsdb->pFS); + TSDB_CHECK_CODE(code, lino, _exit); } else { - // lock - taosThreadRwlockWrlock(&pTsdb->rwLock); + taosThreadRwlockWrlock(&writer[0]->tsdb->rwLock); - code = tsdbFSCommit(pWriter->pTsdb); + code = tsdbFSEditCommit(writer[0]->tsdb->pFS); if (code) { - taosThreadRwlockUnlock(&pTsdb->rwLock); + taosThreadRwlockUnlock(&writer[0]->tsdb->rwLock); TSDB_CHECK_CODE(code, lino, _exit); } - // unlock - taosThreadRwlockUnlock(&pTsdb->rwLock); + taosThreadRwlockUnlock(&writer[0]->tsdb->rwLock); } + tsdbFSEnableBgTask(tsdb->pFS); + + tsdbIterMergerClose(&writer[0]->ctx->tombIterMerger); + tsdbIterMergerClose(&writer[0]->ctx->dataIterMerger); + TARRAY2_DESTROY(writer[0]->ctx->tombIterArr, tsdbIterClose); + TARRAY2_DESTROY(writer[0]->ctx->dataIterArr, tsdbIterClose); + TARRAY2_DESTROY(writer[0]->ctx->sttReaderArr, tsdbSttFileReaderClose); + tsdbDataFileReaderClose(&writer[0]->ctx->dataReader); - // SNAP_DATA_DEL - taosArrayDestroy(pWriter->aDelData); - taosArrayDestroy(pWriter->aDelIdx); - - // SNAP_DATA_TSDB - tBlockDataDestroy(&pWriter->sData); - tBlockDataDestroy(&pWriter->bData); - taosArrayDestroy(pWriter->aSttBlk); - tMapDataClear(&pWriter->mDataBlk); - taosArrayDestroy(pWriter->aBlockIdx); - tDestroyTSchema(pWriter->skmTable.pTSchema); - tBlockDataDestroy(&pWriter->inData); - - for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { - tFree(pWriter->aBuf[iBuf]); + TARRAY2_DESTROY(writer[0]->fopArr, NULL); + tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr); + + for (int32_t i = 0; i < ARRAY_SIZE(writer[0]->aBuf); ++i) { + tFree(writer[0]->aBuf[i]); } - tsdbFSDestroy(&pWriter->fs); - taosMemoryFree(pWriter); - *ppWriter = NULL; + + taosMemoryFree(writer[0]); + writer[0] = NULL; _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); } else { - tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); } return code; } -int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr) { +int32_t tsdbSnapWrite(STsdbSnapWriter* writer, SSnapDataHdr* hdr) { int32_t code = 0; int32_t lino = 0; - if (pHdr->type == SNAP_DATA_TSDB) { - code = tsdbSnapWriteTimeSeriesData(pWriter, pHdr); + if (hdr->type == SNAP_DATA_TSDB) { + code = tsdbSnapWriteTimeSeriesData(writer, hdr); TSDB_CHECK_CODE(code, lino, _exit); - goto _exit; - } else if (pWriter->pDataFWriter) { - code = tsdbSnapWriteFileDataEnd(pWriter); - TSDB_CHECK_CODE(code, lino, _exit); - } - - if (pHdr->type == SNAP_DATA_DEL) { - code = tsdbSnapWriteDelData(pWriter, pHdr); + } else if (hdr->type == SNAP_DATA_DEL) { + code = tsdbSnapWriteTombData(writer, hdr); TSDB_CHECK_CODE(code, lino, _exit); - goto _exit; + } else { + ASSERT(0); } _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s, type:%d index:%" PRId64 " size:%" PRId64, - TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code), pHdr->type, pHdr->index, pHdr->size); + TD_VID(writer->tsdb->pVnode), __func__, lino, tstrerror(code), hdr->type, hdr->index, hdr->size); } else { - tsdbDebug("vgId:%d %s done, type:%d index:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), __func__, - pHdr->type, pHdr->index, pHdr->size); + tsdbDebug("vgId:%d %s done, type:%d index:%" PRId64 " size:%" PRId64, TD_VID(writer->tsdb->pVnode), __func__, + hdr->type, hdr->index, hdr->size); } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c new file mode 100644 index 0000000000000000000000000000000000000000..db8d14d2281455bad88eef366c68e4057fd8c5c0 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -0,0 +1,982 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbSttFileRW.h" + +// SSttFReader ============================================================ +struct SSttFileReader { + SSttFileReaderConfig config[1]; + STsdbFD *fd; + SSttFooter footer[1]; + struct { + bool sttBlkLoaded; + bool statisBlkLoaded; + bool tombBlkLoaded; + } ctx[1]; + TSttBlkArray sttBlkArray[1]; + TStatisBlkArray statisBlkArray[1]; + TTombBlkArray tombBlkArray[1]; + uint8_t *bufArr[5]; +}; + +// SSttFileReader +int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *config, SSttFileReader **reader) { + int32_t code = 0; + int32_t lino = 0; + + reader[0] = taosMemoryCalloc(1, sizeof(*reader[0])); + if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + reader[0]->config[0] = config[0]; + if (reader[0]->config->bufArr == NULL) { + reader[0]->config->bufArr = reader[0]->bufArr; + } + + // open file + if (fname) { + code = tsdbOpenFile(fname, config->szPage, TD_FILE_READ, &reader[0]->fd); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + char fname1[TSDB_FILENAME_LEN]; + tsdbTFileName(config->tsdb, config->file, fname1); + code = tsdbOpenFile(fname1, config->szPage, TD_FILE_READ, &reader[0]->fd); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // // open each segment reader + int64_t offset = config->file->size - sizeof(SSttFooter); + ASSERT(offset >= TSDB_FHDR_SIZE); + + code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter)); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); + tsdbSttFileReaderClose(reader); + } + return code; +} + +int32_t tsdbSttFileReaderClose(SSttFileReader **reader) { + if (reader[0]) { + for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->bufArr); ++i) { + tFree(reader[0]->bufArr[i]); + } + tsdbCloseFile(&reader[0]->fd); + TARRAY2_DESTROY(reader[0]->tombBlkArray, NULL); + TARRAY2_DESTROY(reader[0]->statisBlkArray, NULL); + TARRAY2_DESTROY(reader[0]->sttBlkArray, NULL); + taosMemoryFree(reader[0]); + reader[0] = NULL; + } + return 0; +} + +// SSttFSegReader +int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray **statisBlkArray) { + if (!reader->ctx->statisBlkLoaded) { + if (reader->footer->statisBlkPtr->size > 0) { + ASSERT(reader->footer->statisBlkPtr->size % sizeof(SStatisBlk) == 0); + + int32_t size = reader->footer->statisBlkPtr->size / sizeof(SStatisBlk); + void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size); + if (!data) return TSDB_CODE_OUT_OF_MEMORY; + + int32_t code = + tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size); + if (code) { + taosMemoryFree(data); + return code; + } + + TARRAY2_INIT_EX(reader->statisBlkArray, size, size, data); + } else { + TARRAY2_INIT(reader->statisBlkArray); + } + + reader->ctx->statisBlkLoaded = true; + } + + statisBlkArray[0] = reader->statisBlkArray; + return 0; +} + +int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tombBlkArray) { + if (!reader->ctx->tombBlkLoaded) { + if (reader->footer->tombBlkPtr->size > 0) { + ASSERT(reader->footer->tombBlkPtr->size % sizeof(STombBlk) == 0); + + int32_t size = reader->footer->tombBlkPtr->size / sizeof(STombBlk); + void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size); + if (!data) return TSDB_CODE_OUT_OF_MEMORY; + + int32_t code = + tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size); + if (code) { + taosMemoryFree(data); + return code; + } + + TARRAY2_INIT_EX(reader->tombBlkArray, size, size, data); + } else { + TARRAY2_INIT(reader->tombBlkArray); + } + + reader->ctx->tombBlkLoaded = true; + } + + tombBlkArray[0] = reader->tombBlkArray; + return 0; +} + +int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBlkArray) { + if (!reader->ctx->sttBlkLoaded) { + if (reader->footer->sttBlkPtr->size > 0) { + ASSERT(reader->footer->sttBlkPtr->size % sizeof(SSttBlk) == 0); + + int32_t size = reader->footer->sttBlkPtr->size / sizeof(SSttBlk); + void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size); + if (!data) return TSDB_CODE_OUT_OF_MEMORY; + + int32_t code = tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size); + if (code) { + taosMemoryFree(data); + return code; + } + + TARRAY2_INIT_EX(reader->sttBlkArray, size, size, data); + } else { + TARRAY2_INIT(reader->sttBlkArray); + } + + reader->ctx->sttBlkLoaded = true; + } + + sttBlkArray[0] = reader->sttBlkArray; + return 0; +} + +int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData) { + int32_t code = 0; + int32_t lino = 0; + + code = tRealloc(&reader->config->bufArr[0], sttBlk->bInfo.szBlock); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szBlock); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tDecmprBlockData(reader->config->bufArr[0], sttBlk->bInfo.szBlock, bData, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData, + STSchema *pTSchema, int16_t cids[], int32_t ncid) { + int32_t code = 0; + int32_t lino = 0; + + TABLEID tbid = {.suid = sttBlk->suid}; + if (tbid.suid == 0) { + tbid.uid = sttBlk->minUid; + } else { + tbid.uid = 0; + } + + code = tBlockDataInit(bData, &tbid, pTSchema, cids, ncid); + TSDB_CHECK_CODE(code, lino, _exit); + + // uid + version + tskey + code = tRealloc(&reader->config->bufArr[0], sttBlk->bInfo.szKey); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szKey); + TSDB_CHECK_CODE(code, lino, _exit); + + // hdr + SDiskDataHdr hdr[1]; + int32_t size = 0; + + size += tGetDiskDataHdr(reader->config->bufArr[0] + size, hdr); + + ASSERT(hdr->delimiter == TSDB_FILE_DLMT); + + bData->nRow = hdr->nRow; + bData->uid = hdr->uid; + + // uid + if (hdr->uid == 0) { + ASSERT(hdr->szUid); + code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szUid, TSDB_DATA_TYPE_BIGINT, hdr->cmprAlg, + (uint8_t **)&bData->aUid, sizeof(int64_t) * hdr->nRow, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + ASSERT(hdr->szUid == 0); + } + size += hdr->szUid; + + // version + code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szVer, TSDB_DATA_TYPE_BIGINT, hdr->cmprAlg, + (uint8_t **)&bData->aVersion, sizeof(int64_t) * hdr->nRow, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + size += hdr->szVer; + + // ts + code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr->cmprAlg, + (uint8_t **)&bData->aTSKEY, sizeof(TSKEY) * hdr->nRow, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + size += hdr->szKey; + + ASSERT(size == sttBlk->bInfo.szKey); + + // other columns + if (bData->nColData > 0) { + if (hdr->szBlkCol > 0) { + code = tRealloc(&reader->config->bufArr[0], hdr->szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, reader->config->bufArr[0], + hdr->szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + } + + SBlockCol bc[1] = {{.cid = 0}}; + SBlockCol *blockCol = bc; + + size = 0; + for (int32_t i = 0; i < bData->nColData; i++) { + SColData *colData = tBlockDataGetColDataByIdx(bData, i); + + while (blockCol && blockCol->cid < colData->cid) { + if (size < hdr->szBlkCol) { + size += tGetBlockCol(reader->config->bufArr[0] + size, blockCol); + } else { + ASSERT(size == hdr->szBlkCol); + blockCol = NULL; + } + } + + if (blockCol == NULL || blockCol->cid > colData->cid) { + for (int32_t iRow = 0; iRow < hdr->nRow; iRow++) { + code = tColDataAppendValue(colData, &COL_VAL_NONE(colData->cid, colData->type)); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + ASSERT(blockCol->type == colData->type); + ASSERT(blockCol->flag && blockCol->flag != HAS_NONE); + + if (blockCol->flag == HAS_NULL) { + for (int32_t iRow = 0; iRow < hdr->nRow; iRow++) { + code = tColDataAppendValue(colData, &COL_VAL_NULL(blockCol->cid, blockCol->type)); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + int32_t size1 = blockCol->szBitmap + blockCol->szOffset + blockCol->szValue; + + code = tRealloc(&reader->config->bufArr[1], size1); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr->szBlkCol + blockCol->offset, + reader->config->bufArr[1], size1); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDecmprColData(reader->config->bufArr[1], blockCol, hdr->cmprAlg, hdr->nRow, colData, + &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk, STombBlock *tombBlock) { + int32_t code = 0; + int32_t lino = 0; + + code = tRealloc(&reader->config->bufArr[0], tombBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, tombBlk->dp->offset, reader->config->bufArr[0], tombBlk->dp->size); + if (code) TSDB_CHECK_CODE(code, lino, _exit); + + int64_t size = 0; + tTombBlockClear(tombBlock); + for (int32_t i = 0; i < ARRAY_SIZE(tombBlock->dataArr); ++i) { + code = tsdbDecmprData(reader->config->bufArr[0] + size, tombBlk->size[i], TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, + &reader->config->bufArr[1], sizeof(int64_t) * tombBlk->numRec, &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND_BATCH(&tombBlock->dataArr[i], reader->config->bufArr[1], tombBlk->numRec); + TSDB_CHECK_CODE(code, lino, _exit); + + size += tombBlk->size[i]; + } + + ASSERT(size == tombBlk->dp->size); +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *statisBlk, STbStatisBlock *statisBlock) { + int32_t code = 0; + int32_t lino = 0; + + code = tRealloc(&reader->config->bufArr[0], statisBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, statisBlk->dp->offset, reader->config->bufArr[0], statisBlk->dp->size); + TSDB_CHECK_CODE(code, lino, _exit); + + int64_t size = 0; + tStatisBlockClear(statisBlock); + for (int32_t i = 0; i < ARRAY_SIZE(statisBlock->dataArr); ++i) { + code = + tsdbDecmprData(reader->config->bufArr[0] + size, statisBlk->size[i], TSDB_DATA_TYPE_BIGINT, statisBlk->cmprAlg, + &reader->config->bufArr[1], sizeof(int64_t) * statisBlk->numRec, &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND_BATCH(statisBlock->dataArr + i, reader->config->bufArr[1], statisBlk->numRec); + TSDB_CHECK_CODE(code, lino, _exit); + + size += statisBlk->size[i]; + } + + ASSERT(size == statisBlk->dp->size); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); + } + return code; +} + +// SSttFWriter ============================================================ +struct SSttFileWriter { + SSttFileWriterConfig config[1]; + struct { + bool opened; + TABLEID tbid[1]; + } ctx[1]; + // file + STsdbFD *fd; + STFile file[1]; + // data + SSttFooter footer[1]; + TTombBlkArray tombBlkArray[1]; + TSttBlkArray sttBlkArray[1]; + TStatisBlkArray statisBlkArray[1]; + STombBlock tombBlock[1]; + STbStatisBlock staticBlock[1]; + SBlockData blockData[1]; + // helper data + SSkmInfo skmTb[1]; + SSkmInfo skmRow[1]; + uint8_t *bufArr[5]; +}; + +int32_t tsdbFileDoWriteBlockData(STsdbFD *fd, SBlockData *blockData, int8_t cmprAlg, int64_t *fileSize, + TSttBlkArray *sttBlkArray, uint8_t **bufArr) { + if (blockData->nRow == 0) return 0; + + int32_t code = 0; + + SSttBlk sttBlk[1] = {{ + .suid = blockData->suid, + .minUid = blockData->uid ? blockData->uid : blockData->aUid[0], + .maxUid = blockData->uid ? blockData->uid : blockData->aUid[blockData->nRow - 1], + .minKey = blockData->aTSKEY[0], + .maxKey = blockData->aTSKEY[0], + .minVer = blockData->aVersion[0], + .maxVer = blockData->aVersion[0], + .nRow = blockData->nRow, + }}; + + for (int32_t iRow = 1; iRow < blockData->nRow; iRow++) { + if (sttBlk->minKey > blockData->aTSKEY[iRow]) sttBlk->minKey = blockData->aTSKEY[iRow]; + if (sttBlk->maxKey < blockData->aTSKEY[iRow]) sttBlk->maxKey = blockData->aTSKEY[iRow]; + if (sttBlk->minVer > blockData->aVersion[iRow]) sttBlk->minVer = blockData->aVersion[iRow]; + if (sttBlk->maxVer < blockData->aVersion[iRow]) sttBlk->maxVer = blockData->aVersion[iRow]; + } + + int32_t sizeArr[5] = {0}; + code = tCmprBlockData(blockData, cmprAlg, NULL, NULL, bufArr, sizeArr); + if (code) return code; + + sttBlk->bInfo.offset = *fileSize; + sttBlk->bInfo.szKey = sizeArr[2] + sizeArr[3]; + sttBlk->bInfo.szBlock = sizeArr[0] + sizeArr[1] + sttBlk->bInfo.szKey; + + for (int32_t i = 3; i >= 0; i--) { + if (sizeArr[i]) { + code = tsdbWriteFile(fd, *fileSize, bufArr[i], sizeArr[i]); + if (code) return code; + *fileSize += sizeArr[i]; + } + } + + code = TARRAY2_APPEND_PTR(sttBlkArray, sttBlk); + if (code) return code; + + tBlockDataClear(blockData); + + return 0; +} + +static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) { + if (writer->blockData->nRow == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileDoWriteBlockData(writer->fd, writer->blockData, writer->config->cmprAlg, &writer->file->size, + writer->sttBlkArray, writer->config->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) { + if (STATIS_BLOCK_SIZE(writer->staticBlock) == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + SStatisBlk statisBlk[1] = {{ + .dp[0] = + { + .offset = writer->file->size, + .size = 0, + }, + .minTbid = + { + .suid = TARRAY2_FIRST(writer->staticBlock->suid), + .uid = TARRAY2_FIRST(writer->staticBlock->uid), + }, + .maxTbid = + { + .suid = TARRAY2_LAST(writer->staticBlock->suid), + .uid = TARRAY2_LAST(writer->staticBlock->uid), + }, + .numRec = STATIS_BLOCK_SIZE(writer->staticBlock), + .cmprAlg = writer->config->cmprAlg, + }}; + + for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; i++) { + code = tsdbCmprData((uint8_t *)TARRAY2_DATA(writer->staticBlock->dataArr + i), + TARRAY2_DATA_LEN(&writer->staticBlock->dataArr[i]), TSDB_DATA_TYPE_BIGINT, statisBlk->cmprAlg, + &writer->config->bufArr[0], 0, &statisBlk->size[i], &writer->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbWriteFile(writer->fd, writer->file->size, writer->config->bufArr[0], statisBlk->size[i]); + TSDB_CHECK_CODE(code, lino, _exit); + + statisBlk->dp->size += statisBlk->size[i]; + writer->file->size += statisBlk->size[i]; + } + + code = TARRAY2_APPEND_PTR(writer->statisBlkArray, statisBlk); + TSDB_CHECK_CODE(code, lino, _exit); + + tStatisBlockClear(writer->staticBlock); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, + TTombBlkArray *tombBlkArray, uint8_t **bufArr) { + int32_t code; + + if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0; + + STombBlk tombBlk[1] = {{ + .dp[0] = + { + .offset = *fileSize, + .size = 0, + }, + .minTbid = + { + .suid = TARRAY2_FIRST(tombBlock->suid), + .uid = TARRAY2_FIRST(tombBlock->uid), + }, + .maxTbid = + { + .suid = TARRAY2_LAST(tombBlock->suid), + .uid = TARRAY2_LAST(tombBlock->uid), + }, + .minVer = TARRAY2_FIRST(tombBlock->version), + .maxVer = TARRAY2_FIRST(tombBlock->version), + .numRec = TOMB_BLOCK_SIZE(tombBlock), + .cmprAlg = cmprAlg, + }}; + + for (int32_t i = 1; i < TOMB_BLOCK_SIZE(tombBlock); i++) { + if (tombBlk->minVer > TARRAY2_GET(tombBlock->version, i)) { + tombBlk->minVer = TARRAY2_GET(tombBlock->version, i); + } + if (tombBlk->maxVer < TARRAY2_GET(tombBlock->version, i)) { + tombBlk->maxVer = TARRAY2_GET(tombBlock->version, i); + } + } + + for (int32_t i = 0; i < ARRAY_SIZE(tombBlock->dataArr); i++) { + code = tsdbCmprData((uint8_t *)TARRAY2_DATA(&tombBlock->dataArr[i]), TARRAY2_DATA_LEN(&tombBlock->dataArr[i]), + TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, &bufArr[0], 0, &tombBlk->size[i], &bufArr[1]); + if (code) return code; + + code = tsdbWriteFile(fd, *fileSize, bufArr[0], tombBlk->size[i]); + if (code) return code; + + tombBlk->dp->size += tombBlk->size[i]; + *fileSize += tombBlk->size[i]; + } + + code = TARRAY2_APPEND_PTR(tombBlkArray, tombBlk); + if (code) return code; + + tTombBlockClear(tombBlock); + return 0; +} + +static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) { + if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) return 0; + + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size, + writer->tombBlkArray, writer->config->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize) { + ptr->size = TARRAY2_DATA_LEN(sttBlkArray); + if (ptr->size > 0) { + ptr->offset = *fileSize; + + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size); + if (code) { + return code; + } + + *fileSize += ptr->size; + } + return 0; +} + +static int32_t tsdbSttFileDoWriteSttBlk(SSttFileWriter *writer) { + int32_t code = 0; + int32_t lino; + + code = tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSttFileDoWriteStatisBlk(SSttFileWriter *writer) { + int32_t code = 0; + int32_t lino; + + writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray); + if (writer->footer->statisBlkPtr->size) { + writer->footer->statisBlkPtr->offset = writer->file->size; + code = tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray), + writer->footer->statisBlkPtr->size); + TSDB_CHECK_CODE(code, lino, _exit); + writer->file->size += writer->footer->statisBlkPtr->size; + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize) { + ptr->size = TARRAY2_DATA_LEN(tombBlkArray); + if (ptr->size > 0) { + ptr->offset = *fileSize; + + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size); + if (code) { + return code; + } + + *fileSize += ptr->size; + } + return 0; +} + +static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + code = tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize) { + int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer)); + if (code) return code; + *fileSize += sizeof(*footer); + return 0; +} + +static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) { + return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size); +} + +static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) { + int32_t code = 0; + int32_t lino = 0; + + // set + if (!writer->config->skmTb) writer->config->skmTb = writer->skmTb; + if (!writer->config->skmRow) writer->config->skmRow = writer->skmRow; + if (!writer->config->bufArr) writer->config->bufArr = writer->bufArr; + + writer->file[0] = (STFile){ + .type = TSDB_FTYPE_STT, + .did = writer->config->did, + .fid = writer->config->fid, + .cid = writer->config->cid, + .size = 0, + .stt[0] = + { + .level = writer->config->level, + }, + }; + + // open file + int32_t flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + char fname[TSDB_FILENAME_LEN]; + + tsdbTFileName(writer->config->tsdb, writer->file, fname); + code = tsdbOpenFile(fname, writer->config->szPage, flag, &writer->fd); + TSDB_CHECK_CODE(code, lino, _exit); + + uint8_t hdr[TSDB_FHDR_SIZE] = {0}; + code = tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr)); + TSDB_CHECK_CODE(code, lino, _exit); + writer->file->size += sizeof(hdr); + + writer->ctx->opened = true; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static void tsdbSttFWriterDoClose(SSttFileWriter *writer) { + ASSERT(writer->fd == NULL); + + for (int32_t i = 0; i < ARRAY_SIZE(writer->bufArr); ++i) { + tFree(writer->bufArr[i]); + } + tDestroyTSchema(writer->skmRow->pTSchema); + tDestroyTSchema(writer->skmTb->pTSchema); + tTombBlockDestroy(writer->tombBlock); + tStatisBlockDestroy(writer->staticBlock); + tBlockDataDestroy(writer->blockData); + TARRAY2_DESTROY(writer->tombBlkArray, NULL); + TARRAY2_DESTROY(writer->statisBlkArray, NULL); + TARRAY2_DESTROY(writer->sttBlkArray, NULL); +} + +static int32_t tsdbSttFileDoUpdateHeader(SSttFileWriter *writer) { + // TODO + return 0; +} + +static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *opArray) { + int32_t lino; + int32_t code; + + code = tsdbSttFileDoWriteBlockData(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoWriteStatisBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoWriteTombBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoWriteSttBlk(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoWriteStatisBlk(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoWriteTombBlk(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoWriteFooter(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbSttFileDoUpdateHeader(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFsyncFile(writer->fd); + TSDB_CHECK_CODE(code, lino, _exit); + + tsdbCloseFile(&writer->fd); + + ASSERT(writer->file->size > 0); + STFileOp op = (STFileOp){ + .optype = TSDB_FOP_CREATE, + .fid = writer->config->fid, + .nf = writer->file[0], + }; + + code = TARRAY2_APPEND(opArray, op); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbSttFWriterCloseAbort(SSttFileWriter *writer) { + char fname[TSDB_FILENAME_LEN]; + tsdbTFileName(writer->config->tsdb, writer->file, fname); + tsdbCloseFile(&writer->fd); + taosRemoveFile(fname); + return 0; +} + +int32_t tsdbSttFileWriterOpen(const SSttFileWriterConfig *config, SSttFileWriter **writer) { + writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); + if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + writer[0]->config[0] = config[0]; + writer[0]->ctx->opened = false; + return 0; +} + +int32_t tsdbSttFileWriterClose(SSttFileWriter **writer, int8_t abort, TFileOpArray *opArray) { + int32_t code = 0; + int32_t lino = 0; + + if (writer[0]->ctx->opened) { + if (abort) { + code = tsdbSttFWriterCloseAbort(writer[0]); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbSttFWriterCloseCommit(writer[0], opArray); + TSDB_CHECK_CODE(code, lino, _exit); + } + tsdbSttFWriterDoClose(writer[0]); + } + taosMemoryFree(writer[0]); + writer[0] = NULL; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) { + int32_t code = 0; + int32_t lino = 0; + + if (!writer->ctx->opened) { + code = tsdbSttFWriterDoOpen(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (!TABLE_SAME_SCHEMA(row->suid, row->uid, writer->ctx->tbid->suid, writer->ctx->tbid->uid)) { + code = tsdbSttFileDoWriteBlockData(writer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbUpdateSkmTb(writer->config->tsdb, (TABLEID *)row, writer->config->skmTb); + TSDB_CHECK_CODE(code, lino, _exit); + + TABLEID id = {.suid = row->suid, .uid = row->suid ? 0 : row->uid}; + code = tBlockDataInit(writer->blockData, &id, writer->config->skmTb->pTSchema, NULL, 0); + TSDB_CHECK_CODE(code, lino, _exit); + } + + TSDBKEY key[1]; + if (row->row.type == TSDBROW_ROW_FMT) { + key->ts = row->row.pTSRow->ts; + key->version = row->row.version; + } else { + key->ts = row->row.pBlockData->aTSKEY[row->row.iRow]; + key->version = row->row.pBlockData->aVersion[row->row.iRow]; + } + + if (writer->ctx->tbid->uid != row->uid) { + writer->ctx->tbid->suid = row->suid; + writer->ctx->tbid->uid = row->uid; + + if (STATIS_BLOCK_SIZE(writer->staticBlock) >= writer->config->maxRow) { + code = tsdbSttFileDoWriteStatisBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + STbStatisRecord record = { + .suid = row->suid, + .uid = row->uid, + .firstKey = key->ts, + .lastKey = key->ts, + .count = 1, + }; + code = tStatisBlockPut(writer->staticBlock, &record); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + ASSERT(key->ts >= TARRAY2_LAST(writer->staticBlock->lastKey)); + + if (key->ts > TARRAY2_LAST(writer->staticBlock->lastKey)) { + TARRAY2_LAST(writer->staticBlock->count)++; + TARRAY2_LAST(writer->staticBlock->lastKey) = key->ts; + } + } + + if (row->row.type == TSDBROW_ROW_FMT) { + code = tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, // + TSDBROW_SVERSION(&row->row), writer->config->skmRow); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // row to col conversion + if (key->version <= writer->config->compactVersion // + && writer->blockData->nRow > 0 // + && writer->blockData->aTSKEY[writer->blockData->nRow - 1] == key->ts // + && (writer->blockData->uid // + ? writer->blockData->uid // + : writer->blockData->aUid[writer->blockData->nRow - 1]) == row->uid // + ) { + code = tBlockDataUpdateRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + if (writer->blockData->nRow >= writer->config->maxRow) { + code = tsdbSttFileDoWriteBlockData(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tBlockDataAppendRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema, row->uid); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *bdata) { + int32_t code = 0; + int32_t lino = 0; + + SRowInfo row[1]; + row->suid = bdata->suid; + for (int32_t i = 0; i < bdata->nRow; i++) { + row->uid = bdata->uid ? bdata->uid : bdata->aUid[i]; + row->row = tsdbRowFromBlockData(bdata, i); + + code = tsdbSttFileWriteRow(writer, row); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *record) { + int32_t code; + int32_t lino; + + if (!writer->ctx->opened) { + code = tsdbSttFWriterDoOpen(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + if (writer->blockData->nRow > 0) { + code = tsdbSttFileDoWriteBlockData(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (STATIS_BLOCK_SIZE(writer->staticBlock) > 0) { + code = tsdbSttFileDoWriteStatisBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + code = tTombBlockPut(writer->tombBlock, record); + TSDB_CHECK_CODE(code, lino, _exit); + + if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) { + code = tsdbSttFileDoWriteTombBlock(writer); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); + } + return code; +} + +bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer) { return writer->ctx->opened; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h new file mode 100644 index 0000000000000000000000000000000000000000..242b55795c99be8c416c9238df2cd0356cb89004 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbFS2.h" +#include "tsdbUtil2.h" + +#ifndef _TSDB_STT_FILE_RW_H +#define _TSDB_STT_FILE_RW_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef TARRAY2(SSttBlk) TSttBlkArray; +typedef TARRAY2(SStatisBlk) TStatisBlkArray; + +typedef struct { + SFDataPtr sttBlkPtr[1]; + SFDataPtr statisBlkPtr[1]; + SFDataPtr tombBlkPtr[1]; + SFDataPtr rsrvd[2]; +} SSttFooter; + +// SSttFileReader ========================================== +typedef struct SSttFileReader SSttFileReader; +typedef struct SSttFileReaderConfig SSttFileReaderConfig; +typedef TARRAY2(SSttFileReader *) TSttFileReaderArray; + +// SSttFileReader +int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *config, SSttFileReader **reader); +int32_t tsdbSttFileReaderClose(SSttFileReader **reader); + +// SSttSegReader +int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBlkArray); +int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray **statisBlkArray); +int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **delBlkArray); + +int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData); +int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData, + STSchema *pTSchema, int16_t cids[], int32_t ncid); +int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *statisBlk, STbStatisBlock *sData); +int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *delBlk, STombBlock *dData); + +struct SSttFileReaderConfig { + STsdb *tsdb; + int32_t szPage; + STFile file[1]; + uint8_t **bufArr; +}; + +// SSttFileWriter ========================================== +typedef struct SSttFileWriter SSttFileWriter; +typedef struct SSttFileWriterConfig SSttFileWriterConfig; + +int32_t tsdbSttFileWriterOpen(const SSttFileWriterConfig *config, SSttFileWriter **writer); +int32_t tsdbSttFileWriterClose(SSttFileWriter **writer, int8_t abort, TFileOpArray *opArray); +int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row); +int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *pBlockData); +int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *record); +bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer); + +struct SSttFileWriterConfig { + STsdb *tsdb; + int32_t maxRow; + int32_t szPage; + int8_t cmprAlg; + int64_t compactVersion; + SDiskID did; + int32_t fid; + int64_t cid; + int32_t level; + SSkmInfo *skmTb; + SSkmInfo *skmRow; + uint8_t **bufArr; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_STT_FILE_RW_H*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbUpgrade.c b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c new file mode 100644 index 0000000000000000000000000000000000000000..59ba51c371c9038c396d67a1799c219628032ab1 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbUpgrade.c @@ -0,0 +1,640 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbUpgrade.h" + +// old +extern void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t); +extern int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData); + +// new +extern int32_t save_fs(const TFileSetArray *arr, const char *fname); +extern int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype); +extern int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize, + TBrinBlkArray *brinBlkArray, uint8_t **bufArr); +extern int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize); +extern int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer); +extern int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl); +extern int32_t tsdbSttLvlClear(SSttLvl **lvl); +extern int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize); +extern int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize); +extern int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, + TTombBlkArray *tombBlkArray, uint8_t **bufArr); +extern int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize); +extern int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize); + +static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *reader, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + + // init + struct { + // config + int32_t maxRow; + int8_t cmprAlg; + int32_t szPage; + uint8_t *bufArr[8]; + // reader + SArray *aBlockIdx; + SMapData mDataBlk[1]; + SBlockData blockData[1]; + // writer + STsdbFD *fd; + SBrinBlock brinBlock[1]; + TBrinBlkArray brinBlkArray[1]; + SHeadFooter footer[1]; + } ctx[1] = {{ + .maxRow = tsdb->pVnode->config.tsdbCfg.maxRows, + .cmprAlg = tsdb->pVnode->config.tsdbCfg.compression, + .szPage = tsdb->pVnode->config.tsdbPageSize, + }}; + + // read SBlockIdx array + if ((ctx->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbReadBlockIdx(reader, ctx->aBlockIdx); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosArrayGetSize(ctx->aBlockIdx) > 0) { + // init/open file fd + STFile file = { + .type = TSDB_FTYPE_HEAD, + .did = pDFileSet->diskId, + .fid = fset->fid, + .cid = pDFileSet->pHeadF->commitID, + .size = pDFileSet->pHeadF->size, + }; + + code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_HEAD]); + TSDB_CHECK_CODE(code, lino, _exit); + + // open fd + char fname[TSDB_FILENAME_LEN]; + tsdbTFileName(tsdb, &file, fname); + + code = tsdbOpenFile(fname, ctx->szPage, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd); + TSDB_CHECK_CODE(code, lino, _exit); + + // convert + for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(ctx->aBlockIdx); ++iBlockIdx) { + SBlockIdx *pBlockIdx = taosArrayGet(ctx->aBlockIdx, iBlockIdx); + + code = tsdbReadDataBlk(reader, pBlockIdx, ctx->mDataBlk); + TSDB_CHECK_CODE(code, lino, _exit); + + for (int32_t iDataBlk = 0; iDataBlk < ctx->mDataBlk->nItem; ++iDataBlk) { + SDataBlk dataBlk[1]; + tMapDataGetItemByIdx(ctx->mDataBlk, iDataBlk, dataBlk, tGetDataBlk); + + SBrinRecord record = { + .suid = pBlockIdx->suid, + .uid = pBlockIdx->uid, + .firstKey = dataBlk->minKey.ts, + .firstKeyVer = dataBlk->minKey.version, + .lastKey = dataBlk->maxKey.ts, + .lastKeyVer = dataBlk->maxKey.version, + .minVer = dataBlk->minVer, + .maxVer = dataBlk->maxVer, + .blockOffset = dataBlk->aSubBlock->offset, + .smaOffset = dataBlk->smaInfo.offset, + .blockSize = dataBlk->aSubBlock->szBlock, + .blockKeySize = dataBlk->aSubBlock->szKey, + .smaSize = dataBlk->smaInfo.size, + .numRow = dataBlk->nRow, + .count = dataBlk->nRow, + }; + + if (dataBlk->hasDup) { + code = tsdbReadDataBlockEx(reader, dataBlk, ctx->blockData); + TSDB_CHECK_CODE(code, lino, _exit); + + record.count = 1; + for (int32_t i = 1; i < ctx->blockData->nRow; ++i) { + if (ctx->blockData->aTSKEY[i] != ctx->blockData->aTSKEY[i - 1]) { + record.count++; + } + } + } + + code = tBrinBlockPut(ctx->brinBlock, &record); + TSDB_CHECK_CODE(code, lino, _exit); + + if (BRIN_BLOCK_SIZE(ctx->brinBlock) >= ctx->maxRow) { + code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, + ctx->brinBlkArray, ctx->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } + + if (BRIN_BLOCK_SIZE(ctx->brinBlock) > 0) { + code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, + ctx->brinBlkArray, ctx->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbFileWriteBrinBlk(ctx->fd, ctx->brinBlkArray, ctx->footer->brinBlkPtr, + &fset->farr[TSDB_FTYPE_HEAD]->f->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFsyncFile(ctx->fd); + TSDB_CHECK_CODE(code, lino, _exit); + + tsdbCloseFile(&ctx->fd); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + TARRAY2_DESTROY(ctx->brinBlkArray, NULL); + tBrinBlockDestroy(ctx->brinBlock); + tBlockDataDestroy(ctx->blockData); + tMapDataClear(ctx->mDataBlk); + taosArrayDestroy(ctx->aBlockIdx); + for (int32_t i = 0; i < ARRAY_SIZE(ctx->bufArr); ++i) { + tFree(ctx->bufArr[i]); + } + return code; +} + +static int32_t tsdbUpgradeData(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *reader, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + + if (fset->farr[TSDB_FTYPE_HEAD] == NULL) { + return 0; + } + + STFile file = { + .type = TSDB_FTYPE_DATA, + .did = pDFileSet->diskId, + .fid = fset->fid, + .cid = pDFileSet->pDataF->commitID, + .size = pDFileSet->pDataF->size, + }; + + code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_DATA]); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbUpgradeSma(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *reader, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + + if (fset->farr[TSDB_FTYPE_HEAD] == NULL) { + return 0; + } + + STFile file = { + .type = TSDB_FTYPE_SMA, + .did = pDFileSet->diskId, + .fid = fset->fid, + .cid = pDFileSet->pSmaF->commitID, + .size = pDFileSet->pSmaF->size, + }; + + code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_SMA]); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *reader, STFileSet *fset, + int32_t iStt, SSttLvl *lvl) { + int32_t code = 0; + int32_t lino = 0; + + SArray *aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); + if (aSttBlk == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbReadSttBlk(reader, iStt, aSttBlk); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosArrayGetSize(aSttBlk) > 0) { + SSttFile *pSttF = pDFileSet->aSttF[iStt]; + STFileObj *fobj; + struct { + int32_t szPage; + // writer + STsdbFD *fd; + TSttBlkArray sttBlkArray[1]; + SSttFooter footer[1]; + } ctx[1] = {{ + .szPage = tsdb->pVnode->config.tsdbPageSize, + }}; + + STFile file = { + .type = TSDB_FTYPE_STT, + .did = pDFileSet->diskId, + .fid = fset->fid, + .cid = pSttF->commitID, + .size = pSttF->size, + }; + code = tsdbTFileObjInit(tsdb, &file, &fobj); + TSDB_CHECK_CODE(code, lino, _exit1); + + code = tsdbOpenFile(fobj->fname, ctx->szPage, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd); + TSDB_CHECK_CODE(code, lino, _exit1); + + for (int32_t iSttBlk = 0; iSttBlk < taosArrayGetSize(aSttBlk); iSttBlk++) { + code = TARRAY2_APPEND_PTR(ctx->sttBlkArray, (SSttBlk *)taosArrayGet(aSttBlk, iSttBlk)); + TSDB_CHECK_CODE(code, lino, _exit1); + } + + code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size); + TSDB_CHECK_CODE(code, lino, _exit1); + + code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size); + TSDB_CHECK_CODE(code, lino, _exit1); + + code = tsdbFsyncFile(ctx->fd); + TSDB_CHECK_CODE(code, lino, _exit1); + + tsdbCloseFile(&ctx->fd); + + code = TARRAY2_APPEND(lvl->fobjArr, fobj); + TSDB_CHECK_CODE(code, lino, _exit1); + + _exit1: + TARRAY2_DESTROY(ctx->sttBlkArray, NULL); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + taosArrayDestroy(aSttBlk); + return code; +} + +static int32_t tsdbUpgradeStt(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *reader, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + + if (pDFileSet->nSttF == 0) { + return 0; + } + + SSttLvl *lvl; + code = tsdbSttLvlInit(0, &lvl); + TSDB_CHECK_CODE(code, lino, _exit); + + for (int32_t iStt = 0; iStt < pDFileSet->nSttF; ++iStt) { + code = tsdbUpgradeSttFile(tsdb, pDFileSet, reader, fset, iStt, lvl); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (TARRAY2_SIZE(lvl->fobjArr) > 0) { + code = TARRAY2_APPEND(fset->lvlArr, lvl); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + tsdbSttLvlClear(&lvl); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbUpgradeFileSet(STsdb *tsdb, SDFileSet *pDFileSet, TFileSetArray *fileSetArray) { + int32_t code = 0; + int32_t lino = 0; + + SDataFReader *reader; + STFileSet *fset; + + code = tsdbTFileSetInit(pDFileSet->fid, &fset); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFReaderOpen(&reader, tsdb, pDFileSet); + TSDB_CHECK_CODE(code, lino, _exit); + + // .head + code = tsdbUpgradeHead(tsdb, pDFileSet, reader, fset); + TSDB_CHECK_CODE(code, lino, _exit); + + // .data + code = tsdbUpgradeData(tsdb, pDFileSet, reader, fset); + TSDB_CHECK_CODE(code, lino, _exit); + + // .sma + code = tsdbUpgradeSma(tsdb, pDFileSet, reader, fset); + TSDB_CHECK_CODE(code, lino, _exit); + + // .stt + if (pDFileSet->nSttF > 0) { + code = tsdbUpgradeStt(tsdb, pDFileSet, reader, fset); + TSDB_CHECK_CODE(code, lino, _exit); + } + + tsdbDataFReaderClose(&reader); + + code = TARRAY2_APPEND(fileSetArray, fset); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **fd, STFileObj **fobj, bool *toStt) { + int32_t code = 0; + int32_t lino = 0; + + if (TARRAY2_SIZE(fset->lvlArr) == 0) { // to .tomb file + *toStt = false; + + STFile file = { + .type = TSDB_FTYPE_TOMB, + .did = fset->farr[TSDB_FTYPE_HEAD]->f->did, + .fid = fset->fid, + .cid = 0, + .size = 0, + }; + + code = tsdbTFileObjInit(tsdb, &file, fobj); + TSDB_CHECK_CODE(code, lino, _exit); + + fset->farr[TSDB_FTYPE_TOMB] = *fobj; + } else { // to .stt file + *toStt = true; + SSttLvl *lvl = TARRAY2_GET(fset->lvlArr, 0); + + STFile file = { + .type = TSDB_FTYPE_STT, + .did = TARRAY2_GET(lvl->fobjArr, 0)->f->did, + .fid = fset->fid, + .cid = 0, + .size = 0, + }; + + code = tsdbTFileObjInit(tsdb, &file, fobj); + TSDB_CHECK_CODE(code, lino, _exit); + + code = TARRAY2_APPEND(lvl->fobjArr, fobj[0]); + TSDB_CHECK_CODE(code, lino, _exit); + } + + char fname[TSDB_FILENAME_LEN] = {0}; + code = tsdbOpenFile(fobj[0]->fname, tsdb->pVnode->config.tsdbPageSize, + TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd); + TSDB_CHECK_CODE(code, lino, _exit); + + uint8_t hdr[TSDB_FHDR_SIZE] = {0}; + code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE); + TSDB_CHECK_CODE(code, lino, _exit); + fobj[0]->f->size += TSDB_FHDR_SIZE; + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *aDelIdx, STFileSet *fset) { + int32_t code = 0; + int32_t lino = 0; + + struct { + // context + bool toStt; + int8_t cmprAlg; + int32_t maxRow; + int64_t minKey; + int64_t maxKey; + uint8_t *bufArr[8]; + // reader + SArray *aDelData; + // writer + STsdbFD *fd; + STFileObj *fobj; + STombBlock tombBlock[1]; + TTombBlkArray tombBlkArray[1]; + STombFooter tombFooter[1]; + SSttFooter sttFooter[1]; + } ctx[1] = {{ + .maxRow = tsdb->pVnode->config.tsdbCfg.maxRows, + .cmprAlg = tsdb->pVnode->config.tsdbCfg.compression, + }}; + + tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &ctx->minKey, &ctx->maxKey); + + if ((ctx->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { + SDelIdx *pDelIdx = (SDelIdx *)taosArrayGet(aDelIdx, iDelIdx); + + code = tsdbReadDelData(reader, pDelIdx, ctx->aDelData); + TSDB_CHECK_CODE(code, lino, _exit); + + for (int32_t iDelData = 0; iDelData < taosArrayGetSize(ctx->aDelData); iDelData++) { + SDelData *pDelData = (SDelData *)taosArrayGet(ctx->aDelData, iDelData); + + STombRecord record = { + .suid = pDelIdx->suid, + .uid = pDelIdx->uid, + .version = pDelData->version, + .skey = pDelData->sKey, + .ekey = pDelData->eKey, + }; + + code = tTombBlockPut(ctx->tombBlock, &record); + TSDB_CHECK_CODE(code, lino, _exit); + + if (TOMB_BLOCK_SIZE(ctx->tombBlock) > ctx->maxRow) { + if (ctx->fd == NULL) { + code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); + TSDB_CHECK_CODE(code, lino, _exit); + } + code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, + ctx->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } + + if (TOMB_BLOCK_SIZE(ctx->tombBlock) > 0) { + if (ctx->fd == NULL) { + code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); + TSDB_CHECK_CODE(code, lino, _exit); + } + code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, + ctx->bufArr); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (ctx->fd != NULL) { + if (ctx->toStt) { + code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbFsyncFile(ctx->fd); + TSDB_CHECK_CODE(code, lino, _exit); + + tsdbCloseFile(&ctx->fd); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + for (int32_t i = 0; i < ARRAY_SIZE(ctx->bufArr); i++) { + tFree(ctx->bufArr[i]); + } + TARRAY2_DESTROY(ctx->tombBlkArray, NULL); + tTombBlockDestroy(ctx->tombBlock); + taosArrayDestroy(ctx->aDelData); + return code; +} + +static int32_t tsdbUpgradeTombFile(STsdb *tsdb, SDelFile *pDelFile, TFileSetArray *fileSetArray) { + int32_t code = 0; + int32_t lino = 0; + + SDelFReader *reader = NULL; + SArray *aDelIdx = NULL; + + if ((aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDelFReaderOpen(&reader, pDelFile, tsdb); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadDelIdx(reader, aDelIdx); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosArrayGetSize(aDelIdx) > 0) { + STFileSet *fset; + TARRAY2_FOREACH(fileSetArray, fset) { + code = tsdbDumpTombDataToFSet(tsdb, reader, aDelIdx, fset); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + tsdbDelFReaderClose(&reader); + taosArrayDestroy(aDelIdx); + return code; +} + +static int32_t tsdbDoUpgradeFileSystem(STsdb *tsdb, TFileSetArray *fileSetArray) { + int32_t code = 0; + int32_t lino = 0; + + // upgrade each file set + for (int32_t i = 0; i < taosArrayGetSize(tsdb->fs.aDFileSet); i++) { + code = tsdbUpgradeFileSet(tsdb, taosArrayGet(tsdb->fs.aDFileSet, i), fileSetArray); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // upgrade tomb file + if (tsdb->fs.pDelFile != NULL) { + code = tsdbUpgradeTombFile(tsdb, tsdb->fs.pDelFile, fileSetArray); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + return code; +} + +static int32_t tsdbUpgradeFileSystem(STsdb *tsdb, int8_t rollback) { + int32_t code = 0; + int32_t lino = 0; + + TFileSetArray fileSetArray[1] = {0}; + + // open old file system + code = tsdbFSOpen(tsdb, rollback); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDoUpgradeFileSystem(tsdb, fileSetArray); + TSDB_CHECK_CODE(code, lino, _exit); + + // close file system + code = tsdbFSClose(tsdb); + TSDB_CHECK_CODE(code, lino, _exit); + + // save new file system + char fname[TSDB_FILENAME_LEN]; + current_fname(tsdb, fname, TSDB_FCURRENT); + code = save_fs(fileSetArray, fname); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + } + TARRAY2_DESTROY(fileSetArray, tsdbTFileSetClear); + return code; +} + +int32_t tsdbCheckAndUpgradeFileSystem(STsdb *tsdb, int8_t rollback) { + char fname[TSDB_FILENAME_LEN]; + + tsdbGetCurrentFName(tsdb, fname, NULL); + if (!taosCheckExistFile(fname)) return 0; + + int32_t code = tsdbUpgradeFileSystem(tsdb, rollback); + if (code) return code; + + taosRemoveFile(fname); + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbUpgrade.h b/source/dnode/vnode/src/tsdb/tsdbUpgrade.h new file mode 100644 index 0000000000000000000000000000000000000000..f9aac94e00026bd833f27df5227d7d351664ff63 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbUpgrade.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" +#include "tsdbDataFileRW.h" +#include "tsdbDef.h" +#include "tsdbFS2.h" +#include "tsdbUtil2.h" + +#ifndef _TSDB_UPGRADE_H_ +#define _TSDB_UPGRADE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t tsdbCheckAndUpgradeFileSystem(STsdb *tsdb, int8_t rollback); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_UPGRADE_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 556ec335266e1019470555a0655ce6d6977a7253..9340b24d747e05cc3840a627ae9724c41e3f203e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -528,25 +528,27 @@ void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minK *maxKey = *minKey + tsTickPerMin[precision] * minutes - 1; } -int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now) { +int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t nowSec) { int32_t aFid[3]; TSKEY key; if (pKeepCfg->precision == TSDB_TIME_PRECISION_MILLI) { - now = now * 1000; + nowSec = nowSec * 1000; } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_MICRO) { - now = now * 1000000l; + nowSec = nowSec * 1000000l; } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) { - now = now * 1000000000l; + nowSec = nowSec * 1000000000l; } else { ASSERT(0); } - key = now - pKeepCfg->keep0 * tsTickPerMin[pKeepCfg->precision]; + nowSec = nowSec - tsKeepTimeOffset * tsTickPerHour[pKeepCfg->precision]; + + key = nowSec - pKeepCfg->keep0 * tsTickPerMin[pKeepCfg->precision]; aFid[0] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision); - key = now - pKeepCfg->keep1 * tsTickPerMin[pKeepCfg->precision]; + key = nowSec - pKeepCfg->keep1 * tsTickPerMin[pKeepCfg->precision]; aFid[1] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision); - key = now - pKeepCfg->keep2 * tsTickPerMin[pKeepCfg->precision]; + key = nowSec - pKeepCfg->keep2 * tsTickPerMin[pKeepCfg->precision]; aFid[2] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision); if (fid >= aFid[0]) { @@ -640,7 +642,7 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); - SColVal * pColVal = &(SColVal){0}; + SColVal *pColVal = &(SColVal){0}; STColumn *pTColumn; int32_t iCol, jCol = 1; @@ -764,7 +766,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) } } -int32_t tsdbRowMergerInit(SRowMerger* pMerger, STSchema *pSchema) { +int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pSchema) { pMerger->pTSchema = pSchema; pMerger->pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal)); if (pMerger->pArray == NULL) { @@ -774,7 +776,7 @@ int32_t tsdbRowMergerInit(SRowMerger* pMerger, STSchema *pSchema) { } } -void tsdbRowMergerClear(SRowMerger* pMerger) { +void tsdbRowMergerClear(SRowMerger *pMerger) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); if (IS_VAR_DATA_TYPE(pTColVal->type)) { @@ -785,7 +787,7 @@ void tsdbRowMergerClear(SRowMerger* pMerger) { taosArrayClear(pMerger->pArray); } -void tsdbRowMergerCleanup(SRowMerger* pMerger) { +void tsdbRowMergerCleanup(SRowMerger *pMerger) { int32_t numOfCols = taosArrayGetSize(pMerger->pArray); for (int32_t iCol = 1; iCol < numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); @@ -1041,8 +1043,6 @@ int32_t tsdbBuildDeleteSkyline2(SArray *aDelData, int32_t sidx, int32_t eidx, SA // SBlockData ====================================================== int32_t tBlockDataCreate(SBlockData *pBlockData) { - int32_t code = 0; - pBlockData->suid = 0; pBlockData->uid = 0; pBlockData->nRow = 0; @@ -1051,7 +1051,7 @@ int32_t tBlockDataCreate(SBlockData *pBlockData) { pBlockData->aTSKEY = NULL; pBlockData->nColData = 0; pBlockData->aColData = NULL; - return code; + return 0; } void tBlockDataDestroy(SBlockData *pBlockData) { @@ -1107,8 +1107,8 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int32_t iColumn = 1; STColumn *pTColumn = &pTSchema->columns[iColumn]; for (int32_t iCid = 0; iCid < nCid; iCid++) { - - // aCid array (from taos client catalog) contains columns that does not exist in the pTSchema. the pTSchema is newer + // aCid array (from taos client catalog) contains columns that does not exist in the pTSchema. the pTSchema is + // newer if (pTColumn == NULL) { continue; } @@ -1239,7 +1239,7 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS _exit: return code; } -static int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; // version diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil2.c b/source/dnode/vnode/src/tsdb/tsdbUtil2.c new file mode 100644 index 0000000000000000000000000000000000000000..e938caa1184f34c366dec638b4a5dd9db9eaafb6 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbUtil2.c @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbUtil2.h" + +// SDelBlock ---------- +int32_t tTombBlockInit(STombBlock *tombBlock) { + for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { + TARRAY2_INIT(&tombBlock->dataArr[i]); + } + return 0; +} + +int32_t tTombBlockDestroy(STombBlock *tombBlock) { + for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { + TARRAY2_DESTROY(&tombBlock->dataArr[i], NULL); + } + return 0; +} + +int32_t tTombBlockClear(STombBlock *tombBlock) { + for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { + TARRAY2_CLEAR(&tombBlock->dataArr[i], NULL); + } + return 0; +} + +int32_t tTombBlockPut(STombBlock *tombBlock, const STombRecord *record) { + int32_t code; + for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { + code = TARRAY2_APPEND(&tombBlock->dataArr[i], record->dataArr[i]); + if (code) return code; + } + return 0; +} + +int32_t tTombBlockGet(STombBlock *tombBlock, int32_t idx, STombRecord *record) { + if (idx >= TOMB_BLOCK_SIZE(tombBlock)) return TSDB_CODE_OUT_OF_RANGE; + for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { + record->dataArr[i] = TARRAY2_GET(&tombBlock->dataArr[i], idx); + } + return 0; +} + +int32_t tTombRecordCompare(const STombRecord *r1, const STombRecord *r2) { + if (r1->suid < r2->suid) return -1; + if (r1->suid > r2->suid) return 1; + if (r1->uid < r2->uid) return -1; + if (r1->uid > r2->uid) return 1; + if (r1->version < r2->version) return -1; + if (r1->version > r2->version) return 1; + return 0; +} + +// STbStatisBlock ---------- +int32_t tStatisBlockInit(STbStatisBlock *statisBlock) { + for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; ++i) { + TARRAY2_INIT(&statisBlock->dataArr[i]); + } + return 0; +} + +int32_t tStatisBlockDestroy(STbStatisBlock *statisBlock) { + for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; ++i) { + TARRAY2_DESTROY(&statisBlock->dataArr[i], NULL); + } + return 0; +} + +int32_t tStatisBlockClear(STbStatisBlock *statisBlock) { + for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; ++i) { + TARRAY2_CLEAR(&statisBlock->dataArr[i], NULL); + } + return 0; +} + +int32_t tStatisBlockPut(STbStatisBlock *statisBlock, const STbStatisRecord *record) { + int32_t code; + for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; ++i) { + code = TARRAY2_APPEND(&statisBlock->dataArr[i], record->dataArr[i]); + if (code) return code; + } + return 0; +} + +int32_t tStatisBlockGet(STbStatisBlock *statisBlock, int32_t idx, STbStatisRecord *record) { + if (idx >= STATIS_BLOCK_SIZE(statisBlock)) return TSDB_CODE_OUT_OF_RANGE; + for (int32_t i = 0; i < STATIS_RECORD_NUM_ELEM; ++i) { + record->dataArr[i] = TARRAY2_GET(&statisBlock->dataArr[i], idx); + } + return 0; +} + +// SBrinRecord ---------- +int32_t tBrinBlockInit(SBrinBlock *brinBlock) { + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); ++i) { + TARRAY2_INIT(&brinBlock->dataArr1[i]); + } + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr2); ++i) { + TARRAY2_INIT(&brinBlock->dataArr2[i]); + } + return 0; +} + +int32_t tBrinBlockDestroy(SBrinBlock *brinBlock) { + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); ++i) { + TARRAY2_DESTROY(&brinBlock->dataArr1[i], NULL); + } + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr2); ++i) { + TARRAY2_DESTROY(&brinBlock->dataArr2[i], NULL); + } + return 0; +} + +int32_t tBrinBlockClear(SBrinBlock *brinBlock) { + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); ++i) { + TARRAY2_CLEAR(&brinBlock->dataArr1[i], NULL); + } + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr2); ++i) { + TARRAY2_CLEAR(&brinBlock->dataArr2[i], NULL); + } + return 0; +} + +int32_t tBrinBlockPut(SBrinBlock *brinBlock, const SBrinRecord *record) { + int32_t code; + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); ++i) { + code = TARRAY2_APPEND(&brinBlock->dataArr1[i], record->dataArr1[i]); + if (code) return code; + } + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr2); ++i) { + code = TARRAY2_APPEND(&brinBlock->dataArr2[i], record->dataArr2[i]); + if (code) return code; + } + return 0; +} + +int32_t tBrinBlockGet(SBrinBlock *brinBlock, int32_t idx, SBrinRecord *record) { + if (idx >= BRIN_BLOCK_SIZE(brinBlock)) return TSDB_CODE_OUT_OF_RANGE; + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); ++i) { + record->dataArr1[i] = TARRAY2_GET(&brinBlock->dataArr1[i], idx); + } + for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr2); ++i) { + record->dataArr2[i] = TARRAY2_GET(&brinBlock->dataArr2[i], idx); + } + return 0; +} + +// other apis ---------- +int32_t tsdbUpdateSkmTb(STsdb *pTsdb, const TABLEID *tbid, SSkmInfo *pSkmTb) { + if (tbid->suid) { + if (pSkmTb->suid == tbid->suid) { + pSkmTb->uid = tbid->uid; + return 0; + } + } else if (pSkmTb->uid == tbid->uid) { + return 0; + } + + pSkmTb->suid = tbid->suid; + pSkmTb->uid = tbid->uid; + tDestroyTSchema(pSkmTb->pTSchema); + return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, tbid->suid, tbid->uid, -1, &pSkmTb->pTSchema); +} + +int32_t tsdbUpdateSkmRow(STsdb *pTsdb, const TABLEID *tbid, int32_t sver, SSkmInfo *pSkmRow) { + if (pSkmRow->pTSchema && pSkmRow->suid == tbid->suid) { + if (pSkmRow->suid) { + if (sver == pSkmRow->pTSchema->version) return 0; + } else if (pSkmRow->uid == tbid->uid && pSkmRow->pTSchema->version == sver) { + return 0; + } + } + + pSkmRow->suid = tbid->suid; + pSkmRow->uid = tbid->uid; + tDestroyTSchema(pSkmRow->pTSchema); + return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, tbid->suid, tbid->uid, sver, &pSkmRow->pTSchema); +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil2.h b/source/dnode/vnode/src/tsdb/tsdbUtil2.h new file mode 100644 index 0000000000000000000000000000000000000000..fa0636834155df5d8f8ddfbb3988d6a5a7d86bbf --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbUtil2.h @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TSDB_UTIL_H +#define _TSDB_UTIL_H + +#include "tsdbDef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// STombRecord ---------- +#define TOMB_RECORD_ELEM_NUM 5 +typedef union { + int64_t dataArr[TOMB_RECORD_ELEM_NUM]; + struct { + int64_t suid; + int64_t uid; + int64_t version; + int64_t skey; + int64_t ekey; + }; +} STombRecord; + +typedef union { + TARRAY2(int64_t) dataArr[TOMB_RECORD_ELEM_NUM]; + struct { + TARRAY2(int64_t) suid[1]; + TARRAY2(int64_t) uid[1]; + TARRAY2(int64_t) version[1]; + TARRAY2(int64_t) skey[1]; + TARRAY2(int64_t) ekey[1]; + }; +} STombBlock; + +typedef struct { + SFDataPtr dp[1]; + TABLEID minTbid; + TABLEID maxTbid; + int64_t minVer; + int64_t maxVer; + int32_t numRec; + int32_t size[TOMB_RECORD_ELEM_NUM]; + int8_t cmprAlg; + int8_t rsvd[7]; +} STombBlk; + +typedef TARRAY2(STombBlk) TTombBlkArray; + +#define TOMB_BLOCK_SIZE(db) TARRAY2_SIZE((db)->suid) + +int32_t tTombBlockInit(STombBlock *tombBlock); +int32_t tTombBlockDestroy(STombBlock *tombBlock); +int32_t tTombBlockClear(STombBlock *tombBlock); +int32_t tTombBlockPut(STombBlock *tombBlock, const STombRecord *record); +int32_t tTombBlockGet(STombBlock *tombBlock, int32_t idx, STombRecord *record); +int32_t tTombRecordCompare(const STombRecord *record1, const STombRecord *record2); + +// STbStatisRecord ---------- +#define STATIS_RECORD_NUM_ELEM 5 +typedef union { + int64_t dataArr[STATIS_RECORD_NUM_ELEM]; + struct { + int64_t suid; + int64_t uid; + int64_t firstKey; + int64_t lastKey; + int64_t count; + }; +} STbStatisRecord; + +typedef union { + TARRAY2(int64_t) dataArr[STATIS_RECORD_NUM_ELEM]; + struct { + TARRAY2(int64_t) suid[1]; + TARRAY2(int64_t) uid[1]; + TARRAY2(int64_t) firstKey[1]; + TARRAY2(int64_t) lastKey[1]; + TARRAY2(int64_t) count[1]; + }; +} STbStatisBlock; + +typedef struct { + SFDataPtr dp[1]; + TABLEID minTbid; + TABLEID maxTbid; + int32_t numRec; + int32_t size[STATIS_RECORD_NUM_ELEM]; + int8_t cmprAlg; + int8_t rsvd[7]; +} SStatisBlk; + +#define STATIS_BLOCK_SIZE(db) TARRAY2_SIZE((db)->suid) + +int32_t tStatisBlockInit(STbStatisBlock *statisBlock); +int32_t tStatisBlockDestroy(STbStatisBlock *statisBlock); +int32_t tStatisBlockClear(STbStatisBlock *statisBlock); +int32_t tStatisBlockPut(STbStatisBlock *statisBlock, const STbStatisRecord *record); +int32_t tStatisBlockGet(STbStatisBlock *statisBlock, int32_t idx, STbStatisRecord *record); + +// SBrinRecord ---------- +typedef union { + struct { + int64_t dataArr1[10]; + int32_t dataArr2[5]; + }; + struct { + int64_t suid; + int64_t uid; + int64_t firstKey; + int64_t firstKeyVer; + int64_t lastKey; + int64_t lastKeyVer; + int64_t minVer; + int64_t maxVer; + int64_t blockOffset; + int64_t smaOffset; + int32_t blockSize; + int32_t blockKeySize; + int32_t smaSize; + int32_t numRow; + int32_t count; + }; +} SBrinRecord; + +typedef union { + struct { + TARRAY2(int64_t) dataArr1[10]; + TARRAY2(int32_t) dataArr2[5]; + }; + struct { + TARRAY2(int64_t) suid[1]; + TARRAY2(int64_t) uid[1]; + TARRAY2(int64_t) firstKey[1]; + TARRAY2(int64_t) firstKeyVer[1]; + TARRAY2(int64_t) lastKey[1]; + TARRAY2(int64_t) lastKeyVer[1]; + TARRAY2(int64_t) minVer[1]; + TARRAY2(int64_t) maxVer[1]; + TARRAY2(int64_t) blockOffset[1]; + TARRAY2(int64_t) smaOffset[1]; + TARRAY2(int32_t) blockSize[1]; + TARRAY2(int32_t) blockKeySize[1]; + TARRAY2(int32_t) smaSize[1]; + TARRAY2(int32_t) numRow[1]; + TARRAY2(int32_t) count[1]; + }; +} SBrinBlock; + +typedef struct { + SFDataPtr dp[1]; + TABLEID minTbid; + TABLEID maxTbid; + int64_t minVer; + int64_t maxVer; + int32_t numRec; + int32_t size[15]; + int8_t cmprAlg; + int8_t rsvd[7]; +} SBrinBlk; + +typedef TARRAY2(SBrinBlk) TBrinBlkArray; + +#define BRIN_BLOCK_SIZE(db) TARRAY2_SIZE((db)->suid) + +int32_t tBrinBlockInit(SBrinBlock *brinBlock); +int32_t tBrinBlockDestroy(SBrinBlock *brinBlock); +int32_t tBrinBlockClear(SBrinBlock *brinBlock); +int32_t tBrinBlockPut(SBrinBlock *brinBlock, const SBrinRecord *record); +int32_t tBrinBlockGet(SBrinBlock *brinBlock, int32_t idx, SBrinRecord *record); + +// other apis +int32_t tsdbUpdateSkmTb(STsdb *pTsdb, const TABLEID *tbid, SSkmInfo *pSkmTb); +int32_t tsdbUpdateSkmRow(STsdb *pTsdb, const TABLEID *tbid, int32_t sver, SSkmInfo *pSkmRow); + +#ifdef __cplusplus +} +#endif + +#endif /*_TSDB_UTIL_H*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index efe82e1783c7013645dd05538c439d9db6f3d114..2e161d728ffbb9d11ed794bebdff8affcdc6f429 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -49,7 +49,7 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1, .hashBegin = 0, .hashEnd = 0, .hashMethod = 0, - .sttTrigger = TSDB_DEFAULT_STT_FILE, + .sttTrigger = TSDB_DEFAULT_SST_TRIGGER, .tsdbPageSize = TSDB_DEFAULT_PAGE_SIZE}; int vnodeCheckCfg(const SVnodeCfg *pCfg) { @@ -57,7 +57,7 @@ int vnodeCheckCfg(const SVnodeCfg *pCfg) { return 0; } -const char* vnodeRoleToStr(ESyncRole role) { +const char *vnodeRoleToStr(ESyncRole role) { switch (role) { case TAOS_SYNC_ROLE_VOTER: return "true"; @@ -68,11 +68,11 @@ const char* vnodeRoleToStr(ESyncRole role) { } } -const ESyncRole vnodeStrToRole(char* str) { - if(strcmp(str, "true") == 0){ +const ESyncRole vnodeStrToRole(char *str) { + if (strcmp(str, "true") == 0) { return TAOS_SYNC_ROLE_VOTER; } - if(strcmp(str, "false") == 0){ + if (strcmp(str, "false") == 0) { return TAOS_SYNC_ROLE_LEARNER; } @@ -295,10 +295,9 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { char role[10] = {0}; code = tjsonGetStringValue(info, "isReplica", role); if (code < 0) return -1; - if(strlen(role) != 0){ + if (strlen(role) != 0) { pNode->nodeRole = vnodeStrToRole(role); - } - else{ + } else { pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; } vDebug("vgId:%d, decode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 69f2273bb07ce7de9f5f4a18ca92efbac3a7c989..33c6f9d533099a57ebc0ea29e744f832c3d24734 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -16,6 +16,11 @@ #include "vnd.h" #include "vnodeInt.h" +extern int32_t tsdbPreCommit(STsdb *pTsdb); +extern int32_t tsdbCommitBegin(STsdb *pTsdb, SCommitInfo *pInfo); +extern int32_t tsdbCommitCommit(STsdb *pTsdb); +extern int32_t tsdbCommitAbort(STsdb *pTsdb); + #define VND_INFO_FNAME_TMP "vnode_tmp.json" static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData); @@ -290,11 +295,7 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { pInfo->txn = metaGetTxn(pVnode->pMeta); // save info - if (pVnode->pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); - } + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN); vDebug("vgId:%d, save config while prepare commit", TD_VID(pVnode)); if (vnodeSaveInfo(dir, &pInfo->info) < 0) { @@ -302,7 +303,7 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { TSDB_CHECK_CODE(code, lino, _exit); } - tsdbPrepareCommit(pVnode->pTsdb); + tsdbPreCommit(pVnode->pTsdb); metaPrepareAsyncCommit(pVnode->pMeta); @@ -432,16 +433,11 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { return -1; } - if (pVnode->pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); - } + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN); syncBeginSnapshot(pVnode->sync, pInfo->info.state.committed); - // commit each sub-system - code = tsdbCommit(pVnode->pTsdb, pInfo); + code = tsdbCommitBegin(pVnode->pTsdb, pInfo); TSDB_CHECK_CODE(code, lino, _exit); if (!TSDB_CACHE_NO(pVnode->config)) { @@ -465,7 +461,7 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { TSDB_CHECK_CODE(code, lino, _exit); } - code = tsdbFinishCommit(pVnode->pTsdb); + code = tsdbCommitCommit(pVnode->pTsdb); TSDB_CHECK_CODE(code, lino, _exit); if (VND_IS_RSMA(pVnode)) { @@ -498,16 +494,22 @@ _exit: bool vnodeShouldRollback(SVnode *pVnode) { char tFName[TSDB_FILENAME_LEN] = {0}; - snprintf(tFName, TSDB_FILENAME_LEN, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, - VND_INFO_FNAME_TMP); + int32_t offset = 0; + + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, tFName, TSDB_FILENAME_LEN); + offset = strlen(tFName); + snprintf(tFName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VND_INFO_FNAME_TMP); return taosCheckExistFile(tFName); } void vnodeRollback(SVnode *pVnode) { char tFName[TSDB_FILENAME_LEN] = {0}; - snprintf(tFName, TSDB_FILENAME_LEN, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, - VND_INFO_FNAME_TMP); + int32_t offset = 0; + + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, tFName, TSDB_FILENAME_LEN); + offset = strlen(tFName); + snprintf(tFName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VND_INFO_FNAME_TMP); (void)taosRemoveFile(tFName); } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 447cab4e9ed8fe0597f10526ca9f85940dba8599..3dfaa28c092a8f8e36d942ed581fa85c7b22deb5 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -42,24 +42,24 @@ void initStorageAPI(SStorageAPI* pAPI) { void initTsdbReaderAPI(TsdReader* pReader) { pReader->tsdReaderOpen = (int32_t(*)(void*, SQueryTableDataCond*, void*, int32_t, SSDataBlock*, void**, const char*, - bool, SHashObj**))tsdbReaderOpen; - pReader->tsdReaderClose = tsdbReaderClose; + bool, SHashObj**))tsdbReaderOpen2; + pReader->tsdReaderClose = tsdbReaderClose2; - pReader->tsdNextDataBlock = tsdbNextDataBlock; + pReader->tsdNextDataBlock = tsdbNextDataBlock2; - pReader->tsdReaderRetrieveDataBlock = tsdbRetrieveDataBlock; - pReader->tsdReaderReleaseDataBlock = tsdbReleaseDataBlock; + pReader->tsdReaderRetrieveDataBlock = tsdbRetrieveDataBlock2; + pReader->tsdReaderReleaseDataBlock = tsdbReleaseDataBlock2; - pReader->tsdReaderRetrieveBlockSMAInfo = tsdbRetrieveDatablockSMA; + pReader->tsdReaderRetrieveBlockSMAInfo = tsdbRetrieveDatablockSMA2; pReader->tsdReaderNotifyClosing = tsdbReaderSetCloseFlag; - pReader->tsdReaderResetStatus = tsdbReaderReset; + pReader->tsdReaderResetStatus = tsdbReaderReset2; - pReader->tsdReaderGetDataBlockDistInfo = tsdbGetFileBlocksDistInfo; - pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable; // todo this function should be moved away + pReader->tsdReaderGetDataBlockDistInfo = tsdbGetFileBlocksDistInfo2; + pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable2; // todo this function should be moved away - pReader->tsdSetQueryTableList = tsdbSetTableList; - pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId; + pReader->tsdSetQueryTableList = tsdbSetTableList2; + pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId2; } void initMetadataAPI(SStoreMeta* pMeta) { @@ -203,6 +203,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateCommit = streamStateCommit; pStore->streamStateDestroy = streamStateDestroy; pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; + pStore->streamStateReloadInfo = streamStateReloadInfo; } void initMetaReaderAPI(SStoreMetaReader* pMetaReader) { diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 782ffd788d27d338743447fb67954a9dd7812ba7..74a8d14a86c6340c3acedec831524b8ae765b91e 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -23,26 +23,24 @@ struct SVnodeTask { void* arg; }; -struct SVnodeGlobal { - int8_t init; - int8_t stop; +typedef struct { int nthreads; TdThread* threads; TdThreadMutex mutex; TdThreadCond hasTask; SVnodeTask queue; +} SVnodeThreadPool; + +struct SVnodeGlobal { + int8_t init; + int8_t stop; + SVnodeThreadPool tp[2]; }; struct SVnodeGlobal vnodeGlobal; static void* loop(void* arg); -static tsem_t canCommit = {0}; - -static void vnodeInitCommit() { tsem_init(&canCommit, 0, 4); }; -void vnode_wait_commit() { tsem_wait(&canCommit); } -void vnode_done_commit() { tsem_wait(&canCommit); } - int vnodeInit(int nthreads) { int8_t init; int ret; @@ -51,28 +49,30 @@ int vnodeInit(int nthreads) { if (init) { return 0; } + vnodeGlobal.stop = 0; - taosThreadMutexInit(&vnodeGlobal.mutex, NULL); - taosThreadCondInit(&vnodeGlobal.hasTask, NULL); + for (int32_t i = 0; i < ARRAY_SIZE(vnodeGlobal.tp); i++) { + taosThreadMutexInit(&vnodeGlobal.tp[i].mutex, NULL); + taosThreadCondInit(&vnodeGlobal.tp[i].hasTask, NULL); - taosThreadMutexLock(&vnodeGlobal.mutex); + taosThreadMutexLock(&vnodeGlobal.tp[i].mutex); - vnodeGlobal.stop = 0; - vnodeGlobal.queue.next = &vnodeGlobal.queue; - vnodeGlobal.queue.prev = &vnodeGlobal.queue; + vnodeGlobal.tp[i].queue.next = &vnodeGlobal.tp[i].queue; + vnodeGlobal.tp[i].queue.prev = &vnodeGlobal.tp[i].queue; - taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + taosThreadMutexUnlock(&(vnodeGlobal.tp[i].mutex)); - vnodeGlobal.nthreads = nthreads; - vnodeGlobal.threads = taosMemoryCalloc(nthreads, sizeof(TdThread)); - if (vnodeGlobal.threads == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - vError("failed to init vnode module since:%s", tstrerror(terrno)); - return -1; - } + vnodeGlobal.tp[i].nthreads = nthreads; + vnodeGlobal.tp[i].threads = taosMemoryCalloc(nthreads, sizeof(TdThread)); + if (vnodeGlobal.tp[i].threads == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + vError("failed to init vnode module since:%s", tstrerror(terrno)); + return -1; + } - for (int i = 0; i < nthreads; i++) { - taosThreadCreate(&(vnodeGlobal.threads[i]), NULL, loop, NULL); + for (int j = 0; j < nthreads; j++) { + taosThreadCreate(&(vnodeGlobal.tp[i].threads[j]), NULL, loop, &vnodeGlobal.tp[i]); + } } if (walInit() < 0) { @@ -92,27 +92,29 @@ void vnodeCleanup() { if (init == 0) return; // set stop - taosThreadMutexLock(&(vnodeGlobal.mutex)); vnodeGlobal.stop = 1; - taosThreadCondBroadcast(&(vnodeGlobal.hasTask)); - taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + for (int32_t i = 0; i < ARRAY_SIZE(vnodeGlobal.tp); i++) { + taosThreadMutexLock(&(vnodeGlobal.tp[i].mutex)); + taosThreadCondBroadcast(&(vnodeGlobal.tp[i].hasTask)); + taosThreadMutexUnlock(&(vnodeGlobal.tp[i].mutex)); + + // wait for threads + for (int j = 0; j < vnodeGlobal.tp[i].nthreads; j++) { + taosThreadJoin(vnodeGlobal.tp[i].threads[j], NULL); + } - // wait for threads - for (int i = 0; i < vnodeGlobal.nthreads; i++) { - taosThreadJoin(vnodeGlobal.threads[i], NULL); + // clear source + taosMemoryFreeClear(vnodeGlobal.tp[i].threads); + taosThreadCondDestroy(&(vnodeGlobal.tp[i].hasTask)); + taosThreadMutexDestroy(&(vnodeGlobal.tp[i].mutex)); } - // clear source - taosMemoryFreeClear(vnodeGlobal.threads); - taosThreadCondDestroy(&(vnodeGlobal.hasTask)); - taosThreadMutexDestroy(&(vnodeGlobal.mutex)); - walCleanUp(); tqCleanUp(); smaCleanUp(); } -int vnodeScheduleTask(int (*execute)(void*), void* arg) { +int vnodeScheduleTaskEx(int tpid, int (*execute)(void*), void* arg) { SVnodeTask* pTask; ASSERT(!vnodeGlobal.stop); @@ -126,35 +128,42 @@ int vnodeScheduleTask(int (*execute)(void*), void* arg) { pTask->execute = execute; pTask->arg = arg; - taosThreadMutexLock(&(vnodeGlobal.mutex)); - pTask->next = &vnodeGlobal.queue; - pTask->prev = vnodeGlobal.queue.prev; - vnodeGlobal.queue.prev->next = pTask; - vnodeGlobal.queue.prev = pTask; - taosThreadCondSignal(&(vnodeGlobal.hasTask)); - taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + taosThreadMutexLock(&(vnodeGlobal.tp[tpid].mutex)); + pTask->next = &vnodeGlobal.tp[tpid].queue; + pTask->prev = vnodeGlobal.tp[tpid].queue.prev; + vnodeGlobal.tp[tpid].queue.prev->next = pTask; + vnodeGlobal.tp[tpid].queue.prev = pTask; + taosThreadCondSignal(&(vnodeGlobal.tp[tpid].hasTask)); + taosThreadMutexUnlock(&(vnodeGlobal.tp[tpid].mutex)); return 0; } +int vnodeScheduleTask(int (*execute)(void*), void* arg) { return vnodeScheduleTaskEx(0, execute, arg); } + /* ------------------------ STATIC METHODS ------------------------ */ static void* loop(void* arg) { - SVnodeTask* pTask; - int ret; - - setThreadName("vnode-commit"); + SVnodeThreadPool* tp = (SVnodeThreadPool*)arg; + SVnodeTask* pTask; + int ret; + + if (tp == &vnodeGlobal.tp[0]) { + setThreadName("vnode-commit"); + } else if (tp == &vnodeGlobal.tp[1]) { + setThreadName("vnode-merge"); + } for (;;) { - taosThreadMutexLock(&(vnodeGlobal.mutex)); + taosThreadMutexLock(&(tp->mutex)); for (;;) { - pTask = vnodeGlobal.queue.next; - if (pTask == &vnodeGlobal.queue) { + pTask = tp->queue.next; + if (pTask == &tp->queue) { // no task if (vnodeGlobal.stop) { - taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + taosThreadMutexUnlock(&(tp->mutex)); return NULL; } else { - taosThreadCondWait(&(vnodeGlobal.hasTask), &(vnodeGlobal.mutex)); + taosThreadCondWait(&(tp->hasTask), &(tp->mutex)); } } else { // has task @@ -164,7 +173,7 @@ static void* loop(void* arg) { } } - taosThreadMutexUnlock(&(vnodeGlobal.mutex)); + taosThreadMutexUnlock(&(tp->mutex)); pTask->execute(pTask->arg); taosMemoryFree(pTask); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index c794c7ebd661b598581152587aec8617f878b22b..3b07ef9ea9e8f476e0a428493fdb461734d84871 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -15,7 +15,27 @@ #include "vnd.h" -int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { +int32_t vnodeGetPrimaryDir(const char *relPath, int32_t diskPrimary, STfs *pTfs, char *buf, size_t bufLen) { + if (pTfs) { + SDiskID diskId = {0}; + diskId.id = diskPrimary; + snprintf(buf, bufLen - 1, "%s%s%s", tfsGetDiskPath(pTfs, diskId), TD_DIRSEP, relPath); + } else { + snprintf(buf, bufLen - 1, "%s", relPath); + } + buf[bufLen - 1] = '\0'; + return 0; +} + +static int32_t vnodeMkDir(STfs *pTfs, const char *path) { + if (pTfs) { + return tfsMkdirRecur(pTfs, path); + } else { + return taosMkDir(path); + } +} + +int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs *pTfs) { SVnodeInfo info = {0}; char dir[TSDB_FILENAME_LEN] = {0}; @@ -26,18 +46,11 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { } // create vnode env - if (pTfs) { - if (tfsMkdirAt(pTfs, path, (SDiskID){0}) < 0) { - vError("vgId:%d, failed to create vnode since:%s", pCfg->vgId, tstrerror(terrno)); - return -1; - } - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path); - } else { - if (taosMkDir(path)) { - return TAOS_SYSTEM_ERROR(errno); - } - snprintf(dir, TSDB_FILENAME_LEN, "%s", path); + if (vnodeMkDir(pTfs, path)) { + vError("vgId:%d, failed to prepare vnode dir since %s, path: %s", pCfg->vgId, strerror(errno), path); + return TAOS_SYSTEM_ERROR(errno); } + vnodeGetPrimaryDir(path, diskPrimary, pTfs, dir, TSDB_FILENAME_LEN); if (pCfg) { info.config = *pCfg; @@ -58,16 +71,12 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { return 0; } -int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs) { +int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t diskPrimary, STfs *pTfs) { SVnodeInfo info = {0}; char dir[TSDB_FILENAME_LEN] = {0}; int32_t ret = 0; - if (pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", path); - } + vnodeGetPrimaryDir(path, diskPrimary, pTfs, dir, TSDB_FILENAME_LEN); ret = vnodeLoadInfo(dir, &info); if (ret < 0) { @@ -135,7 +144,8 @@ static int32_t vnodeVgroupIdLen(int32_t vgId) { return strlen(tmp); } -int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, STfs *pTfs) { +int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, + int32_t diskPrimary, STfs *pTfs) { int32_t ret = 0; char oldRname[TSDB_FILENAME_LEN] = {0}; @@ -144,6 +154,7 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr char tsdbFilePrefix[TSDB_FILENAME_LEN] = {0}; snprintf(tsdbPath, TSDB_FILENAME_LEN, "%s%stsdb", srcPath, TD_DIRSEP); snprintf(tsdbFilePrefix, TSDB_FILENAME_LEN, "tsdb%sv", TD_DIRSEP); + int32_t prefixLen = strlen(tsdbFilePrefix); STfsDir *tsdbDir = tfsOpendir(pTfs, tsdbPath); if (tsdbDir == NULL) return 0; @@ -157,15 +168,15 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr char *tsdbFilePrefixPos = strstr(oldRname, tsdbFilePrefix); if (tsdbFilePrefixPos == NULL) continue; - int32_t tsdbFileVgId = atoi(tsdbFilePrefixPos + 6); + int32_t tsdbFileVgId = atoi(tsdbFilePrefixPos + prefixLen); if (tsdbFileVgId == srcVgId) { - char *tsdbFileSurfixPos = tsdbFilePrefixPos + 6 + vnodeVgroupIdLen(srcVgId); + char *tsdbFileSurfixPos = tsdbFilePrefixPos + prefixLen + vnodeVgroupIdLen(srcVgId); - tsdbFilePrefixPos[6] = 0; + tsdbFilePrefixPos[prefixLen] = 0; snprintf(newRname, TSDB_FILENAME_LEN, "%s%d%s", oldRname, dstVgId, tsdbFileSurfixPos); vInfo("vgId:%d, rename file from %s to %s", dstVgId, tsdbFile->rname, newRname); - ret = tfsRename(pTfs, tsdbFile->rname, newRname); + ret = tfsRename(pTfs, diskPrimary, tsdbFile->rname, newRname); if (ret != 0) { vError("vgId:%d, failed to rename file from %s to %s since %s", dstVgId, tsdbFile->rname, newRname, terrstr()); tfsClosedir(tsdbDir); @@ -177,28 +188,20 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr tfsClosedir(tsdbDir); vInfo("vgId:%d, rename dir from %s to %s", dstVgId, srcPath, dstPath); - ret = tfsRename(pTfs, srcPath, dstPath); + ret = tfsRename(pTfs, diskPrimary, srcPath, dstPath); if (ret != 0) { vError("vgId:%d, failed to rename dir from %s to %s since %s", dstVgId, srcPath, dstPath, terrstr()); } return ret; } -int32_t vnodeGetAbsDir(const char *relPath, STfs *pTfs, char *buf, size_t bufLen) { - if (pTfs) { - snprintf(buf, bufLen, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, relPath); - } else { - snprintf(buf, bufLen, "%s", relPath); - } - return 0; -} - -int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq, STfs *pTfs) { +int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq, + int32_t diskPrimary, STfs *pTfs) { SVnodeInfo info = {0}; char dir[TSDB_FILENAME_LEN] = {0}; int32_t ret = 0; - vnodeGetAbsDir(srcPath, pTfs, dir, TSDB_FILENAME_LEN); + vnodeGetPrimaryDir(srcPath, diskPrimary, pTfs, dir, TSDB_FILENAME_LEN); ret = vnodeLoadInfo(dir, &info); if (ret < 0) { @@ -242,7 +245,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod } vInfo("vgId:%d, rename %s to %s", pReq->dstVgId, srcPath, dstPath); - ret = vnodeRenameVgroupId(srcPath, dstPath, pReq->srcVgId, pReq->dstVgId, pTfs); + ret = vnodeRenameVgroupId(srcPath, dstPath, pReq->srcVgId, pReq->dstVgId, diskPrimary, pTfs); if (ret < 0) { vError("vgId:%d, failed to rename vnode from %s to %s since %s", pReq->dstVgId, srcPath, dstPath, tstrerror(terrno)); @@ -253,11 +256,12 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod return 0; } -int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, STfs *pTfs) { +int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, + int32_t diskPrimary, STfs *pTfs) { SVnodeInfo info = {0}; char dir[TSDB_FILENAME_LEN] = {0}; - vnodeGetAbsDir(dstPath, pTfs, dir, TSDB_FILENAME_LEN); + vnodeGetPrimaryDir(dstPath, diskPrimary, pTfs, dir, TSDB_FILENAME_LEN); if (vnodeLoadInfo(dir, &info) == 0) { if (info.config.vgId != dstVgId) { vError("vgId:%d, unexpected vnode config.vgId:%d", dstVgId, info.config.vgId); @@ -266,7 +270,7 @@ int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t s return dstVgId; } - vnodeGetAbsDir(srcPath, pTfs, dir, TSDB_FILENAME_LEN); + vnodeGetPrimaryDir(srcPath, diskPrimary, pTfs, dir, TSDB_FILENAME_LEN); if (vnodeLoadInfo(dir, &info) < 0) { vError("vgId:%d, failed to read vnode config from %s since %s", srcVgId, srcPath, tstrerror(terrno)); return -1; @@ -281,7 +285,7 @@ int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t s } vInfo("vgId:%d, rename %s to %s", dstVgId, srcPath, dstPath); - if (vnodeRenameVgroupId(srcPath, dstPath, srcVgId, dstVgId, pTfs) < 0) { + if (vnodeRenameVgroupId(srcPath, dstPath, srcVgId, dstVgId, diskPrimary, pTfs) < 0) { vError("vgId:%d, failed to rename vnode from %s to %s since %s", dstVgId, srcPath, dstPath, tstrerror(terrno)); return -1; } @@ -294,18 +298,31 @@ void vnodeDestroy(const char *path, STfs *pTfs) { tfsRmdir(pTfs, path); } -SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { +static int32_t vnodeCheckDisk(int32_t diskPrimary, STfs *pTfs) { + int32_t ndisk = 1; + if (pTfs) { + ndisk = tfsGetDisksAtLevel(pTfs, 0); + } + if (diskPrimary < 0 || diskPrimary >= ndisk) { + vError("disk:%d is unavailable from the %d disks mounted at level 0", diskPrimary, ndisk); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + return 0; +} + +SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb) { SVnode *pVnode = NULL; SVnodeInfo info = {0}; char dir[TSDB_FILENAME_LEN] = {0}; char tdir[TSDB_FILENAME_LEN * 2] = {0}; int32_t ret = 0; - if (pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", path); + if (vnodeCheckDisk(diskPrimary, pTfs)) { + vError("failed to open vnode from %s since %s. diskPrimary:%d", path, terrstr(), diskPrimary); + return NULL; } + vnodeGetPrimaryDir(path, diskPrimary, pTfs, dir, TSDB_FILENAME_LEN); info.config = vnodeCfgDefault; @@ -348,6 +365,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { pVnode->state.applied = info.state.committed; pVnode->state.applyTerm = info.state.commitTerm; pVnode->pTfs = pTfs; + pVnode->diskPrimary = diskPrimary; pVnode->msgCb = msgCb; taosThreadMutexInit(&pVnode->lock, NULL); pVnode->blocked = false; @@ -381,12 +399,6 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { goto _err; } - // open sma - if (smaOpen(pVnode, rollback)) { - vError("vgId:%d, failed to open vnode sma since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } - // open wal sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); taosRealPath(tdir, NULL, sizeof(tdir)); @@ -406,6 +418,12 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { goto _err; } + // open sma + if (smaOpen(pVnode, rollback)) { + vError("vgId:%d, failed to open vnode sma since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + // open query if (vnodeQueryOpen(pVnode)) { vError("vgId:%d, failed to open vnode query since %s", TD_VID(pVnode), tstrerror(terrno)); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 022fc4c951cf2b1d27326dc4788affc8bb9caf48..51f4cee40cafd308d6a3dbd601685d9192de5cbb 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -62,7 +62,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { } // query meta - metaReaderInit(&mer1, pVnode->pMeta, 0); + metaReaderDoInit(&mer1, pVnode->pMeta, 0); if (metaGetTableEntryByName(&mer1, infoReq.tbName) < 0) { code = terrno; @@ -79,7 +79,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { schemaTag = mer1.me.stbEntry.schemaTag; metaRsp.suid = mer1.me.uid; } else if (mer1.me.type == TSDB_CHILD_TABLE) { - metaReaderInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); + metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; strcpy(metaRsp.stbName, mer2.me.name); @@ -175,7 +175,7 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { } // query meta - metaReaderInit(&mer1, pVnode->pMeta, 0); + metaReaderDoInit(&mer1, pVnode->pMeta, 0); if (metaGetTableEntryByName(&mer1, cfgReq.tbName) < 0) { code = terrno; @@ -188,7 +188,7 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { code = TSDB_CODE_VND_HASH_MISMATCH; goto _exit; } else if (mer1.me.type == TSDB_CHILD_TABLE) { - metaReaderInit(&mer2, pVnode->pMeta, 0); + metaReaderDoInit(&mer2, pVnode->pMeta, 0); if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; strcpy(cfgRsp.stbName, mer2.me.name); @@ -496,6 +496,30 @@ int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) { return TSDB_CODE_SUCCESS; } +int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void *arg1), + void *arg) { + SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); + if (!pCur) { + return TSDB_CODE_FAILED; + } + + while (1) { + tb_uid_t id = metaStbCursorNext(pCur); + if (id == 0) { + break; + } + + if ((*filter) && (*filter)(arg, &id)) { + continue; + } + + taosArrayPush(list, &id); + } + + metaCloseStbCursor(pCur); + return TSDB_CODE_SUCCESS; +} + int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) { SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid, 0); if (!pCur) { @@ -531,6 +555,58 @@ static int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { return TSDB_CODE_SUCCESS; } +#ifdef TD_ENTERPRISE +#define TK_LOG_STB_NUM 19 +static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", + "data_dir", + "dnodes_info", + "d_info", + "grants_info", + "keeper_monitor", + "logs", + "log_dir", + "log_summary", + "m_info", + "taosadapter_restful_http_request_fail", + "taosadapter_restful_http_request_in_flight", + "taosadapter_restful_http_request_summary_milliseconds", + "taosadapter_restful_http_request_total", + "taosadapter_system_cpu_percent", + "taosadapter_system_mem_percent", + "temp_dir", + "vgroups_info", + "vnodes_role"}; + +// exclude stbs of taoskeeper log +static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { + char *dbName = strchr(pVnode->config.dbname, '.'); + if (!dbName || 0 != strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + return 0; + } + int32_t tbSize = metaSizeOfTbFilterCache(pVnode, 0); + if (tbSize < TK_LOG_STB_NUM) { + for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { + tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]); + if (suid != 0) { + metaPutTbToFilterCache(pVnode, suid, 0); + } + } + tbSize = metaSizeOfTbFilterCache(pVnode, 0); + } + + return tbSize; +} +#endif + +static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { + SVnode *pVnode = (SVnode *)arg1; + + if (metaTbInFilterCache(pVnode, *(tb_uid_t *)(arg2), 0)) { + return true; + } + return false; +} + int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { SArray *suidList = NULL; @@ -539,7 +615,13 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { return TSDB_CODE_FAILED; } - if (vnodeGetStbIdList(pVnode, 0, suidList) < 0) { + int32_t tbFilterSize = 0; + #ifdef TD_ENTERPRISE + tbFilterSize = vnodeGetTimeSeriesBlackList(pVnode); + #endif + + if ((!tbFilterSize && vnodeGetStbIdList(pVnode, 0, suidList) < 0) || + (tbFilterSize && vnodeGetStbIdListByFilter(pVnode, 0, suidList, vnodeTimeSeriesFilter, pVnode) < 0)) { qError("vgId:%d, failed to get stb id list error: %s", TD_VID(pVnode), terrstr()); taosArrayDestroy(suidList); return TSDB_CODE_FAILED; diff --git a/source/dnode/vnode/src/vnd/vnodeRetention.c b/source/dnode/vnode/src/vnd/vnodeRetention.c index 61623f0a69ba976cd31c365cb625efa102b0e2c2..7af1f8e28f3de849a841effffdca279a2cbcef72 100644 --- a/source/dnode/vnode/src/vnd/vnodeRetention.c +++ b/source/dnode/vnode/src/vnd/vnodeRetention.c @@ -15,116 +15,27 @@ #include "vnd.h" -typedef struct { - SVnode *pVnode; - int64_t now; - int64_t commitID; - SVnodeInfo info; -} SRetentionInfo; +extern int32_t tsdbSyncRetention(STsdb *tsdb, int64_t now); +extern int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now, int64_t *taskid); -extern bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now); -extern int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now); -extern int32_t tsdbCommitRetention(STsdb *pTsdb); +int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) { + int32_t code; + int32_t lino; -static int32_t vnodePrepareRentention(SVnode *pVnode, SRetentionInfo *pInfo) { - int32_t code = 0; - int32_t lino = 0; - - tsem_wait(&pVnode->canCommit); - - pInfo->commitID = ++pVnode->state.commitID; - - char dir[TSDB_FILENAME_LEN] = {0}; - if (pVnode->pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); - } - - if (vnodeLoadInfo(dir, &pInfo->info) < 0) { - code = terrno; + if (pVnode->config.sttTrigger == 1) { + tsem_wait(&pVnode->canCommit); + code = tsdbSyncRetention(pVnode->pTsdb, now); TSDB_CHECK_CODE(code, lino, _exit); - } -_exit: - if (code) { - vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + // code = smaDoRetention(pVnode->pSma, now); + // TSDB_CHECK_CODE(code, lino, _exit); tsem_post(&pVnode->canCommit); } else { - vInfo("vgId:%d %s done", TD_VID(pVnode), __func__); - } - return code; -} - -static int32_t vnodeRetentionTask(void *param) { - int32_t code = 0; - int32_t lino = 0; - - SRetentionInfo *pInfo = (SRetentionInfo *)param; - SVnode *pVnode = pInfo->pVnode; - char dir[TSDB_FILENAME_LEN] = {0}; - - if (pVnode->pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); - } - - // save info - pInfo->info.state.commitID = pInfo->commitID; - - if (vnodeSaveInfo(dir, &pInfo->info) < 0) { - code = terrno; + int64_t taskid; + code = tsdbAsyncRetention(pVnode->pTsdb, now, &taskid); TSDB_CHECK_CODE(code, lino, _exit); } - // do job - code = tsdbDoRetention(pInfo->pVnode->pTsdb, pInfo->now); - TSDB_CHECK_CODE(code, lino, _exit); - - // commit info - vnodeCommitInfo(dir); - - // commit sub-job - tsdbCommitRetention(pVnode->pTsdb); - _exit: - if (code) { - vError("vgId:%d %s failed at line %d since %s", TD_VID(pInfo->pVnode), __func__, lino, tstrerror(code)); - } else { - vInfo("vgId:%d %s done", TD_VID(pInfo->pVnode), __func__); - } - tsem_post(&pInfo->pVnode->canCommit); - taosMemoryFree(pInfo); return code; -} - -int32_t vnodeAsyncRentention(SVnode *pVnode, int64_t now) { - int32_t code = 0; - int32_t lino = 0; - - if (!tsdbShouldDoRetention(pVnode->pTsdb, now)) return code; - - SRetentionInfo *pInfo = (SRetentionInfo *)taosMemoryCalloc(1, sizeof(*pInfo)); - if (pInfo == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - pInfo->pVnode = pVnode; - pInfo->now = now; - - code = vnodePrepareRentention(pVnode, pInfo); - TSDB_CHECK_CODE(code, lino, _exit); - - vnodeScheduleTask(vnodeRetentionTask, pInfo); - -_exit: - if (code) { - vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); - if (pInfo) taosMemoryFree(pInfo); - } else { - vInfo("vgId:%d %s done", TD_VID(pInfo->pVnode), __func__); - } - return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 052e4ab2c18e6dad651cf591962bcb4dd49a0a38..d559783c2f284d40f5ea32fc5cfb1552838e46ad 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -86,17 +86,17 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) { int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) { int32_t code = 0; + SVnode *pVnode = pReader->pVnode; // CONFIG ============== // FIXME: if commit multiple times and the config changed? if (!pReader->cfgDone) { char fName[TSDB_FILENAME_LEN]; - if (pReader->pVnode->pTfs) { - snprintf(fName, TSDB_FILENAME_LEN, "%s%s%s%s%s", tfsGetPrimaryPath(pReader->pVnode->pTfs), TD_DIRSEP, - pReader->pVnode->path, TD_DIRSEP, VND_INFO_FNAME); - } else { - snprintf(fName, TSDB_FILENAME_LEN, "%s%s%s", pReader->pVnode->path, TD_DIRSEP, VND_INFO_FNAME); - } + int32_t offset = 0; + + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, fName, TSDB_FILENAME_LEN); + offset = strlen(fName); + snprintf(fName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VND_INFO_FNAME); TdFilePtr pFile = taosOpenFile(fName, TD_FILE_READ); if (NULL == pFile) { @@ -344,11 +344,7 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * .applyTerm = pWriter->info.state.commitTerm}; pVnode->statis = pWriter->info.statis; char dir[TSDB_FILENAME_LEN] = {0}; - if (pWriter->pVnode->pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", pWriter->pVnode->path); - } + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN); vnodeCommitInfo(dir); } else { @@ -386,7 +382,7 @@ _exit: static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { int32_t code = 0; - + SVnode *pVnode = pWriter->pVnode; SSnapDataHdr *pHdr = (SSnapDataHdr *)pData; // decode info @@ -400,15 +396,9 @@ static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_ // modify info as needed char dir[TSDB_FILENAME_LEN] = {0}; - if (pWriter->pVnode->pTfs) { - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pWriter->pVnode->pTfs), TD_DIRSEP, - pWriter->pVnode->path); - } else { - snprintf(dir, TSDB_FILENAME_LEN, "%s", pWriter->pVnode->path); - } + vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN); SVnodeStats vndStats = pWriter->info.config.vndStats; - SVnode *pVnode = pWriter->pVnode; pWriter->info.config = pVnode->config; pWriter->info.config.vndStats = vndStats; vDebug("vgId:%d, save config while write snapshot", pWriter->pVnode->config.vgId); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 204107ee3c89a74fbac7ef4df1e522e13f3403d0..81f87a3e22ad6e3118cf8c883ec22753bc3ddd1c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -402,10 +402,6 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg if (!syncUtilUserCommit(pMsg->msgType)) goto _exit; - if (pMsg->msgType == TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE || pMsg->msgType == TDMT_STREAM_TASK_CHECK_RSP) { - if (tqCheckLogInWal(pVnode->pTq, ver)) return 0; - } - // skip header pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); len = pMsg->contLen - sizeof(SMsgHead); @@ -496,16 +492,6 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg goto _err; } } break; - case TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE: { - if (tqProcessTaskRecover2Req(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { - goto _err; - } - } break; - case TDMT_STREAM_TASK_CHECK_RSP: { - if (tqProcessStreamTaskCheckRsp(pVnode->pTq, ver, pReq, len) < 0) { - goto _err; - } - } break; case TDMT_VND_ALTER_CONFIRM: needCommit = pVnode->config.hashChange; if (vnodeProcessAlterConfirmReq(pVnode, ver, pReq, len, pRsp) < 0) { @@ -638,30 +624,46 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { // return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_VND_TMQ_VG_WALINFO: return tqProcessVgWalInfoReq(pVnode->pTq, pMsg); - case TDMT_VND_TMQ_VG_COMMITTEDINFO: - return tqProcessVgCommittedInfoReq(pVnode->pTq, pMsg); - case TDMT_VND_TMQ_SEEK: - return tqProcessSeekReq(pVnode->pTq, pMsg); + default: + vError("unknown msg type:%d in fetch queue", pMsg->msgType); + return TSDB_CODE_APP_ERROR; + } +} + +int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { + vTrace("vgId:%d, msg:%p in fetch queue is processing", pVnode->config.vgId, pMsg); + if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG || + pMsg->msgType == TDMT_VND_BATCH_META) && + !syncIsReadyForRead(pVnode->sync)) { + vnodeRedirectRpcMsg(pVnode, pMsg, terrno); + return 0; + } + + switch (pMsg->msgType) { case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH: return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true); - case TDMT_STREAM_TASK_CHECK: - return tqProcessStreamTaskCheckReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg); + case TDMT_STREAM_TASK_CHECK: + return tqProcessStreamTaskCheckReq(pVnode->pTq, pMsg); + case TDMT_STREAM_TASK_CHECK_RSP: + return tqProcessStreamTaskCheckRsp(pVnode->pTq, 0, pMsg); case TDMT_STREAM_RETRIEVE: return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg); case TDMT_STREAM_RETRIEVE_RSP: return tqProcessTaskRetrieveRsp(pVnode->pTq, pMsg); - case TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE: - return tqProcessTaskRecover1Req(pVnode->pTq, pMsg); - case TDMT_STREAM_RECOVER_FINISH: - return tqProcessTaskRecoverFinishReq(pVnode->pTq, pMsg); - case TDMT_STREAM_RECOVER_FINISH_RSP: - return tqProcessTaskRecoverFinishRsp(pVnode->pTq, pMsg); + case TDMT_VND_STREAM_SCAN_HISTORY: + return tqProcessTaskScanHistory(pVnode->pTq, pMsg); + case TDMT_STREAM_TRANSFER_STATE: + return tqProcessTaskTransferStateReq(pVnode->pTq, pMsg); + case TDMT_STREAM_SCAN_HISTORY_FINISH: + return tqProcessTaskScanHistoryFinishReq(pVnode->pTq, pMsg); + case TDMT_STREAM_SCAN_HISTORY_FINISH_RSP: + return tqProcessTaskScanHistoryFinishRsp(pVnode->pTq, pMsg); default: - vError("unknown msg type:%d in fetch queue", pMsg->msgType); + vError("unknown msg type:%d in stream queue", pMsg->msgType); return TSDB_CODE_APP_ERROR; } } @@ -682,7 +684,8 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { pMetaRsp->precision = pVnode->config.tsdbCfg.precision; } -extern int32_t vnodeAsyncRentention(SVnode *pVnode, int64_t now); +extern int32_t vnodeDoRetention(SVnode *pVnode, int64_t now); + static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SVTrimDbReq trimReq = {0}; @@ -695,10 +698,7 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int3 vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp); - // process - vnodeAsyncRentention(pVnode, trimReq.timestamp); - tsem_wait(&pVnode->canCommit); - tsem_post(&pVnode->canCommit); + code = vnodeDoRetention(pVnode, trimReq.timestamp); _exit: return code; @@ -723,7 +723,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, tqUpdateTbUidList(pVnode->pTq, tbUids, false); } - vnodeAsyncRentention(pVnode, ttlReq.timestampSec); + vnodeDoRetention(pVnode, ttlReq.timestampSec); end: taosArrayDestroy(tbUids); @@ -1697,7 +1697,7 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pRe tDecodeSBatchDeleteReq(&decoder, &deleteReq); SMetaReader mr = {0}; - metaReaderInit(&mr, pVnode->pMeta, META_READER_NOLOCK); + metaReaderDoInit(&mr, pVnode->pMeta, META_READER_NOLOCK); int32_t sz = taosArrayGetSize(deleteReq.deleteReqs); for (int32_t i = 0; i < sz; i++) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index ff551e6534d1c445c4465e8f2ad3328f03899196..360da41482c43cc13236b104052220c87bd216cf 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -554,7 +554,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) vInfo("vgId:%d, not launch stream tasks, since stream tasks are disabled", pVnode->config.vgId); } else { vInfo("vgId:%d start to launch stream tasks", pVnode->config.vgId); - tqStartStreamTasks(pVnode->pTq); + tqCheckStreamStatus(pVnode->pTq); } } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 5746ea23406a722ec7464a9db07786c5cc5ee410..7d47e82164fdbd5b77a5eb70637957ae048ef7bb 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -938,7 +938,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const void* param); int32_t ctgLaunchJob(SCtgJob* pJob); int32_t ctgMakeAsyncRes(SCtgJob* pJob); -int32_t ctgLaunchSubTask(SCtgTask* pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param); +int32_t ctgLaunchSubTask(SCtgTask** ppTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param); int32_t ctgGetTbCfgCb(SCtgTask* pTask); void ctgFreeHandle(SCatalog* pCatalog); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 562343c9c70b4b8a38c866069e897535c5fdfddd..fb5ecf7ad219b029c6d2d256192dc4b8f5d983b4 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -2090,25 +2090,25 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask* pTask) { } CTG_CACHE_NHIT_INC(CTG_CI_TBL_CFG, 1); - + if (pCtx->tbType <= 0) { CTG_ERR_JRET(ctgReadTbTypeFromCache(pCtg, dbFName, pCtx->pName->tname, &pCtx->tbType)); if (pCtx->tbType <= 0) { SCtgTbMetaParam param; param.pName = pCtx->pName; param.flag = 0; - CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, ¶m)); + CTG_ERR_JRET(ctgLaunchSubTask(&pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, ¶m)); return TSDB_CODE_SUCCESS; } } - if (TSDB_SUPER_TABLE == pCtx->tbType) { + if (TSDB_SUPER_TABLE == pCtx->tbType || TSDB_SYSTEM_TABLE == pCtx->tbType) { CTG_ERR_JRET(ctgGetTableCfgFromMnode(pCtg, pConn, pCtx->pName, NULL, pTask)); } else { if (NULL == pCtx->pVgInfo) { CTG_ERR_JRET(ctgGetTbHashVgroupFromCache(pCtg, pCtx->pName, &pCtx->pVgInfo)); if (NULL == pCtx->pVgInfo) { - CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbCfgCb, dbFName)); + CTG_ERR_JRET(ctgLaunchSubTask(&pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbCfgCb, dbFName)); return TSDB_CODE_SUCCESS; } } @@ -2145,7 +2145,7 @@ int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { if (NULL == pCtx->pVgInfo) { CTG_ERR_JRET(ctgGetTbHashVgroupFromCache(pCtg, pCtx->pName, &pCtx->pVgInfo)); if (NULL == pCtx->pVgInfo) { - CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbTagCb, dbFName)); + CTG_ERR_JRET(ctgLaunchSubTask(&pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbTagCb, dbFName)); return TSDB_CODE_SUCCESS; } } @@ -2331,7 +2331,7 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) { SCtgTbMetaParam param; param.pName = &pCtx->user.tbName; param.flag = CTG_FLAG_SYNC_OP; - CTG_ERR_RET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetUserCb, ¶m)); + CTG_ERR_RET(ctgLaunchSubTask(&pTask, CTG_TASK_GET_TB_META, ctgGetUserCb, ¶m)); } else { CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pCtx->user.user, NULL, pTask)); } @@ -2541,19 +2541,35 @@ _return: CTG_RET(code); } -int32_t ctgLaunchSubTask(SCtgTask* pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param) { - SCtgJob* pJob = pTask->pJob; +SCtgTask* ctgGetTask(SCtgJob* pJob, int32_t taskId) { + int32_t taskNum = taosArrayGetSize(pJob->pTasks); + + for (int32_t i = 0; i < taskNum; ++i) { + SCtgTask* pTask = taosArrayGet(pJob->pTasks, i); + if (pTask->taskId == taskId) { + return pTask; + } + } + + return NULL; +} + + +int32_t ctgLaunchSubTask(SCtgTask** ppTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param) { + SCtgJob* pJob = (*ppTask)->pJob; int32_t subTaskId = -1; bool newTask = false; + int32_t taskId = (*ppTask)->taskId; - ctgClearSubTaskRes(&pTask->subRes); - pTask->subRes.type = type; - pTask->subRes.fp = fp; + ctgClearSubTaskRes(&(*ppTask)->subRes); + (*ppTask)->subRes.type = type; + (*ppTask)->subRes.fp = fp; CTG_ERR_RET(ctgSearchExistingTask(pJob, type, param, &subTaskId)); if (subTaskId < 0) { CTG_ERR_RET(ctgInitTask(pJob, type, param, &subTaskId)); newTask = true; + *ppTask = ctgGetTask(pJob, taskId); } SCtgTask* pSub = taosArrayGet(pJob->pTasks, subTaskId); @@ -2561,10 +2577,10 @@ int32_t ctgLaunchSubTask(SCtgTask* pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, pSub->subTask = true; } - CTG_ERR_RET(ctgSetSubTaskCb(pSub, pTask)); + CTG_ERR_RET(ctgSetSubTaskCb(pSub, *ppTask)); if (newTask) { - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(*ppTask, -1); SCtgMsgCtx* pSubMsgCtx = CTG_GET_TASK_MSGCTX(pSub, -1); pSubMsgCtx->pBatchs = pMsgCtx->pBatchs; @@ -2584,6 +2600,7 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); + pTask = taosArrayGet(pJob->pTasks, i); pTask->status = CTG_TASK_LAUNCHED; } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 605f5efeb417d9c38afcef1ae25bff95ab73b330..44de83b7ef142c5cfc277854d196cde332b3895c 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -773,12 +773,6 @@ int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) { int32_t code = 0; - if (IS_SYS_DBNAME(pReq->tbName.dbname)) { - *inCache = true; - pRes->pRawRes->pass = true; - ctgDebug("sysdb %s, pass", pReq->tbName.dbname); - return TSDB_CODE_SUCCESS; - } SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, pReq->user, strlen(pReq->user)); if (NULL == pUser) { diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 86f6a51d9bcbdec9883b268db2856c7af7fa4f87..dab007aa47729e7e32f4550fca7f3d60e502d149 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1589,6 +1589,12 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { return TSDB_CODE_SUCCESS; } + if (IS_SYS_DBNAME(pReq->tbName.dbname)) { + pRes->pass = true; + ctgDebug("sysdb %s, pass", pReq->tbName.dbname); + return TSDB_CODE_SUCCESS; + } + char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(&pReq->tbName, dbFName); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index b004539e164ca6d3a21960ff62bf354824b05be3..8ddf730d5a884a31377bcb4278ac3927a67e9e31 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -291,21 +291,24 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch hashPrefix = pCfg->hashPrefix + dbFNameLen + 1; } - len += sprintf( - buf2 + VARSTR_HEADER_SIZE, - "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm " - "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " - "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d " - "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " WAL_ROLL_PERIOD %d WAL_SEGMENT_SIZE %" PRId64, - dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile, - pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, - pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, - 1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, - pCfg->walRetentionSize, pCfg->walRollPeriod, pCfg->walSegmentSize); - - if (retentions) { - len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); - taosMemoryFree(retentions); + if (IS_SYS_DBNAME(dbName)) { + len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s`", dbName); + } else { + len += sprintf( + buf2 + VARSTR_HEADER_SIZE, + "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm " + "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " + "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d " + "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64, + dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile, + pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, + pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, + 1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize); + + if (retentions) { + len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); + taosMemoryFree(retentions); + } } (varDataLen(buf2)) = len; @@ -397,7 +400,7 @@ static int32_t setAliveResultIntoDataBlock(int64_t* pConnId, SSDataBlock* pBlock int32_t status = 0; int32_t code = getAliveStatusFromApi(pConnId, dbName, &status); if (code == TSDB_CODE_SUCCESS) { - colDataAppend(pCol1, 0, (const char*)&status, false); + colDataSetVal(pCol1, 0, (const char*)&status, false); } return code; } @@ -612,6 +615,31 @@ void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* if (pCfg->ttl > 0) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " TTL %d", pCfg->ttl); } + + if (TSDB_SUPER_TABLE == pCfg->tableType || TSDB_NORMAL_TABLE == pCfg->tableType) { + int32_t nSma = 0; + for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { + if (IS_BSMA_ON(pCfg->pSchemas + i)) { + ++nSma; + } + } + + if (nSma < pCfg->numOfColumns) { + bool smaOn = false; + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " SMA("); + for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { + if (IS_BSMA_ON(pCfg->pSchemas + i)) { + if (smaOn) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ",`%s`", (pCfg->pSchemas + i)->name); + } else { + smaOn = true; + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "`%s`", (pCfg->pSchemas + i)->name); + } + } + } + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ")"); + } + } } static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* pDbCfg, char* tbName, STableCfg* pCfg) { @@ -774,15 +802,19 @@ static int32_t buildLocalVariablesResultDataBlock(SSDataBlock** pOutput) { pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData)); SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN; taosArrayPush(pBlock->pDataBlock, &infoData); + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN; + taosArrayPush(pBlock->pDataBlock, &infoData); + *pOutput = pBlock; return TSDB_CODE_SUCCESS; } @@ -795,6 +827,7 @@ int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) { SConfigItem* pItem = taosArrayGet(tsCfg->array, i); GRANT_CFG_SKIP; + char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++); @@ -807,6 +840,12 @@ int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { pColInfo = taosArrayGet(pBlock->pDataBlock, c++); colDataSetVal(pColInfo, i, value, false); + char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; + cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen); + varDataSetLen(scope, valueLen); + pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + colDataSetVal(pColInfo, i, scope, false); + numOfRows++; } diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 51731faece7ea65ae4696433d2192cc2662d63d7..9a917adf1b4b8a03b139d26e211b11e519698035 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -80,7 +80,6 @@ enum { STREAM_RECOVER_STEP__PREPARE1, STREAM_RECOVER_STEP__PREPARE2, STREAM_RECOVER_STEP__SCAN1, - STREAM_RECOVER_STEP__SCAN2, }; extern int32_t exchangeObjRefPool; @@ -232,19 +231,20 @@ typedef struct STableMergeScanInfo { int32_t tableEndIndex; bool hasGroupId; uint64_t groupId; - SArray* queryConds; // array of queryTableDataCond STableScanBase base; int32_t bufPageSize; uint32_t sortBufSize; // max buffer size for in-memory sort SArray* pSortInfo; SSortHandle* pSortHandle; SSDataBlock* pSortInputBlock; + SSDataBlock* pReaderBlock; int64_t startTs; // sort start time SArray* sortSourceParams; SLimitInfo limitInfo; int64_t numOfRows; SScanInfo scanInfo; int32_t scanTimes; + int32_t readIdx; SSDataBlock* pResBlock; SSampleExecInfo sample; // sample execution info SSortExecInfo sortExecInfo; @@ -285,6 +285,8 @@ typedef struct SStreamAggSupporter { int16_t stateKeyType; SDiskbasedBuf* pResultBuf; SStateStore stateStore; + STimeWindow winRange; + SStorageAPI* pSessionAPI; } SStreamAggSupporter; typedef struct SWindowSupporter { @@ -364,7 +366,6 @@ typedef struct SStreamScanInfo { SNode* pTagIndexCond; // recover - int32_t blockRecoverContiCnt; int32_t blockRecoverTotCnt; SSDataBlock* pRecoverRes; @@ -503,6 +504,8 @@ typedef struct SStreamSessionAggOperatorInfo { SArray* pUpdated; SSHashObj* pStUpdated; int64_t dataVersion; + SArray* historyWins; + bool isHistoryOp; } SStreamSessionAggOperatorInfo; typedef struct SStreamStateAggOperatorInfo { @@ -522,6 +525,8 @@ typedef struct SStreamStateAggOperatorInfo { SArray* pUpdated; SSHashObj* pSeUpdated; int64_t dataVersion; + bool isHistoryOp; + SArray* historyWins; } SStreamStateAggOperatorInfo; typedef struct SStreamPartitionOperatorInfo { @@ -678,6 +683,8 @@ void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExpr void doClearBufferedBlocks(SStreamScanInfo* pInfo); uint64_t calcGroupId(char* pData, int32_t len); +void streamOpReleaseState(struct SOperatorInfo* pOperator); +void streamOpReloadState(struct SOperatorInfo* pOperator); #ifdef __cplusplus } diff --git a/source/libs/executor/inc/operator.h b/source/libs/executor/inc/operator.h index 1d2685b8c6f98aa8309c2e9900c4d378ff227e48..e6c3405d7ff1b75326601adc0b436ca8295ee78d 100644 --- a/source/libs/executor/inc/operator.h +++ b/source/libs/executor/inc/operator.h @@ -35,6 +35,7 @@ typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr); typedef void (*__optr_close_fn_t)(void* param); typedef int32_t (*__optr_explain_fn_t)(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); typedef int32_t (*__optr_reqBuf_fn_t)(struct SOperatorInfo* pOptr); +typedef void (*__optr_state_fn_t)(struct SOperatorInfo* pOptr); typedef struct SOperatorFpSet { __optr_open_fn_t _openFn; // DO NOT invoke this function directly @@ -45,6 +46,8 @@ typedef struct SOperatorFpSet { __optr_encode_fn_t encodeResultRow; __optr_decode_fn_t decodeResultRow; __optr_explain_fn_t getExplainFn; + __optr_state_fn_t releaseStreamStateFn; + __optr_state_fn_t reloadStreamStateFn; } SOperatorFpSet; enum { @@ -126,13 +129,13 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle); -SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); +SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle); SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); @@ -143,6 +146,7 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain); +void setOperatorStreamStateFn(SOperatorInfo* pOperator, __optr_state_fn_t relaseFn, __optr_state_fn_t reloadFn); int32_t optrDummyOpenFn(SOperatorInfo* pOperator); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); void setOperatorCompleted(SOperatorInfo* pOperator); diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h index 6497bd90b43ee73422c9dbae96f871a27f4e29b0..cdf37bcc6b5a9cd2a06f0398cd17675e2ce62531 100644 --- a/source/libs/executor/inc/querytask.h +++ b/source/libs/executor/inc/querytask.h @@ -62,10 +62,12 @@ typedef struct { SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor int8_t recoverStep; + bool recoverStep1Finished; + bool recoverStep2Finished; int8_t recoverScanFinished; SQueryTableDataCond tableCond; - int64_t fillHistoryVer1; - int64_t fillHistoryVer2; + SVersionRange fillHistoryVer; + STimeWindow fillHistoryWindow; SStreamState* pState; int64_t dataVersion; int64_t checkPointId; diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index 7a0d236a3778faed5cab419ce565b9fff7162312..538a9f18f6012a110b52fa8e2d899f551dd88597 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -26,6 +26,7 @@ extern "C" { enum { SORT_MULTISOURCE_MERGE = 0x1, SORT_SINGLESOURCE_SORT = 0x2, + SORT_BLOCK_TS_MERGE = 0x3 }; typedef struct SMultiMergeSource { @@ -53,6 +54,12 @@ typedef struct SMsortComparParam { int32_t numOfSources; SArray* orderInfo; // SArray bool cmpGroupId; + + int32_t sortType; + // the following field to speed up when sortType == SORT_BLOCK_TS_MERGE + int32_t tsSlotId; + int32_t order; + __compar_fn_t cmpFn; } SMsortComparParam; typedef struct SSortHandle SSortHandle; @@ -64,14 +71,16 @@ typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void* /** * * @param type - * @param maxRows keep maxRows at most - * @param maxTupleLength max len of one tuple, for check if heap sort is applicable + * @param maxRows keep maxRows at most, if 0, pq sort will not be used + * @param maxTupleLength max len of one tuple, for check if pq sort is applicable * @param sortBufSize sort memory buf size, for check if heap sort is applicable * @return */ SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, int32_t type, int32_t pageSize, int32_t numOfPages, - SSDataBlock* pBlock, const char* idstr, uint64_t maxRows, uint32_t maxTupleLength, - uint32_t sortBufSize); + SSDataBlock* pBlock, const char* idstr, uint64_t pqMaxRows, uint32_t pqMaxTupleLength, + uint32_t pqSortBufSize); + +void tsortSetForceUsePQSort(SSortHandle* pHandle); /** * @@ -108,6 +117,10 @@ int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetc */ int32_t tsortSetComparFp(SSortHandle* pHandle, _sort_merge_compar_fn_t fp); +/** + * +*/ +void tsortSetMergeLimit(SSortHandle* pHandle, int64_t mergeLimit); /** * */ diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index be0ad1c2399c1b423bf6a09635d6334581c321fe..176c4b53be828a0ad356953793f722cc39d43893 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -45,6 +45,8 @@ typedef struct SAggOperatorInfo { SGroupResInfo groupResInfo; SExprSupp scalarExprSup; bool groupKeyOptimized; + bool hasValidBlock; + SSDataBlock* pNewGroupBlock; } SAggOperatorInfo; static void destroyAggOperatorInfo(void* param); @@ -53,7 +55,6 @@ static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, u static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock); static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock); -static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator); static int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx); static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator); @@ -111,9 +112,9 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN pInfo->binfo.inputTsOrder = pAggNode->node.inputTsOrder; pInfo->binfo.outputTsOrder = pAggNode->node.outputTsOrder; - setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, - pTaskInfo); - pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, + setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, + !pAggNode->node.forceCreateNonBlockingOptr, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, getAggregateResult, NULL, destroyAggOperatorInfo, optrDefaultBufFn, NULL); if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { @@ -153,28 +154,42 @@ void destroyAggOperatorInfo(void* param) { taosMemoryFreeClear(param); } -// this is a blocking operator -int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { - if (OPTR_IS_OPENED(pOperator)) { - return TSDB_CODE_SUCCESS; - } - +/** + * @brief get blocks from downstream and fill results into groupedRes after aggragation + * @retval false if no more groups + * @retval true if there could have new groups coming + * @note if pOperator.blocking is true, scan all blocks from downstream, all groups are handled + * if false, fill results of ONE GROUP + * */ +static bool nextGroupedResult(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SAggOperatorInfo* pAggInfo = pOperator->info; + if (pOperator->blocking && pAggInfo->hasValidBlock) return false; + SExprSupp* pSup = &pOperator->exprSupp; SOperatorInfo* downstream = pOperator->pDownstream[0]; - int64_t st = taosGetTimestampUs(); - int32_t code = TSDB_CODE_SUCCESS; - int32_t order = pAggInfo->binfo.inputTsOrder; - bool hasValidBlock = false; + int64_t st = taosGetTimestampUs(); + int32_t code = TSDB_CODE_SUCCESS; + int32_t order = pAggInfo->binfo.inputTsOrder; + SSDataBlock* pBlock = pAggInfo->pNewGroupBlock; + if (pBlock) { + pAggInfo->pNewGroupBlock = NULL; + tSimpleHashClear(pAggInfo->aggSup.pResultRowHashTable); + setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId); + setInputDataBlock(pSup, pBlock, order, pBlock->info.scanFlag, true); + code = doAggregateImpl(pOperator, pSup->pCtx); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } + } while (1) { bool blockAllocated = false; - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - if (!hasValidBlock) { + if (!pAggInfo->hasValidBlock) { createDataBlockForEmptyInput(pOperator, &pBlock); if (pBlock == NULL) { break; @@ -184,7 +199,7 @@ int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { break; } } - hasValidBlock = true; + pAggInfo->hasValidBlock = true; pAggInfo->binfo.pRes->info.scanFlag = pBlock->info.scanFlag; // there is an scalar expression that needs to be calculated before apply the group aggregation. @@ -196,7 +211,11 @@ int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } } - + // if non-blocking mode and new group arrived, save the block and break + if (!pOperator->blocking && pAggInfo->groupId != UINT64_MAX && pBlock->info.id.groupId != pAggInfo->groupId) { + pAggInfo->pNewGroupBlock = pBlock; + break; + } // the pDataBlock are always the same one, no need to call this again setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId); setInputDataBlock(pSup, pBlock, order, pBlock->info.scanFlag, true); @@ -215,10 +234,7 @@ int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { } initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0); - OPTR_SET_OPENED(pOperator); - - pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - return pTaskInfo->code; + return pBlock != NULL; } SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { @@ -230,26 +246,25 @@ SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { } SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - setOperatorCompleted(pOperator); - return NULL; - } + bool hasNewGroups = false; + do { + hasNewGroups = nextGroupedResult(pOperator); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - while (1) { - doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); - doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + while (1) { + doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); + doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); - if (!hasRemainResults(&pAggInfo->groupResInfo)) { - setOperatorCompleted(pOperator); - break; - } + if (!hasRemainResults(&pAggInfo->groupResInfo)) { + if (!hasNewGroups) setOperatorCompleted(pOperator); + break; + } - if (pInfo->pRes->info.rows > 0) { - break; + if (pInfo->pRes->info.rows > 0) { + break; + } } - } + } while (pInfo->pRes->info.rows == 0 && hasNewGroups); size_t rows = blockDataGetNumOfRows(pInfo->pRes); pOperator->resultInfo.totalRows += rows; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 832750e967dfd6f996d12281a9ece48b6c6d26c7..e1bf4e7cb0e3b3e2df8cc7931d57c153a952b78b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -127,6 +127,10 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in if (pGroupResInfo->pRows != NULL) { taosArrayDestroy(pGroupResInfo->pRows); } + if (pGroupResInfo->pBuf) { + taosMemoryFree(pGroupResInfo->pBuf); + pGroupResInfo->pBuf = NULL; + } // extract the result rows information from the hash map int32_t size = tSimpleHashGetSize(pHashmap); @@ -2104,6 +2108,8 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* if (groupSort && groupByTbname) { taosArraySort(pTableListInfo->pTableList, orderbyGroupIdComparFn); pTableListInfo->numOfOuputGroups = numOfTables; + } else if (groupByTbname && pScanNode->groupOrderScan){ + pTableListInfo->numOfOuputGroups = numOfTables; } else { pTableListInfo->numOfOuputGroups = 1; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index dfb0c1645a72633bb83c9ee014a415ef798cd081..5a99c1ea9a2fd0f7634daba482cd52d3811a4a11 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -92,6 +92,7 @@ static int32_t doSetStreamOpOpen(SOperatorInfo* pOperator, char* id) { qError("join not supported for stream block scan, %s" PRIx64, id); return TSDB_CODE_APP_ERROR; } + pOperator->status = OP_NOT_OPENED; return doSetStreamOpOpen(pOperator->pDownstream[0], id); } @@ -115,6 +116,17 @@ void resetTaskInfo(qTaskInfo_t tinfo) { clearStreamBlock(pTaskInfo->pRoot); } +void qResetStreamInfoTimeWindow(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo; + if (pTaskInfo == NULL) { + return; + } + + qDebug("%s set stream fill-history window:%" PRId64"-%"PRId64, GET_TASKID(pTaskInfo), INT64_MIN, INT64_MAX); + pTaskInfo->streamInfo.fillHistoryWindow.skey = INT64_MIN; + pTaskInfo->streamInfo.fillHistoryWindow.ekey = INT64_MAX; +} + static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, const char* id) { if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { @@ -130,10 +142,9 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, id); } else { pOperator->status = OP_NOT_OPENED; - SStreamScanInfo* pInfo = pOperator->info; - qDebug("s-task:%s in this batch, all %d blocks need to be processed and dump results", id, (int32_t)numOfBlocks); + qDebug("s-task:%s in this batch, %d blocks need to be processed", id, (int32_t)numOfBlocks); ASSERT(pInfo->validBlockIndex == 0 && taosArrayGetSize(pInfo->pBlockLists) == 0); if (type == STREAM_INPUT__MERGED_SUBMIT) { @@ -265,6 +276,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pTaskInfo->pRoot = createRawScanOperatorInfo(pReaderHandle, pTaskInfo); if (NULL == pTaskInfo->pRoot) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -277,9 +289,8 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 return pTaskInfo; } - struct SSubplan* pPlan = NULL; - - int32_t code = qStringToSubplan(msg, &pPlan); + SSubplan* pPlan = NULL; + int32_t code = qStringToSubplan(msg, &pPlan); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; @@ -314,8 +325,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v return NULL; } - struct SSubplan* pPlan = NULL; - int32_t code = qStringToSubplan(msg, &pPlan); + SSubplan* pPlan = NULL; + int32_t code = qStringToSubplan(msg, &pPlan); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; @@ -324,11 +335,13 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v qTaskInfo_t pTaskInfo = NULL; code = qCreateExecTask(readers, vgId, 0, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); if (code != TSDB_CODE_SUCCESS) { + nodesDestroyNode((SNode*)pPlan); qDestroyTask(pTaskInfo); terrno = code; return NULL; } + qResetStreamInfoTimeWindow(pTaskInfo); return pTaskInfo; } @@ -869,19 +882,41 @@ int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) { } } -int32_t qStreamSourceRecoverStep1(qTaskInfo_t tinfo, int64_t ver) { +int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); - pTaskInfo->streamInfo.fillHistoryVer1 = ver; - pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE1; + + SStreamTaskInfo* pStreamInfo = &pTaskInfo->streamInfo; + + pStreamInfo->fillHistoryVer = *pVerRange; + pStreamInfo->fillHistoryWindow = *pWindow; + pStreamInfo->recoverStep = STREAM_RECOVER_STEP__PREPARE1; + pStreamInfo->recoverStep1Finished = false; + pStreamInfo->recoverStep2Finished = false; + + qDebug("%s step 1. set param for stream scanner for scan-history data, verRange:%" PRId64 " - %" PRId64 ", window:%" PRId64 + " - %" PRId64, + GET_TASKID(pTaskInfo), pStreamInfo->fillHistoryVer.minVer, pStreamInfo->fillHistoryVer.maxVer, pWindow->skey, + pWindow->ekey); return 0; } -int32_t qStreamSourceRecoverStep2(qTaskInfo_t tinfo, int64_t ver) { +int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); - pTaskInfo->streamInfo.fillHistoryVer2 = ver; - pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE2; + + SStreamTaskInfo* pStreamInfo = &pTaskInfo->streamInfo; + + pStreamInfo->fillHistoryVer = *pVerRange; + pStreamInfo->fillHistoryWindow = *pWindow; + pStreamInfo->recoverStep = STREAM_RECOVER_STEP__PREPARE2; + pStreamInfo->recoverStep1Finished = true; + pStreamInfo->recoverStep2Finished = false; + + qDebug("%s step 2. set param for stream scanner for scan-history data, verRange:%" PRId64 " - %" PRId64 + ", window:%" PRId64 " - %" PRId64, + GET_TASKID(pTaskInfo), pStreamInfo->fillHistoryVer.minVer, pStreamInfo->fillHistoryVer.maxVer, pWindow->skey, + pWindow->ekey); return 0; } @@ -892,55 +927,58 @@ int32_t qStreamRecoverFinish(qTaskInfo_t tinfo) { return 0; } -int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) { +int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SOperatorInfo* pOperator = pTaskInfo->pRoot; while (1) { - if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { + int32_t type = pOperator->operatorType; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; - ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || - pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); - ASSERT(pInfo->twAggSup.calTriggerSaved == 0 && pInfo->twAggSup.deleteMarkSaved == 0); + STimeWindowAggSupp* pSup = &pInfo->twAggSup; + + ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); - qInfo("save stream param for interval: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + qInfo("save stream param for interval: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); - pInfo->twAggSup.calTriggerSaved = pInfo->twAggSup.calTrigger; - pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; - pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; - pInfo->twAggSup.deleteMark = INT64_MAX; + pSup->calTriggerSaved = pSup->calTrigger; + pSup->deleteMarkSaved = pSup->deleteMark; + pSup->calTrigger = STREAM_TRIGGER_AT_ONCE; + pSup->deleteMark = INT64_MAX; pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; pInfo->ignoreExpiredData = false; - } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || - pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + STimeWindowAggSupp* pSup = &pInfo->twAggSup; + + ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); - ASSERT(pInfo->twAggSup.calTriggerSaved == 0 && pInfo->twAggSup.deleteMarkSaved == 0); - qInfo("save stream param for session: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + qInfo("save stream param for session: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); - pInfo->twAggSup.calTriggerSaved = pInfo->twAggSup.calTrigger; - pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; - pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; - pInfo->twAggSup.deleteMark = INT64_MAX; + pSup->calTriggerSaved = pSup->calTrigger; + pSup->deleteMarkSaved = pSup->deleteMark; + pSup->calTrigger = STREAM_TRIGGER_AT_ONCE; + pSup->deleteMark = INT64_MAX; pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; pInfo->ignoreExpiredData = false; - } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; - ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || - pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); - ASSERT(pInfo->twAggSup.calTriggerSaved == 0 && pInfo->twAggSup.deleteMarkSaved == 0); + STimeWindowAggSupp* pSup = &pInfo->twAggSup; - qInfo("save stream param for state: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); - pInfo->twAggSup.calTriggerSaved = pInfo->twAggSup.calTrigger; - pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; - pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; - pInfo->twAggSup.deleteMark = INT64_MAX; + qInfo("save stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); + + pSup->calTriggerSaved = pSup->calTrigger; + pSup->deleteMarkSaved = pSup->deleteMark; + pSup->calTrigger = STREAM_TRIGGER_AT_ONCE; + pSup->deleteMark = INT64_MAX; pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; pInfo->ignoreExpiredData = false; } @@ -961,33 +999,37 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) { return 0; } -int32_t qStreamRestoreParam(qTaskInfo_t tinfo) { +int32_t qRestoreStreamOperatorOption(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + const char* id = GET_TASKID(pTaskInfo); SOperatorInfo* pOperator = pTaskInfo->pRoot; while (1) { - if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { + uint16_t type = pOperator->operatorType; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved; - qInfo("restore stream param for interval: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); - } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { + qInfo("%s restore stream agg executors param for interval: %d, %" PRId64, id, pInfo->twAggSup.calTrigger, + pInfo->twAggSup.deleteMark); + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved; - qInfo("restore stream param for session: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); - } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { + qInfo("%s restore stream agg executor param for session: %d, %" PRId64, id, pInfo->twAggSup.calTrigger, + pInfo->twAggSup.deleteMark); + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved; - qInfo("restore stream param for state: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + qInfo("%s restore stream agg executor param for state: %d, %" PRId64, id, pInfo->twAggSup.calTrigger, + pInfo->twAggSup.deleteMark); } // iterate operator tree @@ -1001,7 +1043,6 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) { pOperator = pOperator->pDownstream[0]; } } - return 0; } bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) { @@ -1009,6 +1050,26 @@ bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) { return pTaskInfo->streamInfo.recoverScanFinished; } +bool qStreamRecoverScanStep1Finished(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + return pTaskInfo->streamInfo.recoverStep1Finished; +} + +bool qStreamRecoverScanStep2Finished(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + return pTaskInfo->streamInfo.recoverStep2Finished; +} + +int32_t qStreamRecoverSetAllStepFinished(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + pTaskInfo->streamInfo.recoverStep1Finished = true; + pTaskInfo->streamInfo.recoverStep2Finished = true; + + // reset the time window + pTaskInfo->streamInfo.fillHistoryWindow.skey = INT64_MIN; + return 0; +} + void* qExtractReaderFromStreamScanner(void* scanner) { SStreamScanInfo* pInfo = scanner; return (void*)pInfo->tqReader; @@ -1323,4 +1384,16 @@ SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo) { SOperatorInfo* pOperator = pTaskInfo->pRoot; extractTableList(pArray, pOperator); return pArray; -} \ No newline at end of file +} + +int32_t qStreamOperatorReleaseState(qTaskInfo_t tInfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tInfo; + pTaskInfo->pRoot->fpSet.releaseStreamStateFn(pTaskInfo->pRoot); + return 0; +} + +int32_t qStreamOperatorReloadState(qTaskInfo_t tInfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tInfo; + pTaskInfo->pRoot->fpSet.reloadStreamStateFn(pTaskInfo->pRoot); + return 0; +} diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 0855203e9182d09e73dcd3fea1f96fb878943a25..ebec9aa94e5e8c71b6bc0e6c198cd7234550b0b0 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -544,140 +544,12 @@ _err: void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, int32_t status) { int8_t* pIndicator = (int8_t*)p->pData; - int32_t totalRows = pBlock->info.rows; - if (status == FILTER_RESULT_ALL_QUALIFIED) { // here nothing needs to be done } else if (status == FILTER_RESULT_NONE_QUALIFIED) { pBlock->info.rows = 0; } else if (status == FILTER_RESULT_PARTIAL_QUALIFIED) { - int32_t bmLen = BitmapLen(totalRows); - char* pBitmap = NULL; - int32_t maxRows = 0; - - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); - // it is a reserved column for scalar function, and no data in this column yet. - if (pDst->pData == NULL) { - continue; - } - - int32_t numOfRows = 0; - if (IS_VAR_DATA_TYPE(pDst->info.type)) { - int32_t j = 0; - - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - - if (colDataIsNull_var(pDst, j)) { - colDataSetNull_var(pDst, numOfRows); - } else { - char* p1 = colDataGetVarData(pDst, j); - colDataReassignVal(pDst, numOfRows, j, p1); - } - numOfRows += 1; - j += 1; - } - - if (maxRows < numOfRows) { - maxRows = numOfRows; - } - } else { - if (pBitmap == NULL) { - pBitmap = taosMemoryCalloc(1, bmLen); - } - - memcpy(pBitmap, pDst->nullbitmap, bmLen); - memset(pDst->nullbitmap, 0, bmLen); - - int32_t j = 0; - - switch (pDst->info.type) { - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - - if (colDataIsNull_f(pBitmap, j)) { - colDataSetNull_f(pDst->nullbitmap, numOfRows); - } else { - ((int64_t*)pDst->pData)[numOfRows] = ((int64_t*)pDst->pData)[j]; - } - numOfRows += 1; - j += 1; - } - break; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - if (colDataIsNull_f(pBitmap, j)) { - colDataSetNull_f(pDst->nullbitmap, numOfRows); - } else { - ((int32_t*)pDst->pData)[numOfRows] = ((int32_t*)pDst->pData)[j]; - } - numOfRows += 1; - j += 1; - } - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - if (colDataIsNull_f(pBitmap, j)) { - colDataSetNull_f(pDst->nullbitmap, numOfRows); - } else { - ((int16_t*)pDst->pData)[numOfRows] = ((int16_t*)pDst->pData)[j]; - } - numOfRows += 1; - j += 1; - } - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - if (colDataIsNull_f(pBitmap, j)) { - colDataSetNull_f(pDst->nullbitmap, numOfRows); - } else { - ((int8_t*)pDst->pData)[numOfRows] = ((int8_t*)pDst->pData)[j]; - } - numOfRows += 1; - j += 1; - } - break; - } - } - - if (maxRows < numOfRows) { - maxRows = numOfRows; - } - } - - pBlock->info.rows = maxRows; - if (pBitmap != NULL) { - taosMemoryFree(pBitmap); - } + trimDataBlock(pBlock, pBlock->info.rows, (bool*)pIndicator); } else { qError("unknown filter result type: %d", status); } @@ -1111,9 +983,11 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa int32_t size = 0; void* pVal = NULL; int32_t code = pAPI->stateStore.streamStateSessionGet(pState, pKey, &pVal, &size); - ASSERT(code == 0); + // ASSERT(code == 0); if (code == -1) { - // coverity scan + // for history + qWarn("===stream===not found session result key:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64, pKey->win.skey, + pKey->win.ekey, pKey->groupId); pGroupResInfo->index += 1; continue; } @@ -1183,40 +1057,16 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa return TSDB_CODE_SUCCESS; } -void qStreamCloseTsdbReader(void* task) { - if (task == NULL) { - return; +void streamOpReleaseState(SOperatorInfo* pOperator) { + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.releaseStreamStateFn) { + downstream->fpSet.releaseStreamStateFn(downstream); } +} - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)task; - SOperatorInfo* pOp = pTaskInfo->pRoot; - - qDebug("stream close tsdb reader, reset status uid:%" PRId64 " ts:%" PRId64, pTaskInfo->streamInfo.currentOffset.uid, - pTaskInfo->streamInfo.currentOffset.ts); - - // todo refactor, other thread may already use this read to extract data. - pTaskInfo->streamInfo.currentOffset = (STqOffsetVal){0}; - while (pOp->numOfDownstream == 1 && pOp->pDownstream[0]) { - SOperatorInfo* pDownstreamOp = pOp->pDownstream[0]; - if (pDownstreamOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - SStreamScanInfo* pInfo = pDownstreamOp->info; - if (pInfo->pTableScanOp) { - STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - - setOperatorCompleted(pInfo->pTableScanOp); - while (pTaskInfo->owner != 0) { - taosMsleep(100); - qDebug("wait for the reader stopping"); - } - - pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pTSInfo->base.dataReader); - pTSInfo->base.dataReader = NULL; - - // restore the status, todo refactor. - pInfo->pTableScanOp->status = OP_OPENED; - pTaskInfo->status = TASK_NOT_COMPLETED; - return; - } - } +void streamOpReloadState(SOperatorInfo* pOperator) { + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.reloadStreamStateFn) { + downstream->fpSet.reloadStreamStateFn(downstream); } } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 92152924f9dc196149663069643d588c871918ec..80c88a803effb72588097c4b86acd808dde78354 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -502,9 +502,13 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); tSimpleHashCleanup(pFillSup->pResMap); pFillSup->pResMap = NULL; - releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pFillSup->pAPI->stateStore); //????? - pFillSup->cur.pRowVal = NULL; cleanupExprSupp(&pFillSup->notFillExprSup); + if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) { + taosMemoryFree(pFillSup->cur.pRowVal); + } + taosMemoryFree(pFillSup->prev.pRowVal); + taosMemoryFree(pFillSup->next.pRowVal); + taosMemoryFree(pFillSup->nextNext.pRowVal); taosMemoryFree(pFillSup); return NULL; @@ -546,13 +550,17 @@ static void destroyStreamFillOperatorInfo(void* param) { static void resetFillWindow(SResultRowData* pRowData) { pRowData->key = INT64_MIN; - pRowData->pRowVal = NULL; + taosMemoryFreeClear(pRowData->pRowVal); } void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) { + if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) { + resetFillWindow(&pFillSup->cur); + } else { + pFillSup->cur.key = INT64_MIN; + pFillSup->cur.pRowVal = NULL; + } resetFillWindow(&pFillSup->prev); - releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pAPI->stateStore); //??? - resetFillWindow(&pFillSup->cur); resetFillWindow(&pFillSup->next); resetFillWindow(&pFillSup->nextNext); } @@ -1513,11 +1521,11 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi float v = 0; GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); SET_TYPED_DATA(pCell->pData, pCell->type, v); - } else if (pCell->type == TSDB_DATA_TYPE_DOUBLE) { + } else if (IS_FLOAT_TYPE(pCell->type)) { double v = 0; GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); SET_TYPED_DATA(pCell->pData, pCell->type, v); - } else if (IS_SIGNED_NUMERIC_TYPE(pCell->type)) { + } else if (IS_INTEGER_TYPE(pCell->type)) { int64_t v = 0; GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); SET_TYPED_DATA(pCell->pData, pCell->type, v); @@ -1562,6 +1570,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 57c22d56d38dadf5012a032e7f298f935bbd101e..9228c923a6893ccacbbf51cf3ab90958f7b02bb4 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -1036,7 +1036,7 @@ void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup } void* pGpIdCol = taosArrayGet(pDestBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); - colDataAppend(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); + colDataSetVal(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); pDestBlock->info.rows++; blockDataDestroy(pTmpBlock); } else { @@ -1324,6 +1324,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 730252c7ee2e4ae6fe5bb36d5ab159cfe7cb966f..2db5ea2f1e2b55cd5faf4fe12f47c6f292122079 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -38,11 +38,18 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, .closeFn = closeFn, .reqBufFn = reqBufFn, .getExplainFn = explain, + .releaseStreamStateFn = NULL, + .reloadStreamStateFn = NULL, }; return fpSet; } +void setOperatorStreamStateFn(SOperatorInfo* pOperator, __optr_state_fn_t relaseFn, __optr_state_fn_t reloadFn) { + pOperator->fpSet.releaseStreamStateFn = relaseFn; + pOperator->fpSet.reloadStreamStateFn = reloadFn; +} + int32_t optrDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); pOperator->cost.openCost = 0; @@ -485,13 +492,13 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; pOptr = createSessionAggOperatorInfo(ops[0], pSessionNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION == type) { - pOptr = createStreamSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); + pOptr = createStreamSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION == type) { int32_t children = 0; - pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); + pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children, pHandle); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION == type) { int32_t children = pHandle->numOfVgroups; - pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); + pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children, pHandle); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { pOptr = createPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION == type) { @@ -500,7 +507,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*)pPhyNode; pOptr = createStatewindowOperatorInfo(ops[0], pStateNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE == type) { - pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); + pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN == type) { pOptr = createMergeJoinOperatorInfo(ops, size, (SSortMergeJoinPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) { diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index cd9bbbbb02fc91d79559e181351152e3adbe51b2..1cc377b3ee9320c79e30c34a45d0289936da5918 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -73,6 +73,20 @@ static void destroyIndefinitOperatorInfo(void* param) { taosMemoryFreeClear(param); } +void streamOperatorReleaseState(SOperatorInfo* pOperator) { + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.releaseStreamStateFn) { + downstream->fpSet.releaseStreamStateFn(downstream); + } +} + +void streamOperatorReloadState(SOperatorInfo* pOperator) { + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.reloadStreamStateFn) { + downstream->fpSet.reloadStreamStateFn(downstream); + } +} + SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; @@ -136,6 +150,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doProjectOperation, NULL, destroyProjectOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamOperatorReleaseState, streamOperatorReloadState); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c1c99750ea5b44621b53b935c7d83542672d1fb4..7434db61db28dc6adc2ecaeb88ce9bbc719fb79a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,8 +13,6 @@ * along with this program. If not, see . */ -// clang-format off - #include "executorInt.h" #include "filter.h" #include "function.h" @@ -44,6 +42,7 @@ int32_t scanDebug = 0; #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) #define STREAM_SCAN_OP_NAME "StreamScanOperator" +#define STREAM_SCAN_OP_STATE_NAME "StreamScanFillHistoryState" typedef struct STableMergeScanExecInfo { SFileBlockLoadRecorder blockRecorder; @@ -54,8 +53,7 @@ typedef struct STableMergeScanSortSourceParam { SOperatorInfo* pOperator; int32_t readerIdx; uint64_t uid; - SSDataBlock* inputBlock; - STsdbReader* dataReader; + STsdbReader* reader; } STableMergeScanSortSourceParam; typedef struct STableCountScanOperatorInfo { @@ -490,12 +488,12 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int } int32_t code = 0; + bool freeReader = false; // backup the rows int32_t backupRows = pBlock->info.rows; pBlock->info.rows = rows; - bool freeReader = false; STableCachedVal val = {0}; SMetaReader mr = {0}; @@ -1552,20 +1550,96 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock } } -static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock, bool filter) { +static void doBlockDataWindowFilter(SSDataBlock* pBlock, int32_t tsIndex, STimeWindow* pWindow, const char* id) { + if (pWindow->skey != INT64_MIN || pWindow->ekey != INT64_MAX) { + bool* p = taosMemoryCalloc(pBlock->info.rows, sizeof(bool)); + bool hasUnqualified = false; + + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, tsIndex); + + if (pWindow->skey != INT64_MIN) { + qDebug("%s filter for additional history window, skey:%" PRId64, id, pWindow->skey); + + ASSERT(pCol->pData != NULL); + for (int32_t i = 0; i < pBlock->info.rows; ++i) { + int64_t* ts = (int64_t*)colDataGetData(pCol, i); + p[i] = (*ts >= pWindow->skey); + + if (!p[i]) { + hasUnqualified = true; + } + } + } else if (pWindow->ekey != INT64_MAX) { + qDebug("%s filter for additional history window, ekey:%" PRId64, id, pWindow->ekey); + for (int32_t i = 0; i < pBlock->info.rows; ++i) { + int64_t* ts = (int64_t*)colDataGetData(pCol, i); + p[i] = (*ts <= pWindow->ekey); + + if (!p[i]) { + hasUnqualified = true; + } + } + } + + if (hasUnqualified) { + trimDataBlock(pBlock, pBlock->info.rows, p); + } + + taosMemoryFree(p); + } +} + +// re-build the delete block, ONLY according to the split timestamp +static void rebuildDeleteBlockData(SSDataBlock* pBlock, int64_t skey, const char* id) { + if (skey == INT64_MIN) { + return; + } + + int32_t numOfRows = pBlock->info.rows; + + bool* p = taosMemoryCalloc(numOfRows, sizeof(bool)); + bool hasUnqualified = false; + + SColumnInfoData* pSrcStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + uint64_t* tsStartCol = (uint64_t*)pSrcStartCol->pData; + SColumnInfoData* pSrcEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + uint64_t* tsEndCol = (uint64_t*)pSrcEndCol->pData; + + for (int32_t i = 0; i < numOfRows; i++) { + if (tsStartCol[i] < skey) { + tsStartCol[i] = skey; + } + + if (tsEndCol[i] >= skey) { + p[i] = true; + } else { // this row should be removed, since it is not in this query time window, which is [skey, INT64_MAX] + hasUnqualified = true; + } + } + + if (hasUnqualified) { + trimDataBlock(pBlock, pBlock->info.rows, p); + } + + qDebug("%s re-build delete datablock, start key revised to:%"PRId64", rows:%"PRId64, id, skey, pBlock->info.rows); + taosMemoryFree(p); +} + +static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock, STimeWindow* pTimeWindow, bool filter) { SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; SOperatorInfo* pOperator = pInfo->pStreamScanOp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + const char* id = GET_TASKID(pTaskInfo); blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); - pInfo->pRes->info.rows = pBlock->info.rows; - pInfo->pRes->info.id.uid = pBlock->info.id.uid; - pInfo->pRes->info.type = STREAM_NORMAL; - pInfo->pRes->info.version = pBlock->info.version; + pBlockInfo->rows = pBlock->info.rows; + pBlockInfo->id.uid = pBlock->info.id.uid; + pBlockInfo->type = STREAM_NORMAL; + pBlockInfo->version = pBlock->info.version; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - pInfo->pRes->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); + pBlockInfo->id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); // todo extract method for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) { @@ -1595,7 +1669,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, - pInfo->pRes->info.rows, GET_TASKID(pTaskInfo), &pTableScanInfo->base.metaCache); + pBlockInfo->rows, id, &pTableScanInfo->base.metaCache); // ignore the table not exists error, since this table may have been dropped during the scan procedure. if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_PAR_TABLE_NOT_EXIST) { blockDataFreeRes((SSDataBlock*)pBlock); @@ -1610,9 +1684,14 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); } + // filter the block extracted from WAL files, according to the time window apply additional time window filter + doBlockDataWindowFilter(pInfo->pRes, pInfo->primaryTsIndex, pTimeWindow, id); pInfo->pRes->info.dataLoad = 1; + blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); -// blockDataFreeRes((SSDataBlock*)pBlock); + if (pInfo->pRes->info.rows == 0) { + return 0; + } calBlockTbName(pInfo, pInfo->pRes); return 0; @@ -1669,7 +1748,8 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows, pTaskInfo->streamInfo.currentOffset.version); blockDataCleanup(pInfo->pRes); - setBlockIntoRes(pInfo, pRes, true); + STimeWindow defaultWindow = {.skey = INT64_MIN, .ekey = INT64_MAX}; + setBlockIntoRes(pInfo, pRes, &defaultWindow, true); if (pInfo->pRes->info.rows > 0) { return pInfo->pRes; } @@ -1771,7 +1851,7 @@ void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo) return; } - void* pUpInfo = pInfo->stateStore.updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); + void* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); int32_t code = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo); if (code == TSDB_CODE_SUCCESS) { pInfo->pUpdateInfo = pUpInfo; @@ -1785,25 +1865,31 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { SStorageAPI* pAPI = &pTaskInfo->storageAPI; SStreamScanInfo* pInfo = pOperator->info; + SStreamTaskInfo* pStreamInfo = &pTaskInfo->streamInfo; - qDebug("stream scan started, %s", GET_TASKID(pTaskInfo)); + qDebug("stream scan started, %s", id); - if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1 || - pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE2) { + if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__PREPARE1 || pStreamInfo->recoverStep == STREAM_RECOVER_STEP__PREPARE2) { STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - memcpy(&pTSInfo->base.cond, &pTaskInfo->streamInfo.tableCond, sizeof(SQueryTableDataCond)); - if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1) { - pTSInfo->base.cond.startVersion = 0; - pTSInfo->base.cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer1; - qDebug("stream recover step1, verRange:%" PRId64 " - %" PRId64, pTSInfo->base.cond.startVersion, - pTSInfo->base.cond.endVersion); - pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN1; + memcpy(&pTSInfo->base.cond, &pStreamInfo->tableCond, sizeof(SQueryTableDataCond)); + + if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__PREPARE1) { + pTSInfo->base.cond.startVersion = pStreamInfo->fillHistoryVer.minVer; + pTSInfo->base.cond.endVersion = pStreamInfo->fillHistoryVer.maxVer; + + pTSInfo->base.cond.twindows = pStreamInfo->fillHistoryWindow; + qDebug("stream recover step1, verRange:%" PRId64 "-%" PRId64 " window:%"PRId64"-%"PRId64", %s", pTSInfo->base.cond.startVersion, + pTSInfo->base.cond.endVersion, pTSInfo->base.cond.twindows.skey, pTSInfo->base.cond.twindows.ekey, id); + pStreamInfo->recoverStep = STREAM_RECOVER_STEP__SCAN1; + pStreamInfo->recoverScanFinished = false; } else { - pTSInfo->base.cond.startVersion = pTaskInfo->streamInfo.fillHistoryVer1 + 1; - pTSInfo->base.cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer2; - qDebug("stream recover step2, verRange:%" PRId64 " - %" PRId64, pTSInfo->base.cond.startVersion, - pTSInfo->base.cond.endVersion); - pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN2; + pTSInfo->base.cond.startVersion = pStreamInfo->fillHistoryVer.minVer; + pTSInfo->base.cond.endVersion = pStreamInfo->fillHistoryVer.maxVer; + pTSInfo->base.cond.twindows = pStreamInfo->fillHistoryWindow; + qDebug("stream recover step2, verRange:%" PRId64 " - %" PRId64 ", window:%" PRId64 "-%" PRId64 ", %s", + pTSInfo->base.cond.startVersion, pTSInfo->base.cond.endVersion, pTSInfo->base.cond.twindows.skey, + pTSInfo->base.cond.twindows.ekey, id); + pStreamInfo->recoverStep = STREAM_RECOVER_STEP__NONE; } pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); @@ -1813,14 +1899,10 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTSInfo->scanTimes = 0; pTSInfo->currentGroupId = -1; - pTaskInfo->streamInfo.recoverScanFinished = false; } - if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN1 || - pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN2) { - if (pInfo->blockRecoverContiCnt > 100) { - pInfo->blockRecoverTotCnt += pInfo->blockRecoverContiCnt; - pInfo->blockRecoverContiCnt = 0; + if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__SCAN1) { + if (isTaskKilled(pTaskInfo)) { return NULL; } @@ -1830,51 +1912,50 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { printDataBlock(pInfo->pRecoverRes, "scan recover"); return pInfo->pRecoverRes; } break; - case STREAM_SCAN_FROM_UPDATERES: { - generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); - prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; - printDataBlock(pInfo->pUpdateRes, "recover update"); - return pInfo->pUpdateRes; - } break; - case STREAM_SCAN_FROM_DELETE_DATA: { - generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); - prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; - copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); - pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; - printDataBlock(pInfo->pDeleteDataRes, "recover delete"); - return pInfo->pDeleteDataRes; - } break; - case STREAM_SCAN_FROM_DATAREADER_RANGE: { - SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); - if (pSDB) { - STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA; - checkUpdateData(pInfo, true, pSDB, false); - printDataBlock(pSDB, "scan recover update"); - calBlockTbName(pInfo, pSDB); - return pSDB; - } - blockDataCleanup(pInfo->pUpdateDataRes); - pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - } break; + // case STREAM_SCAN_FROM_UPDATERES: { + // generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); + // prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + // pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + // printDataBlock(pInfo->pUpdateRes, "recover update"); + // return pInfo->pUpdateRes; + // } break; + // case STREAM_SCAN_FROM_DELETE_DATA: { + // generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); + // prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + // pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + // copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); + // pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; + // printDataBlock(pInfo->pDeleteDataRes, "recover delete"); + // return pInfo->pDeleteDataRes; + // } break; + // case STREAM_SCAN_FROM_DATAREADER_RANGE: { + // SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); + // if (pSDB) { + // STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; + // pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA; + // checkUpdateData(pInfo, true, pSDB, false); + // printDataBlock(pSDB, "scan recover update"); + // calBlockTbName(pInfo, pSDB); + // return pSDB; + // } + // blockDataCleanup(pInfo->pUpdateDataRes); + // pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + // } break; default: break; } pInfo->pRecoverRes = doTableScan(pInfo->pTableScanOp); if (pInfo->pRecoverRes != NULL) { - pInfo->blockRecoverContiCnt++; calBlockTbName(pInfo, pInfo->pRecoverRes); if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { - if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN1) { - TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); - } else { - pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pTaskInfo->streamInfo.fillHistoryVer2); - doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes); - } + // if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__SCAN1) { + TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); + // } else { + // pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pStreamInfo->fillHistoryVer.maxVer); + // doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes); + // } } if (pInfo->pCreateTbRes->info.rows > 0) { pInfo->scanMode = STREAM_SCAN_FROM_RES; @@ -1886,7 +1967,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { printDataBlock(pInfo->pRecoverRes, "scan recover"); return pInfo->pRecoverRes; } - pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; + pStreamInfo->recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); @@ -1895,7 +1976,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTSInfo->base.cond.startVersion = -1; pTSInfo->base.cond.endVersion = -1; - pTaskInfo->streamInfo.recoverScanFinished = true; + pStreamInfo->recoverScanFinished = true; return NULL; } @@ -1914,7 +1995,7 @@ FETCH_NEXT_BLOCK: SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); SSDataBlock* pBlock = pPacked->pDataBlock; if (pBlock->info.parTbName[0]) { - pAPI->stateStore.streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); + pAPI->stateStore.streamStatePutParName(pStreamInfo->pState, pBlock->info.id.groupId, pBlock->info.parTbName); } // TODO move into scan @@ -1924,6 +2005,7 @@ FETCH_NEXT_BLOCK: if (pInfo->pUpdateInfo) { pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); } + blockDataUpdateTsWindow(pBlock, 0); switch (pBlock->info.type) { case STREAM_NORMAL: @@ -1946,7 +2028,9 @@ FETCH_NEXT_BLOCK: } else { pDelBlock = pBlock; } + setBlockGroupIdByUid(pInfo, pDelBlock); + rebuildDeleteBlockData(pDelBlock, pStreamInfo->fillHistoryWindow.skey, id); printDataBlock(pDelBlock, "stream scan delete recv filtered"); if (pDelBlock->info.rows == 0) { if (pInfo->tqReader) { @@ -1954,6 +2038,7 @@ FETCH_NEXT_BLOCK: } goto FETCH_NEXT_BLOCK; } + if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) { generateDeleteResultBlock(pInfo, pDelBlock, pInfo->pDeleteDataRes); pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; @@ -2045,8 +2130,7 @@ FETCH_NEXT_BLOCK: return pInfo->pUpdateRes; } - SSDataBlock* pBlock = pInfo->pRes; - SDataBlockInfo* pBlockInfo = &pBlock->info; + SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; int32_t totalBlocks = taosArrayGetSize(pInfo->pBlockLists); NEXT_SUBMIT_BLK: @@ -2070,21 +2154,23 @@ FETCH_NEXT_BLOCK: } } - blockDataCleanup(pBlock); + blockDataCleanup(pInfo->pRes); while (pAPI->tqReaderFn.tqNextBlockImpl(pInfo->tqReader, id)) { SSDataBlock* pRes = NULL; int32_t code = pAPI->tqReaderFn.tqRetrieveBlock(pInfo->tqReader, &pRes, id); - qDebug("retrieve data from submit completed code:%s, rows:%" PRId64 " %s", tstrerror(code), pRes->info.rows, - id); + qDebug("retrieve data from submit completed code:%s rows:%" PRId64 " %s", tstrerror(code), pRes->info.rows, id); if (code != TSDB_CODE_SUCCESS || pRes->info.rows == 0) { qDebug("retrieve data failed, try next block in submit block, %s", id); continue; } - setBlockIntoRes(pInfo, pRes, false); + setBlockIntoRes(pInfo, pRes, &pStreamInfo->fillHistoryWindow, false); + if (pInfo->pRes->info.rows == 0) { + continue; + } if (pInfo->pCreateTbRes->info.rows > 0) { pInfo->scanMode = STREAM_SCAN_FROM_RES; @@ -2092,14 +2178,12 @@ FETCH_NEXT_BLOCK: return pInfo->pCreateTbRes; } - doCheckUpdate(pInfo, pBlockInfo->window.ekey, pBlock); - doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL); - pBlock->info.dataLoad = 1; - blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex); + doCheckUpdate(pInfo, pBlockInfo->window.ekey, pInfo->pRes); + doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); - qDebug("%" PRId64 " rows in datablock, update res:%" PRId64 " %s", pBlockInfo->rows, - pInfo->pUpdateDataRes->info.rows, id); - if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { + int64_t numOfUpdateRes = pInfo->pUpdateDataRes->info.rows; + qDebug("%s %" PRId64 " rows in datablock, update res:%" PRId64, id, pBlockInfo->rows, numOfUpdateRes); + if (pBlockInfo->rows > 0 || numOfUpdateRes > 0) { break; } } @@ -2117,7 +2201,7 @@ FETCH_NEXT_BLOCK: qDebug("stream scan completed, and return source rows:%" PRId64", %s", pBlockInfo->rows, id); if (pBlockInfo->rows > 0) { - return pBlock; + return pInfo->pRes; } if (pInfo->pUpdateDataRes->info.rows > 0) { @@ -2156,7 +2240,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { qDebug("tmqsnap doRawScan called"); if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { bool hasNext = false; - if (pInfo->dataReader) { + if (pInfo->dataReader && pInfo->sContext->withMeta != ONLY_META) { code = pAPI->tsdReader.tsdNextDataBlock(pInfo->dataReader, &hasNext); if (code) { pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->dataReader); @@ -2182,7 +2266,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { SMetaTableInfo mtInfo = pAPI->snapshotFn.getMetaTableInfoFromSnapshot(pInfo->sContext); STqOffsetVal offset = {0}; - if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal + if (mtInfo.uid == 0 || pInfo->sContext->withMeta == ONLY_META) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); tqOffsetResetToLog(&offset, pInfo->sContext->snapVersion + 1); } else { @@ -2296,6 +2380,57 @@ static void destroyStreamScanOperatorInfo(void* param) { taosMemoryFree(pStreamScan); } +void streamScanReleaseState(SOperatorInfo* pOperator) { + SStreamScanInfo* pInfo = pOperator->info; + if (!pInfo->pState) { + return; + } + if (!pInfo->pUpdateInfo) { + return; + } + int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); + void* pBuff = taosMemoryCalloc(1, len); + pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo); + pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), pBuff, len); + taosMemoryFree(pBuff); +} + +void streamScanReloadState(SOperatorInfo* pOperator) { + SStreamScanInfo* pInfo = pOperator->info; + if (!pInfo->pState) { + return; + } + void* pBuff = NULL; + int32_t len = 0; + pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), &pBuff, &len); + SUpdateInfo* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); + int32_t code = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo); + taosMemoryFree(pBuff); + if (code == TSDB_CODE_SUCCESS && pInfo->pUpdateInfo) { + if (pInfo->pUpdateInfo->minTS < 0) { + pInfo->stateStore.updateInfoDestroy(pInfo->pUpdateInfo); + pInfo->pUpdateInfo = pUpInfo; + } else { + pInfo->pUpdateInfo->minTS = TMAX(pInfo->pUpdateInfo->minTS, pUpInfo->minTS); + pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pUpInfo->maxDataVersion); + SHashObj* curMap = pInfo->pUpdateInfo->pMap; + void *pIte = taosHashIterate(curMap, NULL); + while (pIte != NULL) { + size_t keySize = 0; + int64_t* pUid = taosHashGetKey(pIte, &keySize); + taosHashPut(pUpInfo->pMap, pUid, sizeof(int64_t), pIte, sizeof(TSKEY)); + pIte = taosHashIterate(curMap, pIte); + } + taosHashCleanup(curMap); + pInfo->pUpdateInfo->pMap = pUpInfo->pMap; + pUpInfo->pMap = NULL; + pInfo->stateStore.updateInfoDestroy(pUpInfo); + } + } else { + pInfo->stateStore.updateInfoDestroy(pUpInfo); + } +} + SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { SArray* pColIds = NULL; @@ -2457,7 +2592,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->igCheckUpdate = pTableScanNode->igCheckUpdate; pInfo->igExpired = pTableScanNode->igExpired; pInfo->twAggSup.maxTs = INT64_MIN; - pInfo->pState = NULL; + pInfo->pState = pTaskInfo->streamInfo.pState; pInfo->stateStore = pTaskInfo->storageAPI.stateStore; pInfo->readerFn = pTaskInfo->storageAPI.tqReaderFn; @@ -2477,6 +2612,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys __optr_fn_t nextFn = (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) ? doStreamScan : doQueueScan; pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamScanReleaseState, streamScanReloadState); return pOperator; @@ -2648,39 +2784,25 @@ _error: return NULL; } -static SSDataBlock* getTableDataBlockImpl(void* param) { +static SSDataBlock* getBlockForTableMergeScan(void* param) { STableMergeScanSortSourceParam* source = param; SOperatorInfo* pOperator = source->pOperator; STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - int32_t readIdx = source->readerIdx; - SSDataBlock* pBlock = source->inputBlock; + SSDataBlock* pBlock = pInfo->pReaderBlock; int32_t code = 0; - SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx); - int64_t st = taosGetTimestampUs(); - void* p = tableListGetInfo(pInfo->base.pTableListInfo, readIdx + pInfo->tableStartIndex); - SReadHandle* pHandle = &pInfo->base.readHandle; - if (NULL == source->dataReader) { - code = pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, (void**)&source->dataReader, GET_TASKID(pTaskInfo), false, NULL); - if (code != 0) { - T_LONG_JMP(pTaskInfo->env, code); - } - } - - pInfo->base.dataReader = source->dataReader; - STsdbReader* reader = pInfo->base.dataReader; bool hasNext = false; - qTrace("tsdb/read-table-data: %p, enter next reader", reader); + STsdbReader* reader = pInfo->base.dataReader; while (true) { code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext); if (code != 0) { pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); - pInfo->base.dataReader = NULL; + qError("table merge scan fetch next data block error code: %d, %s", code, GET_TASKID(pTaskInfo)); T_LONG_JMP(pTaskInfo->env, code); } @@ -2689,9 +2811,9 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } if (isTaskKilled(pTaskInfo)) { + qInfo("table merge scan fetch next data block found task killed. %s", GET_TASKID(pTaskInfo)); pAPI->tsdReader.tsdReaderReleaseDataBlock(reader); - pInfo->base.dataReader = NULL; - T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + break; } // process this data block based on the probabilities @@ -2700,16 +2822,11 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { continue; } - if (pQueryCond->order == TSDB_ORDER_ASC) { - pQueryCond->twindows.skey = pBlock->info.window.ekey + 1; - } else { - pQueryCond->twindows.ekey = pBlock->info.window.skey - 1; - } - uint32_t status = 0; code = loadDataBlock(pOperator, &pInfo->base, pBlock, &status); // code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { + qInfo("table merge scan load datablock code %d, %s", code, GET_TASKID(pTaskInfo)); T_LONG_JMP(pTaskInfo->env, code); } @@ -2727,16 +2844,9 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; - qTrace("tsdb/read-table-data: %p, close reader", reader); - pInfo->base.dataReader = NULL; return pBlock; } - pAPI->tsdReader.tsdReaderClose(source->dataReader); - source->dataReader = NULL; - pInfo->base.dataReader = NULL; - blockDataDestroy(source->inputBlock); - source->inputBlock = NULL; return NULL; } @@ -2772,6 +2882,8 @@ int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SReadHandle* pHandle = &pInfo->base.readHandle; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; { size_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); @@ -2788,53 +2900,38 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t tableStartIdx = pInfo->tableStartIndex; int32_t tableEndIdx = pInfo->tableEndIndex; - pInfo->base.dataReader = NULL; - - // todo the total available buffer should be determined by total capacity of buffer of this task. - // the additional one is reserved for merge result - // pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1); - int32_t kWay = (TSDB_MAX_BYTES_PER_ROW * 2) / (pInfo->pResBlock->info.rowSize); - if (kWay >= 128) { - kWay = 128; - } else if (kWay <= 2) { - kWay = 2; + bool hasLimit = pInfo->limitInfo.limit.limit != -1 || pInfo->limitInfo.limit.offset != -1; + int64_t mergeLimit = -1; + if (hasLimit) { + mergeLimit = pInfo->limitInfo.limit.limit + pInfo->limitInfo.limit.offset; + } + size_t szRow = blockDataGetRowSize(pInfo->pResBlock); + if (hasLimit) { + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, + NULL, pTaskInfo->id.str, mergeLimit, szRow+8, tsPQSortMemThreshold * 1024* 1024); } else { - int i = 2; - while (i * 2 <= kWay) i = i * 2; - kWay = i; + pInfo->sortBufSize = 2048 * pInfo->bufPageSize; + int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, + pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); + + tsortSetMergeLimit(pInfo->pSortHandle, mergeLimit); } - pInfo->sortBufSize = pInfo->bufPageSize * (kWay + 1); - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, - pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); - - tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlockImpl, NULL, NULL); + tsortSetFetchRawDataFp(pInfo->pSortHandle, getBlockForTableMergeScan, NULL, NULL); // one table has one data block int32_t numOfTable = tableEndIdx - tableStartIdx + 1; - pInfo->queryConds = taosArrayInit(numOfTable, sizeof(SQueryTableDataCond)); - for (int32_t i = 0; i < numOfTable; ++i) { - STableMergeScanSortSourceParam param = {0}; - param.readerIdx = i; - param.pOperator = pOperator; - param.inputBlock = createOneDataBlock(pInfo->pResBlock, false); + STableMergeScanSortSourceParam param = {0}; + param.pOperator = pOperator; + STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx); + pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, NULL); - taosArrayPush(pInfo->sortSourceParams, ¶m); - - SQueryTableDataCond cond; - dumpQueryTableCond(&pInfo->base.cond, &cond); - taosArrayPush(pInfo->queryConds, &cond); - } - - for (int32_t i = 0; i < numOfTable; ++i) { - SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); - STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); - ps->param = param; - ps->onlyRef = true; - tsortAddSource(pInfo->pSortHandle, ps); - } + SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); + ps->param = ¶m; + ps->onlyRef = true; + tsortAddSource(pInfo->pSortHandle, ps); int32_t code = tsortOpen(pInfo->pSortHandle); @@ -2850,8 +2947,6 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; - int32_t numOfTable = taosArrayGetSize(pInfo->queryConds); - SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle); pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod; pInfo->sortExecInfo.sortBuffer = sortExecInfo.sortBuffer; @@ -2859,24 +2954,14 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes; pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes; - for (int32_t i = 0; i < numOfTable; ++i) { - STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); - blockDataDestroy(param->inputBlock); - pAPI->tsdReader.tsdReaderClose(param->dataReader); - param->dataReader = NULL; + if (pInfo->base.dataReader != NULL) { + pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader); + pInfo->base.dataReader = NULL; } - taosArrayClear(pInfo->sortSourceParams); tsortDestroySortHandle(pInfo->pSortHandle); pInfo->pSortHandle = NULL; - for (int32_t i = 0; i < taosArrayGetSize(pInfo->queryConds); i++) { - SQueryTableDataCond* cond = taosArrayGet(pInfo->queryConds, i); - taosMemoryFree(cond->colList); - } - taosArrayDestroy(pInfo->queryConds); - pInfo->queryConds = NULL; - resetLimitInfoForNextGroup(&pInfo->limitInfo); return TSDB_CODE_SUCCESS; } @@ -2889,28 +2974,32 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; blockDataCleanup(pResBlock); - + STupleHandle* pTupleHandle = NULL; while (1) { - STupleHandle* pTupleHandle = tsortNextTuple(pHandle); - if (pTupleHandle == NULL) { - break; + while (1) { + pTupleHandle = tsortNextTuple(pHandle); + if (pTupleHandle == NULL) { + break; + } + + appendOneRowToDataBlock(pResBlock, pTupleHandle); + if (pResBlock->info.rows >= capacity) { + break; + } } - appendOneRowToDataBlock(pResBlock, pTupleHandle); - if (pResBlock->info.rows >= capacity) { - break; + if (tsortIsClosed(pHandle)) { + terrno = TSDB_CODE_TSC_QUERY_CANCELLED; + T_LONG_JMP(pOperator->pTaskInfo->env, terrno); } - } - if (tsortIsClosed(pHandle)) { - terrno = TSDB_CODE_TSC_QUERY_CANCELLED; - T_LONG_JMP(pOperator->pTaskInfo->env, terrno); + bool limitReached = applyLimitOffset(&pInfo->limitInfo, pResBlock, pTaskInfo); + qDebug("%s get sorted row block, rows:%" PRId64 ", limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows, + pInfo->limitInfo.numOfOutputRows); + if (pTupleHandle == NULL || limitReached || pResBlock->info.rows > 0) { + break; + } } - - bool limitReached = applyLimitOffset(&pInfo->limitInfo, pResBlock, pTaskInfo); - qDebug("%s get sorted row block, rows:%" PRId64 ", limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows, - pInfo->limitInfo.numOfOutputRows); - return (pResBlock->info.rows > 0) ? pResBlock : NULL; } @@ -2974,14 +3063,7 @@ void destroyTableMergeScanOperatorInfo(void* param) { STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param; cleanupQueryTableDataCond(&pTableScanInfo->base.cond); - int32_t numOfTable = taosArrayGetSize(pTableScanInfo->queryConds); - - for (int32_t i = 0; i < numOfTable; i++) { - STableMergeScanSortSourceParam* p = taosArrayGet(pTableScanInfo->sortSourceParams, i); - blockDataDestroy(p->inputBlock); - pTableScanInfo->base.readerAPI.tsdReaderClose(p->dataReader); - p->dataReader = NULL; - } + int32_t numOfTable = taosArrayGetSize(pTableScanInfo->sortSourceParams); pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; @@ -2990,16 +3072,11 @@ void destroyTableMergeScanOperatorInfo(void* param) { tsortDestroySortHandle(pTableScanInfo->pSortHandle); pTableScanInfo->pSortHandle = NULL; - for (int i = 0; i < taosArrayGetSize(pTableScanInfo->queryConds); i++) { - SQueryTableDataCond* pCond = taosArrayGet(pTableScanInfo->queryConds, i); - taosMemoryFree(pCond->colList); - } - - taosArrayDestroy(pTableScanInfo->queryConds); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); + pTableScanInfo->pReaderBlock = blockDataDestroy(pTableScanInfo->pReaderBlock); taosArrayDestroy(pTableScanInfo->pSortInfo); taosMemoryFreeClear(param); @@ -3061,6 +3138,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->base.scanFlag = MAIN_SCAN; pInfo->base.readHandle = *readHandle; + pInfo->readIdx = -1; + pInfo->base.limitInfo.limit.limit = -1; pInfo->base.limitInfo.slimit.limit = -1; pInfo->base.pTableListInfo = pTableListInfo; @@ -3083,6 +3162,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false); initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo); + pInfo->pReaderBlock = createOneDataBlock(pInfo->pResBlock, false); + int32_t rowSize = pInfo->pResBlock->info.rowSize; uint32_t nCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock); pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols); @@ -3491,6 +3572,4 @@ static void destoryTableCountScanOperator(void* param) { taosArrayDestroy(pTableCountScanInfo->stbUidList); taosMemoryFreeClear(param); -} - -// clang-format on +} \ No newline at end of file diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 20fb588a026d7c287fa005188faa854d3a6ebb1f..459474d06e79ea0c8407509a765d887c2cd3d469 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -54,15 +54,19 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* int32_t numOfCols = 0; pOperator->exprSupp.pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols); pOperator->exprSupp.numOfExprs = numOfCols; - calcSortOperMaxTupleLength(pInfo, pSortNode->pSortKeys); - pInfo->maxRows = pSortNode->maxRows; - int32_t numOfOutputCols = 0; int32_t code = extractColMatchInfo(pSortNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } + + calcSortOperMaxTupleLength(pInfo, pSortNode->pSortKeys); + pInfo->maxRows = -1; + if (pSortNode->node.pLimit) { + SLimitNode* pLimit = (SLimitNode*)pSortNode->node.pLimit; + if (pLimit->limit > 0) pInfo->maxRows = pLimit->limit + pLimit->offset; + } pOperator->exprSupp.pCtx = createSqlFunctionCtx(pOperator->exprSupp.pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); @@ -718,7 +722,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData resetLimitInfoForNextGroup(&pInfo->limitInfo); } - if (p->info.rows > 0) { + if (p->info.rows > 0 || limitReached) { break; } } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 70fe42595e63f1135df3dbf6cab7a27163f0a6c5..a1f83dda2f051f7ea09b4048b0fa5a2634c49d22 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -966,20 +966,20 @@ static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo, // table name pColInfoData = taosArrayGet(dataBlock->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, tName, false); + colDataSetVal(pColInfoData, numOfRows, tName, false); // database name pColInfoData = taosArrayGet(dataBlock->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); + colDataSetVal(pColInfoData, numOfRows, dbname, false); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, tableType, false); + colDataSetVal(pColInfoData, numOfRows, tableType, false); // col name char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(colName, schemaRow->pSchema[i].name); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, colName, false); + colDataSetVal(pColInfoData, numOfRows, colName, false); // col type int8_t colType = schemaRow->pSchema[i].type; @@ -994,10 +994,10 @@ static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo, (int32_t)((schemaRow->pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(colTypeStr, colTypeLen); - colDataAppend(pColInfoData, numOfRows, (char*)colTypeStr, false); + colDataSetVal(pColInfoData, numOfRows, (char*)colTypeStr, false); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (const char*)&schemaRow->pSchema[i].bytes, false); + colDataSetVal(pColInfoData, numOfRows, (const char*)&schemaRow->pSchema[i].bytes, false); for (int32_t j = 6; j <= 8; ++j) { pColInfoData = taosArrayGet(dataBlock->pDataBlock, j); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 7cc50a70ab799184de5b350f72526ee44ac23295..4e0dff9d4f3474c2b2335d79b0cde362f566dfa1 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -66,20 +66,25 @@ static void setNullRow(SSDataBlock* pBlock, SFillInfo* pFillInfo, int32_t rowInd } static void doSetUserSpecifiedValue(SColumnInfoData* pDst, SVariant* pVar, int32_t rowIndex, int64_t currentKey) { + bool isNull = (TSDB_DATA_TYPE_NULL == pVar->nType) ? true : false; if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { float v = 0; - GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); - colDataSetVal(pDst, rowIndex, (char*)&v, false); + GET_TYPED_DATA(v, float, pVar->nType, &pVar->f); + colDataSetVal(pDst, rowIndex, (char*)&v, isNull); } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { double v = 0; - GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); - colDataSetVal(pDst, rowIndex, (char*)&v, false); + GET_TYPED_DATA(v, double, pVar->nType, &pVar->d); + colDataSetVal(pDst, rowIndex, (char*)&v, isNull); } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { int64_t v = 0; GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); - colDataSetVal(pDst, rowIndex, (char*)&v, false); + colDataSetVal(pDst, rowIndex, (char*)&v, isNull); + } else if (IS_UNSIGNED_NUMERIC_TYPE(pDst->info.type)) { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pVar->nType, &pVar->u); + colDataSetVal(pDst, rowIndex, (char*)&v, isNull); } else if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - colDataSetVal(pDst, rowIndex, (const char*)¤tKey, false); + colDataSetVal(pDst, rowIndex, (const char*)¤tKey, isNull); } else { // varchar/nchar data colDataSetNULL(pDst, rowIndex); } @@ -247,7 +252,7 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo) { static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull); -static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVal* pRowVal) { +static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVal* pRowVal, bool reset) { SColumnInfoData* pTsCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, pFillInfo->srcTsSlotId); pRowVal->key = ((int64_t*)pTsCol->pData)[rowIndex]; @@ -268,7 +273,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVa bool isNull = colDataIsNull_s(pSrcCol, rowIndex); char* p = colDataGetData(pSrcCol, rowIndex); - saveColData(pRowVal->pRowVal, i, p, isNull); + saveColData(pRowVal->pRowVal, i, p, reset ? true : isNull); } else { ASSERT(0); } @@ -293,10 +298,10 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t // set the next value for interpolation if (pFillInfo->currentKey < ts && ascFill) { SRowVal* pRVal = pFillInfo->type == TSDB_FILL_NEXT ? &pFillInfo->next : &pFillInfo->prev; - copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal); + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal, false); } else if (pFillInfo->currentKey > ts && !ascFill) { SRowVal* pRVal = pFillInfo->type == TSDB_FILL_NEXT ? &pFillInfo->prev : &pFillInfo->next; - copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal); + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal, false); } if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && @@ -316,9 +321,14 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t ASSERT(pFillInfo->currentKey == ts); int32_t index = pBlock->info.rows; - if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) { + if (pFillInfo->type == TSDB_FILL_NEXT) { int32_t nextRowIndex = pFillInfo->index + 1; - copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next); + if ((pFillInfo->index + 1) < pFillInfo->numOfRows) { + copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next, false); + } else { + // reset to null after last row + copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next, true); + } } // copy rows to dst buffer diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index db1eed851a0ed36f5b3dd3e8acbc93fe40e59d06..b01998564513d20d17267a64776d0d20f2461c96 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -272,7 +272,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp continue; } else if (isIsfilledPseudoColumn(pExprInfo)) { bool isFilled = true; - colDataAppend(pDst, pResBlock->info.rows, (char*)&isFilled, false); + colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false); continue; } else if (!isInterpFunc(pExprInfo)) { if (isGroupKeyFunc(pExprInfo)) { @@ -312,6 +312,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp case TSDB_FILL_SET_VALUE_F: { SVariant* pVar = &pSliceInfo->pFillColInfo[fillColIndex].fillVal; + bool isNull = (TSDB_DATA_TYPE_NULL == pVar->nType) ? true : false; if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { float v = 0; if (!IS_VAR_DATA_TYPE(pVar->nType)) { @@ -319,7 +320,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else { v = taosStr2Float(varDataVal(pVar->pz), NULL); } - colDataSetVal(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, isNull); } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { double v = 0; if (!IS_VAR_DATA_TYPE(pVar->nType)) { @@ -327,7 +328,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else { v = taosStr2Double(varDataVal(pVar->pz), NULL); } - colDataSetVal(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, isNull); } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { int64_t v = 0; if (!IS_VAR_DATA_TYPE(pVar->nType)) { @@ -335,7 +336,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else { v = taosStr2Int64(varDataVal(pVar->pz), NULL, 10); } - colDataSetVal(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, isNull); } else if (IS_UNSIGNED_NUMERIC_TYPE(pDst->info.type)) { uint64_t v = 0; if (!IS_VAR_DATA_TYPE(pVar->nType)) { @@ -343,7 +344,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else { v = taosStr2UInt64(varDataVal(pVar->pz), NULL, 10); } - colDataSetVal(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, isNull); } else if (IS_BOOLEAN_TYPE(pDst->info.type)) { bool v = false; if (!IS_VAR_DATA_TYPE(pVar->nType)) { @@ -351,7 +352,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } else { v = taosStr2Int8(varDataVal(pVar->pz), NULL, 10); } - colDataSetVal(pDst, rows, (char*)&v, false); + colDataSetVal(pDst, rows, (char*)&v, isNull); } ++fillColIndex; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 5fd46f572a045cc02bcae279918bd3a695be848e..93a84c6f040e32accb1a98ddeace7a2bb8b1eb2d 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -28,6 +28,9 @@ #define IS_FINAL_OP(op) ((op)->isFinal) #define DEAULT_DELETE_MARK (1000LL * 60LL * 60LL * 24LL * 365LL * 10LL); +#define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState" +#define STREAM_SESSION_OP_STATE_NAME "StreamSessionHistoryState" +#define STREAM_STATE_OP_STATE_NAME "StreamStateHistoryState" typedef struct SStateWindowInfo { SResultWindowInfo winInfo; @@ -2726,6 +2729,39 @@ int32_t getMaxFunResSize(SExprSupp* pSup, int32_t numOfCols) { return size; } +void streamIntervalReleaseState(SOperatorInfo* pOperator) { + if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { + SStreamIntervalOperatorInfo* pInfo = pOperator->info; + int32_t resSize = sizeof(TSKEY); + pInfo->statestore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, strlen(STREAM_INTERVAL_OP_STATE_NAME), &pInfo->twAggSup.maxTs, resSize); + } + SStreamIntervalOperatorInfo* pInfo = pOperator->info; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + pAPI->stateStore.streamStateCommit(pInfo->pState); + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.releaseStreamStateFn) { + downstream->fpSet.releaseStreamStateFn(downstream); + } +} + +void streamIntervalReloadState(SOperatorInfo* pOperator) { + if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { + SStreamIntervalOperatorInfo* pInfo = pOperator->info; + int32_t size = 0; + void* pBuf = NULL; + int32_t code = pInfo->statestore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, + strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); + TSKEY ts = *(TSKEY*)pBuf; + taosMemoryFree(pBuf); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); + pInfo->statestore.streamStateReloadInfo(pInfo->pState, ts); + } + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.reloadStreamStateFn) { + downstream->fpSet.reloadStreamStateFn(downstream); + } +} + SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; @@ -2823,7 +2859,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pUpdatedMap = NULL; int32_t funResSize= getMaxFunResSize(&pOperator->exprSupp, numOfCols); pInfo->pState->pFileState = pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, - compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); + compareTs, pInfo->pState, pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo)); pInfo->dataVersion = 0; pInfo->statestore = pTaskInfo->storageAPI.stateStore; pInfo->recvGetAll = false; @@ -2835,6 +2871,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { initIntervalDownStream(downstream, pPhyNode->type, pInfo); } @@ -2873,12 +2910,15 @@ void destroyStreamSessionAggOperatorInfo(void* param) { } taosArrayDestroy(pInfo->pChildren); } + colDataDestroy(&pInfo->twAggSup.timeWindowData); blockDataDestroy(pInfo->pDelRes); blockDataDestroy(pInfo->pWinBlock); blockDataDestroy(pInfo->pUpdateRes); + tSimpleHashCleanup(pInfo->pStUpdated); tSimpleHashCleanup(pInfo->pStDeleted); + taosArrayDestroy(pInfo->historyWins); taosMemoryFreeClear(param); } @@ -2928,7 +2968,7 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin } int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int64_t gap, - SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore) { + SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore, SReadHandle* pHandle, SStorageAPI* pApi) { pSup->resultRowSize = keySize + getResultRowSize(pCtx, numOfOutput); pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR); pSup->gap = gap; @@ -2970,6 +3010,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, pCtx[i].saveHandle.pBuf = pSup->pResultBuf; } + pSup->pSessionAPI = pApi; + return TSDB_CODE_SUCCESS; } @@ -2997,6 +3039,13 @@ void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT bool isInvalidSessionWin(SResultWindowInfo* pWinInfo) { return pWinInfo->sessionWin.win.skey == 0; } +bool inWinRange(STimeWindow* range, STimeWindow* cur) { + if (cur->skey >= range->skey && cur->ekey <= range->ekey) { + return true; + } + return false; +} + void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, SResultWindowInfo* pCurWin) { pCurWin->sessionWin.groupId = groupId; @@ -3005,8 +3054,15 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT int32_t size = pAggSup->resultRowSize; int32_t code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap, &pCurWin->pOutputBuf, &size); + if (code == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) { + code = TSDB_CODE_FAILED; + releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)pCurWin->pOutputBuf, &pAggSup->pSessionAPI->stateStore); + pCurWin->pOutputBuf = taosMemoryCalloc(1, size); + } + if (code == TSDB_CODE_SUCCESS) { pCurWin->isOutput = true; + pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->sessionWin); } else { pCurWin->sessionWin.win.skey = startTs; pCurWin->sessionWin.win.ekey = endTs; @@ -3136,11 +3192,12 @@ SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* return pCur; } -static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated, - SSHashObj* pStDeleted) { - SExprSupp* pSup = &pOperator->exprSupp; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; +static int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated, + SSHashObj* pStDeleted, bool addGap) { + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + int32_t winNum = 0; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SResultRow* pCurResult = NULL; @@ -3151,7 +3208,8 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC while (1) { SResultWindowInfo winInfo = {0}; SStreamStateCur* pCur = getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo); - if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap)) { + if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap) || + !inWinRange(&pAggSup->winRange, &winInfo.sessionWin.win)) { taosMemoryFree(winInfo.pOutputBuf); pAPI->stateStore.streamStateFreeCur(pCur); break; @@ -3160,7 +3218,7 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC initSessionOutputBuf(&winInfo, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset); pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, winInfo.sessionWin.win.ekey); int64_t winDelta = 0; - if (IS_FINAL_OP(pInfo)) { + if (addGap) { winDelta = pAggSup->gap; } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->sessionWin.win, winDelta); @@ -3173,11 +3231,14 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC doDeleteSessionWindow(pAggSup, &winInfo.sessionWin); pAPI->stateStore.streamStateFreeCur(pCur); taosMemoryFree(winInfo.pOutputBuf); + winNum++; } + return winNum; } int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) { saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize, &pAggSup->stateStore); + pWinInfo->pOutputBuf = NULL; return TSDB_CODE_SUCCESS; } @@ -3191,8 +3252,13 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData SResultRow* pResult = NULL; int32_t rows = pSDataBlock->info.rows; int32_t winRows = 0; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); + pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; + if (pAggSup->winRange.ekey <= 0) { + pAggSup->winRange.ekey = INT64_MAX; + } SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; @@ -3204,7 +3270,6 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData } TSKEY* endTsCols = (int64_t*)pEndTsCol->pData; - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; for (int32_t i = 0; i < rows;) { if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) { i++; @@ -3229,7 +3294,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData if (code != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } - compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted); + compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted, addGap); saveSessionOutputBuf(pAggSup, &winInfo); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { @@ -3375,8 +3440,12 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS SResultWindowInfo childWin = {0}; childWin.sessionWin = *pWinKey; int32_t code = getSessionWinBuf(pChAggSup, pCur, &childWin); - if (code == TSDB_CODE_SUCCESS && pWinKey->win.skey <= childWin.sessionWin.win.skey && - childWin.sessionWin.win.ekey <= pWinKey->win.ekey) { + + if (code == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &childWin.sessionWin.win)) { + continue; + } + + if (code == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) { if (num == 0) { setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin); code = initSessionOutputBuf(&parentWin, &pResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); @@ -3389,7 +3458,7 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput, pChild->exprSupp.rowEntryInfoOffset); compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); - compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL); + compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL, true); saveResult(parentWin, pStUpdated); } else { break; @@ -3475,6 +3544,29 @@ void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* // clear the existed group id pBlock->info.id.groupId = 0; buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo); + if (pBlock->info.rows == 0) { + cleanupGroupResInfo(pGroupResInfo); + } +} +void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins) { + int32_t size = taosArrayGetSize(pAllWins); + if (size == 0) { + return; + } + + SSessionKey* pSeKey = taosArrayGet(pAllWins, size - 1); + taosArrayPush(pMaxWins, pSeKey); + if (pSeKey->groupId == 0) { + return; + } + uint64_t preGpId = pSeKey->groupId; + for (int32_t i = size - 2; i >= 0; i--) { + pSeKey = taosArrayGet(pAllWins, i); + if (preGpId != pSeKey->groupId) { + taosArrayPush(pMaxWins, pSeKey); + preGpId = pSeKey->groupId; + } + } } static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { @@ -3554,7 +3646,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { // if chIndex + 1 - size > 0, add new child for (int32_t i = 0; i < chIndex + 1 - size; i++) { SOperatorInfo* pChildOp = - createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0); + createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0, NULL); if (!pChildOp) { T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } @@ -3576,6 +3668,9 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { removeSessionResults(pInfo->pStDeleted, pInfo->pUpdated); tSimpleHashCleanup(pInfo->pStUpdated); pInfo->pStUpdated = NULL; + if(pInfo->isHistoryOp) { + getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + } initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); @@ -3602,8 +3697,68 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return NULL; } +void streamSessionReleaseState(SOperatorInfo* pOperator) { + if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION) { + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; + int32_t resSize = taosArrayGetSize(pInfo->historyWins) * sizeof(SSessionKey); + pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_STATE_NAME, strlen(STREAM_SESSION_OP_STATE_NAME), pInfo->historyWins->pData, resSize); + } + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.releaseStreamStateFn) { + downstream->fpSet.releaseStreamStateFn(downstream); + } +} + +void resetWinRange(STimeWindow* winRange) { + winRange->skey = INT64_MIN; + winRange->ekey = INT64_MAX; +} + +void streamSessionReloadState(SOperatorInfo* pOperator) { + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + resetWinRange(&pAggSup->winRange); + + SResultWindowInfo winInfo = {0}; + int32_t size = 0; + void* pBuf = NULL; + int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + int32_t num = size / sizeof(SSessionKey); + SSessionKey* pSeKeyBuf = (SSessionKey*) pBuf; + ASSERT(size == num * sizeof(SSessionKey)); + if (!pInfo->pStUpdated && num > 0) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pStUpdated = tSimpleHashInit(64, hashFn); + } + for (int32_t i = 0; i < num; i++) { + SResultWindowInfo winInfo = {0}; + setSessionOutputBuf(pAggSup, pSeKeyBuf[i].win.skey, pSeKeyBuf[i].win.ekey, pSeKeyBuf[i].groupId, &winInfo); + int32_t winNum = compactSessionWindow(pOperator, &winInfo, pInfo->pStUpdated, pInfo->pStDeleted, true); + if (winNum > 0) { + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { + saveResult(winInfo, pInfo->pStUpdated); + } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + if (!isCloseWindow(&winInfo.sessionWin.win, &pInfo->twAggSup)) { + saveDeleteRes(pInfo->pStDeleted, winInfo.sessionWin); + } + SSessionKey key = {0}; + getSessionHashKey(&winInfo.sessionWin, &key); + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); + } + } + saveSessionOutputBuf(pAggSup, &winInfo); + } + taosMemoryFree(pBuf); + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.reloadStreamStateFn) { + downstream->fpSet.reloadStreamStateFn(downstream); + } +} + SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo) { + SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; int32_t numOfCols = 0; int32_t code = TSDB_CODE_OUT_OF_MEMORY; @@ -3634,7 +3789,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh } code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, pSessionNode->gap, - pTaskInfo->streamInfo.pState, 0, 0, &pTaskInfo->storageAPI.stateStore); + pTaskInfo->streamInfo.pState, 0, 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -3666,11 +3821,19 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->pUpdated = NULL; pInfo->pStUpdated = NULL; pInfo->dataVersion = 0; + pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); + if (!pInfo->historyWins) { + goto _error; + } + if (pHandle) { + pInfo->isHistoryOp = pHandle->fillHistory; + } setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); if (downstream) { initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); @@ -3778,7 +3941,6 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { removeSessionResults(pInfo->pStDeleted, pInfo->pUpdated); tSimpleHashCleanup(pInfo->pStUpdated); pInfo->pStUpdated = NULL; - initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); @@ -3809,9 +3971,9 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, int32_t numOfChild) { + SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle) { int32_t code = TSDB_CODE_OUT_OF_MEMORY; - SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo); + SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle); if (pOperator == NULL) { goto _error; } @@ -3828,14 +3990,14 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL); } - + setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); setOperatorInfo(pOperator, name, pPhyNode->type, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->operatorType = pPhyNode->type; if (numOfChild > 0) { pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*)); for (int32_t i = 0; i < numOfChild; i++) { - SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0); + SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0, NULL); if (pChildOp == NULL) { goto _error; } @@ -3876,6 +4038,8 @@ void destroyStreamStateOperatorInfo(void* param) { } colDataDestroy(&pInfo->twAggSup.timeWindowData); blockDataDestroy(pInfo->pDelRes); + taosArrayDestroy(pInfo->historyWins); + tSimpleHashCleanup(pInfo->pSeUpdated); tSimpleHashCleanup(pInfo->pSeDeleted); taosMemoryFreeClear(param); } @@ -3892,6 +4056,9 @@ bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) { } bool compareStateKey(void* data, void* key) { + if (!data || !key) { + return false; + } SStateKeys* stateKey = (SStateKeys*)key; stateKey->pData = (char*)key + sizeof(SStateKeys); return compareVal(data, stateKey); @@ -3913,9 +4080,22 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys); pCurWin->pStateKey->isNull = false; + if (code == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->winInfo.sessionWin.win)) { + code = TSDB_CODE_FAILED; + releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)pCurWin->winInfo.pOutputBuf, &pAggSup->pSessionAPI->stateStore); + pCurWin->winInfo.pOutputBuf = taosMemoryCalloc(1, size); + pCurWin->pStateKey = + (SStateKeys*)((char*)pCurWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize)); + pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys); + pCurWin->pStateKey->type = pAggSup->stateKeyType; + pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys); + pCurWin->pStateKey->isNull = false; + } + if (code == TSDB_CODE_SUCCESS) { pCurWin->winInfo.isOutput = true; - } else { + pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin); + } else if (pKeyData) { if (IS_VAR_DATA_TYPE(pAggSup->stateKeyType)) { varDataCopy(pCurWin->pStateKey->pData, pKeyData); } else { @@ -3924,11 +4104,19 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, } pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin; - pNextWin->winInfo.pOutputBuf = NULL; - SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); - code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0); + SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin); + int32_t nextSize = pAggSup->resultRowSize; + code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, &pNextWin->winInfo.pOutputBuf, &nextSize); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_INVALID(pNextWin->winInfo); + } else { + pNextWin->pStateKey = + (SStateKeys*)((char*)pNextWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize)); + pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys); + pNextWin->pStateKey->type = pAggSup->stateKeyType; + pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys); + pNextWin->pStateKey->isNull = false; + pNextWin->winInfo.isOutput = true; } pAggSup->stateStore.streamStateFreeCur(pCur); } @@ -3979,8 +4167,13 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t winRows = 0; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); + pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; + if (pAggSup->winRange.ekey <= 0) { + pAggSup->winRange.ekey = INT64_MAX; + } if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); @@ -3989,7 +4182,6 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl return; } - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; int32_t rows = pSDataBlock->info.rows; blockDataEnsureCapacity(pAggSup->pScanBlock, rows); SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); @@ -4004,6 +4196,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl SStateWindowInfo curWin = {0}; SStateWindowInfo nextWin = {0}; setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin); + if (IS_VALID_SESSION_WIN(nextWin.winInfo)) { + releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)nextWin.winInfo.pOutputBuf, &pAPI->stateStore); + } setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); winRows = updateStateWindowInfo(&curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual, pAggSup->pResultRows, pSeUpdated, pStDeleted); @@ -4113,6 +4308,10 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { tSimpleHashCleanup(pInfo->pSeUpdated); pInfo->pSeUpdated = NULL; + if(pInfo->isHistoryOp) { + getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + } + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); @@ -4138,8 +4337,95 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return NULL; } +void streamStateReleaseState(SOperatorInfo* pOperator) { + SStreamStateAggOperatorInfo* pInfo = pOperator->info; + int32_t resSize = taosArrayGetSize(pInfo->historyWins) * sizeof(SSessionKey); + pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_STATE_NAME, strlen(STREAM_STATE_OP_STATE_NAME), pInfo->historyWins->pData, resSize); + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.releaseStreamStateFn) { + downstream->fpSet.releaseStreamStateFn(downstream); + } +} + +static void compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, + SSHashObj* pStUpdated, SSHashObj* pStDeleted) { + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + + SStreamStateAggOperatorInfo* pInfo = pOperator->info; + SResultRow* pCurResult = NULL; + int32_t numOfOutput = pOperator->exprSupp.numOfExprs; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + SResultRow* pWinResult = NULL; + initSessionOutputBuf(pNextWin, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset); + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->sessionWin.win, 1); + compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); + tSimpleHashRemove(pStUpdated, &pNextWin->sessionWin, sizeof(SSessionKey)); + if (pNextWin->isOutput && pStDeleted) { + saveDeleteRes(pStDeleted, pNextWin->sessionWin); + } + removeSessionResult(pStUpdated, pAggSup->pResultRows, pNextWin->sessionWin); + doDeleteSessionWindow(pAggSup, &pNextWin->sessionWin); + taosMemoryFree(pNextWin->pOutputBuf); +} + +void streamStateReloadState(SOperatorInfo* pOperator) { + SStreamStateAggOperatorInfo* pInfo = pOperator->info; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + resetWinRange(&pAggSup->winRange); + + SSessionKey seKey = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0}; + int32_t size = 0; + void* pBuf = NULL; + int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME, + strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); + int32_t num = size / sizeof(SSessionKey); + SSessionKey* pSeKeyBuf = (SSessionKey*) pBuf; + ASSERT(size == num * sizeof(SSessionKey)); + if (!pInfo->pSeUpdated && num > 0) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pSeUpdated = tSimpleHashInit(64, hashFn); + } + for (int32_t i = 0; i < num; i++) { + SStateWindowInfo curInfo = {0}; + SStateWindowInfo nextInfo = {0}; + SStateWindowInfo dummy = {0}; + setStateOutputBuf(pAggSup, pSeKeyBuf[i].win.skey, pSeKeyBuf[i].groupId, NULL, &curInfo, &nextInfo); + if (compareStateKey(curInfo.pStateKey,nextInfo.pStateKey)) { + compactStateWindow(pOperator, &curInfo.winInfo, &nextInfo.winInfo, pInfo->pSeUpdated, pInfo->pSeUpdated); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { + saveResult(curInfo.winInfo, pInfo->pSeUpdated); + } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) { + saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin); + } + SSessionKey key = {0}; + getSessionHashKey(&curInfo.winInfo.sessionWin, &key); + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo)); + } + } + + if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { + saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + } + + if (IS_VALID_SESSION_WIN(nextInfo.winInfo)) { + releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)nextInfo.winInfo.pOutputBuf, &pAggSup->pSessionAPI->stateStore); + } + } + taosMemoryFree(pBuf); + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.reloadStreamStateFn) { + downstream->fpSet.reloadStreamStateFn(downstream); + } +} + SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo) { + SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) { SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode; int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; @@ -4183,7 +4469,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes; int16_t type = pColNode->node.resType.type; code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, - type, &pTaskInfo->storageAPI.stateStore); + type, &pTaskInfo->storageAPI.stateStore, pHandle, &pTaskInfo->storageAPI); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4199,11 +4485,19 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pUpdated = NULL; pInfo->pSeUpdated = NULL; pInfo->dataVersion = 0; + pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); + if (!pInfo->historyWins) { + goto _error; + } + if (pHandle) { + pInfo->isHistoryOp = pHandle->fillHistory; + } setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState); initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4601,7 +4895,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* doWindowBorderInterpolation(iaInfo, pBlock, pResult, &win, startPos, forwardRows, pExprSup); } - updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true); + updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, 1); applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, numOfOutput); doCloseWindow(pResultRowInfo, iaInfo, pResult); @@ -5023,13 +5317,16 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pUpdated = NULL; pInfo->pUpdatedMap = NULL; int32_t funResSize= getMaxFunResSize(pSup, numOfCols); - pInfo->pState->pFileState = pTaskInfo->storageAPI.stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, - compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); + + pInfo->pState->pFileState = pTaskInfo->storageAPI.stateStore.streamFileStateInit( + tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState, + pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo)); setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); + setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState); pInfo->statestore = pTaskInfo->storageAPI.stateStore; pInfo->recvGetAll = false; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index daf06c81d13d0d85aa6d47f3147d822a0e915311..0a8d7ee376b9a92a7c4bc5fb7544de0f6bb77030 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -24,6 +24,7 @@ #include "tpagedbuf.h" #include "tsort.h" #include "tutil.h" +#include "tsimplehash.h" struct STupleHandle { SSDataBlock* pBlock; @@ -42,12 +43,16 @@ struct SSortHandle { int64_t startTs; uint64_t totalElapsed; - uint64_t maxRows; - uint32_t maxTupleLength; - uint32_t sortBufSize; + uint64_t pqMaxRows; + uint32_t pqMaxTupleLength; + uint32_t pqSortBufSize; + bool forceUsePQSort; BoundedQueue* pBoundedQueue; uint32_t tmpRowIdx; + int64_t mergeLimit; + int64_t currMergeLimitTs; + int32_t sourceId; SSDataBlock* pDataBlock; SMsortComparParam cmpParam; @@ -73,7 +78,7 @@ static void* createTuple(uint32_t columnNum, uint32_t tupleLen) { uint32_t totalLen = sizeof(uint32_t) * columnNum + BitmapLen(columnNum) + tupleLen; return taosMemoryCalloc(1, totalLen); } -static void destoryTuple(void* t) { taosMemoryFree(t); } +static void destoryAllocatedTuple(void* t) { taosMemoryFree(t); } #define tupleOffset(tuple, colIdx) ((uint32_t*)(tuple + sizeof(uint32_t) * colIdx)) #define tupleSetOffset(tuple, colIdx, offset) (*tupleOffset(tuple, colIdx) = offset) @@ -107,20 +112,73 @@ static void* tupleGetField(char* t, uint32_t colIdx, uint32_t colNum) { return t + *tupleOffset(t, colIdx); } -static int32_t colDataComparFn(const void* pLeft, const void* pRight, void* param); - SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle) { return createOneDataBlock(pSortHandle->pDataBlock, false); } +#define AllocatedTupleType 0 +#define ReferencedTupleType 1 // tuple references to one row in pDataBlock +typedef struct TupleDesc { + uint8_t type; + char* data; // if type is AllocatedTuple, then points to the created tuple, otherwise points to the DataBlock +} TupleDesc; + +typedef struct ReferencedTuple { + TupleDesc desc; + size_t rowIndex; +} ReferencedTuple; + +static TupleDesc* createAllocatedTuple(SSDataBlock* pBlock, size_t colNum, uint32_t tupleLen, size_t rowIdx) { + TupleDesc* t = taosMemoryCalloc(1, sizeof(TupleDesc)); + void* pTuple = createTuple(colNum, tupleLen); + if (!pTuple) { + taosMemoryFree(t); + return NULL; + } + size_t colLen = 0; + uint32_t offset = tupleGetDataStartOffset(colNum); + for (size_t colIdx = 0; colIdx < colNum; ++colIdx) { + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, colIdx); + if (colDataIsNull_s(pCol, rowIdx)) { + offset = tupleAddField((char**)&pTuple, colNum, offset, colIdx, 0, 0, true, tupleLen); + } else { + colLen = colDataGetRowLength(pCol, rowIdx); + offset = + tupleAddField((char**)&pTuple, colNum, offset, colIdx, colDataGetData(pCol, rowIdx), colLen, false, tupleLen); + } + } + t->type = AllocatedTupleType; + t->data = pTuple; + return t; +} + +void* tupleDescGetField(const TupleDesc* pDesc, int32_t colIdx, uint32_t colNum) { + if (pDesc->type == ReferencedTupleType) { + ReferencedTuple* pRefTuple = (ReferencedTuple*)pDesc; + SColumnInfoData* pCol = taosArrayGet(((SSDataBlock*)pDesc->data)->pDataBlock, colIdx); + if (colDataIsNull_s(pCol, pRefTuple->rowIndex)) return NULL; + return colDataGetData(pCol, pRefTuple->rowIndex); + } else { + return tupleGetField(pDesc->data, colIdx, colNum); + } +} + +void destroyTuple(void* t) { + TupleDesc* pDesc = t; + if (pDesc->type == AllocatedTupleType) { + destoryAllocatedTuple(pDesc->data); + taosMemoryFree(pDesc); + } +} + /** * * @param type * @return */ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t pageSize, int32_t numOfPages, - SSDataBlock* pBlock, const char* idstr, uint64_t maxRows, uint32_t maxTupleLength, - uint32_t sortBufSize) { + SSDataBlock* pBlock, const char* idstr, uint64_t pqMaxRows, uint32_t pqMaxTupleLength, + uint32_t pqSortBufSize) { SSortHandle* pSortHandle = taosMemoryCalloc(1, sizeof(SSortHandle)); pSortHandle->type = type; @@ -129,21 +187,29 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page pSortHandle->pSortInfo = pSortInfo; pSortHandle->loops = 0; - pSortHandle->maxTupleLength = maxTupleLength; - if (maxRows < 0) - pSortHandle->sortBufSize = 0; - else - pSortHandle->sortBufSize = sortBufSize; - pSortHandle->maxRows = maxRows; + pSortHandle->pqMaxTupleLength = pqMaxTupleLength; + if (pqMaxRows != 0) { + pSortHandle->pqSortBufSize = pqSortBufSize; + pSortHandle->pqMaxRows = pqMaxRows; + } + pSortHandle->forceUsePQSort = false; if (pBlock != NULL) { pSortHandle->pDataBlock = createOneDataBlock(pBlock, false); } + pSortHandle->mergeLimit = -1; + pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES); pSortHandle->cmpParam.orderInfo = pSortInfo; pSortHandle->cmpParam.cmpGroupId = false; - + pSortHandle->cmpParam.sortType = type; + if (type == SORT_BLOCK_TS_MERGE) { + SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pSortInfo, 0); + pSortHandle->cmpParam.tsSlotId = pOrder->slotId; + pSortHandle->cmpParam.order = pOrder->order; + pSortHandle->cmpParam.cmpFn = (pOrder->order == TSDB_ORDER_ASC) ? compareInt64Val : compareInt64ValDesc; + } tsortSetComparFp(pSortHandle, msortComparFn); if (idstr != NULL) { @@ -415,11 +481,14 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT if (pHandle->type == SORT_SINGLESOURCE_SORT) { pSource->pageIndex++; if (pSource->pageIndex >= taosArrayGetSize(pSource->pageIdList)) { + qDebug("adjust merge tree. %d source completed %d", *numOfCompleted, pSource->pageIndex); (*numOfCompleted) += 1; pSource->src.rowIndex = -1; pSource->pageIndex = -1; pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock); } else { + if (pSource->pageIndex % 512 == 0) qDebug("begin source %p page %d", pSource, pSource->pageIndex); + int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex); void* pPage = getBufPage(pHandle->pBuf, *pPgId); @@ -432,7 +501,6 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT if (code != TSDB_CODE_SUCCESS) { return code; } - releaseBufPage(pHandle->pBuf, pPage); } } else { @@ -443,6 +511,7 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT if (pSource->src.pBlock == NULL) { (*numOfCompleted) += 1; pSource->src.rowIndex = -1; + qDebug("adjust merge tree. %d source completed", *numOfCompleted); } } } @@ -523,53 +592,63 @@ int32_t msortComparFn(const void* pLeft, const void* pRight, void* param) { } } - for (int32_t i = 0; i < pInfo->size; ++i) { - SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); - SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId); + if (pParam->sortType == SORT_BLOCK_TS_MERGE) { + SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pParam->tsSlotId); + SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pParam->tsSlotId); + int64_t* left1 = (int64_t*)(pLeftColInfoData->pData) + pLeftSource->src.rowIndex; + int64_t* right1 = (int64_t*)(pRightColInfoData->pData) + pRightSource->src.rowIndex; - bool leftNull = false; - if (pLeftColInfoData->hasNull) { - if (pLeftBlock->pBlockAgg == NULL) { - leftNull = colDataIsNull_s(pLeftColInfoData, pLeftSource->src.rowIndex); - } else { - leftNull = - colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg[i]); + int ret = pParam->cmpFn(left1, right1); + return ret; + } else { + for (int32_t i = 0; i < pInfo->size; ++i) { + SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); + SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId); + SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId); + + bool leftNull = false; + if (pLeftColInfoData->hasNull) { + if (pLeftBlock->pBlockAgg == NULL) { + leftNull = colDataIsNull_s(pLeftColInfoData, pLeftSource->src.rowIndex); + } else { + leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, + pLeftBlock->pBlockAgg[i]); + } } - } - SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId); - bool rightNull = false; - if (pRightColInfoData->hasNull) { - if (pRightBlock->pBlockAgg == NULL) { - rightNull = colDataIsNull_s(pRightColInfoData, pRightSource->src.rowIndex); - } else { - rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, - pRightBlock->pBlockAgg[i]); + bool rightNull = false; + if (pRightColInfoData->hasNull) { + if (pRightBlock->pBlockAgg == NULL) { + rightNull = colDataIsNull_s(pRightColInfoData, pRightSource->src.rowIndex); + } else { + rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, + pRightBlock->pBlockAgg[i]); + } } - } - if (leftNull && rightNull) { - continue; // continue to next slot - } + if (leftNull && rightNull) { + continue; // continue to next slot + } - if (rightNull) { - return pOrder->nullFirst ? 1 : -1; - } + if (rightNull) { + return pOrder->nullFirst ? 1 : -1; + } - if (leftNull) { - return pOrder->nullFirst ? -1 : 1; - } + if (leftNull) { + return pOrder->nullFirst ? -1 : 1; + } - void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex); - void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex); + void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex); + void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex); - __compar_fn_t fn = getKeyComparFunc(pLeftColInfoData->info.type, pOrder->order); + __compar_fn_t fn = getKeyComparFunc(pLeftColInfoData->info.type, pOrder->order); - int ret = fn(left1, right1); - if (ret == 0) { - continue; - } else { - return ret; + int ret = fn(left1, right1); + if (ret == 0) { + continue; + } else { + return ret; + } } } return 0; @@ -614,6 +693,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { // Only *numOfInputSources* can be loaded into buffer to perform the external sort. for (int32_t i = 0; i < sortGroup; ++i) { + qDebug("internal merge sort pass %d group %d. num input sources %d ", t, i, numOfInputSources); pHandle->sourceId += 1; int32_t end = (i + 1) * numOfInputSources - 1; @@ -636,13 +716,15 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { return code; } + int nMergedRows = 0; + SArray* pPageIdList = taosArrayInit(4, sizeof(int32_t)); while (1) { if (tsortIsClosed(pHandle)) { code = terrno = TSDB_CODE_TSC_QUERY_CANCELLED; return code; } - + SSDataBlock* pDataBlock = getSortedBlockDataInner(pHandle, &pHandle->cmpParam, numOfRows); if (pDataBlock == NULL) { break; @@ -666,8 +748,12 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); + nMergedRows += pDataBlock->info.rows; blockDataCleanup(pDataBlock); + if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { + break; + } } sortComparCleanup(&pHandle->cmpParam); @@ -715,113 +801,395 @@ int32_t getProperSortPageSize(size_t rowSize, uint32_t numOfCols) { return pgSize; } -static int32_t createInitialSources(SSortHandle* pHandle) { - size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize; - int32_t code = 0; +static int32_t createPageBuf(SSortHandle* pHandle) { + if (pHandle->pBuf == NULL) { + if (!osTempSpaceAvailable()) { + terrno = TSDB_CODE_NO_DISKSPACE; + qError("create page buf failed since %s, tempDir:%s", terrstr(), tsTempDir); + return terrno; + } - if (pHandle->type == SORT_SINGLESOURCE_SORT) { - SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0); - SSortSource* source = *pSource; - *pSource = NULL; + int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, + "tableBlocksBuf", tsTempDir); + dBufSetPrintInfo(pHandle->pBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + return 0; +} - tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); +typedef struct SBlkMergeSupport { + int64_t** aTs; + int32_t* aRowIdx; + int32_t order; +} SBlkMergeSupport; - while (1) { - SSDataBlock* pBlock = pHandle->fetchfp(source->param); - if (pBlock == NULL) { - break; - } +static int32_t blockCompareTsFn(const void* pLeft, const void* pRight, void* param) { + int32_t left = *(int32_t*)pLeft; + int32_t right = *(int32_t*)pRight; + + SBlkMergeSupport* pSup = (SBlkMergeSupport*)param; + if (pSup->aRowIdx[left] == -1) { + return 1; + } else if (pSup->aRowIdx[right] == -1) { + return -1; + } - if (pHandle->pDataBlock == NULL) { - uint32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - pHandle->pageSize = getProperSortPageSize(blockDataGetRowSize(pBlock), numOfCols); + int64_t leftTs = pSup->aTs[left][pSup->aRowIdx[left]]; + int64_t rightTs = pSup->aTs[right][pSup->aRowIdx[right]]; - // todo, number of pages are set according to the total available sort buffer - pHandle->numOfPages = 1024; - sortBufSize = pHandle->numOfPages * pHandle->pageSize; - pHandle->pDataBlock = createOneDataBlock(pBlock, false); - } + int32_t ret = leftTs>rightTs ? 1 : ((leftTs < rightTs) ? -1 : 0); + if (pSup->order == TSDB_ORDER_DESC) { + ret = -1 * ret; + } + return ret; +} - if (pHandle->beforeFp != NULL) { - pHandle->beforeFp(pBlock, pHandle->param); - } +static int32_t appendDataBlockToPageBuf(SSortHandle* pHandle, SSDataBlock* blk, SArray* aPgId) { + int32_t pageId = -1; + void* pPage = getNewBufPage(pHandle->pBuf, &pageId); + taosArrayPush(aPgId, &pageId); - code = blockDataMerge(pHandle->pDataBlock, pBlock); - if (code != TSDB_CODE_SUCCESS) { - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); + int32_t size = blockDataGetSize(blk) + sizeof(int32_t) + taosArrayGetSize(blk->pDataBlock) * sizeof(int32_t); + ASSERT(size <= getBufPageSize(pHandle->pBuf)); + + blockDataToBuf(pPage, blk); + + setBufPageDirty(pPage, true); + releaseBufPage(pHandle->pBuf, pPage); + + return 0; +} + +static int32_t getPageBufIncForRow(SSDataBlock* blk, int32_t row, int32_t rowIdxInPage) { + int sz = 0; + int numCols = taosArrayGetSize(blk->pDataBlock); + if (!blk->info.hasVarCol) { + sz += numCols * ((rowIdxInPage & 0x7) == 0 ? 1: 0); + sz += blockDataGetRowSize(blk); + } else { + for (int32_t i = 0; i < numCols; ++i) { + SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(blk->pDataBlock, i); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + if (pColInfoData->varmeta.offset[row] != -1) { + char* p = colDataGetData(pColInfoData, row); + sz += varDataTLen(p); } - if (!source->onlyRef && source->src.pBlock) { - blockDataDestroy(source->src.pBlock); - source->src.pBlock = NULL; + + sz += sizeof(pColInfoData->varmeta.offset[0]); + } else { + sz += pColInfoData->info.bytes; + + if (((rowIdxInPage) & 0x07) == 0) { + sz += 1; // bitmap } - taosMemoryFree(source); - return code; } + } + } + return sz; +} + +static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockOrderInfo* order, SArray* aExtSrc) { + int pgHeaderSz = sizeof(int32_t) + sizeof(int32_t) * taosArrayGetSize(pHandle->pDataBlock->pDataBlock); + int32_t rowCap = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize, pgHeaderSz); + blockDataEnsureCapacity(pHandle->pDataBlock, rowCap); + blockDataCleanup(pHandle->pDataBlock); + int32_t numBlks = taosArrayGetSize(aBlk); + + SBlkMergeSupport sup; + sup.aRowIdx = taosMemoryCalloc(numBlks, sizeof(int32_t)); + sup.aTs = taosMemoryCalloc(numBlks, sizeof(int64_t*)); + sup.order = order->order; + for (int i = 0; i < numBlks; ++i) { + SSDataBlock* blk = taosArrayGetP(aBlk, i); + SColumnInfoData* col = taosArrayGet(blk->pDataBlock, order->slotId); + sup.aTs[i] = (int64_t*)col->pData; + sup.aRowIdx[i] = 0; + } - size_t size = blockDataGetSize(pHandle->pDataBlock); - if (size > sortBufSize) { - // Perform the in-memory sort and then flush data in the buffer into disk. - int64_t p = taosGetTimestampUs(); - code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); - if (code != 0) { - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); + int32_t totalRows = 0; + for (int i = 0; i < numBlks; ++i) { + SSDataBlock* blk = taosArrayGetP(aBlk, i); + totalRows += blk->info.rows; + } + + SArray* aPgId = taosArrayInit(8, sizeof(int32_t)); + + SMultiwayMergeTreeInfo* pTree = NULL; + tMergeTreeCreate(&pTree, taosArrayGetSize(aBlk), &sup, blockCompareTsFn); + int32_t nRows = 0; + int32_t nMergedRows = 0; + bool mergeLimitReached = false; + size_t blkPgSz = pgHeaderSz; + int64_t lastPageBufTs = (order->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN; + int64_t currTs = (order->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN; + while (nRows < totalRows) { + int32_t minIdx = tMergeTreeGetChosenIndex(pTree); + SSDataBlock* minBlk = taosArrayGetP(aBlk, minIdx); + int32_t minRow = sup.aRowIdx[minIdx]; + int32_t bufInc = getPageBufIncForRow(minBlk, minRow, pHandle->pDataBlock->info.rows); + + if (blkPgSz <= pHandle->pageSize && blkPgSz + bufInc > pHandle->pageSize) { + SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); + lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; + appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + nMergedRows += pHandle->pDataBlock->info.rows; + + blockDataCleanup(pHandle->pDataBlock); + blkPgSz = pgHeaderSz; + bufInc = getPageBufIncForRow(minBlk, minRow, 0); + + if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { + mergeLimitReached = true; + if ((lastPageBufTs < pHandle->currMergeLimitTs && order->order == TSDB_ORDER_ASC) || + (lastPageBufTs > pHandle->currMergeLimitTs && order->order == TSDB_ORDER_DESC)) { + pHandle->currMergeLimitTs = lastPageBufTs; } - if (!source->onlyRef && source->src.pBlock) { - blockDataDestroy(source->src.pBlock); - source->src.pBlock = NULL; + break; + } + } + blockDataEnsureCapacity(pHandle->pDataBlock, pHandle->pDataBlock->info.rows + 1); + appendOneRowToDataBlock(pHandle->pDataBlock, minBlk, &minRow); + blkPgSz += bufInc; + + ++nRows; + + if (sup.aRowIdx[minIdx] == minBlk->info.rows - 1) { + sup.aRowIdx[minIdx] = -1; + } else { + ++sup.aRowIdx[minIdx]; + } + tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree)); + } + if (pHandle->pDataBlock->info.rows > 0) { + if (!mergeLimitReached) { + SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId); + lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; + appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); + nMergedRows += pHandle->pDataBlock->info.rows; + if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) { + mergeLimitReached = true; + if ((lastPageBufTs < pHandle->currMergeLimitTs && order->order == TSDB_ORDER_ASC) || + (lastPageBufTs > pHandle->currMergeLimitTs && order->order == TSDB_ORDER_DESC)) { + pHandle->currMergeLimitTs = lastPageBufTs; } + } + } + blockDataCleanup(pHandle->pDataBlock); + } + SSDataBlock* pMemSrcBlk = createOneDataBlock(pHandle->pDataBlock, false); + doAddNewExternalMemSource(pHandle->pBuf, aExtSrc, pMemSrcBlk, &pHandle->sourceId, aPgId); - taosMemoryFree(source); - return code; - } + taosMemoryFree(sup.aRowIdx); + taosMemoryFree(sup.aTs); - int64_t el = taosGetTimestampUs() - p; - pHandle->sortElapsed += el; + tMergeTreeDestroy(&pTree); - code = doAddToBuf(pHandle->pDataBlock, pHandle); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + return 0; +} + +static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { + SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0); + size_t nSrc = taosArrayGetSize(pHandle->pOrderedSource); + SArray* aExtSrc = taosArrayInit(nSrc, POINTER_BYTES); + + size_t maxBufSize = pHandle->numOfPages * pHandle->pageSize; + createPageBuf(pHandle); + + SSortSource* pSrc = taosArrayGetP(pHandle->pOrderedSource, 0); + int32_t szSort = 0; + + if (pOrder->order == TSDB_ORDER_ASC) { + pHandle->currMergeLimitTs = INT64_MAX; + } else { + pHandle->currMergeLimitTs = INT64_MIN; + } + + SArray* aBlkSort = taosArrayInit(8, POINTER_BYTES); + SSHashObj* mUidBlk = tSimpleHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT)); + while (1) { + SSDataBlock* pBlk = pHandle->fetchfp(pSrc->param); + if (pBlk != NULL) { + SColumnInfoData* tsCol = taosArrayGet(pBlk->pDataBlock, pOrder->slotId); + int64_t firstRowTs = *(int64_t*)tsCol->pData; + if ((pOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs) || + (pOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) { + continue; + } + } + if (pBlk != NULL) { + szSort += blockDataGetSize(pBlk); + + void* ppBlk = tSimpleHashGet(mUidBlk, &pBlk->info.id.uid, sizeof(pBlk->info.id.uid)); + if (ppBlk != NULL) { + SSDataBlock* tBlk = *(SSDataBlock**)(ppBlk); + blockDataMerge(tBlk, pBlk); + } else { + SSDataBlock* tBlk = createOneDataBlock(pBlk, true); + tSimpleHashPut(mUidBlk, &pBlk->info.id.uid, sizeof(pBlk->info.id.uid), &tBlk, POINTER_BYTES); + taosArrayPush(aBlkSort, &tBlk); } } - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); + if ((pBlk != NULL && szSort > maxBufSize) || (pBlk == NULL && szSort > 0)) { + tSimpleHashClear(mUidBlk); + + int64_t p = taosGetTimestampUs(); + sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc); + int64_t el = taosGetTimestampUs() - p; + pHandle->sortElapsed += el; + + for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) { + blockDataDestroy(taosArrayGetP(aBlkSort, i)); + } + taosArrayClear(aBlkSort); + szSort = 0; + qDebug("source %zu created", taosArrayGetSize(aExtSrc)); + } + if (pBlk == NULL) { + break; } - taosMemoryFree(source); + if (tsortIsClosed(pHandle)) { + tSimpleHashClear(mUidBlk); + for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) { + blockDataDestroy(taosArrayGetP(aBlkSort, i)); + } + taosArrayClear(aBlkSort); + break; + } + } + + tSimpleHashCleanup(mUidBlk); + taosArrayDestroy(aBlkSort); + tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); + if (!tsortIsClosed(pHandle)) { + taosArrayAddAll(pHandle->pOrderedSource, aExtSrc); + } + taosArrayDestroy(aExtSrc); + + pHandle->type = SORT_SINGLESOURCE_SORT; + return 0; +} + +static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { + int32_t code = 0; + size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize; + + SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0); + SSortSource* source = *pSource; + *pSource = NULL; + + tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); + + while (1) { + SSDataBlock* pBlock = pHandle->fetchfp(source->param); + if (pBlock == NULL) { + break; + } + + if (pHandle->pDataBlock == NULL) { + uint32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + pHandle->pageSize = getProperSortPageSize(blockDataGetRowSize(pBlock), numOfCols); + + // todo, number of pages are set according to the total available sort buffer + pHandle->numOfPages = 1024; + sortBufSize = pHandle->numOfPages * pHandle->pageSize; + pHandle->pDataBlock = createOneDataBlock(pBlock, false); + } + + if (pHandle->beforeFp != NULL) { + pHandle->beforeFp(pBlock, pHandle->param); + } - if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) { - size_t size = blockDataGetSize(pHandle->pDataBlock); + code = blockDataMerge(pHandle->pDataBlock, pBlock); + if (code != TSDB_CODE_SUCCESS) { + if (source->param && !source->onlyRef) { + taosMemoryFree(source->param); + } + if (!source->onlyRef && source->src.pBlock) { + blockDataDestroy(source->src.pBlock); + source->src.pBlock = NULL; + } + taosMemoryFree(source); + return code; + } + size_t size = blockDataGetSize(pHandle->pDataBlock); + if (size > sortBufSize) { // Perform the in-memory sort and then flush data in the buffer into disk. int64_t p = taosGetTimestampUs(); - code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); if (code != 0) { + if (source->param && !source->onlyRef) { + taosMemoryFree(source->param); + } + if (!source->onlyRef && source->src.pBlock) { + blockDataDestroy(source->src.pBlock); + source->src.pBlock = NULL; + } + + taosMemoryFree(source); return code; } int64_t el = taosGetTimestampUs() - p; pHandle->sortElapsed += el; + if (pHandle->pqMaxRows > 0) blockDataKeepFirstNRows(pHandle->pDataBlock, pHandle->pqMaxRows); + code = doAddToBuf(pHandle->pDataBlock, pHandle); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } - // All sorted data can fit in memory, external memory sort is not needed. Return to directly - if (size <= sortBufSize && pHandle->pBuf == NULL) { - pHandle->cmpParam.numOfSources = 1; - pHandle->inMemSort = true; + if (source->param && !source->onlyRef) { + taosMemoryFree(source->param); + } - pHandle->loops = 1; - pHandle->tupleHandle.rowIndex = -1; - pHandle->tupleHandle.pBlock = pHandle->pDataBlock; - return 0; - } else { - code = doAddToBuf(pHandle->pDataBlock, pHandle); - } + taosMemoryFree(source); + + if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) { + size_t size = blockDataGetSize(pHandle->pDataBlock); + + // Perform the in-memory sort and then flush data in the buffer into disk. + int64_t p = taosGetTimestampUs(); + + code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); + if (code != 0) { + return code; + } + + if (pHandle->pqMaxRows > 0) blockDataKeepFirstNRows(pHandle->pDataBlock, pHandle->pqMaxRows); + int64_t el = taosGetTimestampUs() - p; + pHandle->sortElapsed += el; + + // All sorted data can fit in memory, external memory sort is not needed. Return to directly + if (size <= sortBufSize && pHandle->pBuf == NULL) { + pHandle->cmpParam.numOfSources = 1; + pHandle->inMemSort = true; + + pHandle->loops = 1; + pHandle->tupleHandle.rowIndex = -1; + pHandle->tupleHandle.pBlock = pHandle->pDataBlock; + return 0; + } else { + code = doAddToBuf(pHandle->pDataBlock, pHandle); } } + return code; +} +static int32_t createInitialSources(SSortHandle* pHandle) { + int32_t code = 0; + + if (pHandle->type == SORT_SINGLESOURCE_SORT) { + code = createBlocksQuickSortInitialSources(pHandle); + } else if (pHandle->type == SORT_BLOCK_TS_MERGE) { + code = createBlocksMergeSortInitialSources(pHandle); + } + qDebug("%zu sources created", taosArrayGetSize(pHandle->pOrderedSource)); return code; } @@ -868,6 +1236,10 @@ void tsortSetClosed(SSortHandle* pHandle) { atomic_store_8(&pHandle->closed, 2); } +void tsortSetMergeLimit(SSortHandle* pHandle, int64_t mergeLimit) { + pHandle->mergeLimit = mergeLimit; +} + int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetchFp, void (*fp)(SSDataBlock*, void*), void* param) { pHandle->fetchfp = fetchFp; @@ -936,10 +1308,19 @@ static STupleHandle* tsortBufMergeSortNextTuple(SSortHandle* pHandle) { return &pHandle->tupleHandle; } +static bool tsortIsForceUsePQSort(SSortHandle* pHandle) { + return pHandle->forceUsePQSort == true; +} + +void tsortSetForceUsePQSort(SSortHandle* pHandle) { + pHandle->forceUsePQSort = true; +} + static bool tsortIsPQSortApplicable(SSortHandle* pHandle) { if (pHandle->type != SORT_SINGLESOURCE_SORT) return false; - uint64_t maxRowsFitInMemory = pHandle->sortBufSize / (pHandle->maxTupleLength + sizeof(char*)); - return maxRowsFitInMemory > pHandle->maxRows; + if (tsortIsForceUsePQSort(pHandle)) return true; + uint64_t maxRowsFitInMemory = pHandle->pqSortBufSize / (pHandle->pqMaxTupleLength + sizeof(char*)); + return maxRowsFitInMemory > pHandle->pqMaxRows; } static bool tsortPQCompFn(void* a, void* b, void* param) { @@ -956,16 +1337,17 @@ static bool tsortPQComFnReverse(void*a, void* b, void* param) { return 0; } -static int32_t colDataComparFn(const void* pLeft, const void* pRight, void* param) { - char* pLTuple = (char*)pLeft; - char* pRTuple = (char*)pRight; +static int32_t tupleComparFn(const void* pLeft, const void* pRight, void* param) { + TupleDesc* pLeftDesc = (TupleDesc*)pLeft; + TupleDesc* pRightDesc = (TupleDesc*)pRight; + SSortHandle* pHandle = (SSortHandle*)param; SArray* orderInfo = (SArray*)pHandle->pSortInfo; uint32_t colNum = blockDataGetNumOfCols(pHandle->pDataBlock); for (int32_t i = 0; i < orderInfo->size; ++i) { SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(orderInfo, i); - void *lData = tupleGetField(pLTuple, pOrder->slotId, colNum); - void *rData = tupleGetField(pRTuple, pOrder->slotId, colNum); + void *lData = tupleDescGetField(pLeftDesc, pOrder->slotId, colNum); + void *rData = tupleDescGetField(pRightDesc, pOrder->slotId, colNum); if (!lData && !rData) continue; if (!lData) return pOrder->nullFirst ? -1 : 1; if (!rData) return pOrder->nullFirst ? 1 : -1; @@ -984,9 +1366,9 @@ static int32_t colDataComparFn(const void* pLeft, const void* pRight, void* para } static int32_t tsortOpenForPQSort(SSortHandle* pHandle) { - pHandle->pBoundedQueue = createBoundedQueue(pHandle->maxRows, tsortPQCompFn, destoryTuple, pHandle); + pHandle->pBoundedQueue = createBoundedQueue(pHandle->pqMaxRows, tsortPQCompFn, destroyTuple, pHandle); if (NULL == pHandle->pBoundedQueue) return TSDB_CODE_OUT_OF_MEMORY; - tsortSetComparFp(pHandle, colDataComparFn); + tsortSetComparFp(pHandle, tupleComparFn); SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0); SSortSource* source = *pSource; @@ -1018,33 +1400,29 @@ static int32_t tsortOpenForPQSort(SSortHandle* pHandle) { } } } - size_t colLen = 0; + ReferencedTuple refTuple = {.desc.data = (char*)pBlock, .desc.type = ReferencedTupleType, .rowIndex = 0}; for (size_t rowIdx = 0; rowIdx < pBlock->info.rows; ++rowIdx) { - void* pTuple = createTuple(colNum, tupleLen); - if (pTuple == NULL) return TSDB_CODE_OUT_OF_MEMORY; - - uint32_t offset = tupleGetDataStartOffset(colNum); - for (size_t colIdx = 0; colIdx < colNum; ++colIdx) { - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, colIdx); - if (colDataIsNull_s(pCol, rowIdx)) { - offset = tupleAddField((char**)&pTuple, colNum, offset, colIdx, 0, 0, true, tupleLen); - } else { - colLen = colDataGetRowLength(pCol, rowIdx); - offset = tupleAddField((char**)&pTuple, colNum, offset, colIdx, colDataGetData(pCol, rowIdx), colLen, false, - tupleLen); - } + refTuple.rowIndex = rowIdx; + pqNode.data = &refTuple; + PriorityQueueNode* pPushedNode = taosBQPush(pHandle->pBoundedQueue, &pqNode); + if (!pPushedNode) { + // do nothing if push failed + } else { + pPushedNode->data = createAllocatedTuple(pBlock, colNum, tupleLen, rowIdx); + if (pPushedNode->data == NULL) return TSDB_CODE_OUT_OF_MEMORY; } - pqNode.data = pTuple; - taosBQPush(pHandle->pBoundedQueue, &pqNode); } } return TSDB_CODE_SUCCESS; } static STupleHandle* tsortPQSortNextTuple(SSortHandle* pHandle) { + if (pHandle->pDataBlock == NULL) { // when no input stream datablock + return NULL; + } blockDataCleanup(pHandle->pDataBlock); blockDataEnsureCapacity(pHandle->pDataBlock, 1); - // abondan the top tuple if queue size bigger than max size + // abandon the top tuple if queue size bigger than max size if (taosBQSize(pHandle->pBoundedQueue) == taosBQMaxSize(pHandle->pBoundedQueue) + 1) { taosBQPop(pHandle->pBoundedQueue); } @@ -1056,14 +1434,14 @@ static STupleHandle* tsortPQSortNextTuple(SSortHandle* pHandle) { if (taosBQSize(pHandle->pBoundedQueue) > 0) { uint32_t colNum = blockDataGetNumOfCols(pHandle->pDataBlock); PriorityQueueNode* node = taosBQTop(pHandle->pBoundedQueue); - char* pTuple = (char*)node->data; + char* pTuple = ((TupleDesc*)node->data)->data; for (uint32_t i = 0; i < colNum; ++i) { void* pData = tupleGetField(pTuple, i, colNum); if (!pData) { colDataSetNULL(bdGetColumnInfoData(pHandle->pDataBlock, i), 0); } else { - colDataAppend(bdGetColumnInfoData(pHandle->pDataBlock, i), 0, pData, false); + colDataSetVal(bdGetColumnInfoData(pHandle->pDataBlock, i), 0, pData, false); } } pHandle->pDataBlock->info.rows++; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index b996c630315613829643f3ccc8a62bc3d9b10134..fad8c9ca5bb4869a9d3d869ba6a36d127c0613fa 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -920,6 +920,7 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { *pDestPos = *pSourcePos; } +#define COMPARE_MINMAX_DATA(type) (( (*(type*)&pDBuf->v) < (*(type*)&pSBuf->v) ) ^ isMinFunc) int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t isMinFunc) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SMinmaxResInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); @@ -927,18 +928,57 @@ int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SMinmaxResInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type; - if (IS_FLOAT_TYPE(type)) { - if (pSBuf->assign && ((((*(double*)&pDBuf->v) < (*(double*)&pSBuf->v)) ^ isMinFunc) || !pDBuf->assign)) { - *(double*)&pDBuf->v = *(double*)&pSBuf->v; - replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); - pDBuf->assign = true; - } - } else { - if (pSBuf->assign && (((pDBuf->v < pSBuf->v) ^ isMinFunc) || !pDBuf->assign)) { - pDBuf->v = pSBuf->v; - replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); - pDBuf->assign = true; + + switch (type) { + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: + if (pSBuf->assign && (COMPARE_MINMAX_DATA(int64_t) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: + if (pSBuf->assign && (COMPARE_MINMAX_DATA(int32_t) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_SMALLINT: + if (pSBuf->assign && (COMPARE_MINMAX_DATA(int16_t) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: + if (pSBuf->assign && (COMPARE_MINMAX_DATA(int8_t) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; + case TSDB_DATA_TYPE_FLOAT: { + if (pSBuf->assign && (COMPARE_MINMAX_DATA(double) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; } + default: + if (pSBuf->assign && (strcmp((char*)&pDBuf->v, (char*)&pSBuf->v) || !pDBuf->assign)) { + pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); + pDBuf->assign = true; + } + break; } pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); pDResInfo->isNullRes &= pSResInfo->isNullRes; @@ -1712,7 +1752,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); varDataSetLen(buf, len); - colDataAppend(pCol, pBlock->info.rows, buf, false); + colDataSetVal(pCol, pBlock->info.rows, buf, false); tMemBucketDestroy(pMemBucket); return pResInfo->numOfRes; diff --git a/source/libs/geometry/src/geomFunc.c b/source/libs/geometry/src/geomFunc.c index 0e2bac584db4ef8602394a57e0d1fbcfb21fa79a..3588bf8b7d932b9ffc2ce94a47971883380cea4a 100644 --- a/source/libs/geometry/src/geomFunc.c +++ b/source/libs/geometry/src/geomFunc.c @@ -145,7 +145,7 @@ int32_t executeMakePointFunc(SColumnInfoData *pInputData[], int32_t iLeft, int32 goto _exit; } - colDataAppend(pOutputData, TMAX(iLeft, iRight), output, (output == NULL)); + colDataSetVal(pOutputData, TMAX(iLeft, iRight), output, (output == NULL)); _exit: if (output) { @@ -165,7 +165,7 @@ int32_t executeGeomFromTextFunc(SColumnInfoData *pInputData, int32_t i, SColumnI goto _exit; } - colDataAppend(pOutputData, i, output, (output == NULL)); + colDataSetVal(pOutputData, i, output, (output == NULL)); _exit: if (output) { @@ -185,7 +185,7 @@ int32_t executeAsTextFunc(SColumnInfoData *pInputData, int32_t i, SColumnInfoDat goto _exit; } - colDataAppend(pOutputData, i, output, (output == NULL)); + colDataSetVal(pOutputData, i, output, (output == NULL)); _exit: if (output) { @@ -213,7 +213,7 @@ int32_t executeRelationFunc(const GEOSGeometry *geom1, const GEOSPreparedGeometr } } - colDataAppend(pOutputData, i, &res, (res==-1)); + colDataSetVal(pOutputData, i, &res, (res==-1)); return code; } diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index dd83083ec9600fab9459e14e8079093bfd4a4578..993178e2b02f3ba75a875df39edb861f5f1f240a 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -173,6 +173,7 @@ int32_t initCtxAsText() { if (geosCtx->WKTWriter) { GEOSWKTWriter_setRoundingPrecision_r(geosCtx->handle, geosCtx->WKTWriter, 6); + GEOSWKTWriter_setTrim_r(geosCtx->handle, geosCtx->WKTWriter, 0); } else { return code; } diff --git a/source/libs/geometry/test/geomFuncTestUtil.cpp b/source/libs/geometry/test/geomFuncTestUtil.cpp index cb59ea098fdf20112382123196517b404f60ac51..0918781499e98ef32e9ffa2a9a1e717d9266c4eb 100644 --- a/source/libs/geometry/test/geomFuncTestUtil.cpp +++ b/source/libs/geometry/test/geomFuncTestUtil.cpp @@ -84,7 +84,7 @@ void setScalarParam(SScalarParam *sclParam, int32_t type, void *valueArray, TDRo } else { const char *val = (const char *)valueArray + (i * bytes); - colDataAppend(sclParam->columnData, i, val, false); + colDataSetVal(sclParam->columnData, i, val, false); } } } diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 8f94bfdb963a8e5342f118bd7ae739c115c1a37d..56cf0a2b51002f8878eda24d0aaf538e1a6a627f 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -468,9 +468,6 @@ static void monGenLogJson(SMonInfo *pMonitor) { return; } - SJson *pLogsJson = tjsonAddArrayToObject(pJson, "logs"); - if (pLogsJson == NULL) return; - SMonLogs *logs[6]; logs[0] = &pMonitor->log; logs[1] = &pMonitor->mmInfo.log; @@ -490,22 +487,6 @@ static void monGenLogJson(SMonInfo *pMonitor) { numOfInfoLogs += pLog->numOfInfoLogs; numOfDebugLogs += pLog->numOfDebugLogs; numOfTraceLogs += pLog->numOfTraceLogs; - - for (int32_t i = 0; i < taosArrayGetSize(pLog->logs); ++i) { - SJson *pLogJson = tjsonCreateObject(); - if (pLogJson == NULL) continue; - - SMonLogItem *pLogItem = taosArrayGet(pLog->logs, i); - - char buf[40] = {0}; - taosFormatUtcTime(buf, sizeof(buf), pLogItem->ts, TSDB_TIME_PRECISION_MILLI); - - tjsonAddStringToObject(pLogJson, "ts", buf); - tjsonAddStringToObject(pLogJson, "level", monLogLevelStr(pLogItem->level)); - tjsonAddStringToObject(pLogJson, "content", pLogItem->content); - - if (tjsonAddItemToArray(pLogsJson, pLogJson) != 0) tjsonDelete(pLogJson); - } } SJson *pSummaryJson = tjsonAddArrayToObject(pJson, "summary"); @@ -547,7 +528,7 @@ void monSendReport() { monGenGrantJson(pMonitor); monGenDnodeJson(pMonitor); monGenDiskJson(pMonitor); - //monGenLogJson(pMonitor); // TS-3691 + monGenLogJson(pMonitor); char *pCont = tjsonToString(pMonitor->pJson); // uDebugL("report cont:%s\n", pCont); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 8305daa45e2782ac203c2376b9032c3464b66578..f5eacf0bd5d1c15bb7c773ee60caea7abc0dc0b5 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -361,6 +361,7 @@ static int32_t logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { COPY_SCALAR_FIELD(groupAction); COPY_SCALAR_FIELD(inputTsOrder); COPY_SCALAR_FIELD(outputTsOrder); + COPY_SCALAR_FIELD(forceCreateNonBlockingOptr); return TSDB_CODE_SUCCESS; } @@ -397,6 +398,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { CLONE_NODE_LIST_FIELD(pTags); CLONE_NODE_FIELD(pSubtable); COPY_SCALAR_FIELD(igLastNull); + COPY_SCALAR_FIELD(groupOrderScan); return TSDB_CODE_SUCCESS; } @@ -502,7 +504,6 @@ static int32_t logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pSortKeys); COPY_SCALAR_FIELD(groupSort); - COPY_SCALAR_FIELD(maxRows); return TSDB_CODE_SUCCESS; } @@ -546,6 +547,7 @@ static int32_t physiNodeCopy(const SPhysiNode* pSrc, SPhysiNode* pDst) { CLONE_NODE_LIST_FIELD(pChildren); COPY_SCALAR_FIELD(inputTsOrder); COPY_SCALAR_FIELD(outputTsOrder); + COPY_SCALAR_FIELD(forceCreateNonBlockingOptr); return TSDB_CODE_SUCCESS; } @@ -557,6 +559,7 @@ static int32_t physiScanCopy(const SScanPhysiNode* pSrc, SScanPhysiNode* pDst) { COPY_SCALAR_FIELD(suid); COPY_SCALAR_FIELD(tableType); COPY_OBJECT_FIELD(tableName, sizeof(SName)); + COPY_SCALAR_FIELD(groupOrderScan); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 99790e0a93f67401719ae0a5343d77bb365d0cfa..f25616065eb064d0160209320ecb744ba6ac23d8 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1559,6 +1559,7 @@ static const char* jkScanPhysiPlanTableId = "TableId"; static const char* jkScanPhysiPlanSTableId = "STableId"; static const char* jkScanPhysiPlanTableType = "TableType"; static const char* jkScanPhysiPlanTableName = "TableName"; +static const char* jkScanPhysiPlanGroupOrderScan = "GroupOrderScan"; static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj; @@ -1582,6 +1583,9 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanPhysiPlanTableName, nameToJson, &pNode->tableName); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkScanPhysiPlanGroupOrderScan, pNode->groupOrderScan); + } return code; } @@ -1608,6 +1612,9 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, jkScanPhysiPlanTableName, jsonToName, &pNode->tableName); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkScanPhysiPlanGroupOrderScan, &pNode->groupOrderScan); + } return code; } @@ -2115,9 +2122,6 @@ static int32_t physiSortNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkSortPhysiPlanTargets, pNode->pTargets); } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkSortPhysiPlanMaxRows, pNode->maxRows); - } return code; } @@ -2135,9 +2139,6 @@ static int32_t jsonToPhysiSortNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkSortPhysiPlanTargets, &pNode->pTargets); } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBigIntValue(pJson, jkSortPhysiPlanMaxRows, &pNode->maxRows); - } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index e79a520615142b977716cc837f5e31fc4bbd73c3..20e829766dafe75aa2372aa1e4147ac9b856ee71 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -1853,7 +1853,8 @@ enum { PHY_NODE_CODE_LIMIT, PHY_NODE_CODE_SLIMIT, PHY_NODE_CODE_INPUT_TS_ORDER, - PHY_NODE_CODE_OUTPUT_TS_ORDER + PHY_NODE_CODE_OUTPUT_TS_ORDER, + PHY_NODE_CODE_FORCE_NONBLOCKING_OPTR }; static int32_t physiNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -1878,6 +1879,9 @@ static int32_t physiNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeEnum(pEncoder, PHY_NODE_CODE_OUTPUT_TS_ORDER, pNode->outputTsOrder); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_NODE_CODE_FORCE_NONBLOCKING_OPTR, pNode->forceCreateNonBlockingOptr); + } return code; } @@ -1910,6 +1914,8 @@ static int32_t msgToPhysiNode(STlvDecoder* pDecoder, void* pObj) { case PHY_NODE_CODE_OUTPUT_TS_ORDER: code = tlvDecodeEnum(pTlv, &pNode->outputTsOrder, sizeof(pNode->outputTsOrder)); break; + case PHY_NODE_CODE_FORCE_NONBLOCKING_OPTR: + code = tlvDecodeBool(pTlv, &pNode->forceCreateNonBlockingOptr); default: break; } @@ -1925,7 +1931,8 @@ enum { PHY_SCAN_CODE_BASE_UID, PHY_SCAN_CODE_BASE_SUID, PHY_SCAN_CODE_BASE_TABLE_TYPE, - PHY_SCAN_CODE_BASE_TABLE_NAME + PHY_SCAN_CODE_BASE_TABLE_NAME, + PHY_SCAN_CODE_BASE_GROUP_ORDER_SCAN }; static int32_t physiScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -1950,6 +1957,9 @@ static int32_t physiScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeObj(pEncoder, PHY_SCAN_CODE_BASE_TABLE_NAME, nameToMsg, &pNode->tableName); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_SCAN_CODE_BASE_GROUP_ORDER_SCAN, pNode->groupOrderScan); + } return code; } @@ -1982,6 +1992,9 @@ static int32_t msgToPhysiScanNode(STlvDecoder* pDecoder, void* pObj) { case PHY_SCAN_CODE_BASE_TABLE_NAME: code = tlvDecodeObjFromTlv(pTlv, msgToName, &pNode->tableName); break; + case PHY_SCAN_CODE_BASE_GROUP_ORDER_SCAN: + code = tlvDecodeBool(pTlv, &pNode->groupOrderScan); + break; default: break; } @@ -2594,7 +2607,7 @@ static int32_t msgToPhysiMergeNode(STlvDecoder* pDecoder, void* pObj) { return code; } -enum { PHY_SORT_CODE_BASE_NODE = 1, PHY_SORT_CODE_EXPR, PHY_SORT_CODE_SORT_KEYS, PHY_SORT_CODE_TARGETS, PHY_SORT_CODE_MAX_ROWS }; +enum { PHY_SORT_CODE_BASE_NODE = 1, PHY_SORT_CODE_EXPR, PHY_SORT_CODE_SORT_KEYS, PHY_SORT_CODE_TARGETS }; static int32_t physiSortNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { const SSortPhysiNode* pNode = (const SSortPhysiNode*)pObj; @@ -2609,9 +2622,6 @@ static int32_t physiSortNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeObj(pEncoder, PHY_SORT_CODE_TARGETS, nodeListToMsg, pNode->pTargets); } - if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeI64(pEncoder, PHY_SORT_CODE_MAX_ROWS, pNode->maxRows); - } return code; } @@ -2635,9 +2645,6 @@ static int32_t msgToPhysiSortNode(STlvDecoder* pDecoder, void* pObj) { case PHY_SORT_CODE_TARGETS: code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets); break; - case PHY_SORT_CODE_MAX_ROWS: - code = tlvDecodeI64(pTlv, &pNode->maxRows); - break; default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 15232b95b6dcb584ee22d708ca42e3d6a359ad82..c8197721fb2803efa6d4ab40a8ef4dd1f20a46bb 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -907,6 +907,10 @@ void nodesDestroyNode(SNode* pNode) { SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pNode; nodesDestroyNode((SNode*)pStmt->pOptions); nodesDestroyList(pStmt->pCols); + if (pStmt->pReq) { + tFreeSMCreateSmaReq(pStmt->pReq); + taosMemoryFreeClear(pStmt->pReq); + } break; } case QUERY_NODE_DROP_INDEX_STMT: // no pointer field @@ -1053,6 +1057,7 @@ void nodesDestroyNode(SNode* pNode) { } case QUERY_NODE_QUERY: { SQuery* pQuery = (SQuery*)pNode; + nodesDestroyNode(pQuery->pPrevRoot); nodesDestroyNode(pQuery->pRoot); nodesDestroyNode(pQuery->pPostRoot); taosMemoryFreeClear(pQuery->pResSchema); diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index ff394467f6f660054f7eaf1768be3c1fcc2a4483..78422bf746ebcb72c495a2475b8dc26872e7bacc 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -206,9 +206,9 @@ SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, cons SNode* createRestoreComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId); SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pQuery); SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SToken* pSubDbName, - bool withMeta); + int8_t withMeta); SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable, - bool withMeta, SNode* pWhere); + int8_t withMeta, SNode* pWhere); SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pTopicName); SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pCGroupId, SToken* pTopicName); SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue); diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index d79aa84bb81e86386642d1b4bd6e219b2ee49b84..69253e62e23a0e3970c072f4db9de9c568c4d5e0 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -35,6 +35,7 @@ int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMe int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow); +int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 518dd95f23314a5dc5ce115785b19db4575ff299..6c3f589159b8db9903a9fefee9dbb6ead26f96d0 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -447,6 +447,7 @@ cmd ::= SHOW MNODES. cmd ::= SHOW QNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); } cmd ::= SHOW FUNCTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } cmd ::= SHOW INDEXES FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, B, A, OP_TYPE_EQUAL); } +cmd ::= SHOW INDEXES FROM db_name(B) NK_DOT table_name(A). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, createIdentifierValueNode(pCxt, &B), createIdentifierValueNode(pCxt, &A), OP_TYPE_EQUAL); } cmd ::= SHOW STREAMS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } cmd ::= SHOW ACCOUNTS. { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } cmd ::= SHOW APPS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); } @@ -471,7 +472,9 @@ cmd ::= SHOW TABLE DISTRIBUTED full_table_name(A). cmd ::= SHOW CONSUMERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } cmd ::= SHOW SUBSCRIPTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } cmd ::= SHOW TAGS FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, B, A, OP_TYPE_EQUAL); } +cmd ::= SHOW TAGS FROM db_name(B) NK_DOT table_name(A). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, createIdentifierValueNode(pCxt, &B), createIdentifierValueNode(pCxt, &A), OP_TYPE_EQUAL); } cmd ::= SHOW TABLE TAGS tag_list_opt(C) FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowTableTagsStmt(pCxt, A, B, C); } +cmd ::= SHOW TABLE TAGS tag_list_opt(C) FROM db_name(B) NK_DOT table_name(A). { pCxt->pRootNode = createShowTableTagsStmt(pCxt, createIdentifierValueNode(pCxt, &A), createIdentifierValueNode(pCxt, &B), C); } cmd ::= SHOW VNODES NK_INTEGER(A). { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A), NULL); } cmd ::= SHOW VNODES NK_STRING(A). { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &A)); } // show alive @@ -538,14 +541,18 @@ sma_stream_opt(A) ::= sma_stream_opt(B) MAX_DELAY duration_literal(C). sma_stream_opt(A) ::= sma_stream_opt(B) DELETE_MARK duration_literal(C). { ((SStreamOptions*)B)->pDeleteMark = releaseRawExprNode(pCxt, C); A = B; } /************************************************ create/drop topic ***************************************************/ +%type with_meta { int32_t } +%destructor with_meta { } +with_meta(A) ::= AS. { A = 0; } +with_meta(A) ::= WITH META AS. { A = 1; } +with_meta(A) ::= ONLY META AS. { A = 2; } + cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_or_subquery(C). { pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, A, &B, C); } -cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, false); } -cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) - WITH META AS DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, true); } -cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) - AS STABLE full_table_name(C) where_clause_opt(D). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, false, D); } -cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) - WITH META AS STABLE full_table_name(C) where_clause_opt(D). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, true, D); } +cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) with_meta(D) + DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, D); } +cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) with_meta(E) + STABLE full_table_name(C) where_clause_opt(D). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, E, D); } + cmd ::= DROP TOPIC exists_opt(A) topic_name(B). { pCxt->pRootNode = createDropTopicStmt(pCxt, A, &B); } cmd ::= DROP CONSUMER GROUP exists_opt(A) cgroup_name(B) ON topic_name(C). { pCxt->pRootNode = createDropCGroupStmt(pCxt, A, &B, &C); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index e08153c34121c541fab57a3a03b66c8fb9b7152d..f85218c50a767ca41acef02618251b2e0749bceb 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1715,7 +1715,7 @@ SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, } SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SToken* pSubDbName, - bool withMeta) { + int8_t withMeta) { CHECK_PARSER_STATUS(pCxt); if (!checkTopicName(pCxt, pTopicName) || !checkDbName(pCxt, pSubDbName, true)) { return NULL; @@ -1730,7 +1730,7 @@ SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, ST } SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable, - bool withMeta, SNode* pWhere) { + int8_t withMeta, SNode* pWhere) { CHECK_PARSER_STATUS(pCxt); if (!checkTopicName(pCxt, pTopicName)) { return NULL; diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 3af2d440c90939108f97501901560b5644447103..ca7ac1a0b666b556f45b91b3b0dad18e8499561f 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -140,6 +140,7 @@ static SKeyword keywordTable[] = { {"MAX_SPEED", TK_MAX_SPEED}, {"MERGE", TK_MERGE}, {"META", TK_META}, + {"ONLY", TK_ONLY}, {"MINROWS", TK_MINROWS}, {"MINUS", TK_MINUS}, {"MNODE", TK_MNODE}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 00fa7ee7e22b0141bc00bb7b0a33cade1117d88f..ef1bc8788c82dbaf371624a9526da3107537f1da 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -509,6 +509,10 @@ static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int } static int32_t getDBCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo* pInfo) { + if (IS_SYS_DBNAME(pDbName)) { + return TSDB_CODE_SUCCESS; + } + SParseContext* pParCxt = pCxt->pParseCxt; SName name; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pDbName, strlen(pDbName)); @@ -2930,14 +2934,14 @@ static int32_t createMultiResFuncsFromStar(STranslateContext* pCxt, SFunctionNod static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) { if (QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_PC, - "The _TAGS pseudo column can only be used for subtable and supertable queries"); + "The _TAGS pseudo column can only be used for child table and super table queries"); } SRealTableNode* pTable = (SRealTableNode*)(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable); const STableMeta* pMeta = pTable->pMeta; if (TSDB_SUPER_TABLE != pMeta->tableType && TSDB_CHILD_TABLE != pMeta->tableType) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_PC, - "The _TAGS pseudo column can only be used for subtable and supertable queries"); + "The _TAGS pseudo column can only be used for child table and super table queries"); } SSchema* pTagsSchema = getTableTagSchema(pMeta); @@ -3526,6 +3530,10 @@ static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; } + if (pSelect->pFromTable->type == QUERY_NODE_REAL_TABLE && + ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType == TSDB_SYSTEM_TABLE) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED, "WINDOW"); + } pCxt->currClause = SQL_CLAUSE_WINDOW; int32_t code = translateExpr(pCxt, &pSelect->pWindow); if (TSDB_CODE_SUCCESS == code) { @@ -5809,6 +5817,15 @@ static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStm if (TSDB_CODE_SUCCESS == code) { code = getSmaIndexAst(pCxt, pStmt, &pReq->ast, &pReq->astLen, &pReq->expr, &pReq->exprLen); } + if (TSDB_CODE_SUCCESS == code) { + STableMeta* pMetaCache = NULL; + code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + pStmt->pOptions->tsPrecision = pMetaCache->tableInfo.precision; + code = createLastTsSelectStmt(pStmt->dbName, pStmt->tableName, pMetaCache, &pStmt->pPrevQuery); + } + taosMemoryFreeClear(pMetaCache); + } return code; } @@ -5834,15 +5851,60 @@ static int32_t checkCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pS } static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { - SMCreateSmaReq createSmaReq = {0}; int32_t code = checkCreateSmaIndex(pCxt, pStmt); + pStmt->pReq = taosMemoryCalloc(1, sizeof(SMCreateSmaReq)); + if (pStmt->pReq == NULL) code = TSDB_CODE_OUT_OF_MEMORY; if (TSDB_CODE_SUCCESS == code) { - code = buildCreateSmaReq(pCxt, pStmt, &createSmaReq); + code = buildCreateSmaReq(pCxt, pStmt, pStmt->pReq); } + TSWAP(pCxt->pPrevRoot, pStmt->pPrevQuery); + return code; +} + +int32_t createIntervalFromCreateSmaIndexStmt(SCreateIndexStmt* pStmt, SInterval* pInterval) { + pInterval->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; + pInterval->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; + pInterval->offset = NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0; + pInterval->sliding = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pInterval->interval; + pInterval->slidingUnit = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pInterval->intervalUnit; + pInterval->precision = pStmt->pOptions->tsPrecision; + return TSDB_CODE_SUCCESS; +} + +int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void ** pResRow) { + int32_t code = TSDB_CODE_SUCCESS; + SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pQuery->pRoot; + int64_t lastTs = 0; + SInterval interval = {0}; + STranslateContext pCxt = {0}; + code = initTranslateContext(pParseCxt, NULL, &pCxt); if (TSDB_CODE_SUCCESS == code) { - code = buildCmdMsg(pCxt, TDMT_MND_CREATE_SMA, (FSerializeFunc)tSerializeSMCreateSmaReq, &createSmaReq); + code = createIntervalFromCreateSmaIndexStmt(pStmt, &interval); } - tFreeSMCreateSmaReq(&createSmaReq); + if (TSDB_CODE_SUCCESS == code) { + if (pResRow && pResRow[0]) { + lastTs = *(int64_t*)pResRow[0]; + } else if (interval.interval > 0) { + lastTs = convertTimePrecision(taosGetTimestampMs(), TSDB_TIME_PRECISION_MILLI, interval.precision); + } else { + lastTs = taosGetTimestampMs(); + } + } + if (TSDB_CODE_SUCCESS == code) { + if (interval.interval > 0) { + pStmt->pReq->lastTs = taosTimeTruncate(lastTs, &interval); + } else { + pStmt->pReq->lastTs = lastTs; + } + code = buildCmdMsg(&pCxt, TDMT_MND_CREATE_SMA, (FSerializeFunc)tSerializeSMCreateSmaReq, pStmt->pReq); + } + if (TSDB_CODE_SUCCESS == code) { + code = setQuery(&pCxt, pQuery); + } + setRefreshMate(&pCxt, pQuery); + destroyTranslateContext(&pCxt); + tFreeSMCreateSmaReq(pStmt->pReq); + taosMemoryFreeClear(pStmt->pReq); return code; } @@ -6996,7 +7058,7 @@ static int32_t translateCreateStream(STranslateContext* pCxt, SCreateStreamStmt* return code; } -int32_t buildIntervalForCreateStream(SCreateStreamStmt* pStmt, SInterval* pInterval) { +static int32_t buildIntervalForCreateStream(SCreateStreamStmt* pStmt, SInterval* pInterval) { int32_t code = TSDB_CODE_SUCCESS; if (QUERY_NODE_SELECT_STMT != nodeType(pStmt->pQuery)) { return code; @@ -7656,7 +7718,7 @@ static int32_t extractShowCreateTableResultSchema(int32_t* numOfCols, SSchema** } static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pSchema) { - *numOfCols = 2; + *numOfCols = 3; *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); if (NULL == (*pSchema)) { return TSDB_CODE_OUT_OF_MEMORY; @@ -7670,6 +7732,10 @@ static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pS (*pSchema)[1].bytes = TSDB_CONFIG_VALUE_LEN; strcpy((*pSchema)[1].name, "value"); + (*pSchema)[2].type = TSDB_DATA_TYPE_BINARY; + (*pSchema)[2].bytes = TSDB_CONFIG_SCOPE_LEN; + strcpy((*pSchema)[2].name, "scope"); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index f82d56ac563828c1e1fa5d523110f03ebcde4422..42a0d1282a986cc4afaf34a2363020d1317651f2 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -164,6 +164,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "%s function is not supported in fill query"; case TSDB_CODE_PAR_INVALID_WINDOW_PC: return "_WSTART, _WEND and _WDURATION can only be used in window query"; + case TSDB_CODE_PAR_INVALID_TAGS_PC: + return "Tags can only applied to super table and child table"; case TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC: return "%s function is not supported in time window query"; case TSDB_CODE_PAR_STREAM_NOT_ALLOWED_FUNC: @@ -172,6 +174,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "%s function is not supported in group query"; case TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC: return "%s function is not supported in system table query"; + case TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED: + return "%s is not supported in system table query"; case TSDB_CODE_PAR_INVALID_INTERP_CLAUSE: return "Invalid usage of RANGE clause, EVERY clause or FILL clause"; case TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN: diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index cbddaf811592a4570e79ed71b857b4aee575c186..10fda8741b073c2fb8a7fa80e21afbc3680b3ede 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -227,6 +227,8 @@ int32_t qContinueParsePostQuery(SParseContext* pCxt, SQuery* pQuery, void** pRes case QUERY_NODE_CREATE_STREAM_STMT: code = translatePostCreateStream(pCxt, pQuery, pResRow); break; + case QUERY_NODE_CREATE_INDEX_STMT: + code = translatePostCreateSmaIndex(pCxt, pQuery, pResRow); default: break; } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 3f05e3269ac84a60bb6fa752e63165451dbb1e85..a912fb4e71c8aeb0b78298915540fbd52029a597 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -104,27 +104,27 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 485 +#define YYNOCODE 487 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - EOrder yy48; - EOperatorType yy70; - int8_t yy73; - int32_t yy120; - SNodeList* yy174; - EFillMode yy204; - SNode* yy242; - STokenPair yy257; - int64_t yy349; - EJoinType yy482; - SAlterOption yy535; - SToken yy669; - ENullOrder yy687; - bool yy777; - SDataType yy794; + EJoinType yy140; + SDataType yy310; + STokenPair yy347; + EOperatorType yy354; + SAlterOption yy365; + SToken yy371; + ENullOrder yy399; + int32_t yy416; + SNode* yy452; + int8_t yy475; + bool yy667; + EOrder yy690; + int64_t yy729; + SNodeList* yy812; + EFillMode yy844; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -140,18 +140,18 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 794 -#define YYNRULE 596 -#define YYNRULE_WITH_ACTION 596 -#define YYNTOKEN 337 -#define YY_MAX_SHIFT 793 -#define YY_MIN_SHIFTREDUCE 1172 -#define YY_MAX_SHIFTREDUCE 1767 -#define YY_ERROR_ACTION 1768 -#define YY_ACCEPT_ACTION 1769 -#define YY_NO_ACTION 1770 -#define YY_MIN_REDUCE 1771 -#define YY_MAX_REDUCE 2366 +#define YYNSTATE 800 +#define YYNRULE 600 +#define YYNRULE_WITH_ACTION 600 +#define YYNTOKEN 338 +#define YY_MAX_SHIFT 799 +#define YY_MIN_SHIFTREDUCE 1180 +#define YY_MAX_SHIFTREDUCE 1779 +#define YY_ERROR_ACTION 1780 +#define YY_ACCEPT_ACTION 1781 +#define YY_NO_ACTION 1782 +#define YY_MIN_REDUCE 1783 +#define YY_MAX_REDUCE 2382 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -218,795 +218,821 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2730) +#define YY_ACTTAB_COUNT (2858) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 2177, 2067, 170, 218, 1783, 670, 447, 531, 665, 1814, - /* 10 */ 662, 446, 48, 46, 1694, 1937, 2065, 671, 647, 1794, - /* 20 */ 401, 2337, 1543, 41, 40, 410, 409, 47, 45, 44, - /* 30 */ 43, 42, 686, 1624, 2237, 1541, 646, 188, 2195, 41, - /* 40 */ 40, 2338, 648, 47, 45, 44, 43, 42, 1550, 2138, - /* 50 */ 2145, 1571, 700, 622, 1568, 622, 2337, 543, 2337, 2060, - /* 60 */ 2177, 181, 1619, 47, 45, 44, 43, 42, 19, 2145, - /* 70 */ 701, 2343, 188, 2343, 188, 1549, 2338, 648, 2338, 648, - /* 80 */ 220, 368, 2050, 360, 531, 2176, 1814, 2212, 107, 184, - /* 90 */ 110, 2178, 704, 2180, 2181, 699, 622, 694, 2195, 2337, - /* 100 */ 790, 1988, 185, 15, 2265, 143, 87, 641, 397, 2261, - /* 110 */ 2145, 1793, 700, 1940, 2343, 188, 48, 46, 2067, 2338, - /* 120 */ 648, 190, 2155, 370, 401, 2342, 1543, 1653, 2337, 2291, - /* 130 */ 394, 683, 1943, 2064, 671, 2195, 1939, 1624, 205, 1541, - /* 140 */ 1626, 1627, 453, 1318, 2341, 2176, 2159, 2212, 2338, 2340, - /* 150 */ 110, 2178, 704, 2180, 2181, 699, 1317, 694, 1569, 404, - /* 160 */ 145, 2145, 152, 2236, 2265, 181, 1619, 164, 397, 2261, - /* 170 */ 1599, 1609, 19, 2001, 382, 1950, 1625, 1628, 84, 1549, - /* 180 */ 381, 83, 1999, 2161, 1654, 1772, 2051, 347, 1999, 640, - /* 190 */ 1544, 123, 1542, 694, 122, 121, 120, 119, 118, 117, - /* 200 */ 116, 115, 114, 263, 790, 1553, 123, 15, 2177, 122, - /* 210 */ 121, 120, 119, 118, 117, 116, 115, 114, 701, 669, - /* 220 */ 1816, 500, 1547, 1548, 1771, 1598, 1601, 1602, 1603, 1604, - /* 230 */ 1605, 1606, 1607, 1608, 696, 692, 1617, 1618, 1620, 1621, - /* 240 */ 1622, 1623, 2, 642, 1626, 1627, 2195, 1792, 132, 131, - /* 250 */ 130, 129, 128, 127, 126, 125, 124, 670, 2145, 62, - /* 260 */ 700, 37, 399, 1648, 1649, 1650, 1651, 1652, 1656, 1657, - /* 270 */ 1658, 1659, 38, 305, 1599, 1609, 1568, 287, 1398, 1399, - /* 280 */ 1625, 1628, 1205, 683, 41, 40, 684, 1948, 47, 45, - /* 290 */ 44, 43, 42, 2176, 1544, 2212, 1542, 2145, 110, 2178, - /* 300 */ 704, 2180, 2181, 699, 528, 694, 133, 529, 1807, 668, - /* 310 */ 2357, 2060, 2265, 568, 1568, 2177, 397, 2261, 62, 1567, - /* 320 */ 93, 1207, 533, 1210, 1211, 662, 1547, 1548, 530, 1598, - /* 330 */ 1601, 1602, 1603, 1604, 1605, 1606, 1607, 1608, 696, 692, - /* 340 */ 1617, 1618, 1620, 1621, 1622, 1623, 2, 12, 48, 46, - /* 350 */ 12, 637, 10, 2195, 395, 683, 401, 406, 1543, 1363, - /* 360 */ 1994, 1996, 167, 373, 545, 2145, 51, 700, 670, 1624, - /* 370 */ 1950, 1541, 684, 1948, 1354, 729, 728, 727, 1358, 726, - /* 380 */ 1360, 1361, 725, 722, 2131, 1369, 719, 1371, 1372, 716, - /* 390 */ 713, 710, 133, 2177, 8, 2280, 659, 142, 1619, 573, - /* 400 */ 2176, 1791, 2212, 698, 19, 110, 2178, 704, 2180, 2181, - /* 410 */ 699, 1549, 694, 2342, 1230, 191, 1229, 185, 1925, 2265, - /* 420 */ 679, 2277, 2060, 397, 2261, 250, 1600, 659, 142, 138, - /* 430 */ 1570, 2195, 374, 423, 372, 371, 790, 570, 51, 15, - /* 440 */ 643, 638, 631, 2145, 2292, 700, 1570, 1231, 586, 585, - /* 450 */ 584, 2145, 48, 46, 1629, 576, 139, 580, 1764, 572, - /* 460 */ 401, 579, 1543, 571, 1453, 1454, 578, 583, 376, 375, - /* 470 */ 609, 1722, 577, 1624, 191, 1541, 1626, 1627, 2176, 2280, - /* 480 */ 2212, 1995, 1996, 341, 2178, 704, 2180, 2181, 699, 697, - /* 490 */ 694, 685, 2230, 536, 1514, 1515, 529, 1807, 2177, 187, - /* 500 */ 2273, 2274, 1619, 140, 2278, 2276, 1599, 1609, 701, 1230, - /* 510 */ 2312, 1229, 1625, 1628, 1757, 1549, 659, 142, 634, 633, - /* 520 */ 1720, 1721, 1723, 1724, 1725, 491, 1544, 739, 1542, 661, - /* 530 */ 186, 2273, 2274, 525, 140, 2278, 2195, 14, 13, 1568, - /* 540 */ 790, 523, 1231, 49, 519, 515, 659, 142, 2145, 1790, - /* 550 */ 700, 737, 157, 156, 734, 733, 732, 154, 1547, 1548, - /* 560 */ 1763, 1598, 1601, 1602, 1603, 1604, 1605, 1606, 1607, 1608, - /* 570 */ 696, 692, 1617, 1618, 1620, 1621, 1622, 1623, 2, 574, - /* 580 */ 1626, 1627, 285, 2176, 2341, 2212, 209, 208, 110, 2178, - /* 590 */ 704, 2180, 2181, 699, 191, 694, 12, 2001, 285, 2145, - /* 600 */ 2357, 1305, 2265, 101, 391, 2177, 397, 2261, 2110, 490, - /* 610 */ 1599, 1609, 1999, 1687, 600, 701, 1625, 1628, 1924, 283, - /* 620 */ 2273, 658, 62, 134, 657, 66, 2337, 598, 1941, 596, - /* 630 */ 1544, 435, 1542, 737, 157, 156, 734, 733, 732, 154, - /* 640 */ 1667, 646, 188, 2195, 684, 1948, 2338, 648, 1769, 189, - /* 650 */ 2273, 2274, 1549, 140, 2278, 2145, 251, 700, 437, 433, - /* 660 */ 2031, 2139, 1547, 1548, 193, 1598, 1601, 1602, 1603, 1604, - /* 670 */ 1605, 1606, 1607, 1608, 696, 692, 1617, 1618, 1620, 1621, - /* 680 */ 1622, 1623, 2, 48, 46, 739, 169, 1233, 1234, 52, - /* 690 */ 2176, 401, 2212, 1543, 1889, 110, 2178, 704, 2180, 2181, - /* 700 */ 699, 575, 694, 2001, 1624, 2155, 1541, 2240, 622, 2265, - /* 710 */ 396, 2337, 234, 397, 2261, 44, 43, 42, 1999, 2164, - /* 720 */ 416, 1322, 1543, 1303, 62, 415, 2343, 188, 174, 2159, - /* 730 */ 2177, 2338, 648, 1619, 1321, 1541, 562, 558, 554, 550, - /* 740 */ 701, 233, 688, 1789, 2237, 2155, 1549, 41, 40, 1307, - /* 750 */ 34, 47, 45, 44, 43, 42, 41, 40, 1264, 2163, - /* 760 */ 47, 45, 44, 43, 42, 1926, 2161, 622, 2195, 2159, - /* 770 */ 2337, 790, 684, 1948, 49, 1549, 694, 730, 191, 1788, - /* 780 */ 2145, 88, 700, 30, 231, 2343, 188, 48, 46, 1309, - /* 790 */ 2338, 648, 681, 2145, 2177, 401, 1571, 1543, 1265, 502, - /* 800 */ 790, 1787, 564, 563, 701, 749, 2161, 398, 1624, 150, - /* 810 */ 1541, 1626, 1627, 684, 1948, 2176, 694, 2212, 1569, 471, - /* 820 */ 111, 2178, 704, 2180, 2181, 699, 1933, 694, 470, 2145, - /* 830 */ 566, 565, 2195, 57, 2265, 684, 1948, 1619, 2264, 2261, - /* 840 */ 1571, 1599, 1609, 1734, 2145, 191, 700, 1625, 1628, 2046, - /* 850 */ 1549, 2145, 230, 224, 572, 451, 1935, 229, 571, 541, - /* 860 */ 650, 1544, 144, 1542, 249, 2236, 41, 40, 248, 1786, - /* 870 */ 47, 45, 44, 43, 42, 790, 1698, 222, 15, 2176, - /* 880 */ 191, 2212, 1568, 2046, 171, 2178, 704, 2180, 2181, 699, - /* 890 */ 1544, 694, 1542, 1547, 1548, 201, 1598, 1601, 1602, 1603, - /* 900 */ 1604, 1605, 1606, 1607, 1608, 696, 692, 1617, 1618, 1620, - /* 910 */ 1621, 1622, 1623, 2, 1931, 1626, 1627, 582, 581, 2145, - /* 920 */ 684, 1948, 1547, 1548, 623, 2302, 1952, 2001, 90, 203, - /* 930 */ 2280, 355, 2046, 404, 380, 351, 602, 1566, 1470, 1471, - /* 940 */ 452, 167, 2000, 2177, 484, 1599, 1609, 498, 202, 1950, - /* 950 */ 497, 1625, 1628, 701, 256, 629, 2275, 684, 1948, 684, - /* 960 */ 1948, 2342, 1733, 1213, 2337, 1544, 467, 1542, 499, 1567, - /* 970 */ 1785, 684, 1948, 469, 1469, 1472, 36, 461, 207, 476, - /* 980 */ 2341, 2195, 41, 40, 2338, 2339, 47, 45, 44, 43, - /* 990 */ 42, 477, 262, 2145, 695, 700, 731, 1547, 1548, 1992, - /* 1000 */ 1598, 1601, 1602, 1603, 1604, 1605, 1606, 1607, 1608, 696, - /* 1010 */ 692, 1617, 1618, 1620, 1621, 1622, 1623, 2, 1847, 369, - /* 1020 */ 2145, 167, 684, 1948, 684, 1948, 684, 1948, 2176, 1951, - /* 1030 */ 2212, 457, 1600, 110, 2178, 704, 2180, 2181, 699, 54, - /* 1040 */ 694, 3, 544, 259, 1945, 2357, 252, 2265, 1890, 41, - /* 1050 */ 40, 397, 2261, 47, 45, 44, 43, 42, 296, 297, - /* 1060 */ 2128, 495, 2305, 295, 489, 488, 487, 486, 483, 482, - /* 1070 */ 481, 480, 479, 475, 474, 473, 472, 350, 464, 463, - /* 1080 */ 462, 607, 459, 458, 367, 1691, 1782, 1634, 767, 766, - /* 1090 */ 765, 764, 413, 1568, 763, 762, 146, 757, 756, 755, - /* 1100 */ 754, 753, 752, 751, 159, 747, 746, 745, 412, 411, - /* 1110 */ 742, 741, 740, 177, 176, 168, 1850, 454, 41, 40, - /* 1120 */ 326, 1655, 47, 45, 44, 43, 42, 87, 622, 2177, - /* 1130 */ 455, 2337, 1784, 2001, 323, 73, 2145, 155, 72, 701, - /* 1140 */ 405, 2330, 1781, 761, 759, 1710, 2343, 188, 1999, 348, - /* 1150 */ 280, 2338, 648, 1944, 684, 1948, 62, 635, 684, 1948, - /* 1160 */ 216, 510, 508, 505, 137, 2177, 1923, 2195, 1780, 691, - /* 1170 */ 2001, 684, 1948, 735, 260, 701, 1992, 366, 286, 2145, - /* 1180 */ 1888, 700, 684, 1948, 1779, 1999, 586, 585, 584, 684, - /* 1190 */ 1948, 667, 2145, 576, 139, 580, 1778, 56, 35, 579, - /* 1200 */ 62, 651, 300, 2195, 578, 583, 376, 375, 1660, 682, - /* 1210 */ 577, 274, 1210, 1211, 2176, 2145, 2212, 700, 2145, 110, - /* 1220 */ 2178, 704, 2180, 2181, 699, 1777, 694, 654, 1776, 684, - /* 1230 */ 1948, 2357, 1887, 2265, 2145, 2177, 414, 397, 2261, 109, - /* 1240 */ 1775, 1774, 445, 1600, 444, 701, 2145, 2284, 647, 306, - /* 1250 */ 2176, 2337, 2212, 2285, 1687, 172, 2178, 704, 2180, 2181, - /* 1260 */ 699, 407, 694, 684, 1948, 736, 646, 188, 1992, 167, - /* 1270 */ 2196, 2338, 648, 2195, 443, 2145, 430, 1950, 2145, 81, - /* 1280 */ 80, 450, 166, 408, 200, 2145, 2177, 700, 261, 319, - /* 1290 */ 2145, 2145, 1978, 604, 2055, 603, 701, 442, 440, 737, - /* 1300 */ 157, 156, 734, 733, 732, 154, 649, 2358, 349, 2177, - /* 1310 */ 750, 431, 191, 1910, 429, 425, 421, 418, 443, 701, - /* 1320 */ 2176, 2299, 2212, 1645, 2195, 110, 2178, 704, 2180, 2181, - /* 1330 */ 699, 91, 694, 148, 74, 135, 2145, 2357, 700, 2265, - /* 1340 */ 1834, 1690, 1825, 397, 2261, 410, 409, 2195, 239, 241, - /* 1350 */ 243, 237, 240, 242, 245, 1557, 191, 244, 621, 2145, - /* 1360 */ 2177, 700, 587, 155, 589, 1823, 1624, 155, 1550, 50, - /* 1370 */ 701, 2176, 50, 2212, 267, 155, 335, 2178, 704, 2180, - /* 1380 */ 2181, 699, 50, 694, 82, 41, 40, 592, 293, 47, - /* 1390 */ 45, 44, 43, 42, 2176, 1619, 2212, 1552, 2195, 110, - /* 1400 */ 2178, 704, 2180, 2181, 699, 71, 694, 153, 1549, 1551, - /* 1410 */ 2145, 2357, 700, 2265, 1766, 1767, 155, 397, 2261, 644, - /* 1420 */ 2177, 14, 13, 1509, 64, 50, 50, 1512, 708, 1719, - /* 1430 */ 701, 153, 1718, 690, 269, 666, 2166, 155, 652, 743, - /* 1440 */ 136, 744, 1467, 153, 1817, 2176, 106, 2212, 298, 2177, - /* 1450 */ 110, 2178, 704, 2180, 2181, 699, 103, 694, 2195, 701, - /* 1460 */ 1808, 1283, 2238, 1281, 2265, 676, 655, 302, 397, 2261, - /* 1470 */ 2145, 1813, 700, 1989, 2295, 660, 1348, 282, 279, 1, - /* 1480 */ 9, 417, 55, 422, 1661, 1610, 318, 2195, 1376, 364, - /* 1490 */ 1574, 1380, 2168, 438, 785, 439, 195, 1387, 196, 2145, - /* 1500 */ 1385, 700, 441, 158, 198, 2176, 1490, 2212, 313, 206, - /* 1510 */ 110, 2178, 704, 2180, 2181, 699, 456, 694, 1571, 460, - /* 1520 */ 2056, 493, 687, 1558, 2265, 1553, 465, 1566, 397, 2261, - /* 1530 */ 478, 2048, 485, 492, 2176, 504, 2212, 494, 503, 111, - /* 1540 */ 2178, 704, 2180, 2181, 699, 501, 694, 591, 210, 211, - /* 1550 */ 506, 507, 213, 2265, 1555, 1561, 1563, 689, 2261, 509, - /* 1560 */ 511, 1572, 601, 526, 4, 2177, 1554, 537, 692, 1617, - /* 1570 */ 1618, 1620, 1621, 1622, 1623, 701, 247, 527, 534, 535, - /* 1580 */ 221, 1569, 538, 1573, 1575, 223, 539, 540, 226, 542, - /* 1590 */ 228, 2177, 594, 546, 567, 85, 569, 86, 232, 588, - /* 1600 */ 354, 701, 1938, 2195, 112, 246, 236, 1934, 608, 606, - /* 1610 */ 151, 89, 238, 2119, 314, 2145, 253, 700, 612, 613, - /* 1620 */ 611, 255, 257, 160, 161, 2177, 1936, 1932, 162, 2195, - /* 1630 */ 1497, 163, 617, 619, 636, 701, 2311, 674, 2287, 2310, - /* 1640 */ 7, 2145, 645, 700, 2116, 70, 2115, 273, 69, 616, - /* 1650 */ 702, 175, 2212, 626, 627, 111, 2178, 704, 2180, 2181, - /* 1660 */ 699, 632, 694, 2195, 2296, 2306, 386, 265, 639, 2265, - /* 1670 */ 275, 618, 268, 359, 2261, 2145, 2176, 700, 2212, 625, - /* 1680 */ 276, 111, 2178, 704, 2180, 2181, 699, 2177, 694, 624, - /* 1690 */ 278, 387, 2360, 656, 653, 2265, 1687, 701, 141, 1570, - /* 1700 */ 2262, 2281, 663, 664, 390, 288, 96, 2061, 1576, 677, - /* 1710 */ 2176, 2177, 2212, 315, 672, 171, 2178, 704, 2180, 2181, - /* 1720 */ 699, 701, 694, 678, 673, 2195, 2075, 2074, 61, 2073, - /* 1730 */ 384, 277, 316, 317, 98, 2177, 281, 2145, 1949, 700, - /* 1740 */ 2246, 393, 100, 102, 786, 701, 2336, 1993, 309, 2195, - /* 1750 */ 706, 1911, 320, 787, 385, 789, 2303, 329, 53, 324, - /* 1760 */ 2137, 2145, 356, 700, 357, 322, 344, 2136, 2135, 343, - /* 1770 */ 333, 78, 2176, 2195, 2212, 419, 1534, 342, 2178, 704, - /* 1780 */ 2180, 2181, 699, 2132, 694, 2145, 420, 700, 1535, 194, - /* 1790 */ 424, 2130, 426, 427, 428, 2129, 2176, 365, 2212, 2177, - /* 1800 */ 2127, 342, 2178, 704, 2180, 2181, 699, 432, 694, 701, - /* 1810 */ 2126, 434, 2125, 436, 1525, 2106, 197, 2105, 199, 1493, - /* 1820 */ 2176, 79, 2212, 1492, 2087, 172, 2178, 704, 2180, 2181, - /* 1830 */ 699, 2086, 694, 2085, 448, 449, 2084, 2195, 2083, 1444, - /* 1840 */ 2039, 2038, 392, 2036, 147, 2035, 2034, 2037, 2033, 2145, - /* 1850 */ 2032, 700, 2030, 2029, 2028, 204, 466, 2027, 468, 2041, - /* 1860 */ 2026, 2025, 2024, 2177, 149, 2011, 2010, 2009, 2040, 2008, - /* 1870 */ 2007, 1446, 2006, 698, 2023, 2022, 2021, 2359, 2020, 2019, - /* 1880 */ 2018, 2017, 2016, 2015, 2176, 2177, 2212, 2014, 2013, 342, - /* 1890 */ 2178, 704, 2180, 2181, 699, 701, 694, 2012, 2005, 2004, - /* 1900 */ 2003, 2195, 496, 2002, 352, 1853, 1319, 1852, 1323, 2177, - /* 1910 */ 63, 353, 1315, 2145, 1851, 700, 1849, 212, 1846, 701, - /* 1920 */ 214, 1845, 513, 2195, 512, 1838, 516, 1827, 400, 514, - /* 1930 */ 520, 524, 518, 215, 1803, 2145, 1212, 700, 1802, 2104, - /* 1940 */ 522, 517, 2094, 521, 217, 76, 2082, 2195, 2176, 219, - /* 1950 */ 2212, 2165, 402, 341, 2178, 704, 2180, 2181, 699, 2145, - /* 1960 */ 694, 700, 2231, 610, 182, 77, 183, 225, 227, 2081, - /* 1970 */ 2176, 532, 2212, 2059, 1927, 342, 2178, 704, 2180, 2181, - /* 1980 */ 699, 793, 694, 1848, 1257, 1844, 547, 549, 1842, 548, - /* 1990 */ 551, 553, 552, 1840, 2176, 312, 2212, 2177, 555, 342, - /* 2000 */ 2178, 704, 2180, 2181, 699, 556, 694, 701, 1837, 557, - /* 2010 */ 559, 180, 1392, 560, 1822, 561, 2177, 1820, 1821, 783, - /* 2020 */ 779, 775, 771, 1819, 310, 1391, 701, 2177, 1799, 1929, - /* 2030 */ 1928, 758, 1306, 235, 1304, 2195, 1302, 701, 760, 1301, - /* 2040 */ 1300, 1299, 1298, 1293, 1295, 1294, 1292, 2145, 1835, 700, - /* 2050 */ 1826, 590, 1824, 377, 2195, 378, 379, 593, 1798, 595, - /* 2060 */ 1797, 597, 1796, 599, 108, 2195, 2145, 303, 700, 113, - /* 2070 */ 1519, 1521, 1518, 2103, 1499, 1523, 1501, 2145, 2093, 700, - /* 2080 */ 58, 29, 605, 67, 2212, 614, 258, 337, 2178, 704, - /* 2090 */ 2180, 2181, 699, 2080, 694, 2177, 2078, 2342, 17, 20, - /* 2100 */ 680, 2176, 1736, 2212, 165, 701, 327, 2178, 704, 2180, - /* 2110 */ 2181, 699, 2176, 694, 2212, 2177, 615, 325, 2178, 704, - /* 2120 */ 2180, 2181, 699, 31, 694, 701, 2177, 1503, 620, 383, - /* 2130 */ 264, 5, 6, 2195, 21, 290, 701, 65, 272, 628, - /* 2140 */ 289, 630, 22, 271, 266, 2145, 2177, 700, 1717, 173, - /* 2150 */ 270, 2166, 33, 2195, 32, 24, 701, 1709, 1751, 92, - /* 2160 */ 254, 1756, 1750, 1757, 2195, 2145, 388, 700, 1755, 1684, - /* 2170 */ 1754, 389, 1683, 284, 59, 178, 2145, 2079, 700, 2077, - /* 2180 */ 2176, 60, 2212, 2076, 2195, 328, 2178, 704, 2180, 2181, - /* 2190 */ 699, 2058, 694, 95, 23, 18, 2145, 291, 700, 292, - /* 2200 */ 2176, 1715, 2212, 94, 2177, 334, 2178, 704, 2180, 2181, - /* 2210 */ 699, 2176, 694, 2212, 701, 25, 338, 2178, 704, 2180, - /* 2220 */ 2181, 699, 294, 694, 299, 2177, 2057, 68, 97, 103, - /* 2230 */ 26, 2176, 99, 2212, 301, 701, 330, 2178, 704, 2180, - /* 2240 */ 2181, 699, 2195, 694, 675, 304, 1636, 1635, 13, 1559, - /* 2250 */ 2177, 2215, 179, 1614, 2145, 693, 700, 11, 192, 1612, - /* 2260 */ 701, 39, 16, 2195, 1611, 1646, 27, 1591, 1583, 703, - /* 2270 */ 28, 705, 1377, 707, 403, 2145, 2177, 700, 709, 711, - /* 2280 */ 1374, 712, 714, 717, 1373, 715, 701, 718, 2195, 2176, - /* 2290 */ 720, 2212, 1370, 721, 339, 2178, 704, 2180, 2181, 699, - /* 2300 */ 2145, 694, 700, 1364, 1362, 723, 1368, 724, 307, 1386, - /* 2310 */ 2176, 104, 2212, 105, 2195, 331, 2178, 704, 2180, 2181, - /* 2320 */ 699, 1367, 694, 75, 1382, 1255, 2145, 1366, 700, 738, - /* 2330 */ 1287, 1365, 1286, 1285, 1284, 2176, 2177, 2212, 1282, 1280, - /* 2340 */ 340, 2178, 704, 2180, 2181, 699, 701, 694, 1279, 1278, - /* 2350 */ 1313, 748, 2177, 308, 1276, 1275, 1274, 1273, 1272, 1271, - /* 2360 */ 1270, 2176, 701, 2212, 1310, 2177, 332, 2178, 704, 2180, - /* 2370 */ 2181, 699, 1308, 694, 2195, 701, 1267, 1266, 1263, 1262, - /* 2380 */ 1261, 1260, 1843, 768, 770, 769, 2145, 1841, 700, 772, - /* 2390 */ 2195, 774, 1839, 1836, 776, 778, 773, 780, 777, 781, - /* 2400 */ 782, 1818, 2145, 2195, 700, 784, 1202, 1795, 311, 788, - /* 2410 */ 1770, 1545, 321, 1770, 792, 2145, 791, 700, 1770, 1770, - /* 2420 */ 1770, 2176, 1770, 2212, 1770, 2177, 345, 2178, 704, 2180, - /* 2430 */ 2181, 699, 1770, 694, 1770, 701, 1770, 2176, 1770, 2212, - /* 2440 */ 1770, 2177, 346, 2178, 704, 2180, 2181, 699, 1770, 694, - /* 2450 */ 2176, 701, 2212, 1770, 2177, 2189, 2178, 704, 2180, 2181, - /* 2460 */ 699, 1770, 694, 2195, 701, 1770, 1770, 1770, 1770, 1770, - /* 2470 */ 1770, 1770, 1770, 1770, 1770, 2145, 1770, 700, 1770, 2195, - /* 2480 */ 1770, 1770, 1770, 1770, 1770, 1770, 1770, 1770, 1770, 1770, - /* 2490 */ 1770, 2145, 2195, 700, 1770, 1770, 1770, 1770, 1770, 1770, - /* 2500 */ 1770, 1770, 1770, 1770, 2145, 1770, 700, 1770, 1770, 1770, - /* 2510 */ 2176, 1770, 2212, 1770, 1770, 2188, 2178, 704, 2180, 2181, - /* 2520 */ 699, 1770, 694, 1770, 1770, 1770, 2176, 2177, 2212, 1770, - /* 2530 */ 1770, 2187, 2178, 704, 2180, 2181, 699, 701, 694, 2176, - /* 2540 */ 1770, 2212, 1770, 2177, 361, 2178, 704, 2180, 2181, 699, - /* 2550 */ 1770, 694, 1770, 701, 2177, 1770, 1770, 1770, 1770, 1770, - /* 2560 */ 1770, 1770, 1770, 1770, 701, 2195, 1770, 1770, 1770, 1770, - /* 2570 */ 1770, 1770, 1770, 1770, 1770, 1770, 1770, 2145, 1770, 700, - /* 2580 */ 1770, 2195, 1770, 1770, 1770, 1770, 1770, 1770, 1770, 1770, - /* 2590 */ 1770, 1770, 2195, 2145, 1770, 700, 1770, 1770, 1770, 1770, - /* 2600 */ 1770, 1770, 1770, 1770, 2145, 1770, 700, 1770, 1770, 1770, - /* 2610 */ 1770, 1770, 2176, 1770, 2212, 1770, 2177, 362, 2178, 704, - /* 2620 */ 2180, 2181, 699, 1770, 694, 1770, 701, 1770, 2176, 1770, - /* 2630 */ 2212, 1770, 2177, 358, 2178, 704, 2180, 2181, 699, 2176, - /* 2640 */ 694, 2212, 701, 1770, 363, 2178, 704, 2180, 2181, 699, - /* 2650 */ 1770, 694, 1770, 1770, 2195, 1770, 1770, 1770, 1770, 1770, - /* 2660 */ 1770, 1770, 1770, 1770, 1770, 1770, 2145, 1770, 700, 1770, - /* 2670 */ 2195, 1770, 1770, 1770, 1770, 1770, 1770, 1770, 1770, 1770, - /* 2680 */ 1770, 1770, 2145, 1770, 700, 1770, 1770, 1770, 1770, 1770, - /* 2690 */ 1770, 1770, 1770, 1770, 1770, 1770, 1770, 1770, 1770, 1770, - /* 2700 */ 1770, 702, 1770, 2212, 1770, 1770, 337, 2178, 704, 2180, - /* 2710 */ 2181, 699, 1770, 694, 1770, 1770, 1770, 2176, 1770, 2212, - /* 2720 */ 1770, 1770, 336, 2178, 704, 2180, 2181, 699, 1770, 694, + /* 0 */ 2126, 2193, 2171, 2082, 221, 690, 1960, 2154, 537, 1949, + /* 10 */ 1826, 670, 48, 46, 1706, 394, 2179, 1213, 2079, 677, + /* 20 */ 401, 2358, 1555, 41, 40, 135, 2175, 47, 45, 44, + /* 30 */ 43, 42, 574, 1636, 453, 1553, 2171, 539, 1582, 2211, + /* 40 */ 41, 40, 1781, 536, 47, 45, 44, 43, 42, 254, + /* 50 */ 1951, 2161, 1945, 706, 630, 531, 1215, 2353, 1218, 1219, + /* 60 */ 2175, 181, 1631, 529, 2177, 398, 525, 521, 19, 1238, + /* 70 */ 66, 1237, 2359, 188, 700, 1561, 30, 2354, 656, 348, + /* 80 */ 689, 369, 2065, 361, 140, 690, 1960, 2192, 1580, 2228, + /* 90 */ 667, 144, 112, 2194, 710, 2196, 2197, 705, 2177, 700, + /* 100 */ 796, 168, 1239, 15, 185, 135, 2281, 103, 700, 1901, + /* 110 */ 397, 2277, 579, 497, 2082, 416, 48, 46, 690, 1960, + /* 120 */ 415, 689, 1769, 190, 401, 264, 1555, 1665, 1371, 2080, + /* 130 */ 677, 2307, 1953, 1580, 38, 306, 1746, 1636, 193, 1553, + /* 140 */ 1638, 1639, 1806, 1362, 735, 734, 733, 1366, 732, 1368, + /* 150 */ 1369, 731, 728, 1805, 1377, 725, 1379, 1380, 722, 719, + /* 160 */ 716, 184, 630, 51, 655, 2353, 1631, 2353, 94, 62, + /* 170 */ 1611, 1621, 19, 2000, 212, 211, 1637, 1640, 675, 1561, + /* 180 */ 2359, 188, 654, 188, 1666, 2354, 656, 2354, 656, 2296, + /* 190 */ 288, 1556, 2161, 1554, 286, 2289, 666, 496, 136, 665, + /* 200 */ 169, 2353, 1795, 2161, 796, 41, 40, 15, 2193, 47, + /* 210 */ 45, 44, 43, 42, 62, 2293, 654, 188, 707, 1315, + /* 220 */ 435, 2354, 656, 1559, 1560, 1783, 1610, 1613, 1614, 1615, + /* 230 */ 1616, 1617, 1618, 1619, 1620, 702, 698, 1629, 1630, 1632, + /* 240 */ 1633, 1634, 1635, 2, 1638, 1639, 2211, 437, 433, 134, + /* 250 */ 133, 132, 131, 130, 129, 128, 127, 126, 2161, 1317, + /* 260 */ 706, 1784, 37, 399, 1660, 1661, 1662, 1663, 1664, 1668, + /* 270 */ 1669, 1670, 1671, 534, 1611, 1621, 535, 1819, 551, 1580, + /* 280 */ 1637, 1640, 125, 1464, 1465, 124, 123, 122, 121, 120, + /* 290 */ 119, 118, 117, 116, 2192, 1556, 2228, 1554, 645, 112, + /* 300 */ 2194, 710, 2196, 2197, 705, 650, 700, 2044, 395, 147, + /* 310 */ 1580, 151, 2252, 2281, 1581, 2193, 166, 397, 2277, 1238, + /* 320 */ 191, 1237, 667, 144, 1962, 670, 191, 1559, 1560, 1703, + /* 330 */ 1610, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1620, 702, + /* 340 */ 698, 1629, 1630, 1632, 1633, 1634, 1635, 2, 12, 48, + /* 350 */ 46, 745, 1239, 2211, 2013, 410, 409, 401, 2182, 1555, + /* 360 */ 2358, 367, 62, 2353, 181, 2161, 1582, 706, 615, 2011, + /* 370 */ 1636, 191, 1553, 592, 591, 590, 690, 1960, 1562, 2357, + /* 380 */ 582, 141, 586, 2354, 2356, 2066, 585, 649, 651, 646, + /* 390 */ 639, 584, 589, 377, 376, 146, 56, 583, 2252, 1631, + /* 400 */ 252, 2192, 629, 2228, 251, 19, 112, 2194, 710, 2196, + /* 410 */ 2197, 705, 1561, 700, 2184, 2211, 297, 298, 185, 542, + /* 420 */ 2281, 296, 535, 1819, 397, 2277, 187, 2289, 2290, 2193, + /* 430 */ 142, 2294, 1221, 655, 1406, 1407, 2353, 796, 1579, 707, + /* 440 */ 15, 1828, 41, 40, 1272, 2308, 47, 45, 44, 43, + /* 450 */ 42, 654, 188, 48, 46, 1641, 2354, 656, 223, 2193, + /* 460 */ 1612, 401, 537, 1555, 1826, 93, 475, 2211, 356, 707, + /* 470 */ 648, 381, 166, 608, 1636, 474, 1553, 1638, 1639, 2161, + /* 480 */ 1963, 706, 2358, 125, 1273, 2353, 124, 123, 122, 121, + /* 490 */ 120, 119, 118, 117, 116, 2013, 62, 2211, 1804, 667, + /* 500 */ 144, 2357, 382, 1631, 12, 2354, 2355, 1611, 1621, 2161, + /* 510 */ 2011, 706, 109, 1637, 1640, 2192, 1561, 2228, 288, 191, + /* 520 */ 112, 2194, 710, 2196, 2197, 705, 60, 700, 1556, 145, + /* 530 */ 1554, 578, 2373, 627, 2281, 577, 1565, 1952, 397, 2277, + /* 540 */ 1710, 796, 690, 1960, 49, 2192, 1580, 2228, 2161, 2193, + /* 550 */ 170, 2194, 710, 2196, 2197, 705, 12, 700, 10, 707, + /* 560 */ 1559, 1560, 451, 1610, 1613, 1614, 1615, 1616, 1617, 1618, + /* 570 */ 1619, 1620, 702, 698, 1629, 1630, 1632, 1633, 1634, 1635, + /* 580 */ 2, 1638, 1639, 445, 1326, 444, 1702, 2211, 41, 40, + /* 590 */ 631, 2318, 47, 45, 44, 43, 42, 1325, 2296, 2161, + /* 600 */ 1583, 706, 669, 186, 2289, 2290, 165, 142, 2294, 1561, + /* 610 */ 1803, 1611, 1621, 690, 1960, 443, 406, 1637, 1640, 2006, + /* 620 */ 2008, 41, 40, 404, 2292, 47, 45, 44, 43, 42, + /* 630 */ 2296, 163, 1556, 452, 1554, 2192, 676, 2228, 383, 1962, + /* 640 */ 112, 2194, 710, 2196, 2197, 705, 2011, 700, 253, 689, + /* 650 */ 447, 2193, 2256, 191, 2281, 446, 2291, 2155, 397, 2277, + /* 660 */ 2161, 707, 506, 2315, 1559, 1560, 154, 1610, 1613, 1614, + /* 670 */ 1615, 1616, 1617, 1618, 1619, 1620, 702, 698, 1629, 1630, + /* 680 */ 1632, 1633, 1634, 1635, 2, 48, 46, 1937, 549, 2211, + /* 690 */ 2075, 457, 2061, 401, 755, 1555, 1612, 630, 667, 144, + /* 700 */ 2353, 2161, 2013, 706, 630, 1936, 1636, 2353, 1553, 391, + /* 710 */ 47, 45, 44, 43, 42, 2359, 188, 2011, 1526, 1527, + /* 720 */ 2354, 656, 2359, 188, 463, 2061, 55, 2354, 656, 14, + /* 730 */ 13, 1734, 51, 690, 1960, 1631, 1947, 2192, 202, 2228, + /* 740 */ 597, 676, 112, 2194, 710, 2196, 2197, 705, 1561, 700, + /* 750 */ 690, 1960, 266, 465, 2373, 607, 2281, 1862, 41, 40, + /* 760 */ 397, 2277, 47, 45, 44, 43, 42, 2193, 1802, 250, + /* 770 */ 480, 205, 745, 796, 690, 1960, 49, 704, 642, 641, + /* 780 */ 1732, 1733, 1735, 1736, 1737, 600, 490, 2061, 2357, 48, + /* 790 */ 46, 1935, 594, 674, 481, 2075, 1776, 401, 249, 1555, + /* 800 */ 404, 1581, 189, 2289, 2290, 2211, 142, 2294, 166, 692, + /* 810 */ 1636, 2253, 1553, 1638, 1639, 1846, 1962, 2161, 2161, 706, + /* 820 */ 743, 156, 155, 740, 739, 738, 153, 592, 591, 590, + /* 830 */ 2007, 2008, 1679, 210, 582, 141, 586, 593, 70, 1631, + /* 840 */ 585, 69, 52, 1611, 1621, 584, 589, 377, 376, 1637, + /* 850 */ 1640, 583, 1561, 2192, 1745, 2228, 1241, 1242, 342, 2194, + /* 860 */ 710, 2196, 2197, 705, 1556, 700, 1554, 2247, 41, 40, + /* 870 */ 1801, 1943, 47, 45, 44, 43, 42, 796, 570, 569, + /* 880 */ 15, 2193, 743, 156, 155, 740, 739, 738, 153, 1800, + /* 890 */ 613, 707, 207, 2328, 1799, 1964, 1559, 1560, 1775, 1610, + /* 900 */ 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1620, 702, 698, + /* 910 */ 1629, 1630, 1632, 1633, 1634, 1635, 2, 1638, 1639, 2211, + /* 920 */ 2161, 1481, 1482, 606, 743, 156, 155, 740, 739, 738, + /* 930 */ 153, 2161, 86, 706, 2013, 85, 604, 630, 602, 2161, + /* 940 */ 2353, 396, 690, 1960, 2161, 690, 1960, 1611, 1621, 2011, + /* 950 */ 736, 690, 1960, 1637, 1640, 2359, 188, 1480, 1483, 259, + /* 960 */ 2354, 656, 550, 737, 1722, 1957, 2004, 2192, 1556, 2228, + /* 970 */ 1554, 255, 112, 2194, 710, 2196, 2197, 705, 9, 700, + /* 980 */ 588, 587, 34, 2147, 2373, 1330, 2281, 676, 41, 40, + /* 990 */ 397, 2277, 47, 45, 44, 43, 42, 1798, 1329, 191, + /* 1000 */ 1559, 1560, 1859, 1610, 1613, 1614, 1615, 1616, 1617, 1618, + /* 1010 */ 1619, 1620, 702, 698, 1629, 1630, 1632, 1633, 1634, 1635, + /* 1020 */ 2, 1646, 352, 167, 1578, 690, 1960, 1580, 327, 690, + /* 1030 */ 1960, 488, 423, 90, 504, 572, 571, 503, 2171, 685, + /* 1040 */ 407, 2075, 324, 73, 2144, 263, 72, 2161, 166, 673, + /* 1050 */ 371, 2013, 2180, 471, 671, 505, 1962, 349, 405, 1955, + /* 1060 */ 473, 701, 2175, 508, 1797, 658, 2011, 1794, 219, 516, + /* 1070 */ 514, 511, 773, 772, 771, 770, 413, 1938, 769, 768, + /* 1080 */ 148, 763, 762, 761, 760, 759, 758, 757, 158, 753, + /* 1090 */ 752, 751, 412, 411, 748, 747, 746, 176, 175, 1583, + /* 1100 */ 2177, 630, 690, 1960, 2353, 1583, 370, 237, 62, 374, + /* 1110 */ 700, 44, 43, 42, 2161, 690, 1960, 2161, 459, 2359, + /* 1120 */ 188, 149, 301, 173, 2354, 656, 690, 1960, 262, 690, + /* 1130 */ 1960, 568, 564, 560, 556, 687, 236, 1793, 662, 694, + /* 1140 */ 1667, 2253, 767, 765, 1792, 1791, 688, 111, 501, 307, + /* 1150 */ 1790, 495, 494, 493, 492, 487, 486, 485, 484, 483, + /* 1160 */ 479, 478, 477, 476, 351, 468, 467, 466, 659, 461, + /* 1170 */ 460, 368, 690, 1960, 2301, 1699, 91, 1612, 375, 234, + /* 1180 */ 373, 372, 1837, 576, 90, 1789, 1788, 2161, 81, 80, + /* 1190 */ 450, 2193, 408, 200, 2161, 2161, 1787, 2013, 1699, 741, + /* 1200 */ 2161, 707, 2004, 637, 595, 578, 442, 440, 1786, 577, + /* 1210 */ 1956, 742, 2012, 320, 2004, 1902, 1990, 350, 35, 54, + /* 1220 */ 431, 3, 2193, 429, 425, 421, 418, 443, 1672, 2211, + /* 1230 */ 756, 454, 707, 1922, 2346, 2161, 2161, 1218, 1219, 154, + /* 1240 */ 83, 2161, 137, 706, 455, 74, 2161, 233, 227, 580, + /* 1250 */ 242, 203, 2193, 240, 232, 547, 244, 265, 2161, 243, + /* 1260 */ 2211, 430, 707, 246, 2300, 191, 245, 248, 154, 581, + /* 1270 */ 247, 1313, 2161, 225, 706, 1835, 50, 2192, 610, 2228, + /* 1280 */ 609, 50, 112, 2194, 710, 2196, 2197, 705, 697, 700, + /* 1290 */ 2211, 1311, 1778, 1779, 2373, 84, 2281, 598, 1796, 1521, + /* 1300 */ 397, 2277, 2161, 643, 706, 1564, 270, 154, 2192, 108, + /* 1310 */ 2228, 2321, 283, 112, 2194, 710, 2196, 2197, 705, 105, + /* 1320 */ 700, 50, 294, 1563, 1555, 2373, 71, 2281, 1524, 152, + /* 1330 */ 154, 397, 2277, 14, 13, 64, 1731, 1553, 2192, 50, + /* 1340 */ 2228, 1730, 2193, 112, 2194, 710, 2196, 2197, 705, 50, + /* 1350 */ 700, 1829, 707, 410, 409, 2373, 277, 2281, 36, 714, + /* 1360 */ 139, 397, 2277, 1569, 41, 40, 272, 672, 47, 45, + /* 1370 */ 44, 43, 42, 2193, 1636, 1900, 1562, 1561, 663, 152, + /* 1380 */ 2211, 1478, 299, 707, 154, 749, 682, 1899, 2212, 303, + /* 1390 */ 1356, 750, 2161, 138, 706, 1673, 384, 152, 2070, 1622, + /* 1400 */ 414, 791, 796, 1631, 1820, 1825, 660, 1291, 2001, 319, + /* 1410 */ 2193, 2211, 2311, 1289, 668, 285, 1561, 282, 422, 1384, + /* 1420 */ 707, 1, 417, 2161, 5, 706, 365, 1586, 2192, 438, + /* 1430 */ 2228, 1502, 196, 112, 2194, 710, 2196, 2197, 705, 1388, + /* 1440 */ 700, 696, 439, 1657, 1395, 2254, 441, 2281, 2211, 195, + /* 1450 */ 198, 397, 2277, 1393, 314, 1579, 456, 157, 209, 2192, + /* 1460 */ 2161, 2228, 706, 1567, 112, 2194, 710, 2196, 2197, 705, + /* 1470 */ 1583, 700, 462, 458, 469, 499, 693, 2071, 2281, 464, + /* 1480 */ 1578, 1566, 397, 2277, 482, 491, 489, 2063, 498, 500, + /* 1490 */ 509, 510, 507, 1556, 213, 1554, 2192, 214, 2228, 2193, + /* 1500 */ 512, 113, 2194, 710, 2196, 2197, 705, 513, 700, 707, + /* 1510 */ 1584, 216, 532, 515, 517, 2281, 4, 533, 540, 2280, + /* 1520 */ 2277, 541, 543, 1581, 224, 1559, 1560, 226, 1585, 544, + /* 1530 */ 545, 1587, 1570, 546, 1565, 229, 548, 2211, 231, 88, + /* 1540 */ 89, 552, 235, 573, 355, 575, 1950, 114, 2193, 2161, + /* 1550 */ 239, 706, 1946, 612, 614, 92, 150, 618, 707, 315, + /* 1560 */ 256, 619, 617, 2135, 1573, 1575, 241, 159, 160, 1948, + /* 1570 */ 258, 260, 1944, 161, 2193, 162, 2132, 698, 1629, 1630, + /* 1580 */ 1632, 1633, 1634, 1635, 704, 2192, 2211, 2228, 625, 1509, + /* 1590 */ 113, 2194, 710, 2196, 2197, 705, 2131, 700, 2161, 644, + /* 1600 */ 706, 622, 634, 8, 2281, 2312, 2327, 680, 695, 2277, + /* 1610 */ 640, 2322, 2211, 387, 624, 2326, 647, 623, 268, 271, + /* 1620 */ 2303, 653, 276, 635, 2161, 2193, 706, 632, 633, 281, + /* 1630 */ 2376, 388, 1699, 661, 708, 707, 2228, 664, 2193, 113, + /* 1640 */ 2194, 710, 2196, 2197, 705, 143, 700, 1582, 707, 289, + /* 1650 */ 178, 1588, 278, 2281, 2076, 316, 98, 360, 2277, 2297, + /* 1660 */ 2192, 317, 2228, 2211, 678, 342, 2194, 710, 2196, 2197, + /* 1670 */ 705, 703, 700, 691, 2246, 2161, 2211, 706, 679, 683, + /* 1680 */ 2090, 279, 174, 684, 2089, 280, 100, 2193, 2161, 318, + /* 1690 */ 706, 2088, 393, 102, 61, 2262, 104, 707, 2352, 1961, + /* 1700 */ 712, 2005, 1923, 792, 2193, 284, 793, 321, 795, 2153, + /* 1710 */ 357, 2192, 310, 2228, 707, 358, 171, 2194, 710, 2196, + /* 1720 */ 2197, 705, 53, 700, 2192, 2211, 2228, 325, 323, 113, + /* 1730 */ 2194, 710, 2196, 2197, 705, 345, 700, 2161, 2152, 706, + /* 1740 */ 330, 344, 2211, 2281, 334, 2151, 78, 385, 2278, 2148, + /* 1750 */ 419, 420, 1546, 2193, 2161, 1547, 706, 194, 424, 2146, + /* 1760 */ 426, 427, 428, 707, 2145, 366, 2143, 657, 2374, 432, + /* 1770 */ 2142, 2141, 434, 2192, 436, 2228, 1537, 2193, 170, 2194, + /* 1780 */ 710, 2196, 2197, 705, 2122, 700, 197, 707, 2121, 199, + /* 1790 */ 2192, 2211, 2228, 1505, 79, 343, 2194, 710, 2196, 2197, + /* 1800 */ 705, 1504, 700, 2161, 2193, 706, 2103, 2102, 2101, 448, + /* 1810 */ 449, 2100, 2099, 2054, 707, 2211, 1455, 2053, 2050, 2319, + /* 1820 */ 386, 201, 2049, 82, 2193, 2048, 2047, 2161, 2052, 706, + /* 1830 */ 204, 2051, 2046, 2045, 707, 2043, 2042, 2041, 206, 2192, + /* 1840 */ 470, 2228, 2211, 2040, 336, 2194, 710, 2196, 2197, 705, + /* 1850 */ 472, 700, 2056, 2039, 2161, 2038, 706, 2037, 2036, 2035, + /* 1860 */ 2034, 2033, 2211, 2192, 2032, 2228, 2031, 392, 343, 2194, + /* 1870 */ 710, 2196, 2197, 705, 2161, 700, 706, 208, 2024, 2023, + /* 1880 */ 87, 2022, 2021, 2055, 2020, 2019, 215, 652, 2030, 2029, + /* 1890 */ 2192, 2193, 2228, 2028, 2027, 171, 2194, 710, 2196, 2197, + /* 1900 */ 705, 707, 700, 2026, 2025, 2018, 2017, 2016, 1457, 2015, + /* 1910 */ 2192, 502, 2228, 2014, 1327, 343, 2194, 710, 2196, 2197, + /* 1920 */ 705, 2193, 700, 353, 354, 1865, 1323, 1864, 1863, 2211, + /* 1930 */ 1331, 707, 217, 218, 400, 1861, 1858, 520, 1857, 519, + /* 1940 */ 524, 2161, 1850, 706, 523, 518, 522, 2375, 527, 526, + /* 1950 */ 1839, 528, 530, 1815, 1220, 76, 1814, 220, 2120, 2211, + /* 1960 */ 2110, 77, 182, 222, 402, 2098, 2181, 183, 538, 228, + /* 1970 */ 2097, 2161, 230, 706, 553, 554, 555, 2192, 2074, 2228, + /* 1980 */ 2193, 1939, 343, 2194, 710, 2196, 2197, 705, 616, 700, + /* 1990 */ 707, 1860, 1856, 1265, 1854, 558, 557, 1852, 559, 561, + /* 2000 */ 562, 563, 1849, 565, 566, 567, 799, 2192, 1834, 2228, + /* 2010 */ 1832, 1833, 343, 2194, 710, 2196, 2197, 705, 2211, 700, + /* 2020 */ 313, 1831, 1811, 1941, 1940, 1400, 1399, 764, 1314, 766, + /* 2030 */ 2161, 1312, 706, 1310, 1309, 1308, 180, 1847, 1307, 1301, + /* 2040 */ 1306, 63, 238, 1838, 789, 785, 781, 777, 1303, 311, + /* 2050 */ 1302, 1300, 378, 2193, 379, 1836, 380, 596, 1810, 1809, + /* 2060 */ 599, 601, 603, 707, 1808, 2119, 611, 605, 2228, 115, + /* 2070 */ 1531, 338, 2194, 710, 2196, 2197, 705, 1533, 700, 1530, + /* 2080 */ 1535, 1511, 29, 67, 2193, 1515, 2109, 2096, 1513, 110, + /* 2090 */ 164, 2211, 304, 620, 707, 2095, 2358, 20, 17, 1748, + /* 2100 */ 6, 21, 65, 2161, 31, 706, 57, 261, 7, 626, + /* 2110 */ 275, 638, 267, 621, 22, 1490, 1489, 274, 269, 2193, + /* 2120 */ 636, 172, 2211, 628, 1729, 686, 2182, 33, 24, 707, + /* 2130 */ 58, 273, 32, 23, 2161, 1721, 706, 1768, 18, 2192, + /* 2140 */ 1769, 2228, 95, 1763, 328, 2194, 710, 2196, 2197, 705, + /* 2150 */ 1762, 700, 389, 1767, 1766, 390, 2193, 2211, 287, 177, + /* 2160 */ 291, 2094, 2073, 292, 97, 1696, 707, 290, 59, 2161, + /* 2170 */ 2192, 706, 2228, 1695, 2072, 326, 2194, 710, 2196, 2197, + /* 2180 */ 705, 2193, 700, 96, 25, 295, 257, 99, 105, 293, + /* 2190 */ 305, 707, 1727, 300, 2211, 68, 26, 101, 1648, 11, + /* 2200 */ 13, 1647, 1571, 179, 2231, 2192, 2161, 2228, 706, 1658, + /* 2210 */ 329, 2194, 710, 2196, 2197, 705, 681, 700, 2193, 2211, + /* 2220 */ 302, 1603, 192, 711, 713, 1626, 1624, 403, 707, 699, + /* 2230 */ 39, 2161, 1623, 706, 16, 27, 717, 1595, 28, 720, + /* 2240 */ 1385, 715, 2192, 2193, 2228, 1382, 1381, 335, 2194, 710, + /* 2250 */ 2196, 2197, 705, 707, 700, 1378, 2211, 718, 721, 723, + /* 2260 */ 724, 726, 1372, 709, 1370, 729, 727, 2192, 2161, 2228, + /* 2270 */ 706, 730, 339, 2194, 710, 2196, 2197, 705, 1376, 700, + /* 2280 */ 106, 2211, 308, 1394, 1375, 1390, 107, 75, 1263, 1374, + /* 2290 */ 1373, 744, 1295, 2161, 1294, 706, 1293, 1292, 309, 1290, + /* 2300 */ 1288, 1287, 1286, 1321, 2192, 754, 2228, 1284, 1283, 331, + /* 2310 */ 2194, 710, 2196, 2197, 705, 2193, 700, 1282, 1281, 1280, + /* 2320 */ 1279, 1278, 1318, 1316, 1275, 707, 1274, 1271, 1270, 2192, + /* 2330 */ 2193, 2228, 1269, 1268, 340, 2194, 710, 2196, 2197, 705, + /* 2340 */ 707, 700, 1855, 774, 775, 776, 1853, 778, 2193, 779, + /* 2350 */ 1851, 782, 780, 2211, 783, 1848, 784, 786, 707, 788, + /* 2360 */ 1830, 790, 1210, 787, 1807, 2161, 312, 706, 2211, 794, + /* 2370 */ 1782, 1557, 798, 322, 797, 1782, 1782, 1782, 1782, 1782, + /* 2380 */ 2161, 1782, 706, 1782, 1782, 1782, 2211, 1782, 1782, 1782, + /* 2390 */ 1782, 1782, 1782, 1782, 1782, 2193, 1782, 1782, 2161, 1782, + /* 2400 */ 706, 2192, 1782, 2228, 1782, 707, 332, 2194, 710, 2196, + /* 2410 */ 2197, 705, 1782, 700, 1782, 1782, 2192, 2193, 2228, 1782, + /* 2420 */ 1782, 341, 2194, 710, 2196, 2197, 705, 707, 700, 1782, + /* 2430 */ 1782, 1782, 1782, 2211, 2192, 1782, 2228, 1782, 1782, 333, + /* 2440 */ 2194, 710, 2196, 2197, 705, 2161, 700, 706, 1782, 1782, + /* 2450 */ 1782, 1782, 1782, 1782, 1782, 2211, 1782, 1782, 1782, 1782, + /* 2460 */ 1782, 1782, 1782, 1782, 2193, 1782, 1782, 2161, 1782, 706, + /* 2470 */ 1782, 1782, 1782, 1782, 707, 1782, 1782, 1782, 1782, 1782, + /* 2480 */ 1782, 2192, 1782, 2228, 1782, 1782, 346, 2194, 710, 2196, + /* 2490 */ 2197, 705, 1782, 700, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2500 */ 1782, 1782, 2211, 2192, 1782, 2228, 1782, 1782, 347, 2194, + /* 2510 */ 710, 2196, 2197, 705, 2161, 700, 706, 1782, 1782, 1782, + /* 2520 */ 1782, 1782, 1782, 2193, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2530 */ 1782, 1782, 1782, 707, 1782, 1782, 2193, 1782, 1782, 1782, + /* 2540 */ 1782, 1782, 1782, 1782, 1782, 1782, 707, 1782, 1782, 1782, + /* 2550 */ 2192, 1782, 2228, 2193, 1782, 2205, 2194, 710, 2196, 2197, + /* 2560 */ 705, 2211, 700, 707, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2570 */ 1782, 1782, 1782, 2161, 2211, 706, 1782, 1782, 1782, 1782, + /* 2580 */ 1782, 1782, 1782, 1782, 1782, 2193, 2161, 1782, 706, 1782, + /* 2590 */ 1782, 2211, 1782, 1782, 1782, 707, 1782, 1782, 1782, 1782, + /* 2600 */ 1782, 1782, 1782, 2161, 1782, 706, 1782, 1782, 1782, 2192, + /* 2610 */ 1782, 2228, 1782, 1782, 2204, 2194, 710, 2196, 2197, 705, + /* 2620 */ 1782, 700, 2192, 2211, 2228, 1782, 1782, 2203, 2194, 710, + /* 2630 */ 2196, 2197, 705, 1782, 700, 2161, 1782, 706, 1782, 2192, + /* 2640 */ 1782, 2228, 1782, 1782, 362, 2194, 710, 2196, 2197, 705, + /* 2650 */ 1782, 700, 2193, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2660 */ 1782, 1782, 707, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2670 */ 1782, 2192, 2193, 2228, 1782, 1782, 363, 2194, 710, 2196, + /* 2680 */ 2197, 705, 707, 700, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2690 */ 2211, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2700 */ 1782, 1782, 2161, 1782, 706, 1782, 1782, 1782, 1782, 1782, + /* 2710 */ 2211, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2720 */ 1782, 1782, 2161, 1782, 706, 1782, 1782, 1782, 1782, 1782, + /* 2730 */ 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 2192, 1782, + /* 2740 */ 2228, 1782, 2193, 359, 2194, 710, 2196, 2197, 705, 1782, + /* 2750 */ 700, 1782, 707, 1782, 1782, 1782, 1782, 1782, 2192, 2193, + /* 2760 */ 2228, 1782, 1782, 364, 2194, 710, 2196, 2197, 705, 707, + /* 2770 */ 700, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2780 */ 2211, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2790 */ 1782, 1782, 2161, 1782, 706, 1782, 1782, 2211, 1782, 1782, + /* 2800 */ 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 2161, + /* 2810 */ 1782, 706, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, + /* 2820 */ 1782, 1782, 1782, 1782, 1782, 1782, 1782, 1782, 708, 1782, + /* 2830 */ 2228, 1782, 1782, 338, 2194, 710, 2196, 2197, 705, 1782, + /* 2840 */ 700, 1782, 1782, 1782, 1782, 2192, 1782, 2228, 1782, 1782, + /* 2850 */ 337, 2194, 710, 2196, 2197, 705, 1782, 700, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 340, 392, 339, 345, 341, 349, 409, 349, 409, 351, - /* 10 */ 350, 414, 12, 13, 14, 379, 407, 408, 456, 340, - /* 20 */ 20, 459, 22, 8, 9, 12, 13, 12, 13, 14, - /* 30 */ 15, 16, 441, 33, 443, 35, 474, 475, 378, 8, - /* 40 */ 9, 479, 480, 12, 13, 14, 15, 16, 35, 409, - /* 50 */ 390, 20, 392, 456, 20, 456, 459, 401, 459, 403, - /* 60 */ 340, 378, 62, 12, 13, 14, 15, 16, 68, 390, - /* 70 */ 350, 474, 475, 474, 475, 75, 479, 480, 479, 480, - /* 80 */ 345, 398, 399, 68, 349, 425, 351, 427, 356, 377, - /* 90 */ 430, 431, 432, 433, 434, 435, 456, 437, 378, 459, - /* 100 */ 100, 389, 442, 103, 444, 373, 358, 350, 448, 449, - /* 110 */ 390, 340, 392, 381, 474, 475, 12, 13, 392, 479, - /* 120 */ 480, 461, 366, 375, 20, 456, 22, 112, 459, 469, - /* 130 */ 404, 20, 384, 407, 408, 378, 380, 33, 62, 35, - /* 140 */ 140, 141, 349, 22, 475, 425, 390, 427, 479, 480, - /* 150 */ 430, 431, 432, 433, 434, 435, 35, 437, 20, 370, - /* 160 */ 440, 390, 442, 443, 444, 378, 62, 378, 448, 449, - /* 170 */ 170, 171, 68, 378, 385, 386, 176, 177, 102, 75, - /* 180 */ 385, 105, 393, 427, 169, 0, 399, 394, 393, 432, - /* 190 */ 190, 21, 192, 437, 24, 25, 26, 27, 28, 29, - /* 200 */ 30, 31, 32, 172, 100, 192, 21, 103, 340, 24, - /* 210 */ 25, 26, 27, 28, 29, 30, 31, 32, 350, 20, - /* 220 */ 352, 100, 222, 223, 0, 225, 226, 227, 228, 229, + /* 0 */ 375, 341, 367, 393, 346, 350, 351, 411, 350, 380, + /* 10 */ 352, 351, 12, 13, 14, 405, 381, 4, 408, 409, + /* 20 */ 20, 3, 22, 8, 9, 370, 391, 12, 13, 14, + /* 30 */ 15, 16, 377, 33, 350, 35, 367, 14, 20, 379, + /* 40 */ 8, 9, 338, 20, 12, 13, 14, 15, 16, 424, + /* 50 */ 381, 391, 380, 393, 458, 49, 43, 461, 45, 46, + /* 60 */ 391, 379, 62, 57, 429, 430, 60, 61, 68, 20, + /* 70 */ 4, 22, 476, 477, 439, 75, 44, 481, 482, 395, + /* 80 */ 20, 399, 400, 68, 35, 350, 351, 427, 20, 429, + /* 90 */ 350, 351, 432, 433, 434, 435, 436, 437, 429, 439, + /* 100 */ 100, 360, 53, 103, 444, 370, 446, 357, 439, 368, + /* 110 */ 450, 451, 377, 84, 393, 411, 12, 13, 350, 351, + /* 120 */ 416, 20, 104, 463, 20, 62, 22, 112, 100, 408, + /* 130 */ 409, 471, 382, 20, 447, 448, 104, 33, 370, 35, + /* 140 */ 140, 141, 341, 115, 116, 117, 118, 119, 120, 121, + /* 150 */ 122, 123, 124, 341, 126, 127, 128, 129, 130, 131, + /* 160 */ 132, 378, 458, 103, 458, 461, 62, 461, 105, 103, + /* 170 */ 170, 171, 68, 390, 145, 146, 176, 177, 20, 75, + /* 180 */ 476, 477, 476, 477, 169, 481, 482, 481, 482, 431, + /* 190 */ 172, 191, 391, 193, 454, 455, 456, 168, 458, 459, + /* 200 */ 340, 461, 342, 391, 100, 8, 9, 103, 341, 12, + /* 210 */ 13, 14, 15, 16, 103, 457, 476, 477, 351, 35, + /* 220 */ 186, 481, 482, 223, 224, 0, 226, 227, 228, 229, /* 230 */ 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, - /* 240 */ 240, 241, 242, 20, 140, 141, 378, 340, 24, 25, - /* 250 */ 26, 27, 28, 29, 30, 31, 32, 349, 390, 103, - /* 260 */ 392, 246, 247, 248, 249, 250, 251, 252, 253, 254, - /* 270 */ 255, 256, 445, 446, 170, 171, 20, 62, 140, 141, - /* 280 */ 176, 177, 4, 20, 8, 9, 349, 350, 12, 13, - /* 290 */ 14, 15, 16, 425, 190, 427, 192, 390, 430, 431, - /* 300 */ 432, 433, 434, 435, 344, 437, 369, 347, 348, 401, - /* 310 */ 442, 403, 444, 376, 20, 340, 448, 449, 103, 20, - /* 320 */ 105, 43, 14, 45, 46, 350, 222, 223, 20, 225, + /* 240 */ 240, 241, 242, 243, 140, 141, 379, 213, 214, 24, + /* 250 */ 25, 26, 27, 28, 29, 30, 31, 32, 391, 75, + /* 260 */ 393, 0, 247, 248, 249, 250, 251, 252, 253, 254, + /* 270 */ 255, 256, 257, 345, 170, 171, 348, 349, 67, 20, + /* 280 */ 176, 177, 21, 170, 171, 24, 25, 26, 27, 28, + /* 290 */ 29, 30, 31, 32, 427, 191, 429, 193, 175, 432, + /* 300 */ 433, 434, 435, 436, 437, 20, 439, 0, 371, 442, + /* 310 */ 20, 444, 445, 446, 20, 341, 379, 450, 451, 20, + /* 320 */ 260, 22, 350, 351, 387, 351, 260, 223, 224, 4, /* 330 */ 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, - /* 340 */ 236, 237, 238, 239, 240, 241, 242, 243, 12, 13, - /* 350 */ 243, 175, 245, 378, 370, 20, 20, 388, 22, 100, - /* 360 */ 391, 392, 378, 37, 67, 390, 103, 392, 349, 33, - /* 370 */ 386, 35, 349, 350, 115, 116, 117, 118, 119, 120, - /* 380 */ 121, 122, 123, 124, 0, 126, 127, 128, 129, 130, - /* 390 */ 131, 132, 369, 340, 39, 429, 349, 350, 62, 376, - /* 400 */ 425, 340, 427, 350, 68, 430, 431, 432, 433, 434, - /* 410 */ 435, 75, 437, 3, 20, 259, 22, 442, 0, 444, - /* 420 */ 401, 455, 403, 448, 449, 134, 170, 349, 350, 35, - /* 430 */ 20, 378, 106, 49, 108, 109, 100, 111, 103, 103, - /* 440 */ 264, 265, 266, 390, 469, 392, 20, 53, 70, 71, - /* 450 */ 72, 390, 12, 13, 14, 77, 78, 79, 182, 133, - /* 460 */ 20, 83, 22, 137, 170, 171, 88, 89, 90, 91, - /* 470 */ 114, 222, 94, 33, 259, 35, 140, 141, 425, 429, - /* 480 */ 427, 391, 392, 430, 431, 432, 433, 434, 435, 436, - /* 490 */ 437, 438, 439, 344, 203, 204, 347, 348, 340, 452, - /* 500 */ 453, 454, 62, 456, 457, 455, 170, 171, 350, 20, - /* 510 */ 352, 22, 176, 177, 104, 75, 349, 350, 269, 270, - /* 520 */ 271, 272, 273, 274, 275, 84, 190, 67, 192, 451, - /* 530 */ 452, 453, 454, 49, 456, 457, 378, 1, 2, 20, - /* 540 */ 100, 57, 53, 103, 60, 61, 349, 350, 390, 340, - /* 550 */ 392, 133, 134, 135, 136, 137, 138, 139, 222, 223, - /* 560 */ 284, 225, 226, 227, 228, 229, 230, 231, 232, 233, - /* 570 */ 234, 235, 236, 237, 238, 239, 240, 241, 242, 13, - /* 580 */ 140, 141, 172, 425, 3, 427, 145, 146, 430, 431, - /* 590 */ 432, 433, 434, 435, 259, 437, 243, 378, 172, 390, - /* 600 */ 442, 35, 444, 356, 385, 340, 448, 449, 374, 168, - /* 610 */ 170, 171, 393, 258, 21, 350, 176, 177, 0, 452, - /* 620 */ 453, 454, 103, 456, 457, 4, 459, 34, 381, 36, - /* 630 */ 190, 185, 192, 133, 134, 135, 136, 137, 138, 139, - /* 640 */ 104, 474, 475, 378, 349, 350, 479, 480, 337, 452, - /* 650 */ 453, 454, 75, 456, 457, 390, 422, 392, 212, 213, - /* 660 */ 0, 409, 222, 223, 369, 225, 226, 227, 228, 229, - /* 670 */ 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, - /* 680 */ 240, 241, 242, 12, 13, 67, 359, 54, 55, 103, - /* 690 */ 425, 20, 427, 22, 367, 430, 431, 432, 433, 434, - /* 700 */ 435, 13, 437, 378, 33, 366, 35, 442, 456, 444, - /* 710 */ 385, 459, 33, 448, 449, 14, 15, 16, 393, 380, - /* 720 */ 409, 22, 22, 35, 103, 414, 474, 475, 49, 390, - /* 730 */ 340, 479, 480, 62, 35, 35, 57, 58, 59, 60, - /* 740 */ 350, 62, 441, 340, 443, 366, 75, 8, 9, 35, - /* 750 */ 2, 12, 13, 14, 15, 16, 8, 9, 35, 380, - /* 760 */ 12, 13, 14, 15, 16, 0, 427, 456, 378, 390, - /* 770 */ 459, 100, 349, 350, 103, 75, 437, 114, 259, 340, - /* 780 */ 390, 102, 392, 44, 105, 474, 475, 12, 13, 75, - /* 790 */ 479, 480, 369, 390, 340, 20, 20, 22, 75, 100, - /* 800 */ 100, 340, 354, 355, 350, 75, 427, 428, 33, 44, - /* 810 */ 35, 140, 141, 349, 350, 425, 437, 427, 20, 159, - /* 820 */ 430, 431, 432, 433, 434, 435, 379, 437, 168, 390, - /* 830 */ 354, 355, 378, 369, 444, 349, 350, 62, 448, 449, - /* 840 */ 20, 170, 171, 104, 390, 259, 392, 176, 177, 350, - /* 850 */ 75, 390, 173, 174, 133, 369, 379, 178, 137, 180, - /* 860 */ 279, 190, 440, 192, 135, 443, 8, 9, 139, 340, - /* 870 */ 12, 13, 14, 15, 16, 100, 14, 198, 103, 425, - /* 880 */ 259, 427, 20, 350, 430, 431, 432, 433, 434, 435, - /* 890 */ 190, 437, 192, 222, 223, 396, 225, 226, 227, 228, - /* 900 */ 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, - /* 910 */ 239, 240, 241, 242, 379, 140, 141, 363, 364, 390, - /* 920 */ 349, 350, 222, 223, 470, 471, 379, 378, 199, 396, - /* 930 */ 429, 202, 350, 370, 205, 18, 207, 20, 140, 141, - /* 940 */ 369, 378, 393, 340, 27, 170, 171, 30, 172, 386, - /* 950 */ 33, 176, 177, 350, 379, 352, 455, 349, 350, 349, - /* 960 */ 350, 456, 104, 14, 459, 190, 49, 192, 51, 20, - /* 970 */ 340, 349, 350, 56, 176, 177, 2, 369, 396, 369, - /* 980 */ 475, 378, 8, 9, 479, 480, 12, 13, 14, 15, - /* 990 */ 16, 369, 172, 390, 379, 392, 387, 222, 223, 390, - /* 1000 */ 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, - /* 1010 */ 235, 236, 237, 238, 239, 240, 241, 242, 0, 102, - /* 1020 */ 390, 378, 349, 350, 349, 350, 349, 350, 425, 386, - /* 1030 */ 427, 114, 170, 430, 431, 432, 433, 434, 435, 42, - /* 1040 */ 437, 44, 369, 415, 369, 442, 369, 444, 367, 8, - /* 1050 */ 9, 448, 449, 12, 13, 14, 15, 16, 134, 135, - /* 1060 */ 0, 144, 400, 139, 147, 148, 149, 150, 151, 152, - /* 1070 */ 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, - /* 1080 */ 163, 409, 165, 166, 167, 4, 340, 14, 70, 71, - /* 1090 */ 72, 73, 74, 20, 76, 77, 78, 79, 80, 81, - /* 1100 */ 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, - /* 1110 */ 92, 93, 94, 95, 96, 18, 0, 22, 8, 9, - /* 1120 */ 23, 169, 12, 13, 14, 15, 16, 358, 456, 340, - /* 1130 */ 35, 459, 341, 378, 37, 38, 390, 44, 41, 350, - /* 1140 */ 385, 352, 340, 363, 364, 104, 474, 475, 393, 52, - /* 1150 */ 483, 479, 480, 384, 349, 350, 103, 472, 349, 350, - /* 1160 */ 63, 64, 65, 66, 353, 340, 0, 378, 340, 68, - /* 1170 */ 378, 349, 350, 387, 369, 350, 390, 385, 369, 390, - /* 1180 */ 366, 392, 349, 350, 340, 393, 70, 71, 72, 349, - /* 1190 */ 350, 369, 390, 77, 78, 79, 340, 104, 246, 83, - /* 1200 */ 103, 44, 369, 378, 88, 89, 90, 91, 256, 369, - /* 1210 */ 94, 466, 45, 46, 425, 390, 427, 392, 390, 430, - /* 1220 */ 431, 432, 433, 434, 435, 340, 437, 44, 340, 349, - /* 1230 */ 350, 442, 366, 444, 390, 340, 353, 448, 449, 142, - /* 1240 */ 340, 340, 189, 170, 191, 350, 390, 352, 456, 369, - /* 1250 */ 425, 459, 427, 257, 258, 430, 431, 432, 433, 434, - /* 1260 */ 435, 370, 437, 349, 350, 387, 474, 475, 390, 378, - /* 1270 */ 378, 479, 480, 378, 221, 390, 216, 386, 390, 182, - /* 1280 */ 183, 184, 172, 369, 187, 390, 340, 392, 62, 371, - /* 1290 */ 390, 390, 374, 206, 400, 208, 350, 200, 201, 133, - /* 1300 */ 134, 135, 136, 137, 138, 139, 481, 482, 211, 340, - /* 1310 */ 365, 214, 259, 368, 217, 218, 219, 220, 221, 350, - /* 1320 */ 425, 352, 427, 222, 378, 430, 431, 432, 433, 434, - /* 1330 */ 435, 105, 437, 42, 114, 44, 390, 442, 392, 444, - /* 1340 */ 0, 260, 0, 448, 449, 12, 13, 378, 107, 107, - /* 1350 */ 107, 110, 110, 110, 107, 22, 259, 110, 48, 390, - /* 1360 */ 340, 392, 22, 44, 22, 0, 33, 44, 35, 44, - /* 1370 */ 350, 425, 44, 427, 44, 44, 430, 431, 432, 433, - /* 1380 */ 434, 435, 44, 437, 164, 8, 9, 22, 44, 12, - /* 1390 */ 13, 14, 15, 16, 425, 62, 427, 35, 378, 430, - /* 1400 */ 431, 432, 433, 434, 435, 44, 437, 44, 75, 35, - /* 1410 */ 390, 442, 392, 444, 140, 141, 44, 448, 449, 473, - /* 1420 */ 340, 1, 2, 104, 44, 44, 44, 104, 44, 104, - /* 1430 */ 350, 44, 104, 100, 104, 104, 47, 44, 281, 13, - /* 1440 */ 44, 13, 104, 44, 0, 425, 103, 427, 104, 340, - /* 1450 */ 430, 431, 432, 433, 434, 435, 113, 437, 378, 350, - /* 1460 */ 348, 35, 442, 35, 444, 104, 283, 104, 448, 449, - /* 1470 */ 390, 350, 392, 389, 400, 458, 104, 476, 450, 460, - /* 1480 */ 261, 410, 172, 49, 104, 104, 104, 378, 104, 426, - /* 1490 */ 20, 104, 103, 205, 50, 419, 424, 104, 358, 390, - /* 1500 */ 104, 392, 419, 104, 358, 425, 188, 427, 412, 42, - /* 1510 */ 430, 431, 432, 433, 434, 435, 397, 437, 20, 397, - /* 1520 */ 400, 169, 442, 190, 444, 192, 395, 20, 448, 449, - /* 1530 */ 349, 349, 397, 395, 425, 362, 427, 395, 101, 430, - /* 1540 */ 431, 432, 433, 434, 435, 99, 437, 4, 361, 349, - /* 1550 */ 98, 360, 349, 444, 192, 222, 223, 448, 449, 349, - /* 1560 */ 349, 20, 19, 342, 48, 340, 192, 419, 235, 236, - /* 1570 */ 237, 238, 239, 240, 241, 350, 33, 346, 342, 346, - /* 1580 */ 358, 20, 392, 20, 20, 358, 351, 411, 358, 351, - /* 1590 */ 358, 340, 49, 349, 342, 358, 378, 358, 358, 56, - /* 1600 */ 342, 350, 378, 378, 349, 62, 378, 378, 423, 209, - /* 1610 */ 421, 103, 378, 390, 419, 390, 356, 392, 196, 418, - /* 1620 */ 195, 417, 356, 378, 378, 340, 378, 378, 378, 378, - /* 1630 */ 194, 378, 416, 349, 268, 350, 465, 267, 468, 465, - /* 1640 */ 276, 390, 181, 392, 390, 102, 390, 467, 105, 392, - /* 1650 */ 425, 465, 427, 390, 278, 430, 431, 432, 433, 434, - /* 1660 */ 435, 390, 437, 378, 400, 400, 390, 405, 390, 444, - /* 1670 */ 464, 410, 405, 448, 449, 390, 425, 392, 427, 277, - /* 1680 */ 463, 430, 431, 432, 433, 434, 435, 340, 437, 262, - /* 1690 */ 410, 285, 484, 282, 280, 444, 258, 350, 350, 20, - /* 1700 */ 449, 429, 410, 349, 351, 356, 356, 403, 20, 174, - /* 1710 */ 425, 340, 427, 405, 390, 430, 431, 432, 433, 434, - /* 1720 */ 435, 350, 437, 402, 390, 378, 390, 390, 103, 390, - /* 1730 */ 383, 462, 405, 374, 356, 340, 477, 390, 350, 392, - /* 1740 */ 447, 390, 356, 103, 36, 350, 478, 390, 356, 378, - /* 1750 */ 382, 368, 349, 343, 383, 342, 471, 372, 413, 338, - /* 1760 */ 0, 390, 406, 392, 406, 357, 420, 0, 0, 372, - /* 1770 */ 372, 42, 425, 378, 427, 35, 35, 430, 431, 432, - /* 1780 */ 433, 434, 435, 0, 437, 390, 215, 392, 35, 35, - /* 1790 */ 215, 0, 35, 35, 215, 0, 425, 215, 427, 340, - /* 1800 */ 0, 430, 431, 432, 433, 434, 435, 35, 437, 350, - /* 1810 */ 0, 22, 0, 35, 210, 0, 198, 0, 198, 192, - /* 1820 */ 425, 199, 427, 190, 0, 430, 431, 432, 433, 434, - /* 1830 */ 435, 0, 437, 0, 186, 185, 0, 378, 0, 47, - /* 1840 */ 0, 0, 383, 0, 42, 0, 0, 0, 0, 390, - /* 1850 */ 0, 392, 0, 0, 0, 159, 35, 0, 159, 0, - /* 1860 */ 0, 0, 0, 340, 42, 0, 0, 0, 0, 0, - /* 1870 */ 0, 22, 0, 350, 0, 0, 0, 482, 0, 0, - /* 1880 */ 0, 0, 0, 0, 425, 340, 427, 0, 0, 430, - /* 1890 */ 431, 432, 433, 434, 435, 350, 437, 0, 0, 0, - /* 1900 */ 0, 378, 143, 0, 48, 0, 22, 0, 22, 340, - /* 1910 */ 112, 48, 35, 390, 0, 392, 0, 62, 0, 350, - /* 1920 */ 62, 0, 49, 378, 35, 0, 35, 0, 383, 39, - /* 1930 */ 35, 35, 39, 62, 0, 390, 14, 392, 0, 0, - /* 1940 */ 39, 49, 0, 49, 42, 39, 0, 378, 425, 40, - /* 1950 */ 427, 47, 383, 430, 431, 432, 433, 434, 435, 390, - /* 1960 */ 437, 392, 439, 1, 44, 39, 47, 39, 181, 0, - /* 1970 */ 425, 47, 427, 0, 0, 430, 431, 432, 433, 434, - /* 1980 */ 435, 19, 437, 0, 69, 0, 35, 39, 0, 49, - /* 1990 */ 35, 39, 49, 0, 425, 33, 427, 340, 35, 430, - /* 2000 */ 431, 432, 433, 434, 435, 49, 437, 350, 0, 39, - /* 2010 */ 35, 49, 35, 49, 0, 39, 340, 0, 0, 57, - /* 2020 */ 58, 59, 60, 0, 62, 22, 350, 340, 0, 0, - /* 2030 */ 0, 44, 35, 110, 35, 378, 35, 350, 44, 35, - /* 2040 */ 35, 35, 35, 22, 35, 35, 35, 390, 0, 392, - /* 2050 */ 0, 51, 0, 22, 378, 22, 22, 35, 0, 35, - /* 2060 */ 0, 35, 0, 22, 102, 378, 390, 105, 392, 20, - /* 2070 */ 35, 35, 35, 0, 35, 104, 22, 390, 0, 392, - /* 2080 */ 172, 103, 425, 103, 427, 22, 174, 430, 431, 432, - /* 2090 */ 433, 434, 435, 0, 437, 340, 0, 3, 263, 44, - /* 2100 */ 138, 425, 104, 427, 193, 350, 430, 431, 432, 433, - /* 2110 */ 434, 435, 425, 437, 427, 340, 172, 430, 431, 432, - /* 2120 */ 433, 434, 435, 103, 437, 350, 340, 197, 179, 172, - /* 2130 */ 103, 48, 48, 378, 44, 173, 350, 3, 47, 101, - /* 2140 */ 178, 99, 44, 44, 104, 390, 340, 392, 104, 103, - /* 2150 */ 103, 47, 44, 378, 103, 44, 350, 104, 35, 103, - /* 2160 */ 198, 104, 35, 104, 378, 390, 35, 392, 35, 104, - /* 2170 */ 35, 35, 104, 47, 257, 47, 390, 0, 392, 0, - /* 2180 */ 425, 44, 427, 0, 378, 430, 431, 432, 433, 434, - /* 2190 */ 435, 0, 437, 39, 263, 263, 390, 47, 392, 104, - /* 2200 */ 425, 104, 427, 103, 340, 430, 431, 432, 433, 434, - /* 2210 */ 435, 425, 437, 427, 350, 103, 430, 431, 432, 433, - /* 2220 */ 434, 435, 103, 437, 103, 340, 0, 103, 39, 113, - /* 2230 */ 44, 425, 103, 427, 173, 350, 430, 431, 432, 433, - /* 2240 */ 434, 435, 378, 437, 175, 47, 101, 101, 2, 22, - /* 2250 */ 340, 103, 47, 104, 390, 103, 392, 244, 47, 104, - /* 2260 */ 350, 103, 103, 378, 104, 222, 103, 22, 104, 224, - /* 2270 */ 103, 114, 104, 35, 35, 390, 340, 392, 103, 35, - /* 2280 */ 104, 103, 35, 35, 104, 103, 350, 103, 378, 425, - /* 2290 */ 35, 427, 104, 103, 430, 431, 432, 433, 434, 435, - /* 2300 */ 390, 437, 392, 104, 104, 35, 125, 103, 44, 35, - /* 2310 */ 425, 103, 427, 103, 378, 430, 431, 432, 433, 434, - /* 2320 */ 435, 125, 437, 103, 22, 69, 390, 125, 392, 68, - /* 2330 */ 35, 125, 35, 35, 35, 425, 340, 427, 35, 35, - /* 2340 */ 430, 431, 432, 433, 434, 435, 350, 437, 35, 35, - /* 2350 */ 75, 97, 340, 44, 35, 35, 35, 22, 35, 35, - /* 2360 */ 35, 425, 350, 427, 75, 340, 430, 431, 432, 433, - /* 2370 */ 434, 435, 35, 437, 378, 350, 35, 35, 35, 35, - /* 2380 */ 22, 35, 0, 35, 39, 49, 390, 0, 392, 35, - /* 2390 */ 378, 39, 0, 0, 35, 39, 49, 35, 49, 49, - /* 2400 */ 39, 0, 390, 378, 392, 35, 35, 0, 22, 21, - /* 2410 */ 485, 22, 22, 485, 20, 390, 21, 392, 485, 485, - /* 2420 */ 485, 425, 485, 427, 485, 340, 430, 431, 432, 433, - /* 2430 */ 434, 435, 485, 437, 485, 350, 485, 425, 485, 427, - /* 2440 */ 485, 340, 430, 431, 432, 433, 434, 435, 485, 437, - /* 2450 */ 425, 350, 427, 485, 340, 430, 431, 432, 433, 434, - /* 2460 */ 435, 485, 437, 378, 350, 485, 485, 485, 485, 485, - /* 2470 */ 485, 485, 485, 485, 485, 390, 485, 392, 485, 378, - /* 2480 */ 485, 485, 485, 485, 485, 485, 485, 485, 485, 485, - /* 2490 */ 485, 390, 378, 392, 485, 485, 485, 485, 485, 485, - /* 2500 */ 485, 485, 485, 485, 390, 485, 392, 485, 485, 485, - /* 2510 */ 425, 485, 427, 485, 485, 430, 431, 432, 433, 434, - /* 2520 */ 435, 485, 437, 485, 485, 485, 425, 340, 427, 485, - /* 2530 */ 485, 430, 431, 432, 433, 434, 435, 350, 437, 425, - /* 2540 */ 485, 427, 485, 340, 430, 431, 432, 433, 434, 435, - /* 2550 */ 485, 437, 485, 350, 340, 485, 485, 485, 485, 485, - /* 2560 */ 485, 485, 485, 485, 350, 378, 485, 485, 485, 485, - /* 2570 */ 485, 485, 485, 485, 485, 485, 485, 390, 485, 392, - /* 2580 */ 485, 378, 485, 485, 485, 485, 485, 485, 485, 485, - /* 2590 */ 485, 485, 378, 390, 485, 392, 485, 485, 485, 485, - /* 2600 */ 485, 485, 485, 485, 390, 485, 392, 485, 485, 485, - /* 2610 */ 485, 485, 425, 485, 427, 485, 340, 430, 431, 432, - /* 2620 */ 433, 434, 435, 485, 437, 485, 350, 485, 425, 485, - /* 2630 */ 427, 485, 340, 430, 431, 432, 433, 434, 435, 425, - /* 2640 */ 437, 427, 350, 485, 430, 431, 432, 433, 434, 435, - /* 2650 */ 485, 437, 485, 485, 378, 485, 485, 485, 485, 485, - /* 2660 */ 485, 485, 485, 485, 485, 485, 390, 485, 392, 485, - /* 2670 */ 378, 485, 485, 485, 485, 485, 485, 485, 485, 485, - /* 2680 */ 485, 485, 390, 485, 392, 485, 485, 485, 485, 485, - /* 2690 */ 485, 485, 485, 485, 485, 485, 485, 485, 485, 485, - /* 2700 */ 485, 425, 485, 427, 485, 485, 430, 431, 432, 433, - /* 2710 */ 434, 435, 485, 437, 485, 485, 485, 425, 485, 427, - /* 2720 */ 485, 485, 430, 431, 432, 433, 434, 435, 485, 437, - /* 2730 */ 485, 485, 485, 485, 485, 485, 485, 485, 485, 485, - /* 2740 */ 485, 485, 485, 485, 485, 485, 485, 485, 485, 485, - /* 2750 */ 485, 485, 485, 485, 337, 337, 337, 337, 337, 337, - /* 2760 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2770 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2780 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2790 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2800 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2810 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2820 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2830 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2840 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2850 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2860 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2870 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2880 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2890 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2900 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2910 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2920 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2930 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2940 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2950 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2960 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2970 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2980 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 2990 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 3000 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 3010 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 3020 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 3030 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 3040 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 3050 */ 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, - /* 3060 */ 337, 337, 337, 337, 337, 337, 337, + /* 340 */ 236, 237, 238, 239, 240, 241, 242, 243, 244, 12, + /* 350 */ 13, 67, 53, 379, 379, 12, 13, 20, 47, 22, + /* 360 */ 458, 386, 103, 461, 379, 391, 20, 393, 114, 394, + /* 370 */ 33, 260, 35, 70, 71, 72, 350, 351, 35, 477, + /* 380 */ 77, 78, 79, 481, 482, 400, 83, 351, 265, 266, + /* 390 */ 267, 88, 89, 90, 91, 442, 370, 94, 445, 62, + /* 400 */ 135, 427, 48, 429, 139, 68, 432, 433, 434, 435, + /* 410 */ 436, 437, 75, 439, 103, 379, 134, 135, 444, 345, + /* 420 */ 446, 139, 348, 349, 450, 451, 454, 455, 456, 341, + /* 430 */ 458, 459, 14, 458, 140, 141, 461, 100, 20, 351, + /* 440 */ 103, 353, 8, 9, 35, 471, 12, 13, 14, 15, + /* 450 */ 16, 476, 477, 12, 13, 14, 481, 482, 346, 341, + /* 460 */ 170, 20, 350, 22, 352, 200, 159, 379, 203, 351, + /* 470 */ 434, 206, 379, 208, 33, 168, 35, 140, 141, 391, + /* 480 */ 387, 393, 458, 21, 75, 461, 24, 25, 26, 27, + /* 490 */ 28, 29, 30, 31, 32, 379, 103, 379, 341, 350, + /* 500 */ 351, 477, 386, 62, 244, 481, 482, 170, 171, 391, + /* 510 */ 394, 393, 357, 176, 177, 427, 75, 429, 172, 260, + /* 520 */ 432, 433, 434, 435, 436, 437, 172, 439, 191, 374, + /* 530 */ 193, 133, 444, 179, 446, 137, 193, 382, 450, 451, + /* 540 */ 14, 100, 350, 351, 103, 427, 20, 429, 391, 341, + /* 550 */ 432, 433, 434, 435, 436, 437, 244, 439, 246, 351, + /* 560 */ 223, 224, 370, 226, 227, 228, 229, 230, 231, 232, + /* 570 */ 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, + /* 580 */ 243, 140, 141, 190, 22, 192, 261, 379, 8, 9, + /* 590 */ 472, 473, 12, 13, 14, 15, 16, 35, 431, 391, + /* 600 */ 20, 393, 453, 454, 455, 456, 172, 458, 459, 75, + /* 610 */ 341, 170, 171, 350, 351, 222, 389, 176, 177, 392, + /* 620 */ 393, 8, 9, 371, 457, 12, 13, 14, 15, 16, + /* 630 */ 431, 379, 191, 370, 193, 427, 350, 429, 386, 387, + /* 640 */ 432, 433, 434, 435, 436, 437, 394, 439, 134, 20, + /* 650 */ 411, 341, 444, 260, 446, 416, 457, 411, 450, 451, + /* 660 */ 391, 351, 100, 353, 223, 224, 44, 226, 227, 228, + /* 670 */ 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, + /* 680 */ 239, 240, 241, 242, 243, 12, 13, 0, 402, 379, + /* 690 */ 404, 350, 351, 20, 75, 22, 170, 458, 350, 351, + /* 700 */ 461, 391, 379, 393, 458, 0, 33, 461, 35, 386, + /* 710 */ 12, 13, 14, 15, 16, 476, 477, 394, 204, 205, + /* 720 */ 481, 482, 476, 477, 350, 351, 104, 481, 482, 1, + /* 730 */ 2, 223, 103, 350, 351, 62, 380, 427, 397, 429, + /* 740 */ 4, 350, 432, 433, 434, 435, 436, 437, 75, 439, + /* 750 */ 350, 351, 172, 370, 444, 19, 446, 0, 8, 9, + /* 760 */ 450, 451, 12, 13, 14, 15, 16, 341, 341, 33, + /* 770 */ 370, 397, 67, 100, 350, 351, 103, 351, 270, 271, + /* 780 */ 272, 273, 274, 275, 276, 49, 350, 351, 3, 12, + /* 790 */ 13, 0, 56, 402, 370, 404, 183, 20, 62, 22, + /* 800 */ 371, 20, 454, 455, 456, 379, 458, 459, 379, 443, + /* 810 */ 33, 445, 35, 140, 141, 0, 387, 391, 391, 393, + /* 820 */ 133, 134, 135, 136, 137, 138, 139, 70, 71, 72, + /* 830 */ 392, 393, 104, 397, 77, 78, 79, 22, 102, 62, + /* 840 */ 83, 105, 103, 170, 171, 88, 89, 90, 91, 176, + /* 850 */ 177, 94, 75, 427, 104, 429, 54, 55, 432, 433, + /* 860 */ 434, 435, 436, 437, 191, 439, 193, 441, 8, 9, + /* 870 */ 341, 380, 12, 13, 14, 15, 16, 100, 355, 356, + /* 880 */ 103, 341, 133, 134, 135, 136, 137, 138, 139, 341, + /* 890 */ 411, 351, 62, 353, 341, 380, 223, 224, 285, 226, + /* 900 */ 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, + /* 910 */ 237, 238, 239, 240, 241, 242, 243, 140, 141, 379, + /* 920 */ 391, 140, 141, 21, 133, 134, 135, 136, 137, 138, + /* 930 */ 139, 391, 102, 393, 379, 105, 34, 458, 36, 391, + /* 940 */ 461, 386, 350, 351, 391, 350, 351, 170, 171, 394, + /* 950 */ 114, 350, 351, 176, 177, 476, 477, 176, 177, 380, + /* 960 */ 481, 482, 370, 388, 104, 370, 391, 427, 191, 429, + /* 970 */ 193, 370, 432, 433, 434, 435, 436, 437, 39, 439, + /* 980 */ 364, 365, 2, 0, 444, 22, 446, 350, 8, 9, + /* 990 */ 450, 451, 12, 13, 14, 15, 16, 341, 35, 260, + /* 1000 */ 223, 224, 0, 226, 227, 228, 229, 230, 231, 232, + /* 1010 */ 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, + /* 1020 */ 243, 14, 18, 18, 20, 350, 351, 20, 23, 350, + /* 1030 */ 351, 27, 49, 359, 30, 355, 356, 33, 367, 402, + /* 1040 */ 371, 404, 37, 38, 0, 370, 41, 391, 379, 370, + /* 1050 */ 376, 379, 381, 49, 411, 51, 387, 52, 386, 385, + /* 1060 */ 56, 380, 391, 100, 341, 280, 394, 341, 63, 64, + /* 1070 */ 65, 66, 70, 71, 72, 73, 74, 0, 76, 77, + /* 1080 */ 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, + /* 1090 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 20, + /* 1100 */ 429, 458, 350, 351, 461, 20, 102, 33, 103, 37, + /* 1110 */ 439, 14, 15, 16, 391, 350, 351, 391, 114, 476, + /* 1120 */ 477, 44, 370, 49, 481, 482, 350, 351, 417, 350, + /* 1130 */ 351, 57, 58, 59, 60, 370, 62, 341, 44, 443, + /* 1140 */ 169, 445, 364, 365, 341, 341, 370, 142, 144, 370, + /* 1150 */ 341, 147, 148, 149, 150, 151, 152, 153, 154, 155, + /* 1160 */ 156, 157, 158, 159, 160, 161, 162, 163, 44, 165, + /* 1170 */ 166, 167, 350, 351, 258, 259, 102, 170, 106, 105, + /* 1180 */ 108, 109, 0, 111, 359, 341, 341, 391, 183, 184, + /* 1190 */ 185, 341, 370, 188, 391, 391, 341, 379, 259, 388, + /* 1200 */ 391, 351, 391, 353, 22, 133, 201, 202, 341, 137, + /* 1210 */ 385, 388, 394, 372, 391, 368, 375, 212, 247, 42, + /* 1220 */ 215, 44, 341, 218, 219, 220, 221, 222, 257, 379, + /* 1230 */ 366, 22, 351, 369, 353, 391, 391, 45, 46, 44, + /* 1240 */ 42, 391, 44, 393, 35, 114, 391, 173, 174, 13, + /* 1250 */ 107, 172, 341, 110, 180, 181, 107, 172, 391, 110, + /* 1260 */ 379, 217, 351, 107, 353, 260, 110, 107, 44, 13, + /* 1270 */ 110, 35, 391, 199, 393, 0, 44, 427, 207, 429, + /* 1280 */ 209, 44, 432, 433, 434, 435, 436, 437, 68, 439, + /* 1290 */ 379, 35, 140, 141, 444, 164, 446, 22, 342, 104, + /* 1300 */ 450, 451, 391, 474, 393, 35, 44, 44, 427, 103, + /* 1310 */ 429, 401, 485, 432, 433, 434, 435, 436, 437, 113, + /* 1320 */ 439, 44, 44, 35, 22, 444, 44, 446, 104, 44, + /* 1330 */ 44, 450, 451, 1, 2, 44, 104, 35, 427, 44, + /* 1340 */ 429, 104, 341, 432, 433, 434, 435, 436, 437, 44, + /* 1350 */ 439, 0, 351, 12, 13, 444, 468, 446, 2, 44, + /* 1360 */ 354, 450, 451, 22, 8, 9, 104, 104, 12, 13, + /* 1370 */ 14, 15, 16, 341, 33, 367, 35, 75, 284, 44, + /* 1380 */ 379, 104, 104, 351, 44, 13, 104, 367, 379, 104, + /* 1390 */ 104, 13, 391, 44, 393, 104, 410, 44, 401, 104, + /* 1400 */ 354, 50, 100, 62, 349, 351, 282, 35, 390, 104, + /* 1410 */ 341, 379, 401, 35, 460, 478, 75, 452, 49, 104, + /* 1420 */ 351, 462, 412, 391, 262, 393, 428, 20, 427, 206, + /* 1430 */ 429, 189, 359, 432, 433, 434, 435, 436, 437, 104, + /* 1440 */ 439, 100, 421, 223, 104, 444, 421, 446, 379, 426, + /* 1450 */ 359, 450, 451, 104, 414, 20, 351, 104, 42, 427, + /* 1460 */ 391, 429, 393, 193, 432, 433, 434, 435, 436, 437, + /* 1470 */ 20, 439, 351, 398, 396, 169, 444, 401, 446, 398, + /* 1480 */ 20, 193, 450, 451, 350, 398, 351, 350, 396, 396, + /* 1490 */ 101, 363, 99, 191, 362, 193, 427, 350, 429, 341, + /* 1500 */ 98, 432, 433, 434, 435, 436, 437, 361, 439, 351, + /* 1510 */ 20, 350, 343, 350, 350, 446, 48, 347, 343, 450, + /* 1520 */ 451, 347, 421, 20, 359, 223, 224, 359, 20, 393, + /* 1530 */ 352, 20, 191, 413, 193, 359, 352, 379, 359, 359, + /* 1540 */ 359, 350, 359, 343, 343, 379, 379, 350, 341, 391, + /* 1550 */ 379, 393, 379, 210, 425, 103, 423, 197, 351, 421, + /* 1560 */ 357, 420, 196, 391, 223, 224, 379, 379, 379, 379, + /* 1570 */ 419, 357, 379, 379, 341, 379, 391, 236, 237, 238, + /* 1580 */ 239, 240, 241, 242, 351, 427, 379, 429, 350, 195, + /* 1590 */ 432, 433, 434, 435, 436, 437, 391, 439, 391, 269, + /* 1600 */ 393, 393, 391, 277, 446, 401, 467, 268, 450, 451, + /* 1610 */ 391, 401, 379, 391, 412, 467, 391, 418, 406, 406, + /* 1620 */ 470, 182, 469, 279, 391, 341, 393, 263, 278, 412, + /* 1630 */ 486, 286, 259, 281, 427, 351, 429, 283, 341, 432, + /* 1640 */ 433, 434, 435, 436, 437, 351, 439, 20, 351, 357, + /* 1650 */ 352, 20, 466, 446, 404, 406, 357, 450, 451, 431, + /* 1660 */ 427, 406, 429, 379, 391, 432, 433, 434, 435, 436, + /* 1670 */ 437, 438, 439, 440, 441, 391, 379, 393, 391, 174, + /* 1680 */ 391, 465, 467, 403, 391, 464, 357, 341, 391, 375, + /* 1690 */ 393, 391, 391, 357, 103, 449, 103, 351, 480, 351, + /* 1700 */ 383, 391, 369, 36, 341, 479, 344, 350, 343, 0, + /* 1710 */ 407, 427, 357, 429, 351, 407, 432, 433, 434, 435, + /* 1720 */ 436, 437, 415, 439, 427, 379, 429, 339, 358, 432, + /* 1730 */ 433, 434, 435, 436, 437, 422, 439, 391, 0, 393, + /* 1740 */ 373, 373, 379, 446, 373, 0, 42, 384, 451, 0, + /* 1750 */ 35, 216, 35, 341, 391, 35, 393, 35, 216, 0, + /* 1760 */ 35, 35, 216, 351, 0, 216, 0, 483, 484, 35, + /* 1770 */ 0, 0, 22, 427, 35, 429, 211, 341, 432, 433, + /* 1780 */ 434, 435, 436, 437, 0, 439, 199, 351, 0, 199, + /* 1790 */ 427, 379, 429, 193, 200, 432, 433, 434, 435, 436, + /* 1800 */ 437, 191, 439, 391, 341, 393, 0, 0, 0, 187, + /* 1810 */ 186, 0, 0, 0, 351, 379, 47, 0, 0, 473, + /* 1820 */ 384, 47, 0, 42, 341, 0, 0, 391, 0, 393, + /* 1830 */ 47, 0, 0, 0, 351, 0, 0, 0, 159, 427, + /* 1840 */ 35, 429, 379, 0, 432, 433, 434, 435, 436, 437, + /* 1850 */ 159, 439, 0, 0, 391, 0, 393, 0, 0, 0, + /* 1860 */ 0, 0, 379, 427, 0, 429, 0, 384, 432, 433, + /* 1870 */ 434, 435, 436, 437, 391, 439, 393, 47, 0, 0, + /* 1880 */ 42, 0, 0, 0, 0, 0, 62, 475, 0, 0, + /* 1890 */ 427, 341, 429, 0, 0, 432, 433, 434, 435, 436, + /* 1900 */ 437, 351, 439, 0, 0, 0, 0, 0, 22, 0, + /* 1910 */ 427, 143, 429, 0, 22, 432, 433, 434, 435, 436, + /* 1920 */ 437, 341, 439, 48, 48, 0, 35, 0, 0, 379, + /* 1930 */ 22, 351, 62, 62, 384, 0, 0, 39, 0, 49, + /* 1940 */ 39, 391, 0, 393, 49, 35, 35, 484, 49, 35, + /* 1950 */ 0, 39, 35, 0, 14, 39, 0, 42, 0, 379, + /* 1960 */ 0, 39, 44, 40, 384, 0, 47, 47, 47, 39, + /* 1970 */ 0, 391, 182, 393, 35, 49, 39, 427, 0, 429, + /* 1980 */ 341, 0, 432, 433, 434, 435, 436, 437, 1, 439, + /* 1990 */ 351, 0, 0, 69, 0, 49, 35, 0, 39, 35, + /* 2000 */ 49, 39, 0, 35, 49, 39, 19, 427, 0, 429, + /* 2010 */ 0, 0, 432, 433, 434, 435, 436, 437, 379, 439, + /* 2020 */ 33, 0, 0, 0, 0, 35, 22, 44, 35, 44, + /* 2030 */ 391, 35, 393, 35, 35, 35, 49, 0, 35, 22, + /* 2040 */ 35, 112, 110, 0, 57, 58, 59, 60, 35, 62, + /* 2050 */ 35, 35, 22, 341, 22, 0, 22, 51, 0, 0, + /* 2060 */ 35, 35, 35, 351, 0, 0, 427, 22, 429, 20, + /* 2070 */ 35, 432, 433, 434, 435, 436, 437, 35, 439, 35, + /* 2080 */ 104, 35, 103, 103, 341, 198, 0, 0, 22, 102, + /* 2090 */ 194, 379, 105, 22, 351, 0, 3, 44, 264, 104, + /* 2100 */ 48, 44, 3, 391, 103, 393, 172, 174, 48, 178, + /* 2110 */ 47, 99, 103, 172, 44, 172, 172, 44, 104, 341, + /* 2120 */ 101, 103, 379, 178, 104, 138, 47, 44, 44, 351, + /* 2130 */ 258, 103, 103, 264, 391, 104, 393, 104, 264, 427, + /* 2140 */ 104, 429, 103, 35, 432, 433, 434, 435, 436, 437, + /* 2150 */ 35, 439, 35, 35, 35, 35, 341, 379, 47, 47, + /* 2160 */ 173, 0, 0, 47, 39, 104, 351, 180, 44, 391, + /* 2170 */ 427, 393, 429, 104, 0, 432, 433, 434, 435, 436, + /* 2180 */ 437, 341, 439, 103, 103, 103, 199, 39, 113, 104, + /* 2190 */ 47, 351, 104, 103, 379, 103, 44, 103, 101, 245, + /* 2200 */ 2, 101, 22, 47, 103, 427, 391, 429, 393, 223, + /* 2210 */ 432, 433, 434, 435, 436, 437, 175, 439, 341, 379, + /* 2220 */ 173, 22, 47, 114, 35, 104, 104, 35, 351, 103, + /* 2230 */ 103, 391, 104, 393, 103, 103, 35, 104, 103, 35, + /* 2240 */ 104, 103, 427, 341, 429, 104, 104, 432, 433, 434, + /* 2250 */ 435, 436, 437, 351, 439, 104, 379, 103, 103, 35, + /* 2260 */ 103, 35, 104, 225, 104, 35, 103, 427, 391, 429, + /* 2270 */ 393, 103, 432, 433, 434, 435, 436, 437, 125, 439, + /* 2280 */ 103, 379, 44, 35, 125, 22, 103, 103, 69, 125, + /* 2290 */ 125, 68, 35, 391, 35, 393, 35, 35, 44, 35, + /* 2300 */ 35, 35, 35, 75, 427, 97, 429, 35, 35, 432, + /* 2310 */ 433, 434, 435, 436, 437, 341, 439, 35, 22, 35, + /* 2320 */ 35, 35, 75, 35, 35, 351, 35, 35, 35, 427, + /* 2330 */ 341, 429, 22, 35, 432, 433, 434, 435, 436, 437, + /* 2340 */ 351, 439, 0, 35, 49, 39, 0, 35, 341, 49, + /* 2350 */ 0, 35, 39, 379, 49, 0, 39, 35, 351, 39, + /* 2360 */ 0, 35, 35, 49, 0, 391, 22, 393, 379, 21, + /* 2370 */ 487, 22, 20, 22, 21, 487, 487, 487, 487, 487, + /* 2380 */ 391, 487, 393, 487, 487, 487, 379, 487, 487, 487, + /* 2390 */ 487, 487, 487, 487, 487, 341, 487, 487, 391, 487, + /* 2400 */ 393, 427, 487, 429, 487, 351, 432, 433, 434, 435, + /* 2410 */ 436, 437, 487, 439, 487, 487, 427, 341, 429, 487, + /* 2420 */ 487, 432, 433, 434, 435, 436, 437, 351, 439, 487, + /* 2430 */ 487, 487, 487, 379, 427, 487, 429, 487, 487, 432, + /* 2440 */ 433, 434, 435, 436, 437, 391, 439, 393, 487, 487, + /* 2450 */ 487, 487, 487, 487, 487, 379, 487, 487, 487, 487, + /* 2460 */ 487, 487, 487, 487, 341, 487, 487, 391, 487, 393, + /* 2470 */ 487, 487, 487, 487, 351, 487, 487, 487, 487, 487, + /* 2480 */ 487, 427, 487, 429, 487, 487, 432, 433, 434, 435, + /* 2490 */ 436, 437, 487, 439, 487, 487, 487, 487, 487, 487, + /* 2500 */ 487, 487, 379, 427, 487, 429, 487, 487, 432, 433, + /* 2510 */ 434, 435, 436, 437, 391, 439, 393, 487, 487, 487, + /* 2520 */ 487, 487, 487, 341, 487, 487, 487, 487, 487, 487, + /* 2530 */ 487, 487, 487, 351, 487, 487, 341, 487, 487, 487, + /* 2540 */ 487, 487, 487, 487, 487, 487, 351, 487, 487, 487, + /* 2550 */ 427, 487, 429, 341, 487, 432, 433, 434, 435, 436, + /* 2560 */ 437, 379, 439, 351, 487, 487, 487, 487, 487, 487, + /* 2570 */ 487, 487, 487, 391, 379, 393, 487, 487, 487, 487, + /* 2580 */ 487, 487, 487, 487, 487, 341, 391, 487, 393, 487, + /* 2590 */ 487, 379, 487, 487, 487, 351, 487, 487, 487, 487, + /* 2600 */ 487, 487, 487, 391, 487, 393, 487, 487, 487, 427, + /* 2610 */ 487, 429, 487, 487, 432, 433, 434, 435, 436, 437, + /* 2620 */ 487, 439, 427, 379, 429, 487, 487, 432, 433, 434, + /* 2630 */ 435, 436, 437, 487, 439, 391, 487, 393, 487, 427, + /* 2640 */ 487, 429, 487, 487, 432, 433, 434, 435, 436, 437, + /* 2650 */ 487, 439, 341, 487, 487, 487, 487, 487, 487, 487, + /* 2660 */ 487, 487, 351, 487, 487, 487, 487, 487, 487, 487, + /* 2670 */ 487, 427, 341, 429, 487, 487, 432, 433, 434, 435, + /* 2680 */ 436, 437, 351, 439, 487, 487, 487, 487, 487, 487, + /* 2690 */ 379, 487, 487, 487, 487, 487, 487, 487, 487, 487, + /* 2700 */ 487, 487, 391, 487, 393, 487, 487, 487, 487, 487, + /* 2710 */ 379, 487, 487, 487, 487, 487, 487, 487, 487, 487, + /* 2720 */ 487, 487, 391, 487, 393, 487, 487, 487, 487, 487, + /* 2730 */ 487, 487, 487, 487, 487, 487, 487, 487, 427, 487, + /* 2740 */ 429, 487, 341, 432, 433, 434, 435, 436, 437, 487, + /* 2750 */ 439, 487, 351, 487, 487, 487, 487, 487, 427, 341, + /* 2760 */ 429, 487, 487, 432, 433, 434, 435, 436, 437, 351, + /* 2770 */ 439, 487, 487, 487, 487, 487, 487, 487, 487, 487, + /* 2780 */ 379, 487, 487, 487, 487, 487, 487, 487, 487, 487, + /* 2790 */ 487, 487, 391, 487, 393, 487, 487, 379, 487, 487, + /* 2800 */ 487, 487, 487, 487, 487, 487, 487, 487, 487, 391, + /* 2810 */ 487, 393, 487, 487, 487, 487, 487, 487, 487, 487, + /* 2820 */ 487, 487, 487, 487, 487, 487, 487, 487, 427, 487, + /* 2830 */ 429, 487, 487, 432, 433, 434, 435, 436, 437, 487, + /* 2840 */ 439, 487, 487, 487, 487, 427, 487, 429, 487, 487, + /* 2850 */ 432, 433, 434, 435, 436, 437, 487, 439, 338, 338, + /* 2860 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2870 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2880 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2890 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2900 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2910 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2920 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2930 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2940 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2950 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2960 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2970 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2980 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 2990 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3000 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3010 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3020 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3030 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3040 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3050 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3060 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3070 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3080 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3090 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3100 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3110 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3120 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3130 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3140 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3150 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3160 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3170 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3180 */ 338, 338, 338, 338, 338, 338, 338, 338, 338, 338, + /* 3190 */ 338, 338, 338, 338, 338, 338, }; -#define YY_SHIFT_COUNT (793) +#define YY_SHIFT_COUNT (799) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2407) +#define YY_SHIFT_MAX (2364) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 1097, 0, 104, 0, 336, 336, 336, 336, 336, 336, - /* 10 */ 336, 336, 336, 336, 336, 336, 440, 671, 671, 775, - /* 20 */ 671, 671, 671, 671, 671, 671, 671, 671, 671, 671, - /* 30 */ 671, 671, 671, 671, 671, 671, 671, 671, 671, 671, - /* 40 */ 671, 671, 671, 671, 671, 671, 671, 671, 671, 671, - /* 50 */ 671, 335, 519, 1053, 263, 215, 156, 586, 156, 263, - /* 60 */ 263, 1333, 156, 1333, 1333, 621, 156, 34, 798, 111, - /* 70 */ 111, 798, 278, 278, 294, 138, 308, 308, 111, 111, - /* 80 */ 111, 111, 111, 111, 111, 199, 111, 111, 297, 34, - /* 90 */ 111, 111, 223, 111, 34, 111, 199, 111, 199, 34, - /* 100 */ 111, 111, 34, 111, 34, 34, 34, 111, 460, 917, - /* 110 */ 15, 15, 378, 170, 700, 700, 700, 700, 700, 700, - /* 120 */ 700, 700, 700, 700, 700, 700, 700, 700, 700, 700, - /* 130 */ 700, 700, 700, 326, 410, 294, 138, 633, 633, 714, - /* 140 */ 426, 426, 426, 618, 107, 107, 714, 299, 299, 299, - /* 150 */ 297, 356, 353, 34, 577, 34, 577, 577, 663, 730, - /* 160 */ 259, 259, 259, 259, 259, 259, 259, 259, 1962, 1116, - /* 170 */ 185, 31, 276, 249, 394, 176, 13, 13, 862, 1073, - /* 180 */ 489, 776, 1167, 949, 721, 820, 996, 355, 581, 996, - /* 190 */ 997, 1081, 256, 1219, 1434, 1470, 1288, 297, 1470, 297, - /* 200 */ 1318, 1467, 1498, 1467, 1352, 1507, 1507, 1467, 1352, 1352, - /* 210 */ 1437, 1446, 1507, 1452, 1507, 1507, 1507, 1541, 1516, 1541, - /* 220 */ 1516, 1470, 297, 1561, 297, 1563, 1564, 297, 1563, 297, - /* 230 */ 297, 297, 1507, 297, 1541, 34, 34, 34, 34, 34, - /* 240 */ 34, 34, 34, 34, 34, 34, 1507, 1541, 577, 577, - /* 250 */ 577, 1400, 1508, 1470, 460, 1422, 1425, 1561, 460, 1436, - /* 260 */ 1219, 1507, 1498, 1498, 577, 1366, 1370, 577, 1366, 1370, - /* 270 */ 577, 577, 34, 1364, 1461, 1366, 1376, 1402, 1427, 1219, - /* 280 */ 1406, 1411, 1414, 1438, 299, 1679, 1219, 1507, 1563, 460, - /* 290 */ 460, 1688, 1370, 577, 577, 577, 577, 577, 1370, 577, - /* 300 */ 1535, 460, 663, 460, 299, 1625, 1640, 577, 730, 1507, - /* 310 */ 460, 1708, 1541, 2730, 2730, 2730, 2730, 2730, 2730, 2730, - /* 320 */ 2730, 2730, 1018, 679, 224, 739, 1543, 858, 1041, 418, - /* 330 */ 748, 974, 1110, 1166, 1377, 1377, 1377, 1377, 1377, 1377, - /* 340 */ 1377, 1377, 1377, 500, 729, 51, 51, 441, 484, 446, - /* 350 */ 660, 76, 121, 699, 593, 291, 924, 924, 701, 536, - /* 360 */ 952, 701, 701, 701, 384, 1060, 1093, 1095, 1291, 1220, - /* 370 */ 765, 1241, 1242, 1243, 1247, 566, 688, 1340, 1342, 1365, - /* 380 */ 1087, 1319, 1323, 1226, 1325, 1328, 1330, 1274, 1157, 1183, - /* 390 */ 1310, 1331, 1338, 1344, 1361, 1363, 1372, 1420, 1380, 1101, - /* 400 */ 1381, 1389, 1382, 1384, 1387, 1393, 1396, 1399, 1343, 1362, - /* 410 */ 1374, 1426, 1428, 723, 1444, 1760, 1767, 1768, 1729, 1783, - /* 420 */ 1740, 1571, 1741, 1753, 1754, 1575, 1791, 1757, 1758, 1579, - /* 430 */ 1795, 1582, 1800, 1772, 1810, 1789, 1812, 1778, 1604, 1815, - /* 440 */ 1618, 1817, 1620, 1622, 1627, 1633, 1824, 1831, 1833, 1648, - /* 450 */ 1650, 1836, 1838, 1792, 1840, 1841, 1843, 1802, 1845, 1846, - /* 460 */ 1847, 1848, 1850, 1852, 1853, 1854, 1696, 1821, 1857, 1699, - /* 470 */ 1859, 1860, 1861, 1862, 1874, 1875, 1876, 1878, 1879, 1880, - /* 480 */ 1881, 1882, 1883, 1887, 1888, 1897, 1822, 1865, 1866, 1867, - /* 490 */ 1868, 1869, 1870, 1849, 1872, 1898, 1899, 1759, 1900, 1903, - /* 500 */ 1884, 1856, 1886, 1863, 1905, 1855, 1877, 1907, 1858, 1914, - /* 510 */ 1871, 1916, 1918, 1889, 1873, 1890, 1921, 1891, 1892, 1893, - /* 520 */ 1925, 1895, 1894, 1901, 1927, 1896, 1934, 1902, 1906, 1920, - /* 530 */ 1904, 1919, 1922, 1924, 1938, 1909, 1926, 1939, 1942, 1946, - /* 540 */ 1928, 1787, 1969, 1973, 1974, 1915, 1983, 1985, 1951, 1940, - /* 550 */ 1948, 1988, 1955, 1943, 1952, 1993, 1963, 1956, 1970, 2008, - /* 560 */ 1975, 1964, 1976, 2014, 2017, 2018, 2023, 2028, 2029, 1798, - /* 570 */ 1923, 1977, 2003, 2030, 1997, 1999, 2001, 2004, 2005, 2006, - /* 580 */ 2007, 1987, 1994, 2009, 2010, 2021, 2011, 2048, 2031, 2050, - /* 590 */ 2033, 2000, 2052, 2034, 2022, 2058, 2024, 2060, 2026, 2062, - /* 600 */ 2041, 2049, 2035, 2036, 2037, 1971, 1978, 2073, 1908, 1980, - /* 610 */ 1930, 2039, 2054, 2078, 1911, 2063, 1944, 1912, 2093, 2096, - /* 620 */ 1957, 1949, 2094, 2055, 1835, 2020, 1998, 2027, 2083, 2038, - /* 630 */ 2084, 2042, 2040, 2090, 2098, 2044, 2046, 2047, 2051, 2053, - /* 640 */ 2099, 2091, 2104, 2056, 2108, 1931, 2057, 2059, 2134, 2111, - /* 650 */ 1932, 2123, 2127, 2131, 2133, 2135, 2136, 2065, 2068, 2126, - /* 660 */ 1917, 2137, 2128, 2177, 2179, 2183, 2191, 2100, 2154, 1904, - /* 670 */ 2150, 2112, 2095, 2097, 2119, 2121, 2069, 2124, 2226, 2189, - /* 680 */ 2061, 2129, 2116, 1904, 2198, 2186, 2145, 2013, 2146, 2246, - /* 690 */ 2227, 2043, 2148, 2149, 2152, 2155, 2158, 2160, 2205, 2159, - /* 700 */ 2163, 2211, 2164, 2245, 2045, 2167, 2157, 2168, 2238, 2239, - /* 710 */ 2175, 2176, 2244, 2178, 2180, 2247, 2182, 2188, 2248, 2184, - /* 720 */ 2199, 2255, 2190, 2200, 2270, 2204, 2181, 2196, 2202, 2206, - /* 730 */ 2208, 2264, 2210, 2274, 2220, 2264, 2264, 2302, 2256, 2261, - /* 740 */ 2295, 2297, 2298, 2299, 2303, 2304, 2313, 2314, 2275, 2254, - /* 750 */ 2309, 2319, 2320, 2321, 2335, 2323, 2324, 2325, 2289, 1987, - /* 760 */ 2337, 1994, 2341, 2342, 2343, 2344, 2358, 2346, 2382, 2348, - /* 770 */ 2336, 2345, 2387, 2354, 2347, 2352, 2392, 2359, 2349, 2356, - /* 780 */ 2393, 2362, 2350, 2361, 2401, 2370, 2371, 2407, 2386, 2388, - /* 790 */ 2389, 2390, 2395, 2394, + /* 0 */ 1005, 0, 104, 0, 337, 337, 337, 337, 337, 337, + /* 10 */ 337, 337, 337, 337, 337, 337, 441, 673, 673, 777, + /* 20 */ 673, 673, 673, 673, 673, 673, 673, 673, 673, 673, + /* 30 */ 673, 673, 673, 673, 673, 673, 673, 673, 673, 673, + /* 40 */ 673, 673, 673, 673, 673, 673, 673, 673, 673, 673, + /* 50 */ 673, 60, 259, 393, 629, 111, 739, 111, 629, 629, + /* 60 */ 111, 1341, 111, 1341, 1341, 66, 111, 68, 781, 101, + /* 70 */ 101, 781, 13, 13, 113, 294, 23, 23, 101, 101, + /* 80 */ 101, 101, 101, 101, 101, 101, 101, 101, 158, 101, + /* 90 */ 101, 211, 68, 101, 101, 285, 68, 101, 158, 101, + /* 100 */ 158, 68, 101, 101, 68, 101, 68, 68, 68, 101, + /* 110 */ 284, 1004, 15, 15, 303, 462, 1302, 1302, 1302, 1302, + /* 120 */ 1302, 1302, 1302, 1302, 1302, 1302, 1302, 1302, 1302, 1302, + /* 130 */ 1302, 1302, 1302, 1302, 1302, 1072, 18, 113, 294, 802, + /* 140 */ 802, 184, 346, 346, 346, 705, 312, 312, 184, 211, + /* 150 */ 254, 260, 68, 534, 68, 534, 534, 836, 619, 28, + /* 160 */ 28, 28, 28, 28, 28, 28, 28, 1987, 757, 261, + /* 170 */ 580, 613, 508, 49, 123, 343, 343, 526, 354, 1007, + /* 180 */ 299, 1079, 1192, 418, 398, 1085, 916, 939, 785, 916, + /* 190 */ 1177, 325, 290, 1162, 1369, 1407, 1223, 211, 1407, 211, + /* 200 */ 1242, 1435, 1416, 1450, 1435, 1416, 1306, 1460, 1435, 1460, + /* 210 */ 1416, 1306, 1306, 1389, 1393, 1460, 1402, 1460, 1460, 1460, + /* 220 */ 1490, 1468, 1490, 1468, 1407, 211, 1503, 211, 1508, 1511, + /* 230 */ 211, 1508, 211, 211, 211, 1460, 211, 1490, 68, 68, + /* 240 */ 68, 68, 68, 68, 68, 68, 68, 68, 68, 1460, + /* 250 */ 1490, 534, 534, 534, 1343, 1452, 1407, 284, 1360, 1366, + /* 260 */ 1503, 284, 1394, 1162, 1460, 1450, 1450, 534, 1330, 1339, + /* 270 */ 534, 1330, 1339, 534, 534, 68, 1326, 1439, 1330, 1344, + /* 280 */ 1350, 1364, 1162, 1345, 1354, 1352, 1373, 1435, 1627, 1508, + /* 290 */ 284, 284, 1631, 1339, 534, 534, 534, 534, 534, 1339, + /* 300 */ 534, 1505, 284, 836, 284, 1435, 1591, 1593, 534, 619, + /* 310 */ 1460, 284, 1667, 1490, 2858, 2858, 2858, 2858, 2858, 2858, + /* 320 */ 2858, 2858, 2858, 1002, 1074, 225, 32, 736, 750, 860, + /* 330 */ 687, 980, 1356, 434, 791, 197, 197, 197, 197, 197, + /* 340 */ 197, 197, 197, 197, 749, 265, 698, 698, 29, 6, + /* 350 */ 34, 307, 830, 562, 963, 902, 514, 282, 282, 1097, + /* 360 */ 728, 971, 1097, 1097, 1097, 983, 1044, 622, 1209, 1198, + /* 370 */ 1131, 1077, 1143, 1149, 1156, 1160, 1236, 1256, 815, 1182, + /* 380 */ 1275, 1071, 1195, 1224, 63, 1232, 1237, 1262, 1152, 1124, + /* 390 */ 1094, 1263, 1277, 1278, 1282, 1285, 1286, 1332, 1291, 1220, + /* 400 */ 1295, 311, 1305, 1315, 1335, 1340, 1349, 1353, 1206, 1270, + /* 410 */ 1288, 1372, 1378, 409, 1351, 1709, 1738, 1745, 1704, 1749, + /* 420 */ 1715, 1535, 1717, 1720, 1722, 1542, 1759, 1725, 1726, 1546, + /* 430 */ 1764, 1549, 1766, 1734, 1770, 1750, 1771, 1739, 1565, 1784, + /* 440 */ 1587, 1788, 1590, 1594, 1600, 1610, 1806, 1807, 1808, 1622, + /* 450 */ 1624, 1811, 1812, 1769, 1813, 1817, 1818, 1774, 1822, 1781, + /* 460 */ 1825, 1826, 1828, 1783, 1831, 1832, 1833, 1835, 1836, 1837, + /* 470 */ 1679, 1805, 1843, 1691, 1852, 1853, 1855, 1857, 1858, 1859, + /* 480 */ 1860, 1861, 1864, 1866, 1888, 1889, 1893, 1894, 1903, 1904, + /* 490 */ 1830, 1878, 1838, 1879, 1881, 1882, 1883, 1884, 1885, 1886, + /* 500 */ 1905, 1906, 1907, 1768, 1909, 1913, 1892, 1875, 1908, 1876, + /* 510 */ 1925, 1824, 1891, 1927, 1870, 1928, 1871, 1935, 1936, 1910, + /* 520 */ 1890, 1898, 1938, 1911, 1895, 1901, 1942, 1914, 1899, 1912, + /* 530 */ 1950, 1917, 1953, 1915, 1916, 1918, 1919, 1920, 1940, 1921, + /* 540 */ 1956, 1923, 1922, 1958, 1960, 1965, 1930, 1790, 1970, 1978, + /* 550 */ 1981, 1924, 1991, 1992, 1939, 1926, 1937, 1994, 1961, 1946, + /* 560 */ 1959, 1997, 1964, 1951, 1962, 2002, 1968, 1955, 1966, 2008, + /* 570 */ 2010, 2011, 2021, 2022, 2023, 1929, 1932, 1990, 2004, 2024, + /* 580 */ 1993, 1996, 1998, 1999, 2000, 2003, 2005, 1983, 1985, 2013, + /* 590 */ 2015, 2017, 2016, 2037, 2030, 2043, 2032, 2006, 2055, 2034, + /* 600 */ 2025, 2058, 2026, 2059, 2027, 2064, 2045, 2049, 2035, 2042, + /* 610 */ 2044, 1976, 1979, 2065, 1934, 1980, 1887, 2046, 2066, 2086, + /* 620 */ 1896, 2071, 1941, 1933, 2087, 2095, 1943, 1931, 1944, 1945, + /* 630 */ 2093, 2053, 1834, 2001, 1995, 2009, 2052, 2019, 2060, 2012, + /* 640 */ 2014, 2057, 2070, 2020, 2018, 2028, 2029, 2031, 2073, 2063, + /* 650 */ 2079, 2039, 2083, 1869, 2033, 2036, 2099, 2084, 1874, 2108, + /* 660 */ 2115, 2117, 2118, 2119, 2120, 2061, 2069, 2111, 1872, 2124, + /* 670 */ 2112, 2161, 2162, 2080, 2125, 1919, 2116, 2081, 2085, 2088, + /* 680 */ 2082, 2090, 2041, 2092, 2174, 2148, 2047, 2094, 2075, 1919, + /* 690 */ 2143, 2152, 2097, 1954, 2100, 2198, 2180, 1986, 2101, 2121, + /* 700 */ 2126, 2122, 2127, 2128, 2156, 2131, 2132, 2175, 2133, 2199, + /* 710 */ 2038, 2135, 2109, 2136, 2189, 2192, 2138, 2141, 2201, 2154, + /* 720 */ 2142, 2204, 2155, 2151, 2224, 2157, 2158, 2226, 2163, 2160, + /* 730 */ 2230, 2168, 2153, 2159, 2164, 2165, 2177, 2238, 2183, 2248, + /* 740 */ 2184, 2238, 2238, 2263, 2219, 2223, 2257, 2259, 2261, 2262, + /* 750 */ 2264, 2265, 2266, 2267, 2228, 2208, 2254, 2272, 2273, 2282, + /* 760 */ 2296, 2284, 2285, 2286, 2247, 1983, 2288, 1985, 2289, 2291, + /* 770 */ 2292, 2293, 2310, 2298, 2342, 2308, 2295, 2306, 2346, 2312, + /* 780 */ 2300, 2313, 2350, 2316, 2305, 2317, 2355, 2322, 2314, 2320, + /* 790 */ 2360, 2326, 2327, 2364, 2344, 2348, 2349, 2351, 2353, 2352, }; -#define YY_REDUCE_COUNT (321) -#define YY_REDUCE_MIN (-438) -#define YY_REDUCE_MAX (2292) +#define YY_REDUCE_COUNT (322) +#define YY_REDUCE_MIN (-404) +#define YY_REDUCE_MAX (2418) static const short yy_reduce_ofst[] = { - /* 0 */ 311, -340, -280, -25, -132, 158, 603, 789, 895, 969, - /* 10 */ 265, 1020, 1080, 390, 1109, 1225, 53, 454, 825, 1251, - /* 20 */ 1285, 1347, 1371, 946, 1395, 1459, 1523, 1545, 1569, 1657, - /* 30 */ 1676, 1687, 1755, 1775, 1786, 1806, 1864, 1885, 1910, 1936, - /* 40 */ 1996, 2012, 2025, 2085, 2101, 2114, 2187, 2203, 2214, 2276, - /* 50 */ 2292, 167, 792, -403, 78, -401, -360, 252, 672, 47, - /* 60 */ 197, 379, -438, -244, 339, -331, 505, -211, -274, -63, - /* 70 */ 23, -391, -40, 149, -317, -31, -342, -265, 295, 464, - /* 80 */ 486, 571, 608, 610, 622, -344, 673, 675, -252, -205, - /* 90 */ 677, 805, -243, 809, 219, 822, -92, 833, 19, -16, - /* 100 */ 423, 840, 325, 880, 563, 755, 891, 914, -268, -207, - /* 110 */ -173, -173, 327, -337, -321, -229, -93, 61, 209, 403, - /* 120 */ 439, 461, 529, 630, 746, 802, 828, 844, 856, 885, - /* 130 */ 888, 900, 901, -288, -34, -213, 90, 448, 476, 554, - /* 140 */ -34, 50, 501, 247, -409, 301, 780, 499, 533, 582, - /* 150 */ 769, 234, 422, 643, 609, 549, 786, 878, 918, 945, - /* 160 */ -364, 447, 477, 535, 547, 575, 615, 547, 628, 681, - /* 170 */ 791, 662, 667, 685, 811, 745, 814, 866, 892, 892, - /* 180 */ 883, 894, 1112, 1121, 1084, 1074, 1017, 1017, 1001, 1017, - /* 190 */ 1028, 1019, 892, 1071, 1063, 1076, 1072, 1140, 1083, 1146, - /* 200 */ 1096, 1119, 1120, 1122, 1131, 1181, 1182, 1135, 1138, 1142, - /* 210 */ 1173, 1187, 1200, 1191, 1203, 1210, 1211, 1221, 1231, 1236, - /* 220 */ 1233, 1148, 1222, 1190, 1227, 1235, 1176, 1230, 1238, 1232, - /* 230 */ 1237, 1239, 1244, 1240, 1252, 1218, 1224, 1228, 1229, 1234, - /* 240 */ 1245, 1246, 1248, 1249, 1250, 1253, 1255, 1258, 1223, 1254, - /* 250 */ 1256, 1185, 1189, 1195, 1260, 1201, 1204, 1257, 1266, 1216, - /* 260 */ 1261, 1284, 1264, 1265, 1263, 1171, 1262, 1271, 1174, 1267, - /* 270 */ 1276, 1278, 892, 1170, 1180, 1186, 1206, 1217, 1269, 1280, - /* 280 */ 1208, 1268, 1259, 1017, 1348, 1272, 1292, 1354, 1353, 1349, - /* 290 */ 1350, 1304, 1308, 1324, 1334, 1336, 1337, 1339, 1327, 1351, - /* 300 */ 1321, 1378, 1359, 1386, 1388, 1293, 1368, 1357, 1383, 1403, - /* 310 */ 1392, 1410, 1413, 1345, 1346, 1356, 1358, 1385, 1397, 1398, - /* 320 */ 1408, 1421, + /* 0 */ -296, -340, -133, -26, 88, 310, 540, 850, 881, 911, + /* 10 */ 208, 1001, 1032, 1069, 1158, 1207, 1233, 118, 1284, 1297, + /* 20 */ 1346, 1363, 1436, 1412, 1463, 1483, 426, 1550, 1580, 1639, + /* 30 */ 1712, 1743, 1778, 1815, 1840, 1877, 1902, 1974, 1989, 2007, + /* 40 */ 2054, 2076, 2123, 2182, 2195, 2212, 2244, 2311, 2331, 2401, + /* 50 */ 2418, -260, -25, 239, 149, -404, 246, 479, -28, 348, + /* 60 */ 643, -365, -294, -331, 671, -98, 24, 252, -390, -345, + /* 70 */ -265, -279, -72, 74, -318, 227, -342, 112, -232, 26, + /* 80 */ 192, 263, 341, 374, 383, 400, 424, 436, 286, 592, + /* 90 */ 595, 674, 116, 601, 675, 36, 323, 679, 391, 752, + /* 100 */ 637, -63, 765, 776, 555, 779, 429, 672, 669, 822, + /* 110 */ 155, -316, -313, -313, -259, -140, -199, -188, 157, 269, + /* 120 */ 427, 529, 548, 553, 656, 723, 726, 796, 803, 804, + /* 130 */ 809, 844, 845, 855, 867, -217, -242, -15, 438, 523, + /* 140 */ 680, 616, -242, 167, 199, -250, 366, 696, 778, 825, + /* 150 */ -375, -47, 93, 575, 818, 811, 823, 841, 864, -371, + /* 160 */ -328, 356, 491, 515, 579, 681, 515, 711, 847, 956, + /* 170 */ 910, 827, 829, 1006, 888, 1008, 1020, 1009, 986, 1009, + /* 180 */ 1046, 997, 1055, 1054, 1018, 1011, 954, 954, 937, 954, + /* 190 */ 965, 959, 1009, 1010, 998, 1021, 1023, 1073, 1025, 1091, + /* 200 */ 1040, 1105, 1075, 1076, 1121, 1081, 1078, 1134, 1135, 1137, + /* 210 */ 1087, 1092, 1093, 1128, 1132, 1147, 1146, 1161, 1163, 1164, + /* 220 */ 1169, 1170, 1175, 1174, 1101, 1165, 1136, 1168, 1178, 1120, + /* 230 */ 1176, 1184, 1179, 1180, 1181, 1191, 1183, 1200, 1166, 1167, + /* 240 */ 1171, 1173, 1187, 1188, 1189, 1190, 1193, 1194, 1196, 1197, + /* 250 */ 1201, 1172, 1185, 1205, 1129, 1133, 1138, 1203, 1141, 1151, + /* 260 */ 1208, 1214, 1199, 1202, 1238, 1204, 1210, 1211, 1139, 1212, + /* 270 */ 1219, 1148, 1213, 1222, 1225, 1009, 1150, 1153, 1215, 1186, + /* 280 */ 1216, 1221, 1217, 1144, 1218, 1226, 954, 1294, 1228, 1298, + /* 290 */ 1292, 1299, 1250, 1249, 1273, 1287, 1289, 1293, 1300, 1255, + /* 300 */ 1301, 1280, 1329, 1314, 1336, 1348, 1246, 1317, 1310, 1333, + /* 310 */ 1357, 1355, 1362, 1365, 1307, 1313, 1303, 1308, 1367, 1368, + /* 320 */ 1371, 1370, 1388, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 10 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 20 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 30 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 40 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 50 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 60 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 70 */ 1768, 1768, 1768, 1768, 2049, 1768, 1768, 1768, 1768, 1768, - /* 80 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1857, 1768, - /* 90 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 100 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1855, 2042, - /* 110 */ 2267, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 120 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 130 */ 1768, 1768, 1768, 1768, 2279, 1768, 1768, 1831, 1831, 1768, - /* 140 */ 2279, 2279, 2279, 1855, 2239, 2239, 1768, 1768, 1768, 1768, - /* 150 */ 1857, 2109, 1768, 1768, 1768, 1768, 1768, 1768, 1977, 1768, - /* 160 */ 1768, 1768, 1768, 1768, 2001, 1768, 1768, 1768, 2101, 1768, - /* 170 */ 1768, 2304, 2361, 1768, 1768, 2307, 1768, 1768, 1768, 1768, - /* 180 */ 1768, 2054, 1768, 1768, 1930, 2294, 2271, 2285, 2345, 2272, - /* 190 */ 2269, 2288, 1768, 2298, 1768, 1768, 2123, 1857, 1768, 1857, - /* 200 */ 2088, 2047, 1768, 2047, 2044, 1768, 1768, 2047, 2044, 2044, - /* 210 */ 1919, 1915, 1768, 1913, 1768, 1768, 1768, 1768, 1815, 1768, - /* 220 */ 1815, 1768, 1857, 1768, 1857, 1768, 1768, 1857, 1768, 1857, - /* 230 */ 1857, 1857, 1768, 1857, 1768, 1768, 1768, 1768, 1768, 1768, - /* 240 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 250 */ 1768, 2121, 2107, 1768, 1855, 2099, 2097, 1768, 1855, 2095, - /* 260 */ 2298, 1768, 1768, 1768, 1768, 2315, 2313, 1768, 2315, 2313, - /* 270 */ 1768, 1768, 1768, 2329, 2325, 2315, 2334, 2331, 2300, 2298, - /* 280 */ 2364, 2351, 2347, 2285, 1768, 1768, 2298, 1768, 1768, 1855, - /* 290 */ 1855, 1768, 2313, 1768, 1768, 1768, 1768, 1768, 2313, 1768, - /* 300 */ 1768, 1855, 1768, 1855, 1768, 1768, 1946, 1768, 1768, 1768, - /* 310 */ 1855, 1800, 1768, 2090, 2112, 2072, 2072, 1980, 1980, 1980, - /* 320 */ 1858, 1773, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 330 */ 1768, 1768, 1768, 1768, 2328, 2327, 2194, 1768, 2243, 2242, - /* 340 */ 2241, 2232, 2193, 1942, 1768, 2192, 2191, 1768, 1768, 1768, - /* 350 */ 1768, 1768, 1768, 1768, 1768, 1768, 2063, 2062, 2185, 1768, - /* 360 */ 1768, 2186, 2184, 2183, 1768, 1768, 1768, 1768, 1768, 1768, - /* 370 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 380 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 2348, 2352, - /* 390 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 2268, 1768, 1768, - /* 400 */ 1768, 2167, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 410 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 420 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 430 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 440 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 450 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 460 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 470 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 480 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 490 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 500 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 510 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 520 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1805, - /* 530 */ 2172, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 540 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 550 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 560 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 570 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 580 */ 1768, 1896, 1895, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 590 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 600 */ 1768, 1768, 1768, 1768, 1768, 2176, 1768, 1768, 1768, 1768, - /* 610 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 620 */ 1768, 1768, 2344, 2301, 1768, 1768, 1768, 1768, 1768, 1768, - /* 630 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 640 */ 1768, 1768, 2167, 1768, 2326, 1768, 1768, 2342, 1768, 2346, - /* 650 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 2278, 2274, 1768, - /* 660 */ 1768, 2270, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 2175, - /* 670 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 680 */ 1768, 1768, 1768, 2166, 1768, 2229, 1768, 1768, 1768, 2263, - /* 690 */ 1768, 1768, 2214, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 700 */ 1768, 1768, 2176, 1768, 2179, 1768, 1768, 1768, 1768, 1768, - /* 710 */ 1974, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 720 */ 1768, 1768, 1768, 1768, 1768, 1768, 1958, 1956, 1955, 1954, - /* 730 */ 1768, 1987, 1768, 1768, 1768, 1983, 1982, 1768, 1768, 1768, - /* 740 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 750 */ 1876, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1868, - /* 760 */ 1768, 1867, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 770 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 780 */ 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, 1768, - /* 790 */ 1768, 1768, 1768, 1768, + /* 0 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 10 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 20 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 30 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 40 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 50 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 60 */ 2091, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 70 */ 1780, 1780, 1780, 1780, 2064, 1780, 1780, 1780, 1780, 1780, + /* 80 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 90 */ 1780, 1869, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 100 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 110 */ 1867, 2057, 2283, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 120 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 130 */ 1780, 1780, 1780, 1780, 1780, 1780, 2295, 1780, 1780, 1843, + /* 140 */ 1843, 1780, 2295, 2295, 2295, 1867, 2255, 2255, 1780, 1869, + /* 150 */ 2125, 1780, 1780, 1780, 1780, 1780, 1780, 1989, 1780, 1780, + /* 160 */ 1780, 1780, 1780, 2013, 1780, 1780, 1780, 2117, 1780, 1780, + /* 170 */ 2320, 2377, 1780, 1780, 2323, 1780, 1780, 1780, 1780, 1780, + /* 180 */ 1780, 2069, 1780, 1780, 1942, 2310, 2287, 2301, 2361, 2288, + /* 190 */ 2285, 2304, 1780, 2314, 1780, 1780, 2139, 1869, 1780, 1869, + /* 200 */ 2104, 1780, 2062, 1780, 1780, 2062, 2059, 1780, 1780, 1780, + /* 210 */ 2062, 2059, 2059, 1931, 1927, 1780, 1925, 1780, 1780, 1780, + /* 220 */ 1780, 1827, 1780, 1827, 1780, 1869, 1780, 1869, 1780, 1780, + /* 230 */ 1869, 1780, 1869, 1869, 1869, 1780, 1869, 1780, 1780, 1780, + /* 240 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 250 */ 1780, 1780, 1780, 1780, 2137, 2123, 1780, 1867, 2115, 2113, + /* 260 */ 1780, 1867, 2111, 2314, 1780, 1780, 1780, 1780, 2331, 2329, + /* 270 */ 1780, 2331, 2329, 1780, 1780, 1780, 2345, 2341, 2331, 2350, + /* 280 */ 2347, 2316, 2314, 2380, 2367, 2363, 2301, 1780, 1780, 1780, + /* 290 */ 1867, 1867, 1780, 2329, 1780, 1780, 1780, 1780, 1780, 2329, + /* 300 */ 1780, 1780, 1867, 1780, 1867, 1780, 1780, 1958, 1780, 1780, + /* 310 */ 1780, 1867, 1812, 1780, 2106, 2128, 2087, 2087, 1992, 1992, + /* 320 */ 1992, 1870, 1785, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 330 */ 1780, 1780, 1780, 1780, 1780, 2344, 2343, 2210, 1780, 2259, + /* 340 */ 2258, 2257, 2248, 2209, 1954, 1780, 2208, 2207, 1780, 1780, + /* 350 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 2078, 2077, 2201, + /* 360 */ 1780, 1780, 2202, 2200, 2199, 1780, 1780, 1780, 1780, 1780, + /* 370 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 380 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 2364, + /* 390 */ 2368, 1780, 1780, 1780, 1780, 1780, 1780, 2284, 1780, 1780, + /* 400 */ 1780, 2183, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 410 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 420 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 430 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 440 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 450 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 460 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 470 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 480 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 490 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 500 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 510 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 520 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 530 */ 1780, 1780, 1780, 1780, 1780, 1817, 2188, 1780, 1780, 1780, + /* 540 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 550 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 560 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 570 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 580 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1908, 1907, 1780, + /* 590 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 600 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 610 */ 1780, 2192, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 620 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 630 */ 2360, 2317, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 640 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 650 */ 2183, 1780, 2342, 1780, 1780, 2358, 1780, 2362, 1780, 1780, + /* 660 */ 1780, 1780, 1780, 1780, 1780, 2294, 2290, 1780, 1780, 2286, + /* 670 */ 1780, 1780, 1780, 1780, 1780, 2191, 1780, 1780, 1780, 1780, + /* 680 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 2182, + /* 690 */ 1780, 2245, 1780, 1780, 1780, 2279, 1780, 1780, 2230, 1780, + /* 700 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 2192, 1780, + /* 710 */ 2195, 1780, 1780, 1780, 1780, 1780, 1986, 1780, 1780, 1780, + /* 720 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 730 */ 1780, 1780, 1970, 1968, 1967, 1966, 1780, 1999, 1780, 1780, + /* 740 */ 1780, 1995, 1994, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 750 */ 1780, 1780, 1780, 1780, 1780, 1780, 1888, 1780, 1780, 1780, + /* 760 */ 1780, 1780, 1780, 1780, 1780, 1880, 1780, 1879, 1780, 1780, + /* 770 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 780 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, + /* 790 */ 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, 1780, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1127,7 +1153,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* MAX_SPEED => nothing */ 0, /* START => nothing */ 0, /* TIMESTAMP => nothing */ - 286, /* END => ABORT */ + 287, /* END => ABORT */ 0, /* TABLE => nothing */ 0, /* NK_LP => nothing */ 0, /* NK_RP => nothing */ @@ -1204,8 +1230,9 @@ static const YYCODETYPE yyFallback[] = { 0, /* INTERVAL => nothing */ 0, /* COUNT => nothing */ 0, /* LAST_ROW => nothing */ - 0, /* TOPIC => nothing */ 0, /* META => nothing */ + 0, /* ONLY => nothing */ + 0, /* TOPIC => nothing */ 0, /* CONSUMER => nothing */ 0, /* GROUP => nothing */ 0, /* DESC => nothing */ @@ -1313,56 +1340,56 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASC => nothing */ 0, /* NULLS => nothing */ 0, /* ABORT => nothing */ - 286, /* AFTER => ABORT */ - 286, /* ATTACH => ABORT */ - 286, /* BEFORE => ABORT */ - 286, /* BEGIN => ABORT */ - 286, /* BITAND => ABORT */ - 286, /* BITNOT => ABORT */ - 286, /* BITOR => ABORT */ - 286, /* BLOCKS => ABORT */ - 286, /* CHANGE => ABORT */ - 286, /* COMMA => ABORT */ - 286, /* CONCAT => ABORT */ - 286, /* CONFLICT => ABORT */ - 286, /* COPY => ABORT */ - 286, /* DEFERRED => ABORT */ - 286, /* DELIMITERS => ABORT */ - 286, /* DETACH => ABORT */ - 286, /* DIVIDE => ABORT */ - 286, /* DOT => ABORT */ - 286, /* EACH => ABORT */ - 286, /* FAIL => ABORT */ - 286, /* FILE => ABORT */ - 286, /* FOR => ABORT */ - 286, /* GLOB => ABORT */ - 286, /* ID => ABORT */ - 286, /* IMMEDIATE => ABORT */ - 286, /* IMPORT => ABORT */ - 286, /* INITIALLY => ABORT */ - 286, /* INSTEAD => ABORT */ - 286, /* ISNULL => ABORT */ - 286, /* KEY => ABORT */ - 286, /* MODULES => ABORT */ - 286, /* NK_BITNOT => ABORT */ - 286, /* NK_SEMI => ABORT */ - 286, /* NOTNULL => ABORT */ - 286, /* OF => ABORT */ - 286, /* PLUS => ABORT */ - 286, /* PRIVILEGE => ABORT */ - 286, /* RAISE => ABORT */ - 286, /* RESTRICT => ABORT */ - 286, /* ROW => ABORT */ - 286, /* SEMI => ABORT */ - 286, /* STAR => ABORT */ - 286, /* STATEMENT => ABORT */ - 286, /* STRICT => ABORT */ - 286, /* STRING => ABORT */ - 286, /* TIMES => ABORT */ - 286, /* VALUES => ABORT */ - 286, /* VARIABLE => ABORT */ - 286, /* VIEW => ABORT */ - 286, /* WAL => ABORT */ + 287, /* AFTER => ABORT */ + 287, /* ATTACH => ABORT */ + 287, /* BEFORE => ABORT */ + 287, /* BEGIN => ABORT */ + 287, /* BITAND => ABORT */ + 287, /* BITNOT => ABORT */ + 287, /* BITOR => ABORT */ + 287, /* BLOCKS => ABORT */ + 287, /* CHANGE => ABORT */ + 287, /* COMMA => ABORT */ + 287, /* CONCAT => ABORT */ + 287, /* CONFLICT => ABORT */ + 287, /* COPY => ABORT */ + 287, /* DEFERRED => ABORT */ + 287, /* DELIMITERS => ABORT */ + 287, /* DETACH => ABORT */ + 287, /* DIVIDE => ABORT */ + 287, /* DOT => ABORT */ + 287, /* EACH => ABORT */ + 287, /* FAIL => ABORT */ + 287, /* FILE => ABORT */ + 287, /* FOR => ABORT */ + 287, /* GLOB => ABORT */ + 287, /* ID => ABORT */ + 287, /* IMMEDIATE => ABORT */ + 287, /* IMPORT => ABORT */ + 287, /* INITIALLY => ABORT */ + 287, /* INSTEAD => ABORT */ + 287, /* ISNULL => ABORT */ + 287, /* KEY => ABORT */ + 287, /* MODULES => ABORT */ + 287, /* NK_BITNOT => ABORT */ + 287, /* NK_SEMI => ABORT */ + 287, /* NOTNULL => ABORT */ + 287, /* OF => ABORT */ + 287, /* PLUS => ABORT */ + 287, /* PRIVILEGE => ABORT */ + 287, /* RAISE => ABORT */ + 287, /* RESTRICT => ABORT */ + 287, /* ROW => ABORT */ + 287, /* SEMI => ABORT */ + 287, /* STAR => ABORT */ + 287, /* STATEMENT => ABORT */ + 287, /* STRICT => ABORT */ + 287, /* STRING => ABORT */ + 287, /* TIMES => ABORT */ + 287, /* VALUES => ABORT */ + 287, /* VARIABLE => ABORT */ + 287, /* VIEW => ABORT */ + 287, /* WAL => ABORT */ }; #endif /* YYFALLBACK */ @@ -1628,313 +1655,315 @@ static const char *const yyTokenName[] = { /* 175 */ "INTERVAL", /* 176 */ "COUNT", /* 177 */ "LAST_ROW", - /* 178 */ "TOPIC", - /* 179 */ "META", - /* 180 */ "CONSUMER", - /* 181 */ "GROUP", - /* 182 */ "DESC", - /* 183 */ "DESCRIBE", - /* 184 */ "RESET", - /* 185 */ "QUERY", - /* 186 */ "CACHE", - /* 187 */ "EXPLAIN", - /* 188 */ "ANALYZE", - /* 189 */ "VERBOSE", - /* 190 */ "NK_BOOL", - /* 191 */ "RATIO", - /* 192 */ "NK_FLOAT", - /* 193 */ "OUTPUTTYPE", - /* 194 */ "AGGREGATE", - /* 195 */ "BUFSIZE", - /* 196 */ "LANGUAGE", - /* 197 */ "REPLACE", - /* 198 */ "STREAM", - /* 199 */ "INTO", - /* 200 */ "PAUSE", - /* 201 */ "RESUME", - /* 202 */ "TRIGGER", - /* 203 */ "AT_ONCE", - /* 204 */ "WINDOW_CLOSE", - /* 205 */ "IGNORE", - /* 206 */ "EXPIRED", - /* 207 */ "FILL_HISTORY", - /* 208 */ "UPDATE", - /* 209 */ "SUBTABLE", - /* 210 */ "UNTREATED", - /* 211 */ "KILL", - /* 212 */ "CONNECTION", - /* 213 */ "TRANSACTION", - /* 214 */ "BALANCE", - /* 215 */ "VGROUP", - /* 216 */ "LEADER", - /* 217 */ "MERGE", - /* 218 */ "REDISTRIBUTE", - /* 219 */ "SPLIT", - /* 220 */ "DELETE", - /* 221 */ "INSERT", - /* 222 */ "NULL", - /* 223 */ "NK_QUESTION", - /* 224 */ "NK_ARROW", - /* 225 */ "ROWTS", - /* 226 */ "QSTART", - /* 227 */ "QEND", - /* 228 */ "QDURATION", - /* 229 */ "WSTART", - /* 230 */ "WEND", - /* 231 */ "WDURATION", - /* 232 */ "IROWTS", - /* 233 */ "ISFILLED", - /* 234 */ "CAST", - /* 235 */ "NOW", - /* 236 */ "TODAY", - /* 237 */ "TIMEZONE", - /* 238 */ "CLIENT_VERSION", - /* 239 */ "SERVER_VERSION", - /* 240 */ "SERVER_STATUS", - /* 241 */ "CURRENT_USER", - /* 242 */ "CASE", - /* 243 */ "WHEN", - /* 244 */ "THEN", - /* 245 */ "ELSE", - /* 246 */ "BETWEEN", - /* 247 */ "IS", - /* 248 */ "NK_LT", - /* 249 */ "NK_GT", - /* 250 */ "NK_LE", - /* 251 */ "NK_GE", - /* 252 */ "NK_NE", - /* 253 */ "MATCH", - /* 254 */ "NMATCH", - /* 255 */ "CONTAINS", - /* 256 */ "IN", - /* 257 */ "JOIN", - /* 258 */ "INNER", - /* 259 */ "SELECT", - /* 260 */ "DISTINCT", - /* 261 */ "WHERE", - /* 262 */ "PARTITION", - /* 263 */ "BY", - /* 264 */ "SESSION", - /* 265 */ "STATE_WINDOW", - /* 266 */ "EVENT_WINDOW", - /* 267 */ "SLIDING", - /* 268 */ "FILL", - /* 269 */ "VALUE", - /* 270 */ "VALUE_F", - /* 271 */ "NONE", - /* 272 */ "PREV", - /* 273 */ "NULL_F", - /* 274 */ "LINEAR", - /* 275 */ "NEXT", - /* 276 */ "HAVING", - /* 277 */ "RANGE", - /* 278 */ "EVERY", - /* 279 */ "ORDER", - /* 280 */ "SLIMIT", - /* 281 */ "SOFFSET", - /* 282 */ "LIMIT", - /* 283 */ "OFFSET", - /* 284 */ "ASC", - /* 285 */ "NULLS", - /* 286 */ "ABORT", - /* 287 */ "AFTER", - /* 288 */ "ATTACH", - /* 289 */ "BEFORE", - /* 290 */ "BEGIN", - /* 291 */ "BITAND", - /* 292 */ "BITNOT", - /* 293 */ "BITOR", - /* 294 */ "BLOCKS", - /* 295 */ "CHANGE", - /* 296 */ "COMMA", - /* 297 */ "CONCAT", - /* 298 */ "CONFLICT", - /* 299 */ "COPY", - /* 300 */ "DEFERRED", - /* 301 */ "DELIMITERS", - /* 302 */ "DETACH", - /* 303 */ "DIVIDE", - /* 304 */ "DOT", - /* 305 */ "EACH", - /* 306 */ "FAIL", - /* 307 */ "FILE", - /* 308 */ "FOR", - /* 309 */ "GLOB", - /* 310 */ "ID", - /* 311 */ "IMMEDIATE", - /* 312 */ "IMPORT", - /* 313 */ "INITIALLY", - /* 314 */ "INSTEAD", - /* 315 */ "ISNULL", - /* 316 */ "KEY", - /* 317 */ "MODULES", - /* 318 */ "NK_BITNOT", - /* 319 */ "NK_SEMI", - /* 320 */ "NOTNULL", - /* 321 */ "OF", - /* 322 */ "PLUS", - /* 323 */ "PRIVILEGE", - /* 324 */ "RAISE", - /* 325 */ "RESTRICT", - /* 326 */ "ROW", - /* 327 */ "SEMI", - /* 328 */ "STAR", - /* 329 */ "STATEMENT", - /* 330 */ "STRICT", - /* 331 */ "STRING", - /* 332 */ "TIMES", - /* 333 */ "VALUES", - /* 334 */ "VARIABLE", - /* 335 */ "VIEW", - /* 336 */ "WAL", - /* 337 */ "cmd", - /* 338 */ "account_options", - /* 339 */ "alter_account_options", - /* 340 */ "literal", - /* 341 */ "alter_account_option", - /* 342 */ "user_name", - /* 343 */ "sysinfo_opt", - /* 344 */ "privileges", - /* 345 */ "priv_level", - /* 346 */ "with_opt", - /* 347 */ "priv_type_list", - /* 348 */ "priv_type", - /* 349 */ "db_name", - /* 350 */ "table_name", - /* 351 */ "topic_name", - /* 352 */ "search_condition", - /* 353 */ "dnode_endpoint", - /* 354 */ "force_opt", - /* 355 */ "unsafe_opt", - /* 356 */ "not_exists_opt", - /* 357 */ "db_options", - /* 358 */ "exists_opt", - /* 359 */ "alter_db_options", - /* 360 */ "speed_opt", - /* 361 */ "start_opt", - /* 362 */ "end_opt", - /* 363 */ "integer_list", - /* 364 */ "variable_list", - /* 365 */ "retention_list", - /* 366 */ "signed", - /* 367 */ "alter_db_option", - /* 368 */ "retention", - /* 369 */ "full_table_name", - /* 370 */ "column_def_list", - /* 371 */ "tags_def_opt", - /* 372 */ "table_options", - /* 373 */ "multi_create_clause", - /* 374 */ "tags_def", - /* 375 */ "multi_drop_clause", - /* 376 */ "alter_table_clause", - /* 377 */ "alter_table_options", - /* 378 */ "column_name", - /* 379 */ "type_name", - /* 380 */ "signed_literal", - /* 381 */ "create_subtable_clause", - /* 382 */ "specific_cols_opt", - /* 383 */ "expression_list", - /* 384 */ "drop_table_clause", - /* 385 */ "col_name_list", - /* 386 */ "column_def", - /* 387 */ "duration_list", - /* 388 */ "rollup_func_list", - /* 389 */ "alter_table_option", - /* 390 */ "duration_literal", - /* 391 */ "rollup_func_name", - /* 392 */ "function_name", - /* 393 */ "col_name", - /* 394 */ "db_name_cond_opt", - /* 395 */ "like_pattern_opt", - /* 396 */ "table_name_cond", - /* 397 */ "from_db_opt", - /* 398 */ "tag_list_opt", - /* 399 */ "tag_item", - /* 400 */ "column_alias", - /* 401 */ "full_index_name", - /* 402 */ "index_options", - /* 403 */ "index_name", - /* 404 */ "func_list", - /* 405 */ "sliding_opt", - /* 406 */ "sma_stream_opt", - /* 407 */ "func", - /* 408 */ "sma_func_name", - /* 409 */ "query_or_subquery", - /* 410 */ "where_clause_opt", - /* 411 */ "cgroup_name", - /* 412 */ "analyze_opt", - /* 413 */ "explain_options", - /* 414 */ "insert_query", - /* 415 */ "or_replace_opt", - /* 416 */ "agg_func_opt", - /* 417 */ "bufsize_opt", - /* 418 */ "language_opt", - /* 419 */ "stream_name", - /* 420 */ "stream_options", - /* 421 */ "col_list_opt", - /* 422 */ "tag_def_or_ref_opt", - /* 423 */ "subtable_opt", - /* 424 */ "ignore_opt", - /* 425 */ "expression", - /* 426 */ "dnode_list", - /* 427 */ "literal_func", - /* 428 */ "literal_list", - /* 429 */ "table_alias", - /* 430 */ "expr_or_subquery", - /* 431 */ "pseudo_column", - /* 432 */ "column_reference", - /* 433 */ "function_expression", - /* 434 */ "case_when_expression", - /* 435 */ "star_func", - /* 436 */ "star_func_para_list", - /* 437 */ "noarg_func", - /* 438 */ "other_para_list", - /* 439 */ "star_func_para", - /* 440 */ "when_then_list", - /* 441 */ "case_when_else_opt", - /* 442 */ "common_expression", - /* 443 */ "when_then_expr", - /* 444 */ "predicate", - /* 445 */ "compare_op", - /* 446 */ "in_op", - /* 447 */ "in_predicate_value", - /* 448 */ "boolean_value_expression", - /* 449 */ "boolean_primary", - /* 450 */ "from_clause_opt", - /* 451 */ "table_reference_list", - /* 452 */ "table_reference", - /* 453 */ "table_primary", - /* 454 */ "joined_table", - /* 455 */ "alias_opt", - /* 456 */ "subquery", - /* 457 */ "parenthesized_joined_table", - /* 458 */ "join_type", - /* 459 */ "query_specification", - /* 460 */ "set_quantifier_opt", - /* 461 */ "select_list", - /* 462 */ "partition_by_clause_opt", - /* 463 */ "range_opt", - /* 464 */ "every_opt", - /* 465 */ "fill_opt", - /* 466 */ "twindow_clause_opt", - /* 467 */ "group_by_clause_opt", - /* 468 */ "having_clause_opt", - /* 469 */ "select_item", - /* 470 */ "partition_list", - /* 471 */ "partition_item", - /* 472 */ "fill_mode", - /* 473 */ "group_by_list", - /* 474 */ "query_expression", - /* 475 */ "query_simple", - /* 476 */ "order_by_clause_opt", - /* 477 */ "slimit_clause_opt", - /* 478 */ "limit_clause_opt", - /* 479 */ "union_query_expression", - /* 480 */ "query_simple_or_subquery", - /* 481 */ "sort_specification_list", - /* 482 */ "sort_specification", - /* 483 */ "ordering_specification_opt", - /* 484 */ "null_ordering_opt", + /* 178 */ "META", + /* 179 */ "ONLY", + /* 180 */ "TOPIC", + /* 181 */ "CONSUMER", + /* 182 */ "GROUP", + /* 183 */ "DESC", + /* 184 */ "DESCRIBE", + /* 185 */ "RESET", + /* 186 */ "QUERY", + /* 187 */ "CACHE", + /* 188 */ "EXPLAIN", + /* 189 */ "ANALYZE", + /* 190 */ "VERBOSE", + /* 191 */ "NK_BOOL", + /* 192 */ "RATIO", + /* 193 */ "NK_FLOAT", + /* 194 */ "OUTPUTTYPE", + /* 195 */ "AGGREGATE", + /* 196 */ "BUFSIZE", + /* 197 */ "LANGUAGE", + /* 198 */ "REPLACE", + /* 199 */ "STREAM", + /* 200 */ "INTO", + /* 201 */ "PAUSE", + /* 202 */ "RESUME", + /* 203 */ "TRIGGER", + /* 204 */ "AT_ONCE", + /* 205 */ "WINDOW_CLOSE", + /* 206 */ "IGNORE", + /* 207 */ "EXPIRED", + /* 208 */ "FILL_HISTORY", + /* 209 */ "UPDATE", + /* 210 */ "SUBTABLE", + /* 211 */ "UNTREATED", + /* 212 */ "KILL", + /* 213 */ "CONNECTION", + /* 214 */ "TRANSACTION", + /* 215 */ "BALANCE", + /* 216 */ "VGROUP", + /* 217 */ "LEADER", + /* 218 */ "MERGE", + /* 219 */ "REDISTRIBUTE", + /* 220 */ "SPLIT", + /* 221 */ "DELETE", + /* 222 */ "INSERT", + /* 223 */ "NULL", + /* 224 */ "NK_QUESTION", + /* 225 */ "NK_ARROW", + /* 226 */ "ROWTS", + /* 227 */ "QSTART", + /* 228 */ "QEND", + /* 229 */ "QDURATION", + /* 230 */ "WSTART", + /* 231 */ "WEND", + /* 232 */ "WDURATION", + /* 233 */ "IROWTS", + /* 234 */ "ISFILLED", + /* 235 */ "CAST", + /* 236 */ "NOW", + /* 237 */ "TODAY", + /* 238 */ "TIMEZONE", + /* 239 */ "CLIENT_VERSION", + /* 240 */ "SERVER_VERSION", + /* 241 */ "SERVER_STATUS", + /* 242 */ "CURRENT_USER", + /* 243 */ "CASE", + /* 244 */ "WHEN", + /* 245 */ "THEN", + /* 246 */ "ELSE", + /* 247 */ "BETWEEN", + /* 248 */ "IS", + /* 249 */ "NK_LT", + /* 250 */ "NK_GT", + /* 251 */ "NK_LE", + /* 252 */ "NK_GE", + /* 253 */ "NK_NE", + /* 254 */ "MATCH", + /* 255 */ "NMATCH", + /* 256 */ "CONTAINS", + /* 257 */ "IN", + /* 258 */ "JOIN", + /* 259 */ "INNER", + /* 260 */ "SELECT", + /* 261 */ "DISTINCT", + /* 262 */ "WHERE", + /* 263 */ "PARTITION", + /* 264 */ "BY", + /* 265 */ "SESSION", + /* 266 */ "STATE_WINDOW", + /* 267 */ "EVENT_WINDOW", + /* 268 */ "SLIDING", + /* 269 */ "FILL", + /* 270 */ "VALUE", + /* 271 */ "VALUE_F", + /* 272 */ "NONE", + /* 273 */ "PREV", + /* 274 */ "NULL_F", + /* 275 */ "LINEAR", + /* 276 */ "NEXT", + /* 277 */ "HAVING", + /* 278 */ "RANGE", + /* 279 */ "EVERY", + /* 280 */ "ORDER", + /* 281 */ "SLIMIT", + /* 282 */ "SOFFSET", + /* 283 */ "LIMIT", + /* 284 */ "OFFSET", + /* 285 */ "ASC", + /* 286 */ "NULLS", + /* 287 */ "ABORT", + /* 288 */ "AFTER", + /* 289 */ "ATTACH", + /* 290 */ "BEFORE", + /* 291 */ "BEGIN", + /* 292 */ "BITAND", + /* 293 */ "BITNOT", + /* 294 */ "BITOR", + /* 295 */ "BLOCKS", + /* 296 */ "CHANGE", + /* 297 */ "COMMA", + /* 298 */ "CONCAT", + /* 299 */ "CONFLICT", + /* 300 */ "COPY", + /* 301 */ "DEFERRED", + /* 302 */ "DELIMITERS", + /* 303 */ "DETACH", + /* 304 */ "DIVIDE", + /* 305 */ "DOT", + /* 306 */ "EACH", + /* 307 */ "FAIL", + /* 308 */ "FILE", + /* 309 */ "FOR", + /* 310 */ "GLOB", + /* 311 */ "ID", + /* 312 */ "IMMEDIATE", + /* 313 */ "IMPORT", + /* 314 */ "INITIALLY", + /* 315 */ "INSTEAD", + /* 316 */ "ISNULL", + /* 317 */ "KEY", + /* 318 */ "MODULES", + /* 319 */ "NK_BITNOT", + /* 320 */ "NK_SEMI", + /* 321 */ "NOTNULL", + /* 322 */ "OF", + /* 323 */ "PLUS", + /* 324 */ "PRIVILEGE", + /* 325 */ "RAISE", + /* 326 */ "RESTRICT", + /* 327 */ "ROW", + /* 328 */ "SEMI", + /* 329 */ "STAR", + /* 330 */ "STATEMENT", + /* 331 */ "STRICT", + /* 332 */ "STRING", + /* 333 */ "TIMES", + /* 334 */ "VALUES", + /* 335 */ "VARIABLE", + /* 336 */ "VIEW", + /* 337 */ "WAL", + /* 338 */ "cmd", + /* 339 */ "account_options", + /* 340 */ "alter_account_options", + /* 341 */ "literal", + /* 342 */ "alter_account_option", + /* 343 */ "user_name", + /* 344 */ "sysinfo_opt", + /* 345 */ "privileges", + /* 346 */ "priv_level", + /* 347 */ "with_opt", + /* 348 */ "priv_type_list", + /* 349 */ "priv_type", + /* 350 */ "db_name", + /* 351 */ "table_name", + /* 352 */ "topic_name", + /* 353 */ "search_condition", + /* 354 */ "dnode_endpoint", + /* 355 */ "force_opt", + /* 356 */ "unsafe_opt", + /* 357 */ "not_exists_opt", + /* 358 */ "db_options", + /* 359 */ "exists_opt", + /* 360 */ "alter_db_options", + /* 361 */ "speed_opt", + /* 362 */ "start_opt", + /* 363 */ "end_opt", + /* 364 */ "integer_list", + /* 365 */ "variable_list", + /* 366 */ "retention_list", + /* 367 */ "signed", + /* 368 */ "alter_db_option", + /* 369 */ "retention", + /* 370 */ "full_table_name", + /* 371 */ "column_def_list", + /* 372 */ "tags_def_opt", + /* 373 */ "table_options", + /* 374 */ "multi_create_clause", + /* 375 */ "tags_def", + /* 376 */ "multi_drop_clause", + /* 377 */ "alter_table_clause", + /* 378 */ "alter_table_options", + /* 379 */ "column_name", + /* 380 */ "type_name", + /* 381 */ "signed_literal", + /* 382 */ "create_subtable_clause", + /* 383 */ "specific_cols_opt", + /* 384 */ "expression_list", + /* 385 */ "drop_table_clause", + /* 386 */ "col_name_list", + /* 387 */ "column_def", + /* 388 */ "duration_list", + /* 389 */ "rollup_func_list", + /* 390 */ "alter_table_option", + /* 391 */ "duration_literal", + /* 392 */ "rollup_func_name", + /* 393 */ "function_name", + /* 394 */ "col_name", + /* 395 */ "db_name_cond_opt", + /* 396 */ "like_pattern_opt", + /* 397 */ "table_name_cond", + /* 398 */ "from_db_opt", + /* 399 */ "tag_list_opt", + /* 400 */ "tag_item", + /* 401 */ "column_alias", + /* 402 */ "full_index_name", + /* 403 */ "index_options", + /* 404 */ "index_name", + /* 405 */ "func_list", + /* 406 */ "sliding_opt", + /* 407 */ "sma_stream_opt", + /* 408 */ "func", + /* 409 */ "sma_func_name", + /* 410 */ "with_meta", + /* 411 */ "query_or_subquery", + /* 412 */ "where_clause_opt", + /* 413 */ "cgroup_name", + /* 414 */ "analyze_opt", + /* 415 */ "explain_options", + /* 416 */ "insert_query", + /* 417 */ "or_replace_opt", + /* 418 */ "agg_func_opt", + /* 419 */ "bufsize_opt", + /* 420 */ "language_opt", + /* 421 */ "stream_name", + /* 422 */ "stream_options", + /* 423 */ "col_list_opt", + /* 424 */ "tag_def_or_ref_opt", + /* 425 */ "subtable_opt", + /* 426 */ "ignore_opt", + /* 427 */ "expression", + /* 428 */ "dnode_list", + /* 429 */ "literal_func", + /* 430 */ "literal_list", + /* 431 */ "table_alias", + /* 432 */ "expr_or_subquery", + /* 433 */ "pseudo_column", + /* 434 */ "column_reference", + /* 435 */ "function_expression", + /* 436 */ "case_when_expression", + /* 437 */ "star_func", + /* 438 */ "star_func_para_list", + /* 439 */ "noarg_func", + /* 440 */ "other_para_list", + /* 441 */ "star_func_para", + /* 442 */ "when_then_list", + /* 443 */ "case_when_else_opt", + /* 444 */ "common_expression", + /* 445 */ "when_then_expr", + /* 446 */ "predicate", + /* 447 */ "compare_op", + /* 448 */ "in_op", + /* 449 */ "in_predicate_value", + /* 450 */ "boolean_value_expression", + /* 451 */ "boolean_primary", + /* 452 */ "from_clause_opt", + /* 453 */ "table_reference_list", + /* 454 */ "table_reference", + /* 455 */ "table_primary", + /* 456 */ "joined_table", + /* 457 */ "alias_opt", + /* 458 */ "subquery", + /* 459 */ "parenthesized_joined_table", + /* 460 */ "join_type", + /* 461 */ "query_specification", + /* 462 */ "set_quantifier_opt", + /* 463 */ "select_list", + /* 464 */ "partition_by_clause_opt", + /* 465 */ "range_opt", + /* 466 */ "every_opt", + /* 467 */ "fill_opt", + /* 468 */ "twindow_clause_opt", + /* 469 */ "group_by_clause_opt", + /* 470 */ "having_clause_opt", + /* 471 */ "select_item", + /* 472 */ "partition_list", + /* 473 */ "partition_item", + /* 474 */ "fill_mode", + /* 475 */ "group_by_list", + /* 476 */ "query_expression", + /* 477 */ "query_simple", + /* 478 */ "order_by_clause_opt", + /* 479 */ "slimit_clause_opt", + /* 480 */ "limit_clause_opt", + /* 481 */ "union_query_expression", + /* 482 */ "query_simple_or_subquery", + /* 483 */ "sort_specification_list", + /* 484 */ "sort_specification", + /* 485 */ "ordering_specification_opt", + /* 486 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -2184,360 +2213,364 @@ static const char *const yyRuleName[] = { /* 239 */ "cmd ::= SHOW QNODES", /* 240 */ "cmd ::= SHOW FUNCTIONS", /* 241 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", - /* 242 */ "cmd ::= SHOW STREAMS", - /* 243 */ "cmd ::= SHOW ACCOUNTS", - /* 244 */ "cmd ::= SHOW APPS", - /* 245 */ "cmd ::= SHOW CONNECTIONS", - /* 246 */ "cmd ::= SHOW LICENCES", - /* 247 */ "cmd ::= SHOW GRANTS", - /* 248 */ "cmd ::= SHOW CREATE DATABASE db_name", - /* 249 */ "cmd ::= SHOW CREATE TABLE full_table_name", - /* 250 */ "cmd ::= SHOW CREATE STABLE full_table_name", - /* 251 */ "cmd ::= SHOW QUERIES", - /* 252 */ "cmd ::= SHOW SCORES", - /* 253 */ "cmd ::= SHOW TOPICS", - /* 254 */ "cmd ::= SHOW VARIABLES", - /* 255 */ "cmd ::= SHOW CLUSTER VARIABLES", - /* 256 */ "cmd ::= SHOW LOCAL VARIABLES", - /* 257 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", - /* 258 */ "cmd ::= SHOW BNODES", - /* 259 */ "cmd ::= SHOW SNODES", - /* 260 */ "cmd ::= SHOW CLUSTER", - /* 261 */ "cmd ::= SHOW TRANSACTIONS", - /* 262 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", - /* 263 */ "cmd ::= SHOW CONSUMERS", - /* 264 */ "cmd ::= SHOW SUBSCRIPTIONS", - /* 265 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", - /* 266 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", - /* 267 */ "cmd ::= SHOW VNODES NK_INTEGER", - /* 268 */ "cmd ::= SHOW VNODES NK_STRING", - /* 269 */ "cmd ::= SHOW db_name_cond_opt ALIVE", - /* 270 */ "cmd ::= SHOW CLUSTER ALIVE", - /* 271 */ "db_name_cond_opt ::=", - /* 272 */ "db_name_cond_opt ::= db_name NK_DOT", - /* 273 */ "like_pattern_opt ::=", - /* 274 */ "like_pattern_opt ::= LIKE NK_STRING", - /* 275 */ "table_name_cond ::= table_name", - /* 276 */ "from_db_opt ::=", - /* 277 */ "from_db_opt ::= FROM db_name", - /* 278 */ "tag_list_opt ::=", - /* 279 */ "tag_list_opt ::= tag_item", - /* 280 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", - /* 281 */ "tag_item ::= TBNAME", - /* 282 */ "tag_item ::= QTAGS", - /* 283 */ "tag_item ::= column_name", - /* 284 */ "tag_item ::= column_name column_alias", - /* 285 */ "tag_item ::= column_name AS column_alias", - /* 286 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", - /* 287 */ "cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP", - /* 288 */ "cmd ::= DROP INDEX exists_opt full_index_name", - /* 289 */ "full_index_name ::= index_name", - /* 290 */ "full_index_name ::= db_name NK_DOT index_name", - /* 291 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", - /* 292 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", - /* 293 */ "func_list ::= func", - /* 294 */ "func_list ::= func_list NK_COMMA func", - /* 295 */ "func ::= sma_func_name NK_LP expression_list NK_RP", - /* 296 */ "sma_func_name ::= function_name", - /* 297 */ "sma_func_name ::= COUNT", - /* 298 */ "sma_func_name ::= FIRST", - /* 299 */ "sma_func_name ::= LAST", - /* 300 */ "sma_func_name ::= LAST_ROW", - /* 301 */ "sma_stream_opt ::=", - /* 302 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", - /* 303 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", - /* 304 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", - /* 305 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", - /* 306 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", - /* 307 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", - /* 308 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name where_clause_opt", - /* 309 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name where_clause_opt", - /* 310 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 311 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", - /* 312 */ "cmd ::= DESC full_table_name", - /* 313 */ "cmd ::= DESCRIBE full_table_name", - /* 314 */ "cmd ::= RESET QUERY CACHE", - /* 315 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", - /* 316 */ "cmd ::= EXPLAIN analyze_opt explain_options insert_query", - /* 317 */ "analyze_opt ::=", - /* 318 */ "analyze_opt ::= ANALYZE", - /* 319 */ "explain_options ::=", - /* 320 */ "explain_options ::= explain_options VERBOSE NK_BOOL", - /* 321 */ "explain_options ::= explain_options RATIO NK_FLOAT", - /* 322 */ "cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt", - /* 323 */ "cmd ::= DROP FUNCTION exists_opt function_name", - /* 324 */ "agg_func_opt ::=", - /* 325 */ "agg_func_opt ::= AGGREGATE", - /* 326 */ "bufsize_opt ::=", - /* 327 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", - /* 328 */ "language_opt ::=", - /* 329 */ "language_opt ::= LANGUAGE NK_STRING", - /* 330 */ "or_replace_opt ::=", - /* 331 */ "or_replace_opt ::= OR REPLACE", - /* 332 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery", - /* 333 */ "cmd ::= DROP STREAM exists_opt stream_name", - /* 334 */ "cmd ::= PAUSE STREAM exists_opt stream_name", - /* 335 */ "cmd ::= RESUME STREAM exists_opt ignore_opt stream_name", - /* 336 */ "col_list_opt ::=", - /* 337 */ "col_list_opt ::= NK_LP col_name_list NK_RP", - /* 338 */ "tag_def_or_ref_opt ::=", - /* 339 */ "tag_def_or_ref_opt ::= tags_def", - /* 340 */ "tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP", - /* 341 */ "stream_options ::=", - /* 342 */ "stream_options ::= stream_options TRIGGER AT_ONCE", - /* 343 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", - /* 344 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", - /* 345 */ "stream_options ::= stream_options WATERMARK duration_literal", - /* 346 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", - /* 347 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", - /* 348 */ "stream_options ::= stream_options DELETE_MARK duration_literal", - /* 349 */ "stream_options ::= stream_options IGNORE UPDATE NK_INTEGER", - /* 350 */ "subtable_opt ::=", - /* 351 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", - /* 352 */ "ignore_opt ::=", - /* 353 */ "ignore_opt ::= IGNORE UNTREATED", - /* 354 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 355 */ "cmd ::= KILL QUERY NK_STRING", - /* 356 */ "cmd ::= KILL TRANSACTION NK_INTEGER", - /* 357 */ "cmd ::= BALANCE VGROUP", - /* 358 */ "cmd ::= BALANCE VGROUP LEADER", - /* 359 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", - /* 360 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", - /* 361 */ "cmd ::= SPLIT VGROUP NK_INTEGER", - /* 362 */ "dnode_list ::= DNODE NK_INTEGER", - /* 363 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", - /* 364 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", - /* 365 */ "cmd ::= query_or_subquery", - /* 366 */ "cmd ::= insert_query", - /* 367 */ "insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", - /* 368 */ "insert_query ::= INSERT INTO full_table_name query_or_subquery", - /* 369 */ "literal ::= NK_INTEGER", - /* 370 */ "literal ::= NK_FLOAT", - /* 371 */ "literal ::= NK_STRING", - /* 372 */ "literal ::= NK_BOOL", - /* 373 */ "literal ::= TIMESTAMP NK_STRING", - /* 374 */ "literal ::= duration_literal", - /* 375 */ "literal ::= NULL", - /* 376 */ "literal ::= NK_QUESTION", - /* 377 */ "duration_literal ::= NK_VARIABLE", - /* 378 */ "signed ::= NK_INTEGER", - /* 379 */ "signed ::= NK_PLUS NK_INTEGER", - /* 380 */ "signed ::= NK_MINUS NK_INTEGER", - /* 381 */ "signed ::= NK_FLOAT", - /* 382 */ "signed ::= NK_PLUS NK_FLOAT", - /* 383 */ "signed ::= NK_MINUS NK_FLOAT", - /* 384 */ "signed_literal ::= signed", - /* 385 */ "signed_literal ::= NK_STRING", - /* 386 */ "signed_literal ::= NK_BOOL", - /* 387 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 388 */ "signed_literal ::= duration_literal", - /* 389 */ "signed_literal ::= NULL", - /* 390 */ "signed_literal ::= literal_func", - /* 391 */ "signed_literal ::= NK_QUESTION", - /* 392 */ "literal_list ::= signed_literal", - /* 393 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 394 */ "db_name ::= NK_ID", - /* 395 */ "table_name ::= NK_ID", - /* 396 */ "column_name ::= NK_ID", - /* 397 */ "function_name ::= NK_ID", - /* 398 */ "table_alias ::= NK_ID", - /* 399 */ "column_alias ::= NK_ID", - /* 400 */ "user_name ::= NK_ID", - /* 401 */ "topic_name ::= NK_ID", - /* 402 */ "stream_name ::= NK_ID", - /* 403 */ "cgroup_name ::= NK_ID", - /* 404 */ "index_name ::= NK_ID", - /* 405 */ "expr_or_subquery ::= expression", - /* 406 */ "expression ::= literal", - /* 407 */ "expression ::= pseudo_column", - /* 408 */ "expression ::= column_reference", - /* 409 */ "expression ::= function_expression", - /* 410 */ "expression ::= case_when_expression", - /* 411 */ "expression ::= NK_LP expression NK_RP", - /* 412 */ "expression ::= NK_PLUS expr_or_subquery", - /* 413 */ "expression ::= NK_MINUS expr_or_subquery", - /* 414 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", - /* 415 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", - /* 416 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", - /* 417 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", - /* 418 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", - /* 419 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 420 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", - /* 421 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", - /* 422 */ "expression_list ::= expr_or_subquery", - /* 423 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", - /* 424 */ "column_reference ::= column_name", - /* 425 */ "column_reference ::= table_name NK_DOT column_name", - /* 426 */ "pseudo_column ::= ROWTS", - /* 427 */ "pseudo_column ::= TBNAME", - /* 428 */ "pseudo_column ::= table_name NK_DOT TBNAME", - /* 429 */ "pseudo_column ::= QSTART", - /* 430 */ "pseudo_column ::= QEND", - /* 431 */ "pseudo_column ::= QDURATION", - /* 432 */ "pseudo_column ::= WSTART", - /* 433 */ "pseudo_column ::= WEND", - /* 434 */ "pseudo_column ::= WDURATION", - /* 435 */ "pseudo_column ::= IROWTS", - /* 436 */ "pseudo_column ::= ISFILLED", - /* 437 */ "pseudo_column ::= QTAGS", - /* 438 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 439 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 440 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", - /* 441 */ "function_expression ::= literal_func", - /* 442 */ "literal_func ::= noarg_func NK_LP NK_RP", - /* 443 */ "literal_func ::= NOW", - /* 444 */ "noarg_func ::= NOW", - /* 445 */ "noarg_func ::= TODAY", - /* 446 */ "noarg_func ::= TIMEZONE", - /* 447 */ "noarg_func ::= DATABASE", - /* 448 */ "noarg_func ::= CLIENT_VERSION", - /* 449 */ "noarg_func ::= SERVER_VERSION", - /* 450 */ "noarg_func ::= SERVER_STATUS", - /* 451 */ "noarg_func ::= CURRENT_USER", - /* 452 */ "noarg_func ::= USER", - /* 453 */ "star_func ::= COUNT", - /* 454 */ "star_func ::= FIRST", - /* 455 */ "star_func ::= LAST", - /* 456 */ "star_func ::= LAST_ROW", - /* 457 */ "star_func_para_list ::= NK_STAR", - /* 458 */ "star_func_para_list ::= other_para_list", - /* 459 */ "other_para_list ::= star_func_para", - /* 460 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 461 */ "star_func_para ::= expr_or_subquery", - /* 462 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 463 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", - /* 464 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", - /* 465 */ "when_then_list ::= when_then_expr", - /* 466 */ "when_then_list ::= when_then_list when_then_expr", - /* 467 */ "when_then_expr ::= WHEN common_expression THEN common_expression", - /* 468 */ "case_when_else_opt ::=", - /* 469 */ "case_when_else_opt ::= ELSE common_expression", - /* 470 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", - /* 471 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", - /* 472 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", - /* 473 */ "predicate ::= expr_or_subquery IS NULL", - /* 474 */ "predicate ::= expr_or_subquery IS NOT NULL", - /* 475 */ "predicate ::= expr_or_subquery in_op in_predicate_value", - /* 476 */ "compare_op ::= NK_LT", - /* 477 */ "compare_op ::= NK_GT", - /* 478 */ "compare_op ::= NK_LE", - /* 479 */ "compare_op ::= NK_GE", - /* 480 */ "compare_op ::= NK_NE", - /* 481 */ "compare_op ::= NK_EQ", - /* 482 */ "compare_op ::= LIKE", - /* 483 */ "compare_op ::= NOT LIKE", - /* 484 */ "compare_op ::= MATCH", - /* 485 */ "compare_op ::= NMATCH", - /* 486 */ "compare_op ::= CONTAINS", - /* 487 */ "in_op ::= IN", - /* 488 */ "in_op ::= NOT IN", - /* 489 */ "in_predicate_value ::= NK_LP literal_list NK_RP", - /* 490 */ "boolean_value_expression ::= boolean_primary", - /* 491 */ "boolean_value_expression ::= NOT boolean_primary", - /* 492 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 493 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 494 */ "boolean_primary ::= predicate", - /* 495 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 496 */ "common_expression ::= expr_or_subquery", - /* 497 */ "common_expression ::= boolean_value_expression", - /* 498 */ "from_clause_opt ::=", - /* 499 */ "from_clause_opt ::= FROM table_reference_list", - /* 500 */ "table_reference_list ::= table_reference", - /* 501 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 502 */ "table_reference ::= table_primary", - /* 503 */ "table_reference ::= joined_table", - /* 504 */ "table_primary ::= table_name alias_opt", - /* 505 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 506 */ "table_primary ::= subquery alias_opt", - /* 507 */ "table_primary ::= parenthesized_joined_table", - /* 508 */ "alias_opt ::=", - /* 509 */ "alias_opt ::= table_alias", - /* 510 */ "alias_opt ::= AS table_alias", - /* 511 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 512 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 513 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 514 */ "join_type ::=", - /* 515 */ "join_type ::= INNER", - /* 516 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 517 */ "set_quantifier_opt ::=", - /* 518 */ "set_quantifier_opt ::= DISTINCT", - /* 519 */ "set_quantifier_opt ::= ALL", - /* 520 */ "select_list ::= select_item", - /* 521 */ "select_list ::= select_list NK_COMMA select_item", - /* 522 */ "select_item ::= NK_STAR", - /* 523 */ "select_item ::= common_expression", - /* 524 */ "select_item ::= common_expression column_alias", - /* 525 */ "select_item ::= common_expression AS column_alias", - /* 526 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 527 */ "where_clause_opt ::=", - /* 528 */ "where_clause_opt ::= WHERE search_condition", - /* 529 */ "partition_by_clause_opt ::=", - /* 530 */ "partition_by_clause_opt ::= PARTITION BY partition_list", - /* 531 */ "partition_list ::= partition_item", - /* 532 */ "partition_list ::= partition_list NK_COMMA partition_item", - /* 533 */ "partition_item ::= expr_or_subquery", - /* 534 */ "partition_item ::= expr_or_subquery column_alias", - /* 535 */ "partition_item ::= expr_or_subquery AS column_alias", - /* 536 */ "twindow_clause_opt ::=", - /* 537 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 538 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", - /* 539 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 540 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 541 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", - /* 542 */ "sliding_opt ::=", - /* 543 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 544 */ "fill_opt ::=", - /* 545 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 546 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP", - /* 547 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP", - /* 548 */ "fill_mode ::= NONE", - /* 549 */ "fill_mode ::= PREV", - /* 550 */ "fill_mode ::= NULL", - /* 551 */ "fill_mode ::= NULL_F", - /* 552 */ "fill_mode ::= LINEAR", - /* 553 */ "fill_mode ::= NEXT", - /* 554 */ "group_by_clause_opt ::=", - /* 555 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 556 */ "group_by_list ::= expr_or_subquery", - /* 557 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", - /* 558 */ "having_clause_opt ::=", - /* 559 */ "having_clause_opt ::= HAVING search_condition", - /* 560 */ "range_opt ::=", - /* 561 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", - /* 562 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_RP", - /* 563 */ "every_opt ::=", - /* 564 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 565 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 566 */ "query_simple ::= query_specification", - /* 567 */ "query_simple ::= union_query_expression", - /* 568 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", - /* 569 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", - /* 570 */ "query_simple_or_subquery ::= query_simple", - /* 571 */ "query_simple_or_subquery ::= subquery", - /* 572 */ "query_or_subquery ::= query_expression", - /* 573 */ "query_or_subquery ::= subquery", - /* 574 */ "order_by_clause_opt ::=", - /* 575 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 576 */ "slimit_clause_opt ::=", - /* 577 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 578 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 579 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 580 */ "limit_clause_opt ::=", - /* 581 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 582 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 583 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 584 */ "subquery ::= NK_LP query_expression NK_RP", - /* 585 */ "subquery ::= NK_LP subquery NK_RP", - /* 586 */ "search_condition ::= common_expression", - /* 587 */ "sort_specification_list ::= sort_specification", - /* 588 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 589 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", - /* 590 */ "ordering_specification_opt ::=", - /* 591 */ "ordering_specification_opt ::= ASC", - /* 592 */ "ordering_specification_opt ::= DESC", - /* 593 */ "null_ordering_opt ::=", - /* 594 */ "null_ordering_opt ::= NULLS FIRST", - /* 595 */ "null_ordering_opt ::= NULLS LAST", + /* 242 */ "cmd ::= SHOW INDEXES FROM db_name NK_DOT table_name", + /* 243 */ "cmd ::= SHOW STREAMS", + /* 244 */ "cmd ::= SHOW ACCOUNTS", + /* 245 */ "cmd ::= SHOW APPS", + /* 246 */ "cmd ::= SHOW CONNECTIONS", + /* 247 */ "cmd ::= SHOW LICENCES", + /* 248 */ "cmd ::= SHOW GRANTS", + /* 249 */ "cmd ::= SHOW CREATE DATABASE db_name", + /* 250 */ "cmd ::= SHOW CREATE TABLE full_table_name", + /* 251 */ "cmd ::= SHOW CREATE STABLE full_table_name", + /* 252 */ "cmd ::= SHOW QUERIES", + /* 253 */ "cmd ::= SHOW SCORES", + /* 254 */ "cmd ::= SHOW TOPICS", + /* 255 */ "cmd ::= SHOW VARIABLES", + /* 256 */ "cmd ::= SHOW CLUSTER VARIABLES", + /* 257 */ "cmd ::= SHOW LOCAL VARIABLES", + /* 258 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", + /* 259 */ "cmd ::= SHOW BNODES", + /* 260 */ "cmd ::= SHOW SNODES", + /* 261 */ "cmd ::= SHOW CLUSTER", + /* 262 */ "cmd ::= SHOW TRANSACTIONS", + /* 263 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", + /* 264 */ "cmd ::= SHOW CONSUMERS", + /* 265 */ "cmd ::= SHOW SUBSCRIPTIONS", + /* 266 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", + /* 267 */ "cmd ::= SHOW TAGS FROM db_name NK_DOT table_name", + /* 268 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", + /* 269 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM db_name NK_DOT table_name", + /* 270 */ "cmd ::= SHOW VNODES NK_INTEGER", + /* 271 */ "cmd ::= SHOW VNODES NK_STRING", + /* 272 */ "cmd ::= SHOW db_name_cond_opt ALIVE", + /* 273 */ "cmd ::= SHOW CLUSTER ALIVE", + /* 274 */ "db_name_cond_opt ::=", + /* 275 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 276 */ "like_pattern_opt ::=", + /* 277 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 278 */ "table_name_cond ::= table_name", + /* 279 */ "from_db_opt ::=", + /* 280 */ "from_db_opt ::= FROM db_name", + /* 281 */ "tag_list_opt ::=", + /* 282 */ "tag_list_opt ::= tag_item", + /* 283 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", + /* 284 */ "tag_item ::= TBNAME", + /* 285 */ "tag_item ::= QTAGS", + /* 286 */ "tag_item ::= column_name", + /* 287 */ "tag_item ::= column_name column_alias", + /* 288 */ "tag_item ::= column_name AS column_alias", + /* 289 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", + /* 290 */ "cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP", + /* 291 */ "cmd ::= DROP INDEX exists_opt full_index_name", + /* 292 */ "full_index_name ::= index_name", + /* 293 */ "full_index_name ::= db_name NK_DOT index_name", + /* 294 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", + /* 295 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", + /* 296 */ "func_list ::= func", + /* 297 */ "func_list ::= func_list NK_COMMA func", + /* 298 */ "func ::= sma_func_name NK_LP expression_list NK_RP", + /* 299 */ "sma_func_name ::= function_name", + /* 300 */ "sma_func_name ::= COUNT", + /* 301 */ "sma_func_name ::= FIRST", + /* 302 */ "sma_func_name ::= LAST", + /* 303 */ "sma_func_name ::= LAST_ROW", + /* 304 */ "sma_stream_opt ::=", + /* 305 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", + /* 306 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", + /* 307 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", + /* 308 */ "with_meta ::= AS", + /* 309 */ "with_meta ::= WITH META AS", + /* 310 */ "with_meta ::= ONLY META AS", + /* 311 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", + /* 312 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta DATABASE db_name", + /* 313 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta STABLE full_table_name where_clause_opt", + /* 314 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 315 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", + /* 316 */ "cmd ::= DESC full_table_name", + /* 317 */ "cmd ::= DESCRIBE full_table_name", + /* 318 */ "cmd ::= RESET QUERY CACHE", + /* 319 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", + /* 320 */ "cmd ::= EXPLAIN analyze_opt explain_options insert_query", + /* 321 */ "analyze_opt ::=", + /* 322 */ "analyze_opt ::= ANALYZE", + /* 323 */ "explain_options ::=", + /* 324 */ "explain_options ::= explain_options VERBOSE NK_BOOL", + /* 325 */ "explain_options ::= explain_options RATIO NK_FLOAT", + /* 326 */ "cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt", + /* 327 */ "cmd ::= DROP FUNCTION exists_opt function_name", + /* 328 */ "agg_func_opt ::=", + /* 329 */ "agg_func_opt ::= AGGREGATE", + /* 330 */ "bufsize_opt ::=", + /* 331 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", + /* 332 */ "language_opt ::=", + /* 333 */ "language_opt ::= LANGUAGE NK_STRING", + /* 334 */ "or_replace_opt ::=", + /* 335 */ "or_replace_opt ::= OR REPLACE", + /* 336 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery", + /* 337 */ "cmd ::= DROP STREAM exists_opt stream_name", + /* 338 */ "cmd ::= PAUSE STREAM exists_opt stream_name", + /* 339 */ "cmd ::= RESUME STREAM exists_opt ignore_opt stream_name", + /* 340 */ "col_list_opt ::=", + /* 341 */ "col_list_opt ::= NK_LP col_name_list NK_RP", + /* 342 */ "tag_def_or_ref_opt ::=", + /* 343 */ "tag_def_or_ref_opt ::= tags_def", + /* 344 */ "tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP", + /* 345 */ "stream_options ::=", + /* 346 */ "stream_options ::= stream_options TRIGGER AT_ONCE", + /* 347 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", + /* 348 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", + /* 349 */ "stream_options ::= stream_options WATERMARK duration_literal", + /* 350 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", + /* 351 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", + /* 352 */ "stream_options ::= stream_options DELETE_MARK duration_literal", + /* 353 */ "stream_options ::= stream_options IGNORE UPDATE NK_INTEGER", + /* 354 */ "subtable_opt ::=", + /* 355 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", + /* 356 */ "ignore_opt ::=", + /* 357 */ "ignore_opt ::= IGNORE UNTREATED", + /* 358 */ "cmd ::= KILL CONNECTION NK_INTEGER", + /* 359 */ "cmd ::= KILL QUERY NK_STRING", + /* 360 */ "cmd ::= KILL TRANSACTION NK_INTEGER", + /* 361 */ "cmd ::= BALANCE VGROUP", + /* 362 */ "cmd ::= BALANCE VGROUP LEADER", + /* 363 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", + /* 364 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", + /* 365 */ "cmd ::= SPLIT VGROUP NK_INTEGER", + /* 366 */ "dnode_list ::= DNODE NK_INTEGER", + /* 367 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", + /* 368 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", + /* 369 */ "cmd ::= query_or_subquery", + /* 370 */ "cmd ::= insert_query", + /* 371 */ "insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", + /* 372 */ "insert_query ::= INSERT INTO full_table_name query_or_subquery", + /* 373 */ "literal ::= NK_INTEGER", + /* 374 */ "literal ::= NK_FLOAT", + /* 375 */ "literal ::= NK_STRING", + /* 376 */ "literal ::= NK_BOOL", + /* 377 */ "literal ::= TIMESTAMP NK_STRING", + /* 378 */ "literal ::= duration_literal", + /* 379 */ "literal ::= NULL", + /* 380 */ "literal ::= NK_QUESTION", + /* 381 */ "duration_literal ::= NK_VARIABLE", + /* 382 */ "signed ::= NK_INTEGER", + /* 383 */ "signed ::= NK_PLUS NK_INTEGER", + /* 384 */ "signed ::= NK_MINUS NK_INTEGER", + /* 385 */ "signed ::= NK_FLOAT", + /* 386 */ "signed ::= NK_PLUS NK_FLOAT", + /* 387 */ "signed ::= NK_MINUS NK_FLOAT", + /* 388 */ "signed_literal ::= signed", + /* 389 */ "signed_literal ::= NK_STRING", + /* 390 */ "signed_literal ::= NK_BOOL", + /* 391 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 392 */ "signed_literal ::= duration_literal", + /* 393 */ "signed_literal ::= NULL", + /* 394 */ "signed_literal ::= literal_func", + /* 395 */ "signed_literal ::= NK_QUESTION", + /* 396 */ "literal_list ::= signed_literal", + /* 397 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 398 */ "db_name ::= NK_ID", + /* 399 */ "table_name ::= NK_ID", + /* 400 */ "column_name ::= NK_ID", + /* 401 */ "function_name ::= NK_ID", + /* 402 */ "table_alias ::= NK_ID", + /* 403 */ "column_alias ::= NK_ID", + /* 404 */ "user_name ::= NK_ID", + /* 405 */ "topic_name ::= NK_ID", + /* 406 */ "stream_name ::= NK_ID", + /* 407 */ "cgroup_name ::= NK_ID", + /* 408 */ "index_name ::= NK_ID", + /* 409 */ "expr_or_subquery ::= expression", + /* 410 */ "expression ::= literal", + /* 411 */ "expression ::= pseudo_column", + /* 412 */ "expression ::= column_reference", + /* 413 */ "expression ::= function_expression", + /* 414 */ "expression ::= case_when_expression", + /* 415 */ "expression ::= NK_LP expression NK_RP", + /* 416 */ "expression ::= NK_PLUS expr_or_subquery", + /* 417 */ "expression ::= NK_MINUS expr_or_subquery", + /* 418 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", + /* 419 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", + /* 420 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", + /* 421 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", + /* 422 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", + /* 423 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 424 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", + /* 425 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", + /* 426 */ "expression_list ::= expr_or_subquery", + /* 427 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", + /* 428 */ "column_reference ::= column_name", + /* 429 */ "column_reference ::= table_name NK_DOT column_name", + /* 430 */ "pseudo_column ::= ROWTS", + /* 431 */ "pseudo_column ::= TBNAME", + /* 432 */ "pseudo_column ::= table_name NK_DOT TBNAME", + /* 433 */ "pseudo_column ::= QSTART", + /* 434 */ "pseudo_column ::= QEND", + /* 435 */ "pseudo_column ::= QDURATION", + /* 436 */ "pseudo_column ::= WSTART", + /* 437 */ "pseudo_column ::= WEND", + /* 438 */ "pseudo_column ::= WDURATION", + /* 439 */ "pseudo_column ::= IROWTS", + /* 440 */ "pseudo_column ::= ISFILLED", + /* 441 */ "pseudo_column ::= QTAGS", + /* 442 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 443 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 444 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", + /* 445 */ "function_expression ::= literal_func", + /* 446 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 447 */ "literal_func ::= NOW", + /* 448 */ "noarg_func ::= NOW", + /* 449 */ "noarg_func ::= TODAY", + /* 450 */ "noarg_func ::= TIMEZONE", + /* 451 */ "noarg_func ::= DATABASE", + /* 452 */ "noarg_func ::= CLIENT_VERSION", + /* 453 */ "noarg_func ::= SERVER_VERSION", + /* 454 */ "noarg_func ::= SERVER_STATUS", + /* 455 */ "noarg_func ::= CURRENT_USER", + /* 456 */ "noarg_func ::= USER", + /* 457 */ "star_func ::= COUNT", + /* 458 */ "star_func ::= FIRST", + /* 459 */ "star_func ::= LAST", + /* 460 */ "star_func ::= LAST_ROW", + /* 461 */ "star_func_para_list ::= NK_STAR", + /* 462 */ "star_func_para_list ::= other_para_list", + /* 463 */ "other_para_list ::= star_func_para", + /* 464 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 465 */ "star_func_para ::= expr_or_subquery", + /* 466 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 467 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", + /* 468 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", + /* 469 */ "when_then_list ::= when_then_expr", + /* 470 */ "when_then_list ::= when_then_list when_then_expr", + /* 471 */ "when_then_expr ::= WHEN common_expression THEN common_expression", + /* 472 */ "case_when_else_opt ::=", + /* 473 */ "case_when_else_opt ::= ELSE common_expression", + /* 474 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", + /* 475 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", + /* 476 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", + /* 477 */ "predicate ::= expr_or_subquery IS NULL", + /* 478 */ "predicate ::= expr_or_subquery IS NOT NULL", + /* 479 */ "predicate ::= expr_or_subquery in_op in_predicate_value", + /* 480 */ "compare_op ::= NK_LT", + /* 481 */ "compare_op ::= NK_GT", + /* 482 */ "compare_op ::= NK_LE", + /* 483 */ "compare_op ::= NK_GE", + /* 484 */ "compare_op ::= NK_NE", + /* 485 */ "compare_op ::= NK_EQ", + /* 486 */ "compare_op ::= LIKE", + /* 487 */ "compare_op ::= NOT LIKE", + /* 488 */ "compare_op ::= MATCH", + /* 489 */ "compare_op ::= NMATCH", + /* 490 */ "compare_op ::= CONTAINS", + /* 491 */ "in_op ::= IN", + /* 492 */ "in_op ::= NOT IN", + /* 493 */ "in_predicate_value ::= NK_LP literal_list NK_RP", + /* 494 */ "boolean_value_expression ::= boolean_primary", + /* 495 */ "boolean_value_expression ::= NOT boolean_primary", + /* 496 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 497 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 498 */ "boolean_primary ::= predicate", + /* 499 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 500 */ "common_expression ::= expr_or_subquery", + /* 501 */ "common_expression ::= boolean_value_expression", + /* 502 */ "from_clause_opt ::=", + /* 503 */ "from_clause_opt ::= FROM table_reference_list", + /* 504 */ "table_reference_list ::= table_reference", + /* 505 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 506 */ "table_reference ::= table_primary", + /* 507 */ "table_reference ::= joined_table", + /* 508 */ "table_primary ::= table_name alias_opt", + /* 509 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 510 */ "table_primary ::= subquery alias_opt", + /* 511 */ "table_primary ::= parenthesized_joined_table", + /* 512 */ "alias_opt ::=", + /* 513 */ "alias_opt ::= table_alias", + /* 514 */ "alias_opt ::= AS table_alias", + /* 515 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 516 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 517 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 518 */ "join_type ::=", + /* 519 */ "join_type ::= INNER", + /* 520 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 521 */ "set_quantifier_opt ::=", + /* 522 */ "set_quantifier_opt ::= DISTINCT", + /* 523 */ "set_quantifier_opt ::= ALL", + /* 524 */ "select_list ::= select_item", + /* 525 */ "select_list ::= select_list NK_COMMA select_item", + /* 526 */ "select_item ::= NK_STAR", + /* 527 */ "select_item ::= common_expression", + /* 528 */ "select_item ::= common_expression column_alias", + /* 529 */ "select_item ::= common_expression AS column_alias", + /* 530 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 531 */ "where_clause_opt ::=", + /* 532 */ "where_clause_opt ::= WHERE search_condition", + /* 533 */ "partition_by_clause_opt ::=", + /* 534 */ "partition_by_clause_opt ::= PARTITION BY partition_list", + /* 535 */ "partition_list ::= partition_item", + /* 536 */ "partition_list ::= partition_list NK_COMMA partition_item", + /* 537 */ "partition_item ::= expr_or_subquery", + /* 538 */ "partition_item ::= expr_or_subquery column_alias", + /* 539 */ "partition_item ::= expr_or_subquery AS column_alias", + /* 540 */ "twindow_clause_opt ::=", + /* 541 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 542 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", + /* 543 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 544 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 545 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", + /* 546 */ "sliding_opt ::=", + /* 547 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 548 */ "fill_opt ::=", + /* 549 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 550 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP", + /* 551 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP", + /* 552 */ "fill_mode ::= NONE", + /* 553 */ "fill_mode ::= PREV", + /* 554 */ "fill_mode ::= NULL", + /* 555 */ "fill_mode ::= NULL_F", + /* 556 */ "fill_mode ::= LINEAR", + /* 557 */ "fill_mode ::= NEXT", + /* 558 */ "group_by_clause_opt ::=", + /* 559 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 560 */ "group_by_list ::= expr_or_subquery", + /* 561 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", + /* 562 */ "having_clause_opt ::=", + /* 563 */ "having_clause_opt ::= HAVING search_condition", + /* 564 */ "range_opt ::=", + /* 565 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", + /* 566 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_RP", + /* 567 */ "every_opt ::=", + /* 568 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 569 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 570 */ "query_simple ::= query_specification", + /* 571 */ "query_simple ::= union_query_expression", + /* 572 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", + /* 573 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", + /* 574 */ "query_simple_or_subquery ::= query_simple", + /* 575 */ "query_simple_or_subquery ::= subquery", + /* 576 */ "query_or_subquery ::= query_expression", + /* 577 */ "query_or_subquery ::= subquery", + /* 578 */ "order_by_clause_opt ::=", + /* 579 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 580 */ "slimit_clause_opt ::=", + /* 581 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 582 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 583 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 584 */ "limit_clause_opt ::=", + /* 585 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 586 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 587 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 588 */ "subquery ::= NK_LP query_expression NK_RP", + /* 589 */ "subquery ::= NK_LP subquery NK_RP", + /* 590 */ "search_condition ::= common_expression", + /* 591 */ "sort_specification_list ::= sort_specification", + /* 592 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 593 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", + /* 594 */ "ordering_specification_opt ::=", + /* 595 */ "ordering_specification_opt ::= ASC", + /* 596 */ "ordering_specification_opt ::= DESC", + /* 597 */ "null_ordering_opt ::=", + /* 598 */ "null_ordering_opt ::= NULLS FIRST", + /* 599 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2664,210 +2697,211 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 337: /* cmd */ - case 340: /* literal */ - case 346: /* with_opt */ - case 352: /* search_condition */ - case 357: /* db_options */ - case 359: /* alter_db_options */ - case 361: /* start_opt */ - case 362: /* end_opt */ - case 366: /* signed */ - case 368: /* retention */ - case 369: /* full_table_name */ - case 372: /* table_options */ - case 376: /* alter_table_clause */ - case 377: /* alter_table_options */ - case 380: /* signed_literal */ - case 381: /* create_subtable_clause */ - case 384: /* drop_table_clause */ - case 386: /* column_def */ - case 390: /* duration_literal */ - case 391: /* rollup_func_name */ - case 393: /* col_name */ - case 394: /* db_name_cond_opt */ - case 395: /* like_pattern_opt */ - case 396: /* table_name_cond */ - case 397: /* from_db_opt */ - case 399: /* tag_item */ - case 401: /* full_index_name */ - case 402: /* index_options */ - case 405: /* sliding_opt */ - case 406: /* sma_stream_opt */ - case 407: /* func */ - case 409: /* query_or_subquery */ - case 410: /* where_clause_opt */ - case 413: /* explain_options */ - case 414: /* insert_query */ - case 420: /* stream_options */ - case 423: /* subtable_opt */ - case 425: /* expression */ - case 427: /* literal_func */ - case 430: /* expr_or_subquery */ - case 431: /* pseudo_column */ - case 432: /* column_reference */ - case 433: /* function_expression */ - case 434: /* case_when_expression */ - case 439: /* star_func_para */ - case 441: /* case_when_else_opt */ - case 442: /* common_expression */ - case 443: /* when_then_expr */ - case 444: /* predicate */ - case 447: /* in_predicate_value */ - case 448: /* boolean_value_expression */ - case 449: /* boolean_primary */ - case 450: /* from_clause_opt */ - case 451: /* table_reference_list */ - case 452: /* table_reference */ - case 453: /* table_primary */ - case 454: /* joined_table */ - case 456: /* subquery */ - case 457: /* parenthesized_joined_table */ - case 459: /* query_specification */ - case 463: /* range_opt */ - case 464: /* every_opt */ - case 465: /* fill_opt */ - case 466: /* twindow_clause_opt */ - case 468: /* having_clause_opt */ - case 469: /* select_item */ - case 471: /* partition_item */ - case 474: /* query_expression */ - case 475: /* query_simple */ - case 477: /* slimit_clause_opt */ - case 478: /* limit_clause_opt */ - case 479: /* union_query_expression */ - case 480: /* query_simple_or_subquery */ - case 482: /* sort_specification */ + case 338: /* cmd */ + case 341: /* literal */ + case 347: /* with_opt */ + case 353: /* search_condition */ + case 358: /* db_options */ + case 360: /* alter_db_options */ + case 362: /* start_opt */ + case 363: /* end_opt */ + case 367: /* signed */ + case 369: /* retention */ + case 370: /* full_table_name */ + case 373: /* table_options */ + case 377: /* alter_table_clause */ + case 378: /* alter_table_options */ + case 381: /* signed_literal */ + case 382: /* create_subtable_clause */ + case 385: /* drop_table_clause */ + case 387: /* column_def */ + case 391: /* duration_literal */ + case 392: /* rollup_func_name */ + case 394: /* col_name */ + case 395: /* db_name_cond_opt */ + case 396: /* like_pattern_opt */ + case 397: /* table_name_cond */ + case 398: /* from_db_opt */ + case 400: /* tag_item */ + case 402: /* full_index_name */ + case 403: /* index_options */ + case 406: /* sliding_opt */ + case 407: /* sma_stream_opt */ + case 408: /* func */ + case 411: /* query_or_subquery */ + case 412: /* where_clause_opt */ + case 415: /* explain_options */ + case 416: /* insert_query */ + case 422: /* stream_options */ + case 425: /* subtable_opt */ + case 427: /* expression */ + case 429: /* literal_func */ + case 432: /* expr_or_subquery */ + case 433: /* pseudo_column */ + case 434: /* column_reference */ + case 435: /* function_expression */ + case 436: /* case_when_expression */ + case 441: /* star_func_para */ + case 443: /* case_when_else_opt */ + case 444: /* common_expression */ + case 445: /* when_then_expr */ + case 446: /* predicate */ + case 449: /* in_predicate_value */ + case 450: /* boolean_value_expression */ + case 451: /* boolean_primary */ + case 452: /* from_clause_opt */ + case 453: /* table_reference_list */ + case 454: /* table_reference */ + case 455: /* table_primary */ + case 456: /* joined_table */ + case 458: /* subquery */ + case 459: /* parenthesized_joined_table */ + case 461: /* query_specification */ + case 465: /* range_opt */ + case 466: /* every_opt */ + case 467: /* fill_opt */ + case 468: /* twindow_clause_opt */ + case 470: /* having_clause_opt */ + case 471: /* select_item */ + case 473: /* partition_item */ + case 476: /* query_expression */ + case 477: /* query_simple */ + case 479: /* slimit_clause_opt */ + case 480: /* limit_clause_opt */ + case 481: /* union_query_expression */ + case 482: /* query_simple_or_subquery */ + case 484: /* sort_specification */ { - nodesDestroyNode((yypminor->yy242)); + nodesDestroyNode((yypminor->yy452)); } break; - case 338: /* account_options */ - case 339: /* alter_account_options */ - case 341: /* alter_account_option */ - case 360: /* speed_opt */ - case 417: /* bufsize_opt */ + case 339: /* account_options */ + case 340: /* alter_account_options */ + case 342: /* alter_account_option */ + case 361: /* speed_opt */ + case 410: /* with_meta */ + case 419: /* bufsize_opt */ { } break; - case 342: /* user_name */ - case 349: /* db_name */ - case 350: /* table_name */ - case 351: /* topic_name */ - case 353: /* dnode_endpoint */ - case 378: /* column_name */ - case 392: /* function_name */ - case 400: /* column_alias */ - case 403: /* index_name */ - case 408: /* sma_func_name */ - case 411: /* cgroup_name */ - case 418: /* language_opt */ - case 419: /* stream_name */ - case 429: /* table_alias */ - case 435: /* star_func */ - case 437: /* noarg_func */ - case 455: /* alias_opt */ + case 343: /* user_name */ + case 350: /* db_name */ + case 351: /* table_name */ + case 352: /* topic_name */ + case 354: /* dnode_endpoint */ + case 379: /* column_name */ + case 393: /* function_name */ + case 401: /* column_alias */ + case 404: /* index_name */ + case 409: /* sma_func_name */ + case 413: /* cgroup_name */ + case 420: /* language_opt */ + case 421: /* stream_name */ + case 431: /* table_alias */ + case 437: /* star_func */ + case 439: /* noarg_func */ + case 457: /* alias_opt */ { } break; - case 343: /* sysinfo_opt */ + case 344: /* sysinfo_opt */ { } break; - case 344: /* privileges */ - case 347: /* priv_type_list */ - case 348: /* priv_type */ + case 345: /* privileges */ + case 348: /* priv_type_list */ + case 349: /* priv_type */ { } break; - case 345: /* priv_level */ + case 346: /* priv_level */ { } break; - case 354: /* force_opt */ - case 355: /* unsafe_opt */ - case 356: /* not_exists_opt */ - case 358: /* exists_opt */ - case 412: /* analyze_opt */ - case 415: /* or_replace_opt */ - case 416: /* agg_func_opt */ - case 424: /* ignore_opt */ - case 460: /* set_quantifier_opt */ + case 355: /* force_opt */ + case 356: /* unsafe_opt */ + case 357: /* not_exists_opt */ + case 359: /* exists_opt */ + case 414: /* analyze_opt */ + case 417: /* or_replace_opt */ + case 418: /* agg_func_opt */ + case 426: /* ignore_opt */ + case 462: /* set_quantifier_opt */ { } break; - case 363: /* integer_list */ - case 364: /* variable_list */ - case 365: /* retention_list */ - case 370: /* column_def_list */ - case 371: /* tags_def_opt */ - case 373: /* multi_create_clause */ - case 374: /* tags_def */ - case 375: /* multi_drop_clause */ - case 382: /* specific_cols_opt */ - case 383: /* expression_list */ - case 385: /* col_name_list */ - case 387: /* duration_list */ - case 388: /* rollup_func_list */ - case 398: /* tag_list_opt */ - case 404: /* func_list */ - case 421: /* col_list_opt */ - case 422: /* tag_def_or_ref_opt */ - case 426: /* dnode_list */ - case 428: /* literal_list */ - case 436: /* star_func_para_list */ - case 438: /* other_para_list */ - case 440: /* when_then_list */ - case 461: /* select_list */ - case 462: /* partition_by_clause_opt */ - case 467: /* group_by_clause_opt */ - case 470: /* partition_list */ - case 473: /* group_by_list */ - case 476: /* order_by_clause_opt */ - case 481: /* sort_specification_list */ + case 364: /* integer_list */ + case 365: /* variable_list */ + case 366: /* retention_list */ + case 371: /* column_def_list */ + case 372: /* tags_def_opt */ + case 374: /* multi_create_clause */ + case 375: /* tags_def */ + case 376: /* multi_drop_clause */ + case 383: /* specific_cols_opt */ + case 384: /* expression_list */ + case 386: /* col_name_list */ + case 388: /* duration_list */ + case 389: /* rollup_func_list */ + case 399: /* tag_list_opt */ + case 405: /* func_list */ + case 423: /* col_list_opt */ + case 424: /* tag_def_or_ref_opt */ + case 428: /* dnode_list */ + case 430: /* literal_list */ + case 438: /* star_func_para_list */ + case 440: /* other_para_list */ + case 442: /* when_then_list */ + case 463: /* select_list */ + case 464: /* partition_by_clause_opt */ + case 469: /* group_by_clause_opt */ + case 472: /* partition_list */ + case 475: /* group_by_list */ + case 478: /* order_by_clause_opt */ + case 483: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy174)); + nodesDestroyList((yypminor->yy812)); } break; - case 367: /* alter_db_option */ - case 389: /* alter_table_option */ + case 368: /* alter_db_option */ + case 390: /* alter_table_option */ { } break; - case 379: /* type_name */ + case 380: /* type_name */ { } break; - case 445: /* compare_op */ - case 446: /* in_op */ + case 447: /* compare_op */ + case 448: /* in_op */ { } break; - case 458: /* join_type */ + case 460: /* join_type */ { } break; - case 472: /* fill_mode */ + case 474: /* fill_mode */ { } break; - case 483: /* ordering_specification_opt */ + case 485: /* ordering_specification_opt */ { } break; - case 484: /* null_ordering_opt */ + case 486: /* null_ordering_opt */ { } @@ -3158,602 +3192,606 @@ static void yy_shift( /* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side ** of that rule */ static const YYCODETYPE yyRuleInfoLhs[] = { - 337, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ - 337, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ - 338, /* (2) account_options ::= */ - 338, /* (3) account_options ::= account_options PPS literal */ - 338, /* (4) account_options ::= account_options TSERIES literal */ - 338, /* (5) account_options ::= account_options STORAGE literal */ - 338, /* (6) account_options ::= account_options STREAMS literal */ - 338, /* (7) account_options ::= account_options QTIME literal */ - 338, /* (8) account_options ::= account_options DBS literal */ - 338, /* (9) account_options ::= account_options USERS literal */ - 338, /* (10) account_options ::= account_options CONNS literal */ - 338, /* (11) account_options ::= account_options STATE literal */ - 339, /* (12) alter_account_options ::= alter_account_option */ - 339, /* (13) alter_account_options ::= alter_account_options alter_account_option */ - 341, /* (14) alter_account_option ::= PASS literal */ - 341, /* (15) alter_account_option ::= PPS literal */ - 341, /* (16) alter_account_option ::= TSERIES literal */ - 341, /* (17) alter_account_option ::= STORAGE literal */ - 341, /* (18) alter_account_option ::= STREAMS literal */ - 341, /* (19) alter_account_option ::= QTIME literal */ - 341, /* (20) alter_account_option ::= DBS literal */ - 341, /* (21) alter_account_option ::= USERS literal */ - 341, /* (22) alter_account_option ::= CONNS literal */ - 341, /* (23) alter_account_option ::= STATE literal */ - 337, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ - 337, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ - 337, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ - 337, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ - 337, /* (28) cmd ::= DROP USER user_name */ - 343, /* (29) sysinfo_opt ::= */ - 343, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ - 337, /* (31) cmd ::= GRANT privileges ON priv_level with_opt TO user_name */ - 337, /* (32) cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name */ - 344, /* (33) privileges ::= ALL */ - 344, /* (34) privileges ::= priv_type_list */ - 344, /* (35) privileges ::= SUBSCRIBE */ - 347, /* (36) priv_type_list ::= priv_type */ - 347, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ - 348, /* (38) priv_type ::= READ */ - 348, /* (39) priv_type ::= WRITE */ - 345, /* (40) priv_level ::= NK_STAR NK_DOT NK_STAR */ - 345, /* (41) priv_level ::= db_name NK_DOT NK_STAR */ - 345, /* (42) priv_level ::= db_name NK_DOT table_name */ - 345, /* (43) priv_level ::= topic_name */ - 346, /* (44) with_opt ::= */ - 346, /* (45) with_opt ::= WITH search_condition */ - 337, /* (46) cmd ::= CREATE DNODE dnode_endpoint */ - 337, /* (47) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ - 337, /* (48) cmd ::= DROP DNODE NK_INTEGER force_opt */ - 337, /* (49) cmd ::= DROP DNODE dnode_endpoint force_opt */ - 337, /* (50) cmd ::= DROP DNODE NK_INTEGER unsafe_opt */ - 337, /* (51) cmd ::= DROP DNODE dnode_endpoint unsafe_opt */ - 337, /* (52) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ - 337, /* (53) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ - 337, /* (54) cmd ::= ALTER ALL DNODES NK_STRING */ - 337, /* (55) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ - 337, /* (56) cmd ::= RESTORE DNODE NK_INTEGER */ - 353, /* (57) dnode_endpoint ::= NK_STRING */ - 353, /* (58) dnode_endpoint ::= NK_ID */ - 353, /* (59) dnode_endpoint ::= NK_IPTOKEN */ - 354, /* (60) force_opt ::= */ - 354, /* (61) force_opt ::= FORCE */ - 355, /* (62) unsafe_opt ::= UNSAFE */ - 337, /* (63) cmd ::= ALTER LOCAL NK_STRING */ - 337, /* (64) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ - 337, /* (65) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - 337, /* (66) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - 337, /* (67) cmd ::= RESTORE QNODE ON DNODE NK_INTEGER */ - 337, /* (68) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ - 337, /* (69) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ - 337, /* (70) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ - 337, /* (71) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ - 337, /* (72) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ - 337, /* (73) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ - 337, /* (74) cmd ::= RESTORE MNODE ON DNODE NK_INTEGER */ - 337, /* (75) cmd ::= RESTORE VNODE ON DNODE NK_INTEGER */ - 337, /* (76) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - 337, /* (77) cmd ::= DROP DATABASE exists_opt db_name */ - 337, /* (78) cmd ::= USE db_name */ - 337, /* (79) cmd ::= ALTER DATABASE db_name alter_db_options */ - 337, /* (80) cmd ::= FLUSH DATABASE db_name */ - 337, /* (81) cmd ::= TRIM DATABASE db_name speed_opt */ - 337, /* (82) cmd ::= COMPACT DATABASE db_name start_opt end_opt */ - 356, /* (83) not_exists_opt ::= IF NOT EXISTS */ - 356, /* (84) not_exists_opt ::= */ - 358, /* (85) exists_opt ::= IF EXISTS */ - 358, /* (86) exists_opt ::= */ - 357, /* (87) db_options ::= */ - 357, /* (88) db_options ::= db_options BUFFER NK_INTEGER */ - 357, /* (89) db_options ::= db_options CACHEMODEL NK_STRING */ - 357, /* (90) db_options ::= db_options CACHESIZE NK_INTEGER */ - 357, /* (91) db_options ::= db_options COMP NK_INTEGER */ - 357, /* (92) db_options ::= db_options DURATION NK_INTEGER */ - 357, /* (93) db_options ::= db_options DURATION NK_VARIABLE */ - 357, /* (94) db_options ::= db_options MAXROWS NK_INTEGER */ - 357, /* (95) db_options ::= db_options MINROWS NK_INTEGER */ - 357, /* (96) db_options ::= db_options KEEP integer_list */ - 357, /* (97) db_options ::= db_options KEEP variable_list */ - 357, /* (98) db_options ::= db_options PAGES NK_INTEGER */ - 357, /* (99) db_options ::= db_options PAGESIZE NK_INTEGER */ - 357, /* (100) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ - 357, /* (101) db_options ::= db_options PRECISION NK_STRING */ - 357, /* (102) db_options ::= db_options REPLICA NK_INTEGER */ - 357, /* (103) db_options ::= db_options VGROUPS NK_INTEGER */ - 357, /* (104) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - 357, /* (105) db_options ::= db_options RETENTIONS retention_list */ - 357, /* (106) db_options ::= db_options SCHEMALESS NK_INTEGER */ - 357, /* (107) db_options ::= db_options WAL_LEVEL NK_INTEGER */ - 357, /* (108) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ - 357, /* (109) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ - 357, /* (110) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ - 357, /* (111) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ - 357, /* (112) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ - 357, /* (113) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ - 357, /* (114) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ - 357, /* (115) db_options ::= db_options STT_TRIGGER NK_INTEGER */ - 357, /* (116) db_options ::= db_options TABLE_PREFIX signed */ - 357, /* (117) db_options ::= db_options TABLE_SUFFIX signed */ - 359, /* (118) alter_db_options ::= alter_db_option */ - 359, /* (119) alter_db_options ::= alter_db_options alter_db_option */ - 367, /* (120) alter_db_option ::= BUFFER NK_INTEGER */ - 367, /* (121) alter_db_option ::= CACHEMODEL NK_STRING */ - 367, /* (122) alter_db_option ::= CACHESIZE NK_INTEGER */ - 367, /* (123) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ - 367, /* (124) alter_db_option ::= KEEP integer_list */ - 367, /* (125) alter_db_option ::= KEEP variable_list */ - 367, /* (126) alter_db_option ::= PAGES NK_INTEGER */ - 367, /* (127) alter_db_option ::= REPLICA NK_INTEGER */ - 367, /* (128) alter_db_option ::= WAL_LEVEL NK_INTEGER */ - 367, /* (129) alter_db_option ::= STT_TRIGGER NK_INTEGER */ - 367, /* (130) alter_db_option ::= MINROWS NK_INTEGER */ - 367, /* (131) alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ - 367, /* (132) alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ - 367, /* (133) alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ - 367, /* (134) alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ - 363, /* (135) integer_list ::= NK_INTEGER */ - 363, /* (136) integer_list ::= integer_list NK_COMMA NK_INTEGER */ - 364, /* (137) variable_list ::= NK_VARIABLE */ - 364, /* (138) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ - 365, /* (139) retention_list ::= retention */ - 365, /* (140) retention_list ::= retention_list NK_COMMA retention */ - 368, /* (141) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ - 360, /* (142) speed_opt ::= */ - 360, /* (143) speed_opt ::= MAX_SPEED NK_INTEGER */ - 361, /* (144) start_opt ::= */ - 361, /* (145) start_opt ::= START WITH NK_INTEGER */ - 361, /* (146) start_opt ::= START WITH NK_STRING */ - 361, /* (147) start_opt ::= START WITH TIMESTAMP NK_STRING */ - 362, /* (148) end_opt ::= */ - 362, /* (149) end_opt ::= END WITH NK_INTEGER */ - 362, /* (150) end_opt ::= END WITH NK_STRING */ - 362, /* (151) end_opt ::= END WITH TIMESTAMP NK_STRING */ - 337, /* (152) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - 337, /* (153) cmd ::= CREATE TABLE multi_create_clause */ - 337, /* (154) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - 337, /* (155) cmd ::= DROP TABLE multi_drop_clause */ - 337, /* (156) cmd ::= DROP STABLE exists_opt full_table_name */ - 337, /* (157) cmd ::= ALTER TABLE alter_table_clause */ - 337, /* (158) cmd ::= ALTER STABLE alter_table_clause */ - 376, /* (159) alter_table_clause ::= full_table_name alter_table_options */ - 376, /* (160) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ - 376, /* (161) alter_table_clause ::= full_table_name DROP COLUMN column_name */ - 376, /* (162) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ - 376, /* (163) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ - 376, /* (164) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ - 376, /* (165) alter_table_clause ::= full_table_name DROP TAG column_name */ - 376, /* (166) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ - 376, /* (167) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - 376, /* (168) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ - 373, /* (169) multi_create_clause ::= create_subtable_clause */ - 373, /* (170) multi_create_clause ::= multi_create_clause create_subtable_clause */ - 381, /* (171) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ - 375, /* (172) multi_drop_clause ::= drop_table_clause */ - 375, /* (173) multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ - 384, /* (174) drop_table_clause ::= exists_opt full_table_name */ - 382, /* (175) specific_cols_opt ::= */ - 382, /* (176) specific_cols_opt ::= NK_LP col_name_list NK_RP */ - 369, /* (177) full_table_name ::= table_name */ - 369, /* (178) full_table_name ::= db_name NK_DOT table_name */ - 370, /* (179) column_def_list ::= column_def */ - 370, /* (180) column_def_list ::= column_def_list NK_COMMA column_def */ - 386, /* (181) column_def ::= column_name type_name */ - 379, /* (182) type_name ::= BOOL */ - 379, /* (183) type_name ::= TINYINT */ - 379, /* (184) type_name ::= SMALLINT */ - 379, /* (185) type_name ::= INT */ - 379, /* (186) type_name ::= INTEGER */ - 379, /* (187) type_name ::= BIGINT */ - 379, /* (188) type_name ::= FLOAT */ - 379, /* (189) type_name ::= DOUBLE */ - 379, /* (190) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - 379, /* (191) type_name ::= TIMESTAMP */ - 379, /* (192) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - 379, /* (193) type_name ::= TINYINT UNSIGNED */ - 379, /* (194) type_name ::= SMALLINT UNSIGNED */ - 379, /* (195) type_name ::= INT UNSIGNED */ - 379, /* (196) type_name ::= BIGINT UNSIGNED */ - 379, /* (197) type_name ::= JSON */ - 379, /* (198) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - 379, /* (199) type_name ::= MEDIUMBLOB */ - 379, /* (200) type_name ::= BLOB */ - 379, /* (201) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - 379, /* (202) type_name ::= GEOMETRY NK_LP NK_INTEGER NK_RP */ - 379, /* (203) type_name ::= DECIMAL */ - 379, /* (204) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - 379, /* (205) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - 371, /* (206) tags_def_opt ::= */ - 371, /* (207) tags_def_opt ::= tags_def */ - 374, /* (208) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - 372, /* (209) table_options ::= */ - 372, /* (210) table_options ::= table_options COMMENT NK_STRING */ - 372, /* (211) table_options ::= table_options MAX_DELAY duration_list */ - 372, /* (212) table_options ::= table_options WATERMARK duration_list */ - 372, /* (213) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ - 372, /* (214) table_options ::= table_options TTL NK_INTEGER */ - 372, /* (215) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - 372, /* (216) table_options ::= table_options DELETE_MARK duration_list */ - 377, /* (217) alter_table_options ::= alter_table_option */ - 377, /* (218) alter_table_options ::= alter_table_options alter_table_option */ - 389, /* (219) alter_table_option ::= COMMENT NK_STRING */ - 389, /* (220) alter_table_option ::= TTL NK_INTEGER */ - 387, /* (221) duration_list ::= duration_literal */ - 387, /* (222) duration_list ::= duration_list NK_COMMA duration_literal */ - 388, /* (223) rollup_func_list ::= rollup_func_name */ - 388, /* (224) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ - 391, /* (225) rollup_func_name ::= function_name */ - 391, /* (226) rollup_func_name ::= FIRST */ - 391, /* (227) rollup_func_name ::= LAST */ - 385, /* (228) col_name_list ::= col_name */ - 385, /* (229) col_name_list ::= col_name_list NK_COMMA col_name */ - 393, /* (230) col_name ::= column_name */ - 337, /* (231) cmd ::= SHOW DNODES */ - 337, /* (232) cmd ::= SHOW USERS */ - 337, /* (233) cmd ::= SHOW USER PRIVILEGES */ - 337, /* (234) cmd ::= SHOW DATABASES */ - 337, /* (235) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - 337, /* (236) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - 337, /* (237) cmd ::= SHOW db_name_cond_opt VGROUPS */ - 337, /* (238) cmd ::= SHOW MNODES */ - 337, /* (239) cmd ::= SHOW QNODES */ - 337, /* (240) cmd ::= SHOW FUNCTIONS */ - 337, /* (241) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - 337, /* (242) cmd ::= SHOW STREAMS */ - 337, /* (243) cmd ::= SHOW ACCOUNTS */ - 337, /* (244) cmd ::= SHOW APPS */ - 337, /* (245) cmd ::= SHOW CONNECTIONS */ - 337, /* (246) cmd ::= SHOW LICENCES */ - 337, /* (247) cmd ::= SHOW GRANTS */ - 337, /* (248) cmd ::= SHOW CREATE DATABASE db_name */ - 337, /* (249) cmd ::= SHOW CREATE TABLE full_table_name */ - 337, /* (250) cmd ::= SHOW CREATE STABLE full_table_name */ - 337, /* (251) cmd ::= SHOW QUERIES */ - 337, /* (252) cmd ::= SHOW SCORES */ - 337, /* (253) cmd ::= SHOW TOPICS */ - 337, /* (254) cmd ::= SHOW VARIABLES */ - 337, /* (255) cmd ::= SHOW CLUSTER VARIABLES */ - 337, /* (256) cmd ::= SHOW LOCAL VARIABLES */ - 337, /* (257) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ - 337, /* (258) cmd ::= SHOW BNODES */ - 337, /* (259) cmd ::= SHOW SNODES */ - 337, /* (260) cmd ::= SHOW CLUSTER */ - 337, /* (261) cmd ::= SHOW TRANSACTIONS */ - 337, /* (262) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ - 337, /* (263) cmd ::= SHOW CONSUMERS */ - 337, /* (264) cmd ::= SHOW SUBSCRIPTIONS */ - 337, /* (265) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ - 337, /* (266) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ - 337, /* (267) cmd ::= SHOW VNODES NK_INTEGER */ - 337, /* (268) cmd ::= SHOW VNODES NK_STRING */ - 337, /* (269) cmd ::= SHOW db_name_cond_opt ALIVE */ - 337, /* (270) cmd ::= SHOW CLUSTER ALIVE */ - 394, /* (271) db_name_cond_opt ::= */ - 394, /* (272) db_name_cond_opt ::= db_name NK_DOT */ - 395, /* (273) like_pattern_opt ::= */ - 395, /* (274) like_pattern_opt ::= LIKE NK_STRING */ - 396, /* (275) table_name_cond ::= table_name */ - 397, /* (276) from_db_opt ::= */ - 397, /* (277) from_db_opt ::= FROM db_name */ - 398, /* (278) tag_list_opt ::= */ - 398, /* (279) tag_list_opt ::= tag_item */ - 398, /* (280) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ - 399, /* (281) tag_item ::= TBNAME */ - 399, /* (282) tag_item ::= QTAGS */ - 399, /* (283) tag_item ::= column_name */ - 399, /* (284) tag_item ::= column_name column_alias */ - 399, /* (285) tag_item ::= column_name AS column_alias */ - 337, /* (286) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ - 337, /* (287) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ - 337, /* (288) cmd ::= DROP INDEX exists_opt full_index_name */ - 401, /* (289) full_index_name ::= index_name */ - 401, /* (290) full_index_name ::= db_name NK_DOT index_name */ - 402, /* (291) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ - 402, /* (292) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ - 404, /* (293) func_list ::= func */ - 404, /* (294) func_list ::= func_list NK_COMMA func */ - 407, /* (295) func ::= sma_func_name NK_LP expression_list NK_RP */ - 408, /* (296) sma_func_name ::= function_name */ - 408, /* (297) sma_func_name ::= COUNT */ - 408, /* (298) sma_func_name ::= FIRST */ - 408, /* (299) sma_func_name ::= LAST */ - 408, /* (300) sma_func_name ::= LAST_ROW */ - 406, /* (301) sma_stream_opt ::= */ - 406, /* (302) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ - 406, /* (303) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ - 406, /* (304) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ - 337, /* (305) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ - 337, /* (306) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ - 337, /* (307) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - 337, /* (308) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name where_clause_opt */ - 337, /* (309) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name where_clause_opt */ - 337, /* (310) cmd ::= DROP TOPIC exists_opt topic_name */ - 337, /* (311) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ - 337, /* (312) cmd ::= DESC full_table_name */ - 337, /* (313) cmd ::= DESCRIBE full_table_name */ - 337, /* (314) cmd ::= RESET QUERY CACHE */ - 337, /* (315) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - 337, /* (316) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ - 412, /* (317) analyze_opt ::= */ - 412, /* (318) analyze_opt ::= ANALYZE */ - 413, /* (319) explain_options ::= */ - 413, /* (320) explain_options ::= explain_options VERBOSE NK_BOOL */ - 413, /* (321) explain_options ::= explain_options RATIO NK_FLOAT */ - 337, /* (322) cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ - 337, /* (323) cmd ::= DROP FUNCTION exists_opt function_name */ - 416, /* (324) agg_func_opt ::= */ - 416, /* (325) agg_func_opt ::= AGGREGATE */ - 417, /* (326) bufsize_opt ::= */ - 417, /* (327) bufsize_opt ::= BUFSIZE NK_INTEGER */ - 418, /* (328) language_opt ::= */ - 418, /* (329) language_opt ::= LANGUAGE NK_STRING */ - 415, /* (330) or_replace_opt ::= */ - 415, /* (331) or_replace_opt ::= OR REPLACE */ - 337, /* (332) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ - 337, /* (333) cmd ::= DROP STREAM exists_opt stream_name */ - 337, /* (334) cmd ::= PAUSE STREAM exists_opt stream_name */ - 337, /* (335) cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ - 421, /* (336) col_list_opt ::= */ - 421, /* (337) col_list_opt ::= NK_LP col_name_list NK_RP */ - 422, /* (338) tag_def_or_ref_opt ::= */ - 422, /* (339) tag_def_or_ref_opt ::= tags_def */ - 422, /* (340) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ - 420, /* (341) stream_options ::= */ - 420, /* (342) stream_options ::= stream_options TRIGGER AT_ONCE */ - 420, /* (343) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - 420, /* (344) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - 420, /* (345) stream_options ::= stream_options WATERMARK duration_literal */ - 420, /* (346) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ - 420, /* (347) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ - 420, /* (348) stream_options ::= stream_options DELETE_MARK duration_literal */ - 420, /* (349) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ - 423, /* (350) subtable_opt ::= */ - 423, /* (351) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - 424, /* (352) ignore_opt ::= */ - 424, /* (353) ignore_opt ::= IGNORE UNTREATED */ - 337, /* (354) cmd ::= KILL CONNECTION NK_INTEGER */ - 337, /* (355) cmd ::= KILL QUERY NK_STRING */ - 337, /* (356) cmd ::= KILL TRANSACTION NK_INTEGER */ - 337, /* (357) cmd ::= BALANCE VGROUP */ - 337, /* (358) cmd ::= BALANCE VGROUP LEADER */ - 337, /* (359) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - 337, /* (360) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - 337, /* (361) cmd ::= SPLIT VGROUP NK_INTEGER */ - 426, /* (362) dnode_list ::= DNODE NK_INTEGER */ - 426, /* (363) dnode_list ::= dnode_list DNODE NK_INTEGER */ - 337, /* (364) cmd ::= DELETE FROM full_table_name where_clause_opt */ - 337, /* (365) cmd ::= query_or_subquery */ - 337, /* (366) cmd ::= insert_query */ - 414, /* (367) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ - 414, /* (368) insert_query ::= INSERT INTO full_table_name query_or_subquery */ - 340, /* (369) literal ::= NK_INTEGER */ - 340, /* (370) literal ::= NK_FLOAT */ - 340, /* (371) literal ::= NK_STRING */ - 340, /* (372) literal ::= NK_BOOL */ - 340, /* (373) literal ::= TIMESTAMP NK_STRING */ - 340, /* (374) literal ::= duration_literal */ - 340, /* (375) literal ::= NULL */ - 340, /* (376) literal ::= NK_QUESTION */ - 390, /* (377) duration_literal ::= NK_VARIABLE */ - 366, /* (378) signed ::= NK_INTEGER */ - 366, /* (379) signed ::= NK_PLUS NK_INTEGER */ - 366, /* (380) signed ::= NK_MINUS NK_INTEGER */ - 366, /* (381) signed ::= NK_FLOAT */ - 366, /* (382) signed ::= NK_PLUS NK_FLOAT */ - 366, /* (383) signed ::= NK_MINUS NK_FLOAT */ - 380, /* (384) signed_literal ::= signed */ - 380, /* (385) signed_literal ::= NK_STRING */ - 380, /* (386) signed_literal ::= NK_BOOL */ - 380, /* (387) signed_literal ::= TIMESTAMP NK_STRING */ - 380, /* (388) signed_literal ::= duration_literal */ - 380, /* (389) signed_literal ::= NULL */ - 380, /* (390) signed_literal ::= literal_func */ - 380, /* (391) signed_literal ::= NK_QUESTION */ - 428, /* (392) literal_list ::= signed_literal */ - 428, /* (393) literal_list ::= literal_list NK_COMMA signed_literal */ - 349, /* (394) db_name ::= NK_ID */ - 350, /* (395) table_name ::= NK_ID */ - 378, /* (396) column_name ::= NK_ID */ - 392, /* (397) function_name ::= NK_ID */ - 429, /* (398) table_alias ::= NK_ID */ - 400, /* (399) column_alias ::= NK_ID */ - 342, /* (400) user_name ::= NK_ID */ - 351, /* (401) topic_name ::= NK_ID */ - 419, /* (402) stream_name ::= NK_ID */ - 411, /* (403) cgroup_name ::= NK_ID */ - 403, /* (404) index_name ::= NK_ID */ - 430, /* (405) expr_or_subquery ::= expression */ - 425, /* (406) expression ::= literal */ - 425, /* (407) expression ::= pseudo_column */ - 425, /* (408) expression ::= column_reference */ - 425, /* (409) expression ::= function_expression */ - 425, /* (410) expression ::= case_when_expression */ - 425, /* (411) expression ::= NK_LP expression NK_RP */ - 425, /* (412) expression ::= NK_PLUS expr_or_subquery */ - 425, /* (413) expression ::= NK_MINUS expr_or_subquery */ - 425, /* (414) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ - 425, /* (415) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ - 425, /* (416) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ - 425, /* (417) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ - 425, /* (418) expression ::= expr_or_subquery NK_REM expr_or_subquery */ - 425, /* (419) expression ::= column_reference NK_ARROW NK_STRING */ - 425, /* (420) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ - 425, /* (421) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ - 383, /* (422) expression_list ::= expr_or_subquery */ - 383, /* (423) expression_list ::= expression_list NK_COMMA expr_or_subquery */ - 432, /* (424) column_reference ::= column_name */ - 432, /* (425) column_reference ::= table_name NK_DOT column_name */ - 431, /* (426) pseudo_column ::= ROWTS */ - 431, /* (427) pseudo_column ::= TBNAME */ - 431, /* (428) pseudo_column ::= table_name NK_DOT TBNAME */ - 431, /* (429) pseudo_column ::= QSTART */ - 431, /* (430) pseudo_column ::= QEND */ - 431, /* (431) pseudo_column ::= QDURATION */ - 431, /* (432) pseudo_column ::= WSTART */ - 431, /* (433) pseudo_column ::= WEND */ - 431, /* (434) pseudo_column ::= WDURATION */ - 431, /* (435) pseudo_column ::= IROWTS */ - 431, /* (436) pseudo_column ::= ISFILLED */ - 431, /* (437) pseudo_column ::= QTAGS */ - 433, /* (438) function_expression ::= function_name NK_LP expression_list NK_RP */ - 433, /* (439) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - 433, /* (440) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ - 433, /* (441) function_expression ::= literal_func */ - 427, /* (442) literal_func ::= noarg_func NK_LP NK_RP */ - 427, /* (443) literal_func ::= NOW */ - 437, /* (444) noarg_func ::= NOW */ - 437, /* (445) noarg_func ::= TODAY */ - 437, /* (446) noarg_func ::= TIMEZONE */ - 437, /* (447) noarg_func ::= DATABASE */ - 437, /* (448) noarg_func ::= CLIENT_VERSION */ - 437, /* (449) noarg_func ::= SERVER_VERSION */ - 437, /* (450) noarg_func ::= SERVER_STATUS */ - 437, /* (451) noarg_func ::= CURRENT_USER */ - 437, /* (452) noarg_func ::= USER */ - 435, /* (453) star_func ::= COUNT */ - 435, /* (454) star_func ::= FIRST */ - 435, /* (455) star_func ::= LAST */ - 435, /* (456) star_func ::= LAST_ROW */ - 436, /* (457) star_func_para_list ::= NK_STAR */ - 436, /* (458) star_func_para_list ::= other_para_list */ - 438, /* (459) other_para_list ::= star_func_para */ - 438, /* (460) other_para_list ::= other_para_list NK_COMMA star_func_para */ - 439, /* (461) star_func_para ::= expr_or_subquery */ - 439, /* (462) star_func_para ::= table_name NK_DOT NK_STAR */ - 434, /* (463) case_when_expression ::= CASE when_then_list case_when_else_opt END */ - 434, /* (464) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ - 440, /* (465) when_then_list ::= when_then_expr */ - 440, /* (466) when_then_list ::= when_then_list when_then_expr */ - 443, /* (467) when_then_expr ::= WHEN common_expression THEN common_expression */ - 441, /* (468) case_when_else_opt ::= */ - 441, /* (469) case_when_else_opt ::= ELSE common_expression */ - 444, /* (470) predicate ::= expr_or_subquery compare_op expr_or_subquery */ - 444, /* (471) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ - 444, /* (472) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ - 444, /* (473) predicate ::= expr_or_subquery IS NULL */ - 444, /* (474) predicate ::= expr_or_subquery IS NOT NULL */ - 444, /* (475) predicate ::= expr_or_subquery in_op in_predicate_value */ - 445, /* (476) compare_op ::= NK_LT */ - 445, /* (477) compare_op ::= NK_GT */ - 445, /* (478) compare_op ::= NK_LE */ - 445, /* (479) compare_op ::= NK_GE */ - 445, /* (480) compare_op ::= NK_NE */ - 445, /* (481) compare_op ::= NK_EQ */ - 445, /* (482) compare_op ::= LIKE */ - 445, /* (483) compare_op ::= NOT LIKE */ - 445, /* (484) compare_op ::= MATCH */ - 445, /* (485) compare_op ::= NMATCH */ - 445, /* (486) compare_op ::= CONTAINS */ - 446, /* (487) in_op ::= IN */ - 446, /* (488) in_op ::= NOT IN */ - 447, /* (489) in_predicate_value ::= NK_LP literal_list NK_RP */ - 448, /* (490) boolean_value_expression ::= boolean_primary */ - 448, /* (491) boolean_value_expression ::= NOT boolean_primary */ - 448, /* (492) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - 448, /* (493) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - 449, /* (494) boolean_primary ::= predicate */ - 449, /* (495) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - 442, /* (496) common_expression ::= expr_or_subquery */ - 442, /* (497) common_expression ::= boolean_value_expression */ - 450, /* (498) from_clause_opt ::= */ - 450, /* (499) from_clause_opt ::= FROM table_reference_list */ - 451, /* (500) table_reference_list ::= table_reference */ - 451, /* (501) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - 452, /* (502) table_reference ::= table_primary */ - 452, /* (503) table_reference ::= joined_table */ - 453, /* (504) table_primary ::= table_name alias_opt */ - 453, /* (505) table_primary ::= db_name NK_DOT table_name alias_opt */ - 453, /* (506) table_primary ::= subquery alias_opt */ - 453, /* (507) table_primary ::= parenthesized_joined_table */ - 455, /* (508) alias_opt ::= */ - 455, /* (509) alias_opt ::= table_alias */ - 455, /* (510) alias_opt ::= AS table_alias */ - 457, /* (511) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - 457, /* (512) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - 454, /* (513) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - 458, /* (514) join_type ::= */ - 458, /* (515) join_type ::= INNER */ - 459, /* (516) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - 460, /* (517) set_quantifier_opt ::= */ - 460, /* (518) set_quantifier_opt ::= DISTINCT */ - 460, /* (519) set_quantifier_opt ::= ALL */ - 461, /* (520) select_list ::= select_item */ - 461, /* (521) select_list ::= select_list NK_COMMA select_item */ - 469, /* (522) select_item ::= NK_STAR */ - 469, /* (523) select_item ::= common_expression */ - 469, /* (524) select_item ::= common_expression column_alias */ - 469, /* (525) select_item ::= common_expression AS column_alias */ - 469, /* (526) select_item ::= table_name NK_DOT NK_STAR */ - 410, /* (527) where_clause_opt ::= */ - 410, /* (528) where_clause_opt ::= WHERE search_condition */ - 462, /* (529) partition_by_clause_opt ::= */ - 462, /* (530) partition_by_clause_opt ::= PARTITION BY partition_list */ - 470, /* (531) partition_list ::= partition_item */ - 470, /* (532) partition_list ::= partition_list NK_COMMA partition_item */ - 471, /* (533) partition_item ::= expr_or_subquery */ - 471, /* (534) partition_item ::= expr_or_subquery column_alias */ - 471, /* (535) partition_item ::= expr_or_subquery AS column_alias */ - 466, /* (536) twindow_clause_opt ::= */ - 466, /* (537) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - 466, /* (538) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ - 466, /* (539) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - 466, /* (540) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - 466, /* (541) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ - 405, /* (542) sliding_opt ::= */ - 405, /* (543) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - 465, /* (544) fill_opt ::= */ - 465, /* (545) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - 465, /* (546) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ - 465, /* (547) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ - 472, /* (548) fill_mode ::= NONE */ - 472, /* (549) fill_mode ::= PREV */ - 472, /* (550) fill_mode ::= NULL */ - 472, /* (551) fill_mode ::= NULL_F */ - 472, /* (552) fill_mode ::= LINEAR */ - 472, /* (553) fill_mode ::= NEXT */ - 467, /* (554) group_by_clause_opt ::= */ - 467, /* (555) group_by_clause_opt ::= GROUP BY group_by_list */ - 473, /* (556) group_by_list ::= expr_or_subquery */ - 473, /* (557) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ - 468, /* (558) having_clause_opt ::= */ - 468, /* (559) having_clause_opt ::= HAVING search_condition */ - 463, /* (560) range_opt ::= */ - 463, /* (561) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - 463, /* (562) range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ - 464, /* (563) every_opt ::= */ - 464, /* (564) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - 474, /* (565) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - 475, /* (566) query_simple ::= query_specification */ - 475, /* (567) query_simple ::= union_query_expression */ - 479, /* (568) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - 479, /* (569) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - 480, /* (570) query_simple_or_subquery ::= query_simple */ - 480, /* (571) query_simple_or_subquery ::= subquery */ - 409, /* (572) query_or_subquery ::= query_expression */ - 409, /* (573) query_or_subquery ::= subquery */ - 476, /* (574) order_by_clause_opt ::= */ - 476, /* (575) order_by_clause_opt ::= ORDER BY sort_specification_list */ - 477, /* (576) slimit_clause_opt ::= */ - 477, /* (577) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - 477, /* (578) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - 477, /* (579) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - 478, /* (580) limit_clause_opt ::= */ - 478, /* (581) limit_clause_opt ::= LIMIT NK_INTEGER */ - 478, /* (582) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - 478, /* (583) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - 456, /* (584) subquery ::= NK_LP query_expression NK_RP */ - 456, /* (585) subquery ::= NK_LP subquery NK_RP */ - 352, /* (586) search_condition ::= common_expression */ - 481, /* (587) sort_specification_list ::= sort_specification */ - 481, /* (588) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - 482, /* (589) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - 483, /* (590) ordering_specification_opt ::= */ - 483, /* (591) ordering_specification_opt ::= ASC */ - 483, /* (592) ordering_specification_opt ::= DESC */ - 484, /* (593) null_ordering_opt ::= */ - 484, /* (594) null_ordering_opt ::= NULLS FIRST */ - 484, /* (595) null_ordering_opt ::= NULLS LAST */ + 338, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ + 338, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ + 339, /* (2) account_options ::= */ + 339, /* (3) account_options ::= account_options PPS literal */ + 339, /* (4) account_options ::= account_options TSERIES literal */ + 339, /* (5) account_options ::= account_options STORAGE literal */ + 339, /* (6) account_options ::= account_options STREAMS literal */ + 339, /* (7) account_options ::= account_options QTIME literal */ + 339, /* (8) account_options ::= account_options DBS literal */ + 339, /* (9) account_options ::= account_options USERS literal */ + 339, /* (10) account_options ::= account_options CONNS literal */ + 339, /* (11) account_options ::= account_options STATE literal */ + 340, /* (12) alter_account_options ::= alter_account_option */ + 340, /* (13) alter_account_options ::= alter_account_options alter_account_option */ + 342, /* (14) alter_account_option ::= PASS literal */ + 342, /* (15) alter_account_option ::= PPS literal */ + 342, /* (16) alter_account_option ::= TSERIES literal */ + 342, /* (17) alter_account_option ::= STORAGE literal */ + 342, /* (18) alter_account_option ::= STREAMS literal */ + 342, /* (19) alter_account_option ::= QTIME literal */ + 342, /* (20) alter_account_option ::= DBS literal */ + 342, /* (21) alter_account_option ::= USERS literal */ + 342, /* (22) alter_account_option ::= CONNS literal */ + 342, /* (23) alter_account_option ::= STATE literal */ + 338, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ + 338, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ + 338, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ + 338, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ + 338, /* (28) cmd ::= DROP USER user_name */ + 344, /* (29) sysinfo_opt ::= */ + 344, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ + 338, /* (31) cmd ::= GRANT privileges ON priv_level with_opt TO user_name */ + 338, /* (32) cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name */ + 345, /* (33) privileges ::= ALL */ + 345, /* (34) privileges ::= priv_type_list */ + 345, /* (35) privileges ::= SUBSCRIBE */ + 348, /* (36) priv_type_list ::= priv_type */ + 348, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ + 349, /* (38) priv_type ::= READ */ + 349, /* (39) priv_type ::= WRITE */ + 346, /* (40) priv_level ::= NK_STAR NK_DOT NK_STAR */ + 346, /* (41) priv_level ::= db_name NK_DOT NK_STAR */ + 346, /* (42) priv_level ::= db_name NK_DOT table_name */ + 346, /* (43) priv_level ::= topic_name */ + 347, /* (44) with_opt ::= */ + 347, /* (45) with_opt ::= WITH search_condition */ + 338, /* (46) cmd ::= CREATE DNODE dnode_endpoint */ + 338, /* (47) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ + 338, /* (48) cmd ::= DROP DNODE NK_INTEGER force_opt */ + 338, /* (49) cmd ::= DROP DNODE dnode_endpoint force_opt */ + 338, /* (50) cmd ::= DROP DNODE NK_INTEGER unsafe_opt */ + 338, /* (51) cmd ::= DROP DNODE dnode_endpoint unsafe_opt */ + 338, /* (52) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + 338, /* (53) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + 338, /* (54) cmd ::= ALTER ALL DNODES NK_STRING */ + 338, /* (55) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + 338, /* (56) cmd ::= RESTORE DNODE NK_INTEGER */ + 354, /* (57) dnode_endpoint ::= NK_STRING */ + 354, /* (58) dnode_endpoint ::= NK_ID */ + 354, /* (59) dnode_endpoint ::= NK_IPTOKEN */ + 355, /* (60) force_opt ::= */ + 355, /* (61) force_opt ::= FORCE */ + 356, /* (62) unsafe_opt ::= UNSAFE */ + 338, /* (63) cmd ::= ALTER LOCAL NK_STRING */ + 338, /* (64) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + 338, /* (65) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + 338, /* (66) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + 338, /* (67) cmd ::= RESTORE QNODE ON DNODE NK_INTEGER */ + 338, /* (68) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + 338, /* (69) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + 338, /* (70) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + 338, /* (71) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + 338, /* (72) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + 338, /* (73) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + 338, /* (74) cmd ::= RESTORE MNODE ON DNODE NK_INTEGER */ + 338, /* (75) cmd ::= RESTORE VNODE ON DNODE NK_INTEGER */ + 338, /* (76) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + 338, /* (77) cmd ::= DROP DATABASE exists_opt db_name */ + 338, /* (78) cmd ::= USE db_name */ + 338, /* (79) cmd ::= ALTER DATABASE db_name alter_db_options */ + 338, /* (80) cmd ::= FLUSH DATABASE db_name */ + 338, /* (81) cmd ::= TRIM DATABASE db_name speed_opt */ + 338, /* (82) cmd ::= COMPACT DATABASE db_name start_opt end_opt */ + 357, /* (83) not_exists_opt ::= IF NOT EXISTS */ + 357, /* (84) not_exists_opt ::= */ + 359, /* (85) exists_opt ::= IF EXISTS */ + 359, /* (86) exists_opt ::= */ + 358, /* (87) db_options ::= */ + 358, /* (88) db_options ::= db_options BUFFER NK_INTEGER */ + 358, /* (89) db_options ::= db_options CACHEMODEL NK_STRING */ + 358, /* (90) db_options ::= db_options CACHESIZE NK_INTEGER */ + 358, /* (91) db_options ::= db_options COMP NK_INTEGER */ + 358, /* (92) db_options ::= db_options DURATION NK_INTEGER */ + 358, /* (93) db_options ::= db_options DURATION NK_VARIABLE */ + 358, /* (94) db_options ::= db_options MAXROWS NK_INTEGER */ + 358, /* (95) db_options ::= db_options MINROWS NK_INTEGER */ + 358, /* (96) db_options ::= db_options KEEP integer_list */ + 358, /* (97) db_options ::= db_options KEEP variable_list */ + 358, /* (98) db_options ::= db_options PAGES NK_INTEGER */ + 358, /* (99) db_options ::= db_options PAGESIZE NK_INTEGER */ + 358, /* (100) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ + 358, /* (101) db_options ::= db_options PRECISION NK_STRING */ + 358, /* (102) db_options ::= db_options REPLICA NK_INTEGER */ + 358, /* (103) db_options ::= db_options VGROUPS NK_INTEGER */ + 358, /* (104) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + 358, /* (105) db_options ::= db_options RETENTIONS retention_list */ + 358, /* (106) db_options ::= db_options SCHEMALESS NK_INTEGER */ + 358, /* (107) db_options ::= db_options WAL_LEVEL NK_INTEGER */ + 358, /* (108) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ + 358, /* (109) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ + 358, /* (110) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + 358, /* (111) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ + 358, /* (112) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + 358, /* (113) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ + 358, /* (114) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ + 358, /* (115) db_options ::= db_options STT_TRIGGER NK_INTEGER */ + 358, /* (116) db_options ::= db_options TABLE_PREFIX signed */ + 358, /* (117) db_options ::= db_options TABLE_SUFFIX signed */ + 360, /* (118) alter_db_options ::= alter_db_option */ + 360, /* (119) alter_db_options ::= alter_db_options alter_db_option */ + 368, /* (120) alter_db_option ::= BUFFER NK_INTEGER */ + 368, /* (121) alter_db_option ::= CACHEMODEL NK_STRING */ + 368, /* (122) alter_db_option ::= CACHESIZE NK_INTEGER */ + 368, /* (123) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ + 368, /* (124) alter_db_option ::= KEEP integer_list */ + 368, /* (125) alter_db_option ::= KEEP variable_list */ + 368, /* (126) alter_db_option ::= PAGES NK_INTEGER */ + 368, /* (127) alter_db_option ::= REPLICA NK_INTEGER */ + 368, /* (128) alter_db_option ::= WAL_LEVEL NK_INTEGER */ + 368, /* (129) alter_db_option ::= STT_TRIGGER NK_INTEGER */ + 368, /* (130) alter_db_option ::= MINROWS NK_INTEGER */ + 368, /* (131) alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ + 368, /* (132) alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + 368, /* (133) alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ + 368, /* (134) alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + 364, /* (135) integer_list ::= NK_INTEGER */ + 364, /* (136) integer_list ::= integer_list NK_COMMA NK_INTEGER */ + 365, /* (137) variable_list ::= NK_VARIABLE */ + 365, /* (138) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ + 366, /* (139) retention_list ::= retention */ + 366, /* (140) retention_list ::= retention_list NK_COMMA retention */ + 369, /* (141) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ + 361, /* (142) speed_opt ::= */ + 361, /* (143) speed_opt ::= MAX_SPEED NK_INTEGER */ + 362, /* (144) start_opt ::= */ + 362, /* (145) start_opt ::= START WITH NK_INTEGER */ + 362, /* (146) start_opt ::= START WITH NK_STRING */ + 362, /* (147) start_opt ::= START WITH TIMESTAMP NK_STRING */ + 363, /* (148) end_opt ::= */ + 363, /* (149) end_opt ::= END WITH NK_INTEGER */ + 363, /* (150) end_opt ::= END WITH NK_STRING */ + 363, /* (151) end_opt ::= END WITH TIMESTAMP NK_STRING */ + 338, /* (152) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + 338, /* (153) cmd ::= CREATE TABLE multi_create_clause */ + 338, /* (154) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + 338, /* (155) cmd ::= DROP TABLE multi_drop_clause */ + 338, /* (156) cmd ::= DROP STABLE exists_opt full_table_name */ + 338, /* (157) cmd ::= ALTER TABLE alter_table_clause */ + 338, /* (158) cmd ::= ALTER STABLE alter_table_clause */ + 377, /* (159) alter_table_clause ::= full_table_name alter_table_options */ + 377, /* (160) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + 377, /* (161) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + 377, /* (162) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + 377, /* (163) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + 377, /* (164) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + 377, /* (165) alter_table_clause ::= full_table_name DROP TAG column_name */ + 377, /* (166) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + 377, /* (167) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + 377, /* (168) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ + 374, /* (169) multi_create_clause ::= create_subtable_clause */ + 374, /* (170) multi_create_clause ::= multi_create_clause create_subtable_clause */ + 382, /* (171) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ + 376, /* (172) multi_drop_clause ::= drop_table_clause */ + 376, /* (173) multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ + 385, /* (174) drop_table_clause ::= exists_opt full_table_name */ + 383, /* (175) specific_cols_opt ::= */ + 383, /* (176) specific_cols_opt ::= NK_LP col_name_list NK_RP */ + 370, /* (177) full_table_name ::= table_name */ + 370, /* (178) full_table_name ::= db_name NK_DOT table_name */ + 371, /* (179) column_def_list ::= column_def */ + 371, /* (180) column_def_list ::= column_def_list NK_COMMA column_def */ + 387, /* (181) column_def ::= column_name type_name */ + 380, /* (182) type_name ::= BOOL */ + 380, /* (183) type_name ::= TINYINT */ + 380, /* (184) type_name ::= SMALLINT */ + 380, /* (185) type_name ::= INT */ + 380, /* (186) type_name ::= INTEGER */ + 380, /* (187) type_name ::= BIGINT */ + 380, /* (188) type_name ::= FLOAT */ + 380, /* (189) type_name ::= DOUBLE */ + 380, /* (190) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + 380, /* (191) type_name ::= TIMESTAMP */ + 380, /* (192) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + 380, /* (193) type_name ::= TINYINT UNSIGNED */ + 380, /* (194) type_name ::= SMALLINT UNSIGNED */ + 380, /* (195) type_name ::= INT UNSIGNED */ + 380, /* (196) type_name ::= BIGINT UNSIGNED */ + 380, /* (197) type_name ::= JSON */ + 380, /* (198) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + 380, /* (199) type_name ::= MEDIUMBLOB */ + 380, /* (200) type_name ::= BLOB */ + 380, /* (201) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + 380, /* (202) type_name ::= GEOMETRY NK_LP NK_INTEGER NK_RP */ + 380, /* (203) type_name ::= DECIMAL */ + 380, /* (204) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + 380, /* (205) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + 372, /* (206) tags_def_opt ::= */ + 372, /* (207) tags_def_opt ::= tags_def */ + 375, /* (208) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + 373, /* (209) table_options ::= */ + 373, /* (210) table_options ::= table_options COMMENT NK_STRING */ + 373, /* (211) table_options ::= table_options MAX_DELAY duration_list */ + 373, /* (212) table_options ::= table_options WATERMARK duration_list */ + 373, /* (213) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ + 373, /* (214) table_options ::= table_options TTL NK_INTEGER */ + 373, /* (215) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + 373, /* (216) table_options ::= table_options DELETE_MARK duration_list */ + 378, /* (217) alter_table_options ::= alter_table_option */ + 378, /* (218) alter_table_options ::= alter_table_options alter_table_option */ + 390, /* (219) alter_table_option ::= COMMENT NK_STRING */ + 390, /* (220) alter_table_option ::= TTL NK_INTEGER */ + 388, /* (221) duration_list ::= duration_literal */ + 388, /* (222) duration_list ::= duration_list NK_COMMA duration_literal */ + 389, /* (223) rollup_func_list ::= rollup_func_name */ + 389, /* (224) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ + 392, /* (225) rollup_func_name ::= function_name */ + 392, /* (226) rollup_func_name ::= FIRST */ + 392, /* (227) rollup_func_name ::= LAST */ + 386, /* (228) col_name_list ::= col_name */ + 386, /* (229) col_name_list ::= col_name_list NK_COMMA col_name */ + 394, /* (230) col_name ::= column_name */ + 338, /* (231) cmd ::= SHOW DNODES */ + 338, /* (232) cmd ::= SHOW USERS */ + 338, /* (233) cmd ::= SHOW USER PRIVILEGES */ + 338, /* (234) cmd ::= SHOW DATABASES */ + 338, /* (235) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + 338, /* (236) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + 338, /* (237) cmd ::= SHOW db_name_cond_opt VGROUPS */ + 338, /* (238) cmd ::= SHOW MNODES */ + 338, /* (239) cmd ::= SHOW QNODES */ + 338, /* (240) cmd ::= SHOW FUNCTIONS */ + 338, /* (241) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + 338, /* (242) cmd ::= SHOW INDEXES FROM db_name NK_DOT table_name */ + 338, /* (243) cmd ::= SHOW STREAMS */ + 338, /* (244) cmd ::= SHOW ACCOUNTS */ + 338, /* (245) cmd ::= SHOW APPS */ + 338, /* (246) cmd ::= SHOW CONNECTIONS */ + 338, /* (247) cmd ::= SHOW LICENCES */ + 338, /* (248) cmd ::= SHOW GRANTS */ + 338, /* (249) cmd ::= SHOW CREATE DATABASE db_name */ + 338, /* (250) cmd ::= SHOW CREATE TABLE full_table_name */ + 338, /* (251) cmd ::= SHOW CREATE STABLE full_table_name */ + 338, /* (252) cmd ::= SHOW QUERIES */ + 338, /* (253) cmd ::= SHOW SCORES */ + 338, /* (254) cmd ::= SHOW TOPICS */ + 338, /* (255) cmd ::= SHOW VARIABLES */ + 338, /* (256) cmd ::= SHOW CLUSTER VARIABLES */ + 338, /* (257) cmd ::= SHOW LOCAL VARIABLES */ + 338, /* (258) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ + 338, /* (259) cmd ::= SHOW BNODES */ + 338, /* (260) cmd ::= SHOW SNODES */ + 338, /* (261) cmd ::= SHOW CLUSTER */ + 338, /* (262) cmd ::= SHOW TRANSACTIONS */ + 338, /* (263) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ + 338, /* (264) cmd ::= SHOW CONSUMERS */ + 338, /* (265) cmd ::= SHOW SUBSCRIPTIONS */ + 338, /* (266) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ + 338, /* (267) cmd ::= SHOW TAGS FROM db_name NK_DOT table_name */ + 338, /* (268) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ + 338, /* (269) cmd ::= SHOW TABLE TAGS tag_list_opt FROM db_name NK_DOT table_name */ + 338, /* (270) cmd ::= SHOW VNODES NK_INTEGER */ + 338, /* (271) cmd ::= SHOW VNODES NK_STRING */ + 338, /* (272) cmd ::= SHOW db_name_cond_opt ALIVE */ + 338, /* (273) cmd ::= SHOW CLUSTER ALIVE */ + 395, /* (274) db_name_cond_opt ::= */ + 395, /* (275) db_name_cond_opt ::= db_name NK_DOT */ + 396, /* (276) like_pattern_opt ::= */ + 396, /* (277) like_pattern_opt ::= LIKE NK_STRING */ + 397, /* (278) table_name_cond ::= table_name */ + 398, /* (279) from_db_opt ::= */ + 398, /* (280) from_db_opt ::= FROM db_name */ + 399, /* (281) tag_list_opt ::= */ + 399, /* (282) tag_list_opt ::= tag_item */ + 399, /* (283) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ + 400, /* (284) tag_item ::= TBNAME */ + 400, /* (285) tag_item ::= QTAGS */ + 400, /* (286) tag_item ::= column_name */ + 400, /* (287) tag_item ::= column_name column_alias */ + 400, /* (288) tag_item ::= column_name AS column_alias */ + 338, /* (289) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ + 338, /* (290) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ + 338, /* (291) cmd ::= DROP INDEX exists_opt full_index_name */ + 402, /* (292) full_index_name ::= index_name */ + 402, /* (293) full_index_name ::= db_name NK_DOT index_name */ + 403, /* (294) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ + 403, /* (295) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ + 405, /* (296) func_list ::= func */ + 405, /* (297) func_list ::= func_list NK_COMMA func */ + 408, /* (298) func ::= sma_func_name NK_LP expression_list NK_RP */ + 409, /* (299) sma_func_name ::= function_name */ + 409, /* (300) sma_func_name ::= COUNT */ + 409, /* (301) sma_func_name ::= FIRST */ + 409, /* (302) sma_func_name ::= LAST */ + 409, /* (303) sma_func_name ::= LAST_ROW */ + 407, /* (304) sma_stream_opt ::= */ + 407, /* (305) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + 407, /* (306) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ + 407, /* (307) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ + 410, /* (308) with_meta ::= AS */ + 410, /* (309) with_meta ::= WITH META AS */ + 410, /* (310) with_meta ::= ONLY META AS */ + 338, /* (311) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ + 338, /* (312) cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta DATABASE db_name */ + 338, /* (313) cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta STABLE full_table_name where_clause_opt */ + 338, /* (314) cmd ::= DROP TOPIC exists_opt topic_name */ + 338, /* (315) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ + 338, /* (316) cmd ::= DESC full_table_name */ + 338, /* (317) cmd ::= DESCRIBE full_table_name */ + 338, /* (318) cmd ::= RESET QUERY CACHE */ + 338, /* (319) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + 338, /* (320) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ + 414, /* (321) analyze_opt ::= */ + 414, /* (322) analyze_opt ::= ANALYZE */ + 415, /* (323) explain_options ::= */ + 415, /* (324) explain_options ::= explain_options VERBOSE NK_BOOL */ + 415, /* (325) explain_options ::= explain_options RATIO NK_FLOAT */ + 338, /* (326) cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ + 338, /* (327) cmd ::= DROP FUNCTION exists_opt function_name */ + 418, /* (328) agg_func_opt ::= */ + 418, /* (329) agg_func_opt ::= AGGREGATE */ + 419, /* (330) bufsize_opt ::= */ + 419, /* (331) bufsize_opt ::= BUFSIZE NK_INTEGER */ + 420, /* (332) language_opt ::= */ + 420, /* (333) language_opt ::= LANGUAGE NK_STRING */ + 417, /* (334) or_replace_opt ::= */ + 417, /* (335) or_replace_opt ::= OR REPLACE */ + 338, /* (336) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ + 338, /* (337) cmd ::= DROP STREAM exists_opt stream_name */ + 338, /* (338) cmd ::= PAUSE STREAM exists_opt stream_name */ + 338, /* (339) cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ + 423, /* (340) col_list_opt ::= */ + 423, /* (341) col_list_opt ::= NK_LP col_name_list NK_RP */ + 424, /* (342) tag_def_or_ref_opt ::= */ + 424, /* (343) tag_def_or_ref_opt ::= tags_def */ + 424, /* (344) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ + 422, /* (345) stream_options ::= */ + 422, /* (346) stream_options ::= stream_options TRIGGER AT_ONCE */ + 422, /* (347) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + 422, /* (348) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + 422, /* (349) stream_options ::= stream_options WATERMARK duration_literal */ + 422, /* (350) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + 422, /* (351) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ + 422, /* (352) stream_options ::= stream_options DELETE_MARK duration_literal */ + 422, /* (353) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ + 425, /* (354) subtable_opt ::= */ + 425, /* (355) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + 426, /* (356) ignore_opt ::= */ + 426, /* (357) ignore_opt ::= IGNORE UNTREATED */ + 338, /* (358) cmd ::= KILL CONNECTION NK_INTEGER */ + 338, /* (359) cmd ::= KILL QUERY NK_STRING */ + 338, /* (360) cmd ::= KILL TRANSACTION NK_INTEGER */ + 338, /* (361) cmd ::= BALANCE VGROUP */ + 338, /* (362) cmd ::= BALANCE VGROUP LEADER */ + 338, /* (363) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + 338, /* (364) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + 338, /* (365) cmd ::= SPLIT VGROUP NK_INTEGER */ + 428, /* (366) dnode_list ::= DNODE NK_INTEGER */ + 428, /* (367) dnode_list ::= dnode_list DNODE NK_INTEGER */ + 338, /* (368) cmd ::= DELETE FROM full_table_name where_clause_opt */ + 338, /* (369) cmd ::= query_or_subquery */ + 338, /* (370) cmd ::= insert_query */ + 416, /* (371) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ + 416, /* (372) insert_query ::= INSERT INTO full_table_name query_or_subquery */ + 341, /* (373) literal ::= NK_INTEGER */ + 341, /* (374) literal ::= NK_FLOAT */ + 341, /* (375) literal ::= NK_STRING */ + 341, /* (376) literal ::= NK_BOOL */ + 341, /* (377) literal ::= TIMESTAMP NK_STRING */ + 341, /* (378) literal ::= duration_literal */ + 341, /* (379) literal ::= NULL */ + 341, /* (380) literal ::= NK_QUESTION */ + 391, /* (381) duration_literal ::= NK_VARIABLE */ + 367, /* (382) signed ::= NK_INTEGER */ + 367, /* (383) signed ::= NK_PLUS NK_INTEGER */ + 367, /* (384) signed ::= NK_MINUS NK_INTEGER */ + 367, /* (385) signed ::= NK_FLOAT */ + 367, /* (386) signed ::= NK_PLUS NK_FLOAT */ + 367, /* (387) signed ::= NK_MINUS NK_FLOAT */ + 381, /* (388) signed_literal ::= signed */ + 381, /* (389) signed_literal ::= NK_STRING */ + 381, /* (390) signed_literal ::= NK_BOOL */ + 381, /* (391) signed_literal ::= TIMESTAMP NK_STRING */ + 381, /* (392) signed_literal ::= duration_literal */ + 381, /* (393) signed_literal ::= NULL */ + 381, /* (394) signed_literal ::= literal_func */ + 381, /* (395) signed_literal ::= NK_QUESTION */ + 430, /* (396) literal_list ::= signed_literal */ + 430, /* (397) literal_list ::= literal_list NK_COMMA signed_literal */ + 350, /* (398) db_name ::= NK_ID */ + 351, /* (399) table_name ::= NK_ID */ + 379, /* (400) column_name ::= NK_ID */ + 393, /* (401) function_name ::= NK_ID */ + 431, /* (402) table_alias ::= NK_ID */ + 401, /* (403) column_alias ::= NK_ID */ + 343, /* (404) user_name ::= NK_ID */ + 352, /* (405) topic_name ::= NK_ID */ + 421, /* (406) stream_name ::= NK_ID */ + 413, /* (407) cgroup_name ::= NK_ID */ + 404, /* (408) index_name ::= NK_ID */ + 432, /* (409) expr_or_subquery ::= expression */ + 427, /* (410) expression ::= literal */ + 427, /* (411) expression ::= pseudo_column */ + 427, /* (412) expression ::= column_reference */ + 427, /* (413) expression ::= function_expression */ + 427, /* (414) expression ::= case_when_expression */ + 427, /* (415) expression ::= NK_LP expression NK_RP */ + 427, /* (416) expression ::= NK_PLUS expr_or_subquery */ + 427, /* (417) expression ::= NK_MINUS expr_or_subquery */ + 427, /* (418) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + 427, /* (419) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + 427, /* (420) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + 427, /* (421) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + 427, /* (422) expression ::= expr_or_subquery NK_REM expr_or_subquery */ + 427, /* (423) expression ::= column_reference NK_ARROW NK_STRING */ + 427, /* (424) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + 427, /* (425) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + 384, /* (426) expression_list ::= expr_or_subquery */ + 384, /* (427) expression_list ::= expression_list NK_COMMA expr_or_subquery */ + 434, /* (428) column_reference ::= column_name */ + 434, /* (429) column_reference ::= table_name NK_DOT column_name */ + 433, /* (430) pseudo_column ::= ROWTS */ + 433, /* (431) pseudo_column ::= TBNAME */ + 433, /* (432) pseudo_column ::= table_name NK_DOT TBNAME */ + 433, /* (433) pseudo_column ::= QSTART */ + 433, /* (434) pseudo_column ::= QEND */ + 433, /* (435) pseudo_column ::= QDURATION */ + 433, /* (436) pseudo_column ::= WSTART */ + 433, /* (437) pseudo_column ::= WEND */ + 433, /* (438) pseudo_column ::= WDURATION */ + 433, /* (439) pseudo_column ::= IROWTS */ + 433, /* (440) pseudo_column ::= ISFILLED */ + 433, /* (441) pseudo_column ::= QTAGS */ + 435, /* (442) function_expression ::= function_name NK_LP expression_list NK_RP */ + 435, /* (443) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + 435, /* (444) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ + 435, /* (445) function_expression ::= literal_func */ + 429, /* (446) literal_func ::= noarg_func NK_LP NK_RP */ + 429, /* (447) literal_func ::= NOW */ + 439, /* (448) noarg_func ::= NOW */ + 439, /* (449) noarg_func ::= TODAY */ + 439, /* (450) noarg_func ::= TIMEZONE */ + 439, /* (451) noarg_func ::= DATABASE */ + 439, /* (452) noarg_func ::= CLIENT_VERSION */ + 439, /* (453) noarg_func ::= SERVER_VERSION */ + 439, /* (454) noarg_func ::= SERVER_STATUS */ + 439, /* (455) noarg_func ::= CURRENT_USER */ + 439, /* (456) noarg_func ::= USER */ + 437, /* (457) star_func ::= COUNT */ + 437, /* (458) star_func ::= FIRST */ + 437, /* (459) star_func ::= LAST */ + 437, /* (460) star_func ::= LAST_ROW */ + 438, /* (461) star_func_para_list ::= NK_STAR */ + 438, /* (462) star_func_para_list ::= other_para_list */ + 440, /* (463) other_para_list ::= star_func_para */ + 440, /* (464) other_para_list ::= other_para_list NK_COMMA star_func_para */ + 441, /* (465) star_func_para ::= expr_or_subquery */ + 441, /* (466) star_func_para ::= table_name NK_DOT NK_STAR */ + 436, /* (467) case_when_expression ::= CASE when_then_list case_when_else_opt END */ + 436, /* (468) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ + 442, /* (469) when_then_list ::= when_then_expr */ + 442, /* (470) when_then_list ::= when_then_list when_then_expr */ + 445, /* (471) when_then_expr ::= WHEN common_expression THEN common_expression */ + 443, /* (472) case_when_else_opt ::= */ + 443, /* (473) case_when_else_opt ::= ELSE common_expression */ + 446, /* (474) predicate ::= expr_or_subquery compare_op expr_or_subquery */ + 446, /* (475) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + 446, /* (476) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + 446, /* (477) predicate ::= expr_or_subquery IS NULL */ + 446, /* (478) predicate ::= expr_or_subquery IS NOT NULL */ + 446, /* (479) predicate ::= expr_or_subquery in_op in_predicate_value */ + 447, /* (480) compare_op ::= NK_LT */ + 447, /* (481) compare_op ::= NK_GT */ + 447, /* (482) compare_op ::= NK_LE */ + 447, /* (483) compare_op ::= NK_GE */ + 447, /* (484) compare_op ::= NK_NE */ + 447, /* (485) compare_op ::= NK_EQ */ + 447, /* (486) compare_op ::= LIKE */ + 447, /* (487) compare_op ::= NOT LIKE */ + 447, /* (488) compare_op ::= MATCH */ + 447, /* (489) compare_op ::= NMATCH */ + 447, /* (490) compare_op ::= CONTAINS */ + 448, /* (491) in_op ::= IN */ + 448, /* (492) in_op ::= NOT IN */ + 449, /* (493) in_predicate_value ::= NK_LP literal_list NK_RP */ + 450, /* (494) boolean_value_expression ::= boolean_primary */ + 450, /* (495) boolean_value_expression ::= NOT boolean_primary */ + 450, /* (496) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + 450, /* (497) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + 451, /* (498) boolean_primary ::= predicate */ + 451, /* (499) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + 444, /* (500) common_expression ::= expr_or_subquery */ + 444, /* (501) common_expression ::= boolean_value_expression */ + 452, /* (502) from_clause_opt ::= */ + 452, /* (503) from_clause_opt ::= FROM table_reference_list */ + 453, /* (504) table_reference_list ::= table_reference */ + 453, /* (505) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + 454, /* (506) table_reference ::= table_primary */ + 454, /* (507) table_reference ::= joined_table */ + 455, /* (508) table_primary ::= table_name alias_opt */ + 455, /* (509) table_primary ::= db_name NK_DOT table_name alias_opt */ + 455, /* (510) table_primary ::= subquery alias_opt */ + 455, /* (511) table_primary ::= parenthesized_joined_table */ + 457, /* (512) alias_opt ::= */ + 457, /* (513) alias_opt ::= table_alias */ + 457, /* (514) alias_opt ::= AS table_alias */ + 459, /* (515) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + 459, /* (516) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + 456, /* (517) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + 460, /* (518) join_type ::= */ + 460, /* (519) join_type ::= INNER */ + 461, /* (520) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + 462, /* (521) set_quantifier_opt ::= */ + 462, /* (522) set_quantifier_opt ::= DISTINCT */ + 462, /* (523) set_quantifier_opt ::= ALL */ + 463, /* (524) select_list ::= select_item */ + 463, /* (525) select_list ::= select_list NK_COMMA select_item */ + 471, /* (526) select_item ::= NK_STAR */ + 471, /* (527) select_item ::= common_expression */ + 471, /* (528) select_item ::= common_expression column_alias */ + 471, /* (529) select_item ::= common_expression AS column_alias */ + 471, /* (530) select_item ::= table_name NK_DOT NK_STAR */ + 412, /* (531) where_clause_opt ::= */ + 412, /* (532) where_clause_opt ::= WHERE search_condition */ + 464, /* (533) partition_by_clause_opt ::= */ + 464, /* (534) partition_by_clause_opt ::= PARTITION BY partition_list */ + 472, /* (535) partition_list ::= partition_item */ + 472, /* (536) partition_list ::= partition_list NK_COMMA partition_item */ + 473, /* (537) partition_item ::= expr_or_subquery */ + 473, /* (538) partition_item ::= expr_or_subquery column_alias */ + 473, /* (539) partition_item ::= expr_or_subquery AS column_alias */ + 468, /* (540) twindow_clause_opt ::= */ + 468, /* (541) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + 468, /* (542) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ + 468, /* (543) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + 468, /* (544) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + 468, /* (545) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ + 406, /* (546) sliding_opt ::= */ + 406, /* (547) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + 467, /* (548) fill_opt ::= */ + 467, /* (549) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + 467, /* (550) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ + 467, /* (551) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ + 474, /* (552) fill_mode ::= NONE */ + 474, /* (553) fill_mode ::= PREV */ + 474, /* (554) fill_mode ::= NULL */ + 474, /* (555) fill_mode ::= NULL_F */ + 474, /* (556) fill_mode ::= LINEAR */ + 474, /* (557) fill_mode ::= NEXT */ + 469, /* (558) group_by_clause_opt ::= */ + 469, /* (559) group_by_clause_opt ::= GROUP BY group_by_list */ + 475, /* (560) group_by_list ::= expr_or_subquery */ + 475, /* (561) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ + 470, /* (562) having_clause_opt ::= */ + 470, /* (563) having_clause_opt ::= HAVING search_condition */ + 465, /* (564) range_opt ::= */ + 465, /* (565) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ + 465, /* (566) range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ + 466, /* (567) every_opt ::= */ + 466, /* (568) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + 476, /* (569) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + 477, /* (570) query_simple ::= query_specification */ + 477, /* (571) query_simple ::= union_query_expression */ + 481, /* (572) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + 481, /* (573) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + 482, /* (574) query_simple_or_subquery ::= query_simple */ + 482, /* (575) query_simple_or_subquery ::= subquery */ + 411, /* (576) query_or_subquery ::= query_expression */ + 411, /* (577) query_or_subquery ::= subquery */ + 478, /* (578) order_by_clause_opt ::= */ + 478, /* (579) order_by_clause_opt ::= ORDER BY sort_specification_list */ + 479, /* (580) slimit_clause_opt ::= */ + 479, /* (581) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + 479, /* (582) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + 479, /* (583) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + 480, /* (584) limit_clause_opt ::= */ + 480, /* (585) limit_clause_opt ::= LIMIT NK_INTEGER */ + 480, /* (586) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + 480, /* (587) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + 458, /* (588) subquery ::= NK_LP query_expression NK_RP */ + 458, /* (589) subquery ::= NK_LP subquery NK_RP */ + 353, /* (590) search_condition ::= common_expression */ + 483, /* (591) sort_specification_list ::= sort_specification */ + 483, /* (592) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + 484, /* (593) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + 485, /* (594) ordering_specification_opt ::= */ + 485, /* (595) ordering_specification_opt ::= ASC */ + 485, /* (596) ordering_specification_opt ::= DESC */ + 486, /* (597) null_ordering_opt ::= */ + 486, /* (598) null_ordering_opt ::= NULLS FIRST */ + 486, /* (599) null_ordering_opt ::= NULLS LAST */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -4001,360 +4039,364 @@ static const signed char yyRuleInfoNRhs[] = { -2, /* (239) cmd ::= SHOW QNODES */ -2, /* (240) cmd ::= SHOW FUNCTIONS */ -5, /* (241) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - -2, /* (242) cmd ::= SHOW STREAMS */ - -2, /* (243) cmd ::= SHOW ACCOUNTS */ - -2, /* (244) cmd ::= SHOW APPS */ - -2, /* (245) cmd ::= SHOW CONNECTIONS */ - -2, /* (246) cmd ::= SHOW LICENCES */ - -2, /* (247) cmd ::= SHOW GRANTS */ - -4, /* (248) cmd ::= SHOW CREATE DATABASE db_name */ - -4, /* (249) cmd ::= SHOW CREATE TABLE full_table_name */ - -4, /* (250) cmd ::= SHOW CREATE STABLE full_table_name */ - -2, /* (251) cmd ::= SHOW QUERIES */ - -2, /* (252) cmd ::= SHOW SCORES */ - -2, /* (253) cmd ::= SHOW TOPICS */ - -2, /* (254) cmd ::= SHOW VARIABLES */ - -3, /* (255) cmd ::= SHOW CLUSTER VARIABLES */ - -3, /* (256) cmd ::= SHOW LOCAL VARIABLES */ - -5, /* (257) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ - -2, /* (258) cmd ::= SHOW BNODES */ - -2, /* (259) cmd ::= SHOW SNODES */ - -2, /* (260) cmd ::= SHOW CLUSTER */ - -2, /* (261) cmd ::= SHOW TRANSACTIONS */ - -4, /* (262) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ - -2, /* (263) cmd ::= SHOW CONSUMERS */ - -2, /* (264) cmd ::= SHOW SUBSCRIPTIONS */ - -5, /* (265) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ - -7, /* (266) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ - -3, /* (267) cmd ::= SHOW VNODES NK_INTEGER */ - -3, /* (268) cmd ::= SHOW VNODES NK_STRING */ - -3, /* (269) cmd ::= SHOW db_name_cond_opt ALIVE */ - -3, /* (270) cmd ::= SHOW CLUSTER ALIVE */ - 0, /* (271) db_name_cond_opt ::= */ - -2, /* (272) db_name_cond_opt ::= db_name NK_DOT */ - 0, /* (273) like_pattern_opt ::= */ - -2, /* (274) like_pattern_opt ::= LIKE NK_STRING */ - -1, /* (275) table_name_cond ::= table_name */ - 0, /* (276) from_db_opt ::= */ - -2, /* (277) from_db_opt ::= FROM db_name */ - 0, /* (278) tag_list_opt ::= */ - -1, /* (279) tag_list_opt ::= tag_item */ - -3, /* (280) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ - -1, /* (281) tag_item ::= TBNAME */ - -1, /* (282) tag_item ::= QTAGS */ - -1, /* (283) tag_item ::= column_name */ - -2, /* (284) tag_item ::= column_name column_alias */ - -3, /* (285) tag_item ::= column_name AS column_alias */ - -8, /* (286) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ - -9, /* (287) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ - -4, /* (288) cmd ::= DROP INDEX exists_opt full_index_name */ - -1, /* (289) full_index_name ::= index_name */ - -3, /* (290) full_index_name ::= db_name NK_DOT index_name */ - -10, /* (291) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ - -12, /* (292) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ - -1, /* (293) func_list ::= func */ - -3, /* (294) func_list ::= func_list NK_COMMA func */ - -4, /* (295) func ::= sma_func_name NK_LP expression_list NK_RP */ - -1, /* (296) sma_func_name ::= function_name */ - -1, /* (297) sma_func_name ::= COUNT */ - -1, /* (298) sma_func_name ::= FIRST */ - -1, /* (299) sma_func_name ::= LAST */ - -1, /* (300) sma_func_name ::= LAST_ROW */ - 0, /* (301) sma_stream_opt ::= */ - -3, /* (302) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ - -3, /* (303) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ - -3, /* (304) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ - -6, /* (305) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ - -7, /* (306) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ - -9, /* (307) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - -8, /* (308) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name where_clause_opt */ - -10, /* (309) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name where_clause_opt */ - -4, /* (310) cmd ::= DROP TOPIC exists_opt topic_name */ - -7, /* (311) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ - -2, /* (312) cmd ::= DESC full_table_name */ - -2, /* (313) cmd ::= DESCRIBE full_table_name */ - -3, /* (314) cmd ::= RESET QUERY CACHE */ - -4, /* (315) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - -4, /* (316) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ - 0, /* (317) analyze_opt ::= */ - -1, /* (318) analyze_opt ::= ANALYZE */ - 0, /* (319) explain_options ::= */ - -3, /* (320) explain_options ::= explain_options VERBOSE NK_BOOL */ - -3, /* (321) explain_options ::= explain_options RATIO NK_FLOAT */ - -12, /* (322) cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ - -4, /* (323) cmd ::= DROP FUNCTION exists_opt function_name */ - 0, /* (324) agg_func_opt ::= */ - -1, /* (325) agg_func_opt ::= AGGREGATE */ - 0, /* (326) bufsize_opt ::= */ - -2, /* (327) bufsize_opt ::= BUFSIZE NK_INTEGER */ - 0, /* (328) language_opt ::= */ - -2, /* (329) language_opt ::= LANGUAGE NK_STRING */ - 0, /* (330) or_replace_opt ::= */ - -2, /* (331) or_replace_opt ::= OR REPLACE */ - -12, /* (332) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ - -4, /* (333) cmd ::= DROP STREAM exists_opt stream_name */ - -4, /* (334) cmd ::= PAUSE STREAM exists_opt stream_name */ - -5, /* (335) cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ - 0, /* (336) col_list_opt ::= */ - -3, /* (337) col_list_opt ::= NK_LP col_name_list NK_RP */ - 0, /* (338) tag_def_or_ref_opt ::= */ - -1, /* (339) tag_def_or_ref_opt ::= tags_def */ - -4, /* (340) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ - 0, /* (341) stream_options ::= */ - -3, /* (342) stream_options ::= stream_options TRIGGER AT_ONCE */ - -3, /* (343) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - -4, /* (344) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - -3, /* (345) stream_options ::= stream_options WATERMARK duration_literal */ - -4, /* (346) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ - -3, /* (347) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ - -3, /* (348) stream_options ::= stream_options DELETE_MARK duration_literal */ - -4, /* (349) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ - 0, /* (350) subtable_opt ::= */ - -4, /* (351) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - 0, /* (352) ignore_opt ::= */ - -2, /* (353) ignore_opt ::= IGNORE UNTREATED */ - -3, /* (354) cmd ::= KILL CONNECTION NK_INTEGER */ - -3, /* (355) cmd ::= KILL QUERY NK_STRING */ - -3, /* (356) cmd ::= KILL TRANSACTION NK_INTEGER */ - -2, /* (357) cmd ::= BALANCE VGROUP */ - -3, /* (358) cmd ::= BALANCE VGROUP LEADER */ - -4, /* (359) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - -4, /* (360) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - -3, /* (361) cmd ::= SPLIT VGROUP NK_INTEGER */ - -2, /* (362) dnode_list ::= DNODE NK_INTEGER */ - -3, /* (363) dnode_list ::= dnode_list DNODE NK_INTEGER */ - -4, /* (364) cmd ::= DELETE FROM full_table_name where_clause_opt */ - -1, /* (365) cmd ::= query_or_subquery */ - -1, /* (366) cmd ::= insert_query */ - -7, /* (367) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ - -4, /* (368) insert_query ::= INSERT INTO full_table_name query_or_subquery */ - -1, /* (369) literal ::= NK_INTEGER */ - -1, /* (370) literal ::= NK_FLOAT */ - -1, /* (371) literal ::= NK_STRING */ - -1, /* (372) literal ::= NK_BOOL */ - -2, /* (373) literal ::= TIMESTAMP NK_STRING */ - -1, /* (374) literal ::= duration_literal */ - -1, /* (375) literal ::= NULL */ - -1, /* (376) literal ::= NK_QUESTION */ - -1, /* (377) duration_literal ::= NK_VARIABLE */ - -1, /* (378) signed ::= NK_INTEGER */ - -2, /* (379) signed ::= NK_PLUS NK_INTEGER */ - -2, /* (380) signed ::= NK_MINUS NK_INTEGER */ - -1, /* (381) signed ::= NK_FLOAT */ - -2, /* (382) signed ::= NK_PLUS NK_FLOAT */ - -2, /* (383) signed ::= NK_MINUS NK_FLOAT */ - -1, /* (384) signed_literal ::= signed */ - -1, /* (385) signed_literal ::= NK_STRING */ - -1, /* (386) signed_literal ::= NK_BOOL */ - -2, /* (387) signed_literal ::= TIMESTAMP NK_STRING */ - -1, /* (388) signed_literal ::= duration_literal */ - -1, /* (389) signed_literal ::= NULL */ - -1, /* (390) signed_literal ::= literal_func */ - -1, /* (391) signed_literal ::= NK_QUESTION */ - -1, /* (392) literal_list ::= signed_literal */ - -3, /* (393) literal_list ::= literal_list NK_COMMA signed_literal */ - -1, /* (394) db_name ::= NK_ID */ - -1, /* (395) table_name ::= NK_ID */ - -1, /* (396) column_name ::= NK_ID */ - -1, /* (397) function_name ::= NK_ID */ - -1, /* (398) table_alias ::= NK_ID */ - -1, /* (399) column_alias ::= NK_ID */ - -1, /* (400) user_name ::= NK_ID */ - -1, /* (401) topic_name ::= NK_ID */ - -1, /* (402) stream_name ::= NK_ID */ - -1, /* (403) cgroup_name ::= NK_ID */ - -1, /* (404) index_name ::= NK_ID */ - -1, /* (405) expr_or_subquery ::= expression */ - -1, /* (406) expression ::= literal */ - -1, /* (407) expression ::= pseudo_column */ - -1, /* (408) expression ::= column_reference */ - -1, /* (409) expression ::= function_expression */ - -1, /* (410) expression ::= case_when_expression */ - -3, /* (411) expression ::= NK_LP expression NK_RP */ - -2, /* (412) expression ::= NK_PLUS expr_or_subquery */ - -2, /* (413) expression ::= NK_MINUS expr_or_subquery */ - -3, /* (414) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ - -3, /* (415) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ - -3, /* (416) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ - -3, /* (417) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ - -3, /* (418) expression ::= expr_or_subquery NK_REM expr_or_subquery */ - -3, /* (419) expression ::= column_reference NK_ARROW NK_STRING */ - -3, /* (420) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ - -3, /* (421) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ - -1, /* (422) expression_list ::= expr_or_subquery */ - -3, /* (423) expression_list ::= expression_list NK_COMMA expr_or_subquery */ - -1, /* (424) column_reference ::= column_name */ - -3, /* (425) column_reference ::= table_name NK_DOT column_name */ - -1, /* (426) pseudo_column ::= ROWTS */ - -1, /* (427) pseudo_column ::= TBNAME */ - -3, /* (428) pseudo_column ::= table_name NK_DOT TBNAME */ - -1, /* (429) pseudo_column ::= QSTART */ - -1, /* (430) pseudo_column ::= QEND */ - -1, /* (431) pseudo_column ::= QDURATION */ - -1, /* (432) pseudo_column ::= WSTART */ - -1, /* (433) pseudo_column ::= WEND */ - -1, /* (434) pseudo_column ::= WDURATION */ - -1, /* (435) pseudo_column ::= IROWTS */ - -1, /* (436) pseudo_column ::= ISFILLED */ - -1, /* (437) pseudo_column ::= QTAGS */ - -4, /* (438) function_expression ::= function_name NK_LP expression_list NK_RP */ - -4, /* (439) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - -6, /* (440) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ - -1, /* (441) function_expression ::= literal_func */ - -3, /* (442) literal_func ::= noarg_func NK_LP NK_RP */ - -1, /* (443) literal_func ::= NOW */ - -1, /* (444) noarg_func ::= NOW */ - -1, /* (445) noarg_func ::= TODAY */ - -1, /* (446) noarg_func ::= TIMEZONE */ - -1, /* (447) noarg_func ::= DATABASE */ - -1, /* (448) noarg_func ::= CLIENT_VERSION */ - -1, /* (449) noarg_func ::= SERVER_VERSION */ - -1, /* (450) noarg_func ::= SERVER_STATUS */ - -1, /* (451) noarg_func ::= CURRENT_USER */ - -1, /* (452) noarg_func ::= USER */ - -1, /* (453) star_func ::= COUNT */ - -1, /* (454) star_func ::= FIRST */ - -1, /* (455) star_func ::= LAST */ - -1, /* (456) star_func ::= LAST_ROW */ - -1, /* (457) star_func_para_list ::= NK_STAR */ - -1, /* (458) star_func_para_list ::= other_para_list */ - -1, /* (459) other_para_list ::= star_func_para */ - -3, /* (460) other_para_list ::= other_para_list NK_COMMA star_func_para */ - -1, /* (461) star_func_para ::= expr_or_subquery */ - -3, /* (462) star_func_para ::= table_name NK_DOT NK_STAR */ - -4, /* (463) case_when_expression ::= CASE when_then_list case_when_else_opt END */ - -5, /* (464) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ - -1, /* (465) when_then_list ::= when_then_expr */ - -2, /* (466) when_then_list ::= when_then_list when_then_expr */ - -4, /* (467) when_then_expr ::= WHEN common_expression THEN common_expression */ - 0, /* (468) case_when_else_opt ::= */ - -2, /* (469) case_when_else_opt ::= ELSE common_expression */ - -3, /* (470) predicate ::= expr_or_subquery compare_op expr_or_subquery */ - -5, /* (471) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ - -6, /* (472) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ - -3, /* (473) predicate ::= expr_or_subquery IS NULL */ - -4, /* (474) predicate ::= expr_or_subquery IS NOT NULL */ - -3, /* (475) predicate ::= expr_or_subquery in_op in_predicate_value */ - -1, /* (476) compare_op ::= NK_LT */ - -1, /* (477) compare_op ::= NK_GT */ - -1, /* (478) compare_op ::= NK_LE */ - -1, /* (479) compare_op ::= NK_GE */ - -1, /* (480) compare_op ::= NK_NE */ - -1, /* (481) compare_op ::= NK_EQ */ - -1, /* (482) compare_op ::= LIKE */ - -2, /* (483) compare_op ::= NOT LIKE */ - -1, /* (484) compare_op ::= MATCH */ - -1, /* (485) compare_op ::= NMATCH */ - -1, /* (486) compare_op ::= CONTAINS */ - -1, /* (487) in_op ::= IN */ - -2, /* (488) in_op ::= NOT IN */ - -3, /* (489) in_predicate_value ::= NK_LP literal_list NK_RP */ - -1, /* (490) boolean_value_expression ::= boolean_primary */ - -2, /* (491) boolean_value_expression ::= NOT boolean_primary */ - -3, /* (492) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - -3, /* (493) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - -1, /* (494) boolean_primary ::= predicate */ - -3, /* (495) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - -1, /* (496) common_expression ::= expr_or_subquery */ - -1, /* (497) common_expression ::= boolean_value_expression */ - 0, /* (498) from_clause_opt ::= */ - -2, /* (499) from_clause_opt ::= FROM table_reference_list */ - -1, /* (500) table_reference_list ::= table_reference */ - -3, /* (501) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - -1, /* (502) table_reference ::= table_primary */ - -1, /* (503) table_reference ::= joined_table */ - -2, /* (504) table_primary ::= table_name alias_opt */ - -4, /* (505) table_primary ::= db_name NK_DOT table_name alias_opt */ - -2, /* (506) table_primary ::= subquery alias_opt */ - -1, /* (507) table_primary ::= parenthesized_joined_table */ - 0, /* (508) alias_opt ::= */ - -1, /* (509) alias_opt ::= table_alias */ - -2, /* (510) alias_opt ::= AS table_alias */ - -3, /* (511) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - -3, /* (512) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - -6, /* (513) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - 0, /* (514) join_type ::= */ - -1, /* (515) join_type ::= INNER */ - -12, /* (516) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - 0, /* (517) set_quantifier_opt ::= */ - -1, /* (518) set_quantifier_opt ::= DISTINCT */ - -1, /* (519) set_quantifier_opt ::= ALL */ - -1, /* (520) select_list ::= select_item */ - -3, /* (521) select_list ::= select_list NK_COMMA select_item */ - -1, /* (522) select_item ::= NK_STAR */ - -1, /* (523) select_item ::= common_expression */ - -2, /* (524) select_item ::= common_expression column_alias */ - -3, /* (525) select_item ::= common_expression AS column_alias */ - -3, /* (526) select_item ::= table_name NK_DOT NK_STAR */ - 0, /* (527) where_clause_opt ::= */ - -2, /* (528) where_clause_opt ::= WHERE search_condition */ - 0, /* (529) partition_by_clause_opt ::= */ - -3, /* (530) partition_by_clause_opt ::= PARTITION BY partition_list */ - -1, /* (531) partition_list ::= partition_item */ - -3, /* (532) partition_list ::= partition_list NK_COMMA partition_item */ - -1, /* (533) partition_item ::= expr_or_subquery */ - -2, /* (534) partition_item ::= expr_or_subquery column_alias */ - -3, /* (535) partition_item ::= expr_or_subquery AS column_alias */ - 0, /* (536) twindow_clause_opt ::= */ - -6, /* (537) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - -4, /* (538) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ - -6, /* (539) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - -8, /* (540) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - -7, /* (541) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ - 0, /* (542) sliding_opt ::= */ - -4, /* (543) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - 0, /* (544) fill_opt ::= */ - -4, /* (545) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - -6, /* (546) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ - -6, /* (547) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ - -1, /* (548) fill_mode ::= NONE */ - -1, /* (549) fill_mode ::= PREV */ - -1, /* (550) fill_mode ::= NULL */ - -1, /* (551) fill_mode ::= NULL_F */ - -1, /* (552) fill_mode ::= LINEAR */ - -1, /* (553) fill_mode ::= NEXT */ - 0, /* (554) group_by_clause_opt ::= */ - -3, /* (555) group_by_clause_opt ::= GROUP BY group_by_list */ - -1, /* (556) group_by_list ::= expr_or_subquery */ - -3, /* (557) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ - 0, /* (558) having_clause_opt ::= */ - -2, /* (559) having_clause_opt ::= HAVING search_condition */ - 0, /* (560) range_opt ::= */ - -6, /* (561) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - -4, /* (562) range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ - 0, /* (563) every_opt ::= */ - -4, /* (564) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - -4, /* (565) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - -1, /* (566) query_simple ::= query_specification */ - -1, /* (567) query_simple ::= union_query_expression */ - -4, /* (568) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - -3, /* (569) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - -1, /* (570) query_simple_or_subquery ::= query_simple */ - -1, /* (571) query_simple_or_subquery ::= subquery */ - -1, /* (572) query_or_subquery ::= query_expression */ - -1, /* (573) query_or_subquery ::= subquery */ - 0, /* (574) order_by_clause_opt ::= */ - -3, /* (575) order_by_clause_opt ::= ORDER BY sort_specification_list */ - 0, /* (576) slimit_clause_opt ::= */ - -2, /* (577) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - -4, /* (578) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - -4, /* (579) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - 0, /* (580) limit_clause_opt ::= */ - -2, /* (581) limit_clause_opt ::= LIMIT NK_INTEGER */ - -4, /* (582) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - -4, /* (583) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - -3, /* (584) subquery ::= NK_LP query_expression NK_RP */ - -3, /* (585) subquery ::= NK_LP subquery NK_RP */ - -1, /* (586) search_condition ::= common_expression */ - -1, /* (587) sort_specification_list ::= sort_specification */ - -3, /* (588) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - -3, /* (589) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - 0, /* (590) ordering_specification_opt ::= */ - -1, /* (591) ordering_specification_opt ::= ASC */ - -1, /* (592) ordering_specification_opt ::= DESC */ - 0, /* (593) null_ordering_opt ::= */ - -2, /* (594) null_ordering_opt ::= NULLS FIRST */ - -2, /* (595) null_ordering_opt ::= NULLS LAST */ + -6, /* (242) cmd ::= SHOW INDEXES FROM db_name NK_DOT table_name */ + -2, /* (243) cmd ::= SHOW STREAMS */ + -2, /* (244) cmd ::= SHOW ACCOUNTS */ + -2, /* (245) cmd ::= SHOW APPS */ + -2, /* (246) cmd ::= SHOW CONNECTIONS */ + -2, /* (247) cmd ::= SHOW LICENCES */ + -2, /* (248) cmd ::= SHOW GRANTS */ + -4, /* (249) cmd ::= SHOW CREATE DATABASE db_name */ + -4, /* (250) cmd ::= SHOW CREATE TABLE full_table_name */ + -4, /* (251) cmd ::= SHOW CREATE STABLE full_table_name */ + -2, /* (252) cmd ::= SHOW QUERIES */ + -2, /* (253) cmd ::= SHOW SCORES */ + -2, /* (254) cmd ::= SHOW TOPICS */ + -2, /* (255) cmd ::= SHOW VARIABLES */ + -3, /* (256) cmd ::= SHOW CLUSTER VARIABLES */ + -3, /* (257) cmd ::= SHOW LOCAL VARIABLES */ + -5, /* (258) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ + -2, /* (259) cmd ::= SHOW BNODES */ + -2, /* (260) cmd ::= SHOW SNODES */ + -2, /* (261) cmd ::= SHOW CLUSTER */ + -2, /* (262) cmd ::= SHOW TRANSACTIONS */ + -4, /* (263) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ + -2, /* (264) cmd ::= SHOW CONSUMERS */ + -2, /* (265) cmd ::= SHOW SUBSCRIPTIONS */ + -5, /* (266) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ + -6, /* (267) cmd ::= SHOW TAGS FROM db_name NK_DOT table_name */ + -7, /* (268) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ + -8, /* (269) cmd ::= SHOW TABLE TAGS tag_list_opt FROM db_name NK_DOT table_name */ + -3, /* (270) cmd ::= SHOW VNODES NK_INTEGER */ + -3, /* (271) cmd ::= SHOW VNODES NK_STRING */ + -3, /* (272) cmd ::= SHOW db_name_cond_opt ALIVE */ + -3, /* (273) cmd ::= SHOW CLUSTER ALIVE */ + 0, /* (274) db_name_cond_opt ::= */ + -2, /* (275) db_name_cond_opt ::= db_name NK_DOT */ + 0, /* (276) like_pattern_opt ::= */ + -2, /* (277) like_pattern_opt ::= LIKE NK_STRING */ + -1, /* (278) table_name_cond ::= table_name */ + 0, /* (279) from_db_opt ::= */ + -2, /* (280) from_db_opt ::= FROM db_name */ + 0, /* (281) tag_list_opt ::= */ + -1, /* (282) tag_list_opt ::= tag_item */ + -3, /* (283) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ + -1, /* (284) tag_item ::= TBNAME */ + -1, /* (285) tag_item ::= QTAGS */ + -1, /* (286) tag_item ::= column_name */ + -2, /* (287) tag_item ::= column_name column_alias */ + -3, /* (288) tag_item ::= column_name AS column_alias */ + -8, /* (289) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ + -9, /* (290) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ + -4, /* (291) cmd ::= DROP INDEX exists_opt full_index_name */ + -1, /* (292) full_index_name ::= index_name */ + -3, /* (293) full_index_name ::= db_name NK_DOT index_name */ + -10, /* (294) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ + -12, /* (295) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ + -1, /* (296) func_list ::= func */ + -3, /* (297) func_list ::= func_list NK_COMMA func */ + -4, /* (298) func ::= sma_func_name NK_LP expression_list NK_RP */ + -1, /* (299) sma_func_name ::= function_name */ + -1, /* (300) sma_func_name ::= COUNT */ + -1, /* (301) sma_func_name ::= FIRST */ + -1, /* (302) sma_func_name ::= LAST */ + -1, /* (303) sma_func_name ::= LAST_ROW */ + 0, /* (304) sma_stream_opt ::= */ + -3, /* (305) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + -3, /* (306) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ + -3, /* (307) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ + -1, /* (308) with_meta ::= AS */ + -3, /* (309) with_meta ::= WITH META AS */ + -3, /* (310) with_meta ::= ONLY META AS */ + -6, /* (311) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ + -7, /* (312) cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta DATABASE db_name */ + -8, /* (313) cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta STABLE full_table_name where_clause_opt */ + -4, /* (314) cmd ::= DROP TOPIC exists_opt topic_name */ + -7, /* (315) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ + -2, /* (316) cmd ::= DESC full_table_name */ + -2, /* (317) cmd ::= DESCRIBE full_table_name */ + -3, /* (318) cmd ::= RESET QUERY CACHE */ + -4, /* (319) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + -4, /* (320) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ + 0, /* (321) analyze_opt ::= */ + -1, /* (322) analyze_opt ::= ANALYZE */ + 0, /* (323) explain_options ::= */ + -3, /* (324) explain_options ::= explain_options VERBOSE NK_BOOL */ + -3, /* (325) explain_options ::= explain_options RATIO NK_FLOAT */ + -12, /* (326) cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ + -4, /* (327) cmd ::= DROP FUNCTION exists_opt function_name */ + 0, /* (328) agg_func_opt ::= */ + -1, /* (329) agg_func_opt ::= AGGREGATE */ + 0, /* (330) bufsize_opt ::= */ + -2, /* (331) bufsize_opt ::= BUFSIZE NK_INTEGER */ + 0, /* (332) language_opt ::= */ + -2, /* (333) language_opt ::= LANGUAGE NK_STRING */ + 0, /* (334) or_replace_opt ::= */ + -2, /* (335) or_replace_opt ::= OR REPLACE */ + -12, /* (336) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ + -4, /* (337) cmd ::= DROP STREAM exists_opt stream_name */ + -4, /* (338) cmd ::= PAUSE STREAM exists_opt stream_name */ + -5, /* (339) cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ + 0, /* (340) col_list_opt ::= */ + -3, /* (341) col_list_opt ::= NK_LP col_name_list NK_RP */ + 0, /* (342) tag_def_or_ref_opt ::= */ + -1, /* (343) tag_def_or_ref_opt ::= tags_def */ + -4, /* (344) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ + 0, /* (345) stream_options ::= */ + -3, /* (346) stream_options ::= stream_options TRIGGER AT_ONCE */ + -3, /* (347) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + -4, /* (348) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + -3, /* (349) stream_options ::= stream_options WATERMARK duration_literal */ + -4, /* (350) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + -3, /* (351) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ + -3, /* (352) stream_options ::= stream_options DELETE_MARK duration_literal */ + -4, /* (353) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ + 0, /* (354) subtable_opt ::= */ + -4, /* (355) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + 0, /* (356) ignore_opt ::= */ + -2, /* (357) ignore_opt ::= IGNORE UNTREATED */ + -3, /* (358) cmd ::= KILL CONNECTION NK_INTEGER */ + -3, /* (359) cmd ::= KILL QUERY NK_STRING */ + -3, /* (360) cmd ::= KILL TRANSACTION NK_INTEGER */ + -2, /* (361) cmd ::= BALANCE VGROUP */ + -3, /* (362) cmd ::= BALANCE VGROUP LEADER */ + -4, /* (363) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + -4, /* (364) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + -3, /* (365) cmd ::= SPLIT VGROUP NK_INTEGER */ + -2, /* (366) dnode_list ::= DNODE NK_INTEGER */ + -3, /* (367) dnode_list ::= dnode_list DNODE NK_INTEGER */ + -4, /* (368) cmd ::= DELETE FROM full_table_name where_clause_opt */ + -1, /* (369) cmd ::= query_or_subquery */ + -1, /* (370) cmd ::= insert_query */ + -7, /* (371) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ + -4, /* (372) insert_query ::= INSERT INTO full_table_name query_or_subquery */ + -1, /* (373) literal ::= NK_INTEGER */ + -1, /* (374) literal ::= NK_FLOAT */ + -1, /* (375) literal ::= NK_STRING */ + -1, /* (376) literal ::= NK_BOOL */ + -2, /* (377) literal ::= TIMESTAMP NK_STRING */ + -1, /* (378) literal ::= duration_literal */ + -1, /* (379) literal ::= NULL */ + -1, /* (380) literal ::= NK_QUESTION */ + -1, /* (381) duration_literal ::= NK_VARIABLE */ + -1, /* (382) signed ::= NK_INTEGER */ + -2, /* (383) signed ::= NK_PLUS NK_INTEGER */ + -2, /* (384) signed ::= NK_MINUS NK_INTEGER */ + -1, /* (385) signed ::= NK_FLOAT */ + -2, /* (386) signed ::= NK_PLUS NK_FLOAT */ + -2, /* (387) signed ::= NK_MINUS NK_FLOAT */ + -1, /* (388) signed_literal ::= signed */ + -1, /* (389) signed_literal ::= NK_STRING */ + -1, /* (390) signed_literal ::= NK_BOOL */ + -2, /* (391) signed_literal ::= TIMESTAMP NK_STRING */ + -1, /* (392) signed_literal ::= duration_literal */ + -1, /* (393) signed_literal ::= NULL */ + -1, /* (394) signed_literal ::= literal_func */ + -1, /* (395) signed_literal ::= NK_QUESTION */ + -1, /* (396) literal_list ::= signed_literal */ + -3, /* (397) literal_list ::= literal_list NK_COMMA signed_literal */ + -1, /* (398) db_name ::= NK_ID */ + -1, /* (399) table_name ::= NK_ID */ + -1, /* (400) column_name ::= NK_ID */ + -1, /* (401) function_name ::= NK_ID */ + -1, /* (402) table_alias ::= NK_ID */ + -1, /* (403) column_alias ::= NK_ID */ + -1, /* (404) user_name ::= NK_ID */ + -1, /* (405) topic_name ::= NK_ID */ + -1, /* (406) stream_name ::= NK_ID */ + -1, /* (407) cgroup_name ::= NK_ID */ + -1, /* (408) index_name ::= NK_ID */ + -1, /* (409) expr_or_subquery ::= expression */ + -1, /* (410) expression ::= literal */ + -1, /* (411) expression ::= pseudo_column */ + -1, /* (412) expression ::= column_reference */ + -1, /* (413) expression ::= function_expression */ + -1, /* (414) expression ::= case_when_expression */ + -3, /* (415) expression ::= NK_LP expression NK_RP */ + -2, /* (416) expression ::= NK_PLUS expr_or_subquery */ + -2, /* (417) expression ::= NK_MINUS expr_or_subquery */ + -3, /* (418) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + -3, /* (419) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + -3, /* (420) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + -3, /* (421) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + -3, /* (422) expression ::= expr_or_subquery NK_REM expr_or_subquery */ + -3, /* (423) expression ::= column_reference NK_ARROW NK_STRING */ + -3, /* (424) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + -3, /* (425) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + -1, /* (426) expression_list ::= expr_or_subquery */ + -3, /* (427) expression_list ::= expression_list NK_COMMA expr_or_subquery */ + -1, /* (428) column_reference ::= column_name */ + -3, /* (429) column_reference ::= table_name NK_DOT column_name */ + -1, /* (430) pseudo_column ::= ROWTS */ + -1, /* (431) pseudo_column ::= TBNAME */ + -3, /* (432) pseudo_column ::= table_name NK_DOT TBNAME */ + -1, /* (433) pseudo_column ::= QSTART */ + -1, /* (434) pseudo_column ::= QEND */ + -1, /* (435) pseudo_column ::= QDURATION */ + -1, /* (436) pseudo_column ::= WSTART */ + -1, /* (437) pseudo_column ::= WEND */ + -1, /* (438) pseudo_column ::= WDURATION */ + -1, /* (439) pseudo_column ::= IROWTS */ + -1, /* (440) pseudo_column ::= ISFILLED */ + -1, /* (441) pseudo_column ::= QTAGS */ + -4, /* (442) function_expression ::= function_name NK_LP expression_list NK_RP */ + -4, /* (443) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + -6, /* (444) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ + -1, /* (445) function_expression ::= literal_func */ + -3, /* (446) literal_func ::= noarg_func NK_LP NK_RP */ + -1, /* (447) literal_func ::= NOW */ + -1, /* (448) noarg_func ::= NOW */ + -1, /* (449) noarg_func ::= TODAY */ + -1, /* (450) noarg_func ::= TIMEZONE */ + -1, /* (451) noarg_func ::= DATABASE */ + -1, /* (452) noarg_func ::= CLIENT_VERSION */ + -1, /* (453) noarg_func ::= SERVER_VERSION */ + -1, /* (454) noarg_func ::= SERVER_STATUS */ + -1, /* (455) noarg_func ::= CURRENT_USER */ + -1, /* (456) noarg_func ::= USER */ + -1, /* (457) star_func ::= COUNT */ + -1, /* (458) star_func ::= FIRST */ + -1, /* (459) star_func ::= LAST */ + -1, /* (460) star_func ::= LAST_ROW */ + -1, /* (461) star_func_para_list ::= NK_STAR */ + -1, /* (462) star_func_para_list ::= other_para_list */ + -1, /* (463) other_para_list ::= star_func_para */ + -3, /* (464) other_para_list ::= other_para_list NK_COMMA star_func_para */ + -1, /* (465) star_func_para ::= expr_or_subquery */ + -3, /* (466) star_func_para ::= table_name NK_DOT NK_STAR */ + -4, /* (467) case_when_expression ::= CASE when_then_list case_when_else_opt END */ + -5, /* (468) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ + -1, /* (469) when_then_list ::= when_then_expr */ + -2, /* (470) when_then_list ::= when_then_list when_then_expr */ + -4, /* (471) when_then_expr ::= WHEN common_expression THEN common_expression */ + 0, /* (472) case_when_else_opt ::= */ + -2, /* (473) case_when_else_opt ::= ELSE common_expression */ + -3, /* (474) predicate ::= expr_or_subquery compare_op expr_or_subquery */ + -5, /* (475) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + -6, /* (476) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + -3, /* (477) predicate ::= expr_or_subquery IS NULL */ + -4, /* (478) predicate ::= expr_or_subquery IS NOT NULL */ + -3, /* (479) predicate ::= expr_or_subquery in_op in_predicate_value */ + -1, /* (480) compare_op ::= NK_LT */ + -1, /* (481) compare_op ::= NK_GT */ + -1, /* (482) compare_op ::= NK_LE */ + -1, /* (483) compare_op ::= NK_GE */ + -1, /* (484) compare_op ::= NK_NE */ + -1, /* (485) compare_op ::= NK_EQ */ + -1, /* (486) compare_op ::= LIKE */ + -2, /* (487) compare_op ::= NOT LIKE */ + -1, /* (488) compare_op ::= MATCH */ + -1, /* (489) compare_op ::= NMATCH */ + -1, /* (490) compare_op ::= CONTAINS */ + -1, /* (491) in_op ::= IN */ + -2, /* (492) in_op ::= NOT IN */ + -3, /* (493) in_predicate_value ::= NK_LP literal_list NK_RP */ + -1, /* (494) boolean_value_expression ::= boolean_primary */ + -2, /* (495) boolean_value_expression ::= NOT boolean_primary */ + -3, /* (496) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + -3, /* (497) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + -1, /* (498) boolean_primary ::= predicate */ + -3, /* (499) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + -1, /* (500) common_expression ::= expr_or_subquery */ + -1, /* (501) common_expression ::= boolean_value_expression */ + 0, /* (502) from_clause_opt ::= */ + -2, /* (503) from_clause_opt ::= FROM table_reference_list */ + -1, /* (504) table_reference_list ::= table_reference */ + -3, /* (505) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + -1, /* (506) table_reference ::= table_primary */ + -1, /* (507) table_reference ::= joined_table */ + -2, /* (508) table_primary ::= table_name alias_opt */ + -4, /* (509) table_primary ::= db_name NK_DOT table_name alias_opt */ + -2, /* (510) table_primary ::= subquery alias_opt */ + -1, /* (511) table_primary ::= parenthesized_joined_table */ + 0, /* (512) alias_opt ::= */ + -1, /* (513) alias_opt ::= table_alias */ + -2, /* (514) alias_opt ::= AS table_alias */ + -3, /* (515) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + -3, /* (516) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + -6, /* (517) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + 0, /* (518) join_type ::= */ + -1, /* (519) join_type ::= INNER */ + -12, /* (520) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + 0, /* (521) set_quantifier_opt ::= */ + -1, /* (522) set_quantifier_opt ::= DISTINCT */ + -1, /* (523) set_quantifier_opt ::= ALL */ + -1, /* (524) select_list ::= select_item */ + -3, /* (525) select_list ::= select_list NK_COMMA select_item */ + -1, /* (526) select_item ::= NK_STAR */ + -1, /* (527) select_item ::= common_expression */ + -2, /* (528) select_item ::= common_expression column_alias */ + -3, /* (529) select_item ::= common_expression AS column_alias */ + -3, /* (530) select_item ::= table_name NK_DOT NK_STAR */ + 0, /* (531) where_clause_opt ::= */ + -2, /* (532) where_clause_opt ::= WHERE search_condition */ + 0, /* (533) partition_by_clause_opt ::= */ + -3, /* (534) partition_by_clause_opt ::= PARTITION BY partition_list */ + -1, /* (535) partition_list ::= partition_item */ + -3, /* (536) partition_list ::= partition_list NK_COMMA partition_item */ + -1, /* (537) partition_item ::= expr_or_subquery */ + -2, /* (538) partition_item ::= expr_or_subquery column_alias */ + -3, /* (539) partition_item ::= expr_or_subquery AS column_alias */ + 0, /* (540) twindow_clause_opt ::= */ + -6, /* (541) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + -4, /* (542) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ + -6, /* (543) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + -8, /* (544) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + -7, /* (545) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ + 0, /* (546) sliding_opt ::= */ + -4, /* (547) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + 0, /* (548) fill_opt ::= */ + -4, /* (549) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + -6, /* (550) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ + -6, /* (551) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ + -1, /* (552) fill_mode ::= NONE */ + -1, /* (553) fill_mode ::= PREV */ + -1, /* (554) fill_mode ::= NULL */ + -1, /* (555) fill_mode ::= NULL_F */ + -1, /* (556) fill_mode ::= LINEAR */ + -1, /* (557) fill_mode ::= NEXT */ + 0, /* (558) group_by_clause_opt ::= */ + -3, /* (559) group_by_clause_opt ::= GROUP BY group_by_list */ + -1, /* (560) group_by_list ::= expr_or_subquery */ + -3, /* (561) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ + 0, /* (562) having_clause_opt ::= */ + -2, /* (563) having_clause_opt ::= HAVING search_condition */ + 0, /* (564) range_opt ::= */ + -6, /* (565) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ + -4, /* (566) range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ + 0, /* (567) every_opt ::= */ + -4, /* (568) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + -4, /* (569) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + -1, /* (570) query_simple ::= query_specification */ + -1, /* (571) query_simple ::= union_query_expression */ + -4, /* (572) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + -3, /* (573) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + -1, /* (574) query_simple_or_subquery ::= query_simple */ + -1, /* (575) query_simple_or_subquery ::= subquery */ + -1, /* (576) query_or_subquery ::= query_expression */ + -1, /* (577) query_or_subquery ::= subquery */ + 0, /* (578) order_by_clause_opt ::= */ + -3, /* (579) order_by_clause_opt ::= ORDER BY sort_specification_list */ + 0, /* (580) slimit_clause_opt ::= */ + -2, /* (581) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + -4, /* (582) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + -4, /* (583) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + 0, /* (584) limit_clause_opt ::= */ + -2, /* (585) limit_clause_opt ::= LIMIT NK_INTEGER */ + -4, /* (586) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + -4, /* (587) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + -3, /* (588) subquery ::= NK_LP query_expression NK_RP */ + -3, /* (589) subquery ::= NK_LP subquery NK_RP */ + -1, /* (590) search_condition ::= common_expression */ + -1, /* (591) sort_specification_list ::= sort_specification */ + -3, /* (592) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + -3, /* (593) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + 0, /* (594) ordering_specification_opt ::= */ + -1, /* (595) ordering_specification_opt ::= ASC */ + -1, /* (596) ordering_specification_opt ::= DESC */ + 0, /* (597) null_ordering_opt ::= */ + -2, /* (598) null_ordering_opt ::= NULLS FIRST */ + -2, /* (599) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -4446,11 +4488,11 @@ static YYACTIONTYPE yy_reduce( YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,338,&yymsp[0].minor); + yy_destructor(yypParser,339,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,339,&yymsp[0].minor); + yy_destructor(yypParser,340,&yymsp[0].minor); break; case 2: /* account_options ::= */ { } @@ -4464,20 +4506,20 @@ static YYACTIONTYPE yy_reduce( case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); -{ yy_destructor(yypParser,338,&yymsp[-2].minor); +{ yy_destructor(yypParser,339,&yymsp[-2].minor); { } - yy_destructor(yypParser,340,&yymsp[0].minor); + yy_destructor(yypParser,341,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ -{ yy_destructor(yypParser,341,&yymsp[0].minor); +{ yy_destructor(yypParser,342,&yymsp[0].minor); { } } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ -{ yy_destructor(yypParser,339,&yymsp[-1].minor); +{ yy_destructor(yypParser,340,&yymsp[-1].minor); { } - yy_destructor(yypParser,341,&yymsp[0].minor); + yy_destructor(yypParser,342,&yymsp[0].minor); } break; case 14: /* alter_account_option ::= PASS literal */ @@ -4491,113 +4533,113 @@ static YYACTIONTYPE yy_reduce( case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); { } - yy_destructor(yypParser,340,&yymsp[0].minor); + yy_destructor(yypParser,341,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy669, &yymsp[-1].minor.yy0, yymsp[0].minor.yy73); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy371, &yymsp[-1].minor.yy0, yymsp[0].minor.yy475); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy669, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy371, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy669, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy371, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy669, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy371, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy669); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy371); } break; case 29: /* sysinfo_opt ::= */ -{ yymsp[1].minor.yy73 = 1; } +{ yymsp[1].minor.yy475 = 1; } break; case 30: /* sysinfo_opt ::= SYSINFO NK_INTEGER */ -{ yymsp[-1].minor.yy73 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy475 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } break; case 31: /* cmd ::= GRANT privileges ON priv_level with_opt TO user_name */ -{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-5].minor.yy349, &yymsp[-3].minor.yy257, &yymsp[0].minor.yy669, yymsp[-2].minor.yy242); } +{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-5].minor.yy729, &yymsp[-3].minor.yy347, &yymsp[0].minor.yy371, yymsp[-2].minor.yy452); } break; case 32: /* cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name */ -{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-5].minor.yy349, &yymsp[-3].minor.yy257, &yymsp[0].minor.yy669, yymsp[-2].minor.yy242); } +{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-5].minor.yy729, &yymsp[-3].minor.yy347, &yymsp[0].minor.yy371, yymsp[-2].minor.yy452); } break; case 33: /* privileges ::= ALL */ -{ yymsp[0].minor.yy349 = PRIVILEGE_TYPE_ALL; } +{ yymsp[0].minor.yy729 = PRIVILEGE_TYPE_ALL; } break; case 34: /* privileges ::= priv_type_list */ case 36: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==36); -{ yylhsminor.yy349 = yymsp[0].minor.yy349; } - yymsp[0].minor.yy349 = yylhsminor.yy349; +{ yylhsminor.yy729 = yymsp[0].minor.yy729; } + yymsp[0].minor.yy729 = yylhsminor.yy729; break; case 35: /* privileges ::= SUBSCRIBE */ -{ yymsp[0].minor.yy349 = PRIVILEGE_TYPE_SUBSCRIBE; } +{ yymsp[0].minor.yy729 = PRIVILEGE_TYPE_SUBSCRIBE; } break; case 37: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ -{ yylhsminor.yy349 = yymsp[-2].minor.yy349 | yymsp[0].minor.yy349; } - yymsp[-2].minor.yy349 = yylhsminor.yy349; +{ yylhsminor.yy729 = yymsp[-2].minor.yy729 | yymsp[0].minor.yy729; } + yymsp[-2].minor.yy729 = yylhsminor.yy729; break; case 38: /* priv_type ::= READ */ -{ yymsp[0].minor.yy349 = PRIVILEGE_TYPE_READ; } +{ yymsp[0].minor.yy729 = PRIVILEGE_TYPE_READ; } break; case 39: /* priv_type ::= WRITE */ -{ yymsp[0].minor.yy349 = PRIVILEGE_TYPE_WRITE; } +{ yymsp[0].minor.yy729 = PRIVILEGE_TYPE_WRITE; } break; case 40: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ -{ yylhsminor.yy257.first = yymsp[-2].minor.yy0; yylhsminor.yy257.second = yymsp[0].minor.yy0; } - yymsp[-2].minor.yy257 = yylhsminor.yy257; +{ yylhsminor.yy347.first = yymsp[-2].minor.yy0; yylhsminor.yy347.second = yymsp[0].minor.yy0; } + yymsp[-2].minor.yy347 = yylhsminor.yy347; break; case 41: /* priv_level ::= db_name NK_DOT NK_STAR */ -{ yylhsminor.yy257.first = yymsp[-2].minor.yy669; yylhsminor.yy257.second = yymsp[0].minor.yy0; } - yymsp[-2].minor.yy257 = yylhsminor.yy257; +{ yylhsminor.yy347.first = yymsp[-2].minor.yy371; yylhsminor.yy347.second = yymsp[0].minor.yy0; } + yymsp[-2].minor.yy347 = yylhsminor.yy347; break; case 42: /* priv_level ::= db_name NK_DOT table_name */ -{ yylhsminor.yy257.first = yymsp[-2].minor.yy669; yylhsminor.yy257.second = yymsp[0].minor.yy669; } - yymsp[-2].minor.yy257 = yylhsminor.yy257; +{ yylhsminor.yy347.first = yymsp[-2].minor.yy371; yylhsminor.yy347.second = yymsp[0].minor.yy371; } + yymsp[-2].minor.yy347 = yylhsminor.yy347; break; case 43: /* priv_level ::= topic_name */ -{ yylhsminor.yy257.first = yymsp[0].minor.yy669; yylhsminor.yy257.second = nil_token; } - yymsp[0].minor.yy257 = yylhsminor.yy257; +{ yylhsminor.yy347.first = yymsp[0].minor.yy371; yylhsminor.yy347.second = nil_token; } + yymsp[0].minor.yy347 = yylhsminor.yy347; break; case 44: /* with_opt ::= */ case 144: /* start_opt ::= */ yytestcase(yyruleno==144); case 148: /* end_opt ::= */ yytestcase(yyruleno==148); - case 273: /* like_pattern_opt ::= */ yytestcase(yyruleno==273); - case 350: /* subtable_opt ::= */ yytestcase(yyruleno==350); - case 468: /* case_when_else_opt ::= */ yytestcase(yyruleno==468); - case 498: /* from_clause_opt ::= */ yytestcase(yyruleno==498); - case 527: /* where_clause_opt ::= */ yytestcase(yyruleno==527); - case 536: /* twindow_clause_opt ::= */ yytestcase(yyruleno==536); - case 542: /* sliding_opt ::= */ yytestcase(yyruleno==542); - case 544: /* fill_opt ::= */ yytestcase(yyruleno==544); - case 558: /* having_clause_opt ::= */ yytestcase(yyruleno==558); - case 560: /* range_opt ::= */ yytestcase(yyruleno==560); - case 563: /* every_opt ::= */ yytestcase(yyruleno==563); - case 576: /* slimit_clause_opt ::= */ yytestcase(yyruleno==576); - case 580: /* limit_clause_opt ::= */ yytestcase(yyruleno==580); -{ yymsp[1].minor.yy242 = NULL; } + case 276: /* like_pattern_opt ::= */ yytestcase(yyruleno==276); + case 354: /* subtable_opt ::= */ yytestcase(yyruleno==354); + case 472: /* case_when_else_opt ::= */ yytestcase(yyruleno==472); + case 502: /* from_clause_opt ::= */ yytestcase(yyruleno==502); + case 531: /* where_clause_opt ::= */ yytestcase(yyruleno==531); + case 540: /* twindow_clause_opt ::= */ yytestcase(yyruleno==540); + case 546: /* sliding_opt ::= */ yytestcase(yyruleno==546); + case 548: /* fill_opt ::= */ yytestcase(yyruleno==548); + case 562: /* having_clause_opt ::= */ yytestcase(yyruleno==562); + case 564: /* range_opt ::= */ yytestcase(yyruleno==564); + case 567: /* every_opt ::= */ yytestcase(yyruleno==567); + case 580: /* slimit_clause_opt ::= */ yytestcase(yyruleno==580); + case 584: /* limit_clause_opt ::= */ yytestcase(yyruleno==584); +{ yymsp[1].minor.yy452 = NULL; } break; case 45: /* with_opt ::= WITH search_condition */ - case 499: /* from_clause_opt ::= FROM table_reference_list */ yytestcase(yyruleno==499); - case 528: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==528); - case 559: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==559); -{ yymsp[-1].minor.yy242 = yymsp[0].minor.yy242; } + case 503: /* from_clause_opt ::= FROM table_reference_list */ yytestcase(yyruleno==503); + case 532: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==532); + case 563: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==563); +{ yymsp[-1].minor.yy452 = yymsp[0].minor.yy452; } break; case 46: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy669, NULL); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy371, NULL); } break; case 47: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy669, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy0); } break; case 48: /* cmd ::= DROP DNODE NK_INTEGER force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy777, false); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy667, false); } break; case 49: /* cmd ::= DROP DNODE dnode_endpoint force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy669, yymsp[0].minor.yy777, false); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy371, yymsp[0].minor.yy667, false); } break; case 50: /* cmd ::= DROP DNODE NK_INTEGER unsafe_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, false, yymsp[0].minor.yy777); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, false, yymsp[0].minor.yy667); } break; case 51: /* cmd ::= DROP DNODE dnode_endpoint unsafe_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy669, false, yymsp[0].minor.yy777); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy371, false, yymsp[0].minor.yy667); } break; case 52: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } @@ -4617,53 +4659,53 @@ static YYACTIONTYPE yy_reduce( case 57: /* dnode_endpoint ::= NK_STRING */ case 58: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==58); case 59: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==59); - case 297: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==297); - case 298: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==298); - case 299: /* sma_func_name ::= LAST */ yytestcase(yyruleno==299); - case 300: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==300); - case 394: /* db_name ::= NK_ID */ yytestcase(yyruleno==394); - case 395: /* table_name ::= NK_ID */ yytestcase(yyruleno==395); - case 396: /* column_name ::= NK_ID */ yytestcase(yyruleno==396); - case 397: /* function_name ::= NK_ID */ yytestcase(yyruleno==397); - case 398: /* table_alias ::= NK_ID */ yytestcase(yyruleno==398); - case 399: /* column_alias ::= NK_ID */ yytestcase(yyruleno==399); - case 400: /* user_name ::= NK_ID */ yytestcase(yyruleno==400); - case 401: /* topic_name ::= NK_ID */ yytestcase(yyruleno==401); - case 402: /* stream_name ::= NK_ID */ yytestcase(yyruleno==402); - case 403: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==403); - case 404: /* index_name ::= NK_ID */ yytestcase(yyruleno==404); - case 444: /* noarg_func ::= NOW */ yytestcase(yyruleno==444); - case 445: /* noarg_func ::= TODAY */ yytestcase(yyruleno==445); - case 446: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==446); - case 447: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==447); - case 448: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==448); - case 449: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==449); - case 450: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==450); - case 451: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==451); - case 452: /* noarg_func ::= USER */ yytestcase(yyruleno==452); - case 453: /* star_func ::= COUNT */ yytestcase(yyruleno==453); - case 454: /* star_func ::= FIRST */ yytestcase(yyruleno==454); - case 455: /* star_func ::= LAST */ yytestcase(yyruleno==455); - case 456: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==456); -{ yylhsminor.yy669 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy669 = yylhsminor.yy669; + case 300: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==300); + case 301: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==301); + case 302: /* sma_func_name ::= LAST */ yytestcase(yyruleno==302); + case 303: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==303); + case 398: /* db_name ::= NK_ID */ yytestcase(yyruleno==398); + case 399: /* table_name ::= NK_ID */ yytestcase(yyruleno==399); + case 400: /* column_name ::= NK_ID */ yytestcase(yyruleno==400); + case 401: /* function_name ::= NK_ID */ yytestcase(yyruleno==401); + case 402: /* table_alias ::= NK_ID */ yytestcase(yyruleno==402); + case 403: /* column_alias ::= NK_ID */ yytestcase(yyruleno==403); + case 404: /* user_name ::= NK_ID */ yytestcase(yyruleno==404); + case 405: /* topic_name ::= NK_ID */ yytestcase(yyruleno==405); + case 406: /* stream_name ::= NK_ID */ yytestcase(yyruleno==406); + case 407: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==407); + case 408: /* index_name ::= NK_ID */ yytestcase(yyruleno==408); + case 448: /* noarg_func ::= NOW */ yytestcase(yyruleno==448); + case 449: /* noarg_func ::= TODAY */ yytestcase(yyruleno==449); + case 450: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==450); + case 451: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==451); + case 452: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==452); + case 453: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==453); + case 454: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==454); + case 455: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==455); + case 456: /* noarg_func ::= USER */ yytestcase(yyruleno==456); + case 457: /* star_func ::= COUNT */ yytestcase(yyruleno==457); + case 458: /* star_func ::= FIRST */ yytestcase(yyruleno==458); + case 459: /* star_func ::= LAST */ yytestcase(yyruleno==459); + case 460: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==460); +{ yylhsminor.yy371 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy371 = yylhsminor.yy371; break; case 60: /* force_opt ::= */ case 84: /* not_exists_opt ::= */ yytestcase(yyruleno==84); case 86: /* exists_opt ::= */ yytestcase(yyruleno==86); - case 317: /* analyze_opt ::= */ yytestcase(yyruleno==317); - case 324: /* agg_func_opt ::= */ yytestcase(yyruleno==324); - case 330: /* or_replace_opt ::= */ yytestcase(yyruleno==330); - case 352: /* ignore_opt ::= */ yytestcase(yyruleno==352); - case 517: /* set_quantifier_opt ::= */ yytestcase(yyruleno==517); -{ yymsp[1].minor.yy777 = false; } + case 321: /* analyze_opt ::= */ yytestcase(yyruleno==321); + case 328: /* agg_func_opt ::= */ yytestcase(yyruleno==328); + case 334: /* or_replace_opt ::= */ yytestcase(yyruleno==334); + case 356: /* ignore_opt ::= */ yytestcase(yyruleno==356); + case 521: /* set_quantifier_opt ::= */ yytestcase(yyruleno==521); +{ yymsp[1].minor.yy667 = false; } break; case 61: /* force_opt ::= FORCE */ case 62: /* unsafe_opt ::= UNSAFE */ yytestcase(yyruleno==62); - case 318: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==318); - case 325: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==325); - case 518: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==518); -{ yymsp[0].minor.yy777 = true; } + case 322: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==322); + case 329: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==329); + case 522: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==522); +{ yymsp[0].minor.yy667 = true; } break; case 63: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } @@ -4705,234 +4747,234 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_VNODE_STMT, &yymsp[0].minor.yy0); } break; case 76: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy777, &yymsp[-1].minor.yy669, yymsp[0].minor.yy242); } +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy667, &yymsp[-1].minor.yy371, yymsp[0].minor.yy452); } break; case 77: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy777, &yymsp[0].minor.yy669); } +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy667, &yymsp[0].minor.yy371); } break; case 78: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy669); } +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy371); } break; case 79: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy669, yymsp[0].minor.yy242); } +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy371, yymsp[0].minor.yy452); } break; case 80: /* cmd ::= FLUSH DATABASE db_name */ -{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy669); } +{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy371); } break; case 81: /* cmd ::= TRIM DATABASE db_name speed_opt */ -{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy669, yymsp[0].minor.yy120); } +{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy371, yymsp[0].minor.yy416); } break; case 82: /* cmd ::= COMPACT DATABASE db_name start_opt end_opt */ -{ pCxt->pRootNode = createCompactStmt(pCxt, &yymsp[-2].minor.yy669, yymsp[-1].minor.yy242, yymsp[0].minor.yy242); } +{ pCxt->pRootNode = createCompactStmt(pCxt, &yymsp[-2].minor.yy371, yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } break; case 83: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy777 = true; } +{ yymsp[-2].minor.yy667 = true; } break; case 85: /* exists_opt ::= IF EXISTS */ - case 331: /* or_replace_opt ::= OR REPLACE */ yytestcase(yyruleno==331); - case 353: /* ignore_opt ::= IGNORE UNTREATED */ yytestcase(yyruleno==353); -{ yymsp[-1].minor.yy777 = true; } + case 335: /* or_replace_opt ::= OR REPLACE */ yytestcase(yyruleno==335); + case 357: /* ignore_opt ::= IGNORE UNTREATED */ yytestcase(yyruleno==357); +{ yymsp[-1].minor.yy667 = true; } break; case 87: /* db_options ::= */ -{ yymsp[1].minor.yy242 = createDefaultDatabaseOptions(pCxt); } +{ yymsp[1].minor.yy452 = createDefaultDatabaseOptions(pCxt); } break; case 88: /* db_options ::= db_options BUFFER NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 89: /* db_options ::= db_options CACHEMODEL NK_STRING */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 90: /* db_options ::= db_options CACHESIZE NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 91: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 92: /* db_options ::= db_options DURATION NK_INTEGER */ case 93: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==93); -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 94: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 95: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 96: /* db_options ::= db_options KEEP integer_list */ case 97: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==97); -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_KEEP, yymsp[0].minor.yy174); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_KEEP, yymsp[0].minor.yy812); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 98: /* db_options ::= db_options PAGES NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 99: /* db_options ::= db_options PAGESIZE NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 100: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 101: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 102: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 103: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 104: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 105: /* db_options ::= db_options RETENTIONS retention_list */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_RETENTIONS, yymsp[0].minor.yy174); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_RETENTIONS, yymsp[0].minor.yy812); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 106: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 107: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 108: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 109: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 110: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-3].minor.yy242, DB_OPTION_WAL_RETENTION_PERIOD, &t); + yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-3].minor.yy452, DB_OPTION_WAL_RETENTION_PERIOD, &t); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; + yymsp[-3].minor.yy452 = yylhsminor.yy452; break; case 111: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 112: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-3].minor.yy242, DB_OPTION_WAL_RETENTION_SIZE, &t); + yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-3].minor.yy452, DB_OPTION_WAL_RETENTION_SIZE, &t); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; + yymsp[-3].minor.yy452 = yylhsminor.yy452; break; case 113: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 114: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 115: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 116: /* db_options ::= db_options TABLE_PREFIX signed */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_TABLE_PREFIX, yymsp[0].minor.yy242); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_TABLE_PREFIX, yymsp[0].minor.yy452); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 117: /* db_options ::= db_options TABLE_SUFFIX signed */ -{ yylhsminor.yy242 = setDatabaseOption(pCxt, yymsp[-2].minor.yy242, DB_OPTION_TABLE_SUFFIX, yymsp[0].minor.yy242); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setDatabaseOption(pCxt, yymsp[-2].minor.yy452, DB_OPTION_TABLE_SUFFIX, yymsp[0].minor.yy452); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 118: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy242 = createAlterDatabaseOptions(pCxt); yylhsminor.yy242 = setAlterDatabaseOption(pCxt, yylhsminor.yy242, &yymsp[0].minor.yy535); } - yymsp[0].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createAlterDatabaseOptions(pCxt); yylhsminor.yy452 = setAlterDatabaseOption(pCxt, yylhsminor.yy452, &yymsp[0].minor.yy365); } + yymsp[0].minor.yy452 = yylhsminor.yy452; break; case 119: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy242 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy242, &yymsp[0].minor.yy535); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy452, &yymsp[0].minor.yy365); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; case 120: /* alter_db_option ::= BUFFER NK_INTEGER */ -{ yymsp[-1].minor.yy535.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy535.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy365.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; case 121: /* alter_db_option ::= CACHEMODEL NK_STRING */ -{ yymsp[-1].minor.yy535.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy535.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy365.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; case 122: /* alter_db_option ::= CACHESIZE NK_INTEGER */ -{ yymsp[-1].minor.yy535.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy535.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy365.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; case 123: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ -{ yymsp[-1].minor.yy535.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy535.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy365.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; case 124: /* alter_db_option ::= KEEP integer_list */ case 125: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==125); -{ yymsp[-1].minor.yy535.type = DB_OPTION_KEEP; yymsp[-1].minor.yy535.pList = yymsp[0].minor.yy174; } +{ yymsp[-1].minor.yy365.type = DB_OPTION_KEEP; yymsp[-1].minor.yy365.pList = yymsp[0].minor.yy812; } break; case 126: /* alter_db_option ::= PAGES NK_INTEGER */ -{ yymsp[-1].minor.yy535.type = DB_OPTION_PAGES; yymsp[-1].minor.yy535.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy365.type = DB_OPTION_PAGES; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; case 127: /* alter_db_option ::= REPLICA NK_INTEGER */ -{ yymsp[-1].minor.yy535.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy535.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy365.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; case 128: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ -{ yymsp[-1].minor.yy535.type = DB_OPTION_WAL; yymsp[-1].minor.yy535.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy365.type = DB_OPTION_WAL; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; case 129: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ -{ yymsp[-1].minor.yy535.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy535.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy365.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; case 130: /* alter_db_option ::= MINROWS NK_INTEGER */ -{ yymsp[-1].minor.yy535.type = DB_OPTION_MINROWS; yymsp[-1].minor.yy535.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy365.type = DB_OPTION_MINROWS; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; case 131: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ -{ yymsp[-1].minor.yy535.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-1].minor.yy535.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy365.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; case 132: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yymsp[-2].minor.yy535.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-2].minor.yy535.val = t; + yymsp[-2].minor.yy365.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-2].minor.yy365.val = t; } break; case 133: /* alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ -{ yymsp[-1].minor.yy535.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-1].minor.yy535.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy365.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; case 134: /* alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yymsp[-2].minor.yy535.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-2].minor.yy535.val = t; + yymsp[-2].minor.yy365.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-2].minor.yy365.val = t; } break; case 135: /* integer_list ::= NK_INTEGER */ -{ yylhsminor.yy174 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy174 = yylhsminor.yy174; +{ yylhsminor.yy812 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy812 = yylhsminor.yy812; break; case 136: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ - case 363: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==363); -{ yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy174 = yylhsminor.yy174; + case 367: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==367); +{ yylhsminor.yy812 = addNodeToList(pCxt, yymsp[-2].minor.yy812, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy812 = yylhsminor.yy812; break; case 137: /* variable_list ::= NK_VARIABLE */ -{ yylhsminor.yy174 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy174 = yylhsminor.yy174; +{ yylhsminor.yy812 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy812 = yylhsminor.yy812; break; case 138: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -{ yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy174 = yylhsminor.yy174; +{ yylhsminor.yy812 = addNodeToList(pCxt, yymsp[-2].minor.yy812, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy812 = yylhsminor.yy812; break; case 139: /* retention_list ::= retention */ case 169: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==169); @@ -4940,305 +4982,305 @@ static YYACTIONTYPE yy_reduce( case 179: /* column_def_list ::= column_def */ yytestcase(yyruleno==179); case 223: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==223); case 228: /* col_name_list ::= col_name */ yytestcase(yyruleno==228); - case 279: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==279); - case 293: /* func_list ::= func */ yytestcase(yyruleno==293); - case 392: /* literal_list ::= signed_literal */ yytestcase(yyruleno==392); - case 459: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==459); - case 465: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==465); - case 520: /* select_list ::= select_item */ yytestcase(yyruleno==520); - case 531: /* partition_list ::= partition_item */ yytestcase(yyruleno==531); - case 587: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==587); -{ yylhsminor.yy174 = createNodeList(pCxt, yymsp[0].minor.yy242); } - yymsp[0].minor.yy174 = yylhsminor.yy174; + case 282: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==282); + case 296: /* func_list ::= func */ yytestcase(yyruleno==296); + case 396: /* literal_list ::= signed_literal */ yytestcase(yyruleno==396); + case 463: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==463); + case 469: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==469); + case 524: /* select_list ::= select_item */ yytestcase(yyruleno==524); + case 535: /* partition_list ::= partition_item */ yytestcase(yyruleno==535); + case 591: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==591); +{ yylhsminor.yy812 = createNodeList(pCxt, yymsp[0].minor.yy452); } + yymsp[0].minor.yy812 = yylhsminor.yy812; break; case 140: /* retention_list ::= retention_list NK_COMMA retention */ case 173: /* multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ yytestcase(yyruleno==173); case 180: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==180); case 224: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==224); case 229: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==229); - case 280: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==280); - case 294: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==294); - case 393: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==393); - case 460: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==460); - case 521: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==521); - case 532: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==532); - case 588: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==588); -{ yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, yymsp[0].minor.yy242); } - yymsp[-2].minor.yy174 = yylhsminor.yy174; + case 283: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==283); + case 297: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==297); + case 397: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==397); + case 464: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==464); + case 525: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==525); + case 536: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==536); + case 592: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==592); +{ yylhsminor.yy812 = addNodeToList(pCxt, yymsp[-2].minor.yy812, yymsp[0].minor.yy452); } + yymsp[-2].minor.yy812 = yylhsminor.yy812; break; case 141: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy242 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 142: /* speed_opt ::= */ - case 326: /* bufsize_opt ::= */ yytestcase(yyruleno==326); -{ yymsp[1].minor.yy120 = 0; } + case 330: /* bufsize_opt ::= */ yytestcase(yyruleno==330); +{ yymsp[1].minor.yy416 = 0; } break; case 143: /* speed_opt ::= MAX_SPEED NK_INTEGER */ - case 327: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==327); -{ yymsp[-1].minor.yy120 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } + case 331: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==331); +{ yymsp[-1].minor.yy416 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } break; case 145: /* start_opt ::= START WITH NK_INTEGER */ case 149: /* end_opt ::= END WITH NK_INTEGER */ yytestcase(yyruleno==149); -{ yymsp[-2].minor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-2].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 146: /* start_opt ::= START WITH NK_STRING */ case 150: /* end_opt ::= END WITH NK_STRING */ yytestcase(yyruleno==150); -{ yymsp[-2].minor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } +{ yymsp[-2].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } break; case 147: /* start_opt ::= START WITH TIMESTAMP NK_STRING */ case 151: /* end_opt ::= END WITH TIMESTAMP NK_STRING */ yytestcase(yyruleno==151); -{ yymsp[-3].minor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } +{ yymsp[-3].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } break; case 152: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ case 154: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==154); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy777, yymsp[-5].minor.yy242, yymsp[-3].minor.yy174, yymsp[-1].minor.yy174, yymsp[0].minor.yy242); } +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy667, yymsp[-5].minor.yy452, yymsp[-3].minor.yy812, yymsp[-1].minor.yy812, yymsp[0].minor.yy452); } break; case 153: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy174); } +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy812); } break; case 155: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy174); } +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy812); } break; case 156: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy777, yymsp[0].minor.yy242); } +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy667, yymsp[0].minor.yy452); } break; case 157: /* cmd ::= ALTER TABLE alter_table_clause */ - case 365: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==365); - case 366: /* cmd ::= insert_query */ yytestcase(yyruleno==366); -{ pCxt->pRootNode = yymsp[0].minor.yy242; } + case 369: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==369); + case 370: /* cmd ::= insert_query */ yytestcase(yyruleno==370); +{ pCxt->pRootNode = yymsp[0].minor.yy452; } break; case 158: /* cmd ::= ALTER STABLE alter_table_clause */ -{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy242); } +{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy452); } break; case 159: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy242 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy242, yymsp[0].minor.yy242); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; case 160: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy242 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy242, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy669, yymsp[0].minor.yy794); } - yymsp[-4].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy452, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy371, yymsp[0].minor.yy310); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; break; case 161: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy242 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy242, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy669); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy452, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy371); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; break; case 162: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy242 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy242, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy669, yymsp[0].minor.yy794); } - yymsp[-4].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy452, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy371, yymsp[0].minor.yy310); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; break; case 163: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy242 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy242, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy669, &yymsp[0].minor.yy669); } - yymsp[-4].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy452, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy371, &yymsp[0].minor.yy371); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; break; case 164: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy242 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy242, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy669, yymsp[0].minor.yy794); } - yymsp[-4].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy452, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy371, yymsp[0].minor.yy310); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; break; case 165: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy242 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy242, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy669); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy452, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy371); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; break; case 166: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy242 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy242, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy669, yymsp[0].minor.yy794); } - yymsp[-4].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy452, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy371, yymsp[0].minor.yy310); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; break; case 167: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy242 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy242, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy669, &yymsp[0].minor.yy669); } - yymsp[-4].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy452, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy371, &yymsp[0].minor.yy371); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; break; case 168: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ -{ yylhsminor.yy242 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy242, &yymsp[-2].minor.yy669, yymsp[0].minor.yy242); } - yymsp[-5].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy452, &yymsp[-2].minor.yy371, yymsp[0].minor.yy452); } + yymsp[-5].minor.yy452 = yylhsminor.yy452; break; case 170: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ - case 466: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==466); -{ yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-1].minor.yy174, yymsp[0].minor.yy242); } - yymsp[-1].minor.yy174 = yylhsminor.yy174; + case 470: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==470); +{ yylhsminor.yy812 = addNodeToList(pCxt, yymsp[-1].minor.yy812, yymsp[0].minor.yy452); } + yymsp[-1].minor.yy812 = yylhsminor.yy812; break; case 171: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ -{ yylhsminor.yy242 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy777, yymsp[-8].minor.yy242, yymsp[-6].minor.yy242, yymsp[-5].minor.yy174, yymsp[-2].minor.yy174, yymsp[0].minor.yy242); } - yymsp[-9].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy667, yymsp[-8].minor.yy452, yymsp[-6].minor.yy452, yymsp[-5].minor.yy812, yymsp[-2].minor.yy812, yymsp[0].minor.yy452); } + yymsp[-9].minor.yy452 = yylhsminor.yy452; break; case 174: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy242 = createDropTableClause(pCxt, yymsp[-1].minor.yy777, yymsp[0].minor.yy242); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createDropTableClause(pCxt, yymsp[-1].minor.yy667, yymsp[0].minor.yy452); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; case 175: /* specific_cols_opt ::= */ case 206: /* tags_def_opt ::= */ yytestcase(yyruleno==206); - case 278: /* tag_list_opt ::= */ yytestcase(yyruleno==278); - case 336: /* col_list_opt ::= */ yytestcase(yyruleno==336); - case 338: /* tag_def_or_ref_opt ::= */ yytestcase(yyruleno==338); - case 529: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==529); - case 554: /* group_by_clause_opt ::= */ yytestcase(yyruleno==554); - case 574: /* order_by_clause_opt ::= */ yytestcase(yyruleno==574); -{ yymsp[1].minor.yy174 = NULL; } + case 281: /* tag_list_opt ::= */ yytestcase(yyruleno==281); + case 340: /* col_list_opt ::= */ yytestcase(yyruleno==340); + case 342: /* tag_def_or_ref_opt ::= */ yytestcase(yyruleno==342); + case 533: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==533); + case 558: /* group_by_clause_opt ::= */ yytestcase(yyruleno==558); + case 578: /* order_by_clause_opt ::= */ yytestcase(yyruleno==578); +{ yymsp[1].minor.yy812 = NULL; } break; case 176: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ - case 337: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==337); -{ yymsp[-2].minor.yy174 = yymsp[-1].minor.yy174; } + case 341: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==341); +{ yymsp[-2].minor.yy812 = yymsp[-1].minor.yy812; } break; case 177: /* full_table_name ::= table_name */ -{ yylhsminor.yy242 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy669, NULL); } - yymsp[0].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy371, NULL); } + yymsp[0].minor.yy452 = yylhsminor.yy452; break; case 178: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy242 = createRealTableNode(pCxt, &yymsp[-2].minor.yy669, &yymsp[0].minor.yy669, NULL); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createRealTableNode(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy371, NULL); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 181: /* column_def ::= column_name type_name */ -{ yylhsminor.yy242 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy669, yymsp[0].minor.yy794, NULL); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy371, yymsp[0].minor.yy310, NULL); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; case 182: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy794 = createDataType(TSDB_DATA_TYPE_BOOL); } +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_BOOL); } break; case 183: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy794 = createDataType(TSDB_DATA_TYPE_TINYINT); } +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; case 184: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy794 = createDataType(TSDB_DATA_TYPE_SMALLINT); } +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; case 185: /* type_name ::= INT */ case 186: /* type_name ::= INTEGER */ yytestcase(yyruleno==186); -{ yymsp[0].minor.yy794 = createDataType(TSDB_DATA_TYPE_INT); } +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_INT); } break; case 187: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy794 = createDataType(TSDB_DATA_TYPE_BIGINT); } +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; case 188: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy794 = createDataType(TSDB_DATA_TYPE_FLOAT); } +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; case 189: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy794 = createDataType(TSDB_DATA_TYPE_DOUBLE); } +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; case 190: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy794 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy310 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; case 191: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy794 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; case 192: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy794 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy310 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; case 193: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy794 = createDataType(TSDB_DATA_TYPE_UTINYINT); } +{ yymsp[-1].minor.yy310 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; case 194: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy794 = createDataType(TSDB_DATA_TYPE_USMALLINT); } +{ yymsp[-1].minor.yy310 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; case 195: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy794 = createDataType(TSDB_DATA_TYPE_UINT); } +{ yymsp[-1].minor.yy310 = createDataType(TSDB_DATA_TYPE_UINT); } break; case 196: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy794 = createDataType(TSDB_DATA_TYPE_UBIGINT); } +{ yymsp[-1].minor.yy310 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; case 197: /* type_name ::= JSON */ -{ yymsp[0].minor.yy794 = createDataType(TSDB_DATA_TYPE_JSON); } +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_JSON); } break; case 198: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy794 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy310 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; case 199: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy794 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; case 200: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy794 = createDataType(TSDB_DATA_TYPE_BLOB); } +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_BLOB); } break; case 201: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy794 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy310 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; case 202: /* type_name ::= GEOMETRY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy794 = createVarLenDataType(TSDB_DATA_TYPE_GEOMETRY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy310 = createVarLenDataType(TSDB_DATA_TYPE_GEOMETRY, &yymsp[-1].minor.yy0); } break; case 203: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy794 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[0].minor.yy310 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 204: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy794 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-3].minor.yy310 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 205: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy794 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-5].minor.yy310 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 207: /* tags_def_opt ::= tags_def */ - case 339: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==339); - case 458: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==458); -{ yylhsminor.yy174 = yymsp[0].minor.yy174; } - yymsp[0].minor.yy174 = yylhsminor.yy174; + case 343: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==343); + case 462: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==462); +{ yylhsminor.yy812 = yymsp[0].minor.yy812; } + yymsp[0].minor.yy812 = yylhsminor.yy812; break; case 208: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ - case 340: /* tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ yytestcase(yyruleno==340); -{ yymsp[-3].minor.yy174 = yymsp[-1].minor.yy174; } + case 344: /* tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ yytestcase(yyruleno==344); +{ yymsp[-3].minor.yy812 = yymsp[-1].minor.yy812; } break; case 209: /* table_options ::= */ -{ yymsp[1].minor.yy242 = createDefaultTableOptions(pCxt); } +{ yymsp[1].minor.yy452 = createDefaultTableOptions(pCxt); } break; case 210: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy242 = setTableOption(pCxt, yymsp[-2].minor.yy242, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-2].minor.yy452, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 211: /* table_options ::= table_options MAX_DELAY duration_list */ -{ yylhsminor.yy242 = setTableOption(pCxt, yymsp[-2].minor.yy242, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy174); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-2].minor.yy452, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy812); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 212: /* table_options ::= table_options WATERMARK duration_list */ -{ yylhsminor.yy242 = setTableOption(pCxt, yymsp[-2].minor.yy242, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy174); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-2].minor.yy452, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy812); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 213: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ -{ yylhsminor.yy242 = setTableOption(pCxt, yymsp[-4].minor.yy242, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy174); } - yymsp[-4].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-4].minor.yy452, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy812); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; break; case 214: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy242 = setTableOption(pCxt, yymsp[-2].minor.yy242, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-2].minor.yy452, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 215: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy242 = setTableOption(pCxt, yymsp[-4].minor.yy242, TABLE_OPTION_SMA, yymsp[-1].minor.yy174); } - yymsp[-4].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-4].minor.yy452, TABLE_OPTION_SMA, yymsp[-1].minor.yy812); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; break; case 216: /* table_options ::= table_options DELETE_MARK duration_list */ -{ yylhsminor.yy242 = setTableOption(pCxt, yymsp[-2].minor.yy242, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy174); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-2].minor.yy452, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy812); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; case 217: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy242 = createAlterTableOptions(pCxt); yylhsminor.yy242 = setTableOption(pCxt, yylhsminor.yy242, yymsp[0].minor.yy535.type, &yymsp[0].minor.yy535.val); } - yymsp[0].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createAlterTableOptions(pCxt); yylhsminor.yy452 = setTableOption(pCxt, yylhsminor.yy452, yymsp[0].minor.yy365.type, &yymsp[0].minor.yy365.val); } + yymsp[0].minor.yy452 = yylhsminor.yy452; break; case 218: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy242 = setTableOption(pCxt, yymsp[-1].minor.yy242, yymsp[0].minor.yy535.type, &yymsp[0].minor.yy535.val); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = setTableOption(pCxt, yymsp[-1].minor.yy452, yymsp[0].minor.yy365.type, &yymsp[0].minor.yy365.val); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; case 219: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy535.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy535.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy365.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; case 220: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy535.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy535.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy365.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy365.val = yymsp[0].minor.yy0; } break; case 221: /* duration_list ::= duration_literal */ - case 422: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==422); -{ yylhsminor.yy174 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy242)); } - yymsp[0].minor.yy174 = yylhsminor.yy174; + case 426: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==426); +{ yylhsminor.yy812 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } + yymsp[0].minor.yy812 = yylhsminor.yy812; break; case 222: /* duration_list ::= duration_list NK_COMMA duration_literal */ - case 423: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==423); -{ yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, releaseRawExprNode(pCxt, yymsp[0].minor.yy242)); } - yymsp[-2].minor.yy174 = yylhsminor.yy174; + case 427: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==427); +{ yylhsminor.yy812 = addNodeToList(pCxt, yymsp[-2].minor.yy812, releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } + yymsp[-2].minor.yy812 = yylhsminor.yy812; break; case 225: /* rollup_func_name ::= function_name */ -{ yylhsminor.yy242 = createFunctionNode(pCxt, &yymsp[0].minor.yy669, NULL); } - yymsp[0].minor.yy242 = yylhsminor.yy242; +{ yylhsminor.yy452 = createFunctionNode(pCxt, &yymsp[0].minor.yy371, NULL); } + yymsp[0].minor.yy452 = yylhsminor.yy452; break; case 226: /* rollup_func_name ::= FIRST */ case 227: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==227); - case 282: /* tag_item ::= QTAGS */ yytestcase(yyruleno==282); -{ yylhsminor.yy242 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } - yymsp[0].minor.yy242 = yylhsminor.yy242; + case 285: /* tag_item ::= QTAGS */ yytestcase(yyruleno==285); +{ yylhsminor.yy452 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } + yymsp[0].minor.yy452 = yylhsminor.yy452; break; case 230: /* col_name ::= column_name */ - case 283: /* tag_item ::= column_name */ yytestcase(yyruleno==283); -{ yylhsminor.yy242 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy669); } - yymsp[0].minor.yy242 = yylhsminor.yy242; + case 286: /* tag_item ::= column_name */ yytestcase(yyruleno==286); +{ yylhsminor.yy452 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy371); } + yymsp[0].minor.yy452 = yylhsminor.yy452; break; case 231: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } @@ -5253,13 +5295,13 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); } break; case 235: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy242, yymsp[0].minor.yy242, OP_TYPE_LIKE); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy452, yymsp[0].minor.yy452, OP_TYPE_LIKE); } break; case 236: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy242, yymsp[0].minor.yy242, OP_TYPE_LIKE); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy452, yymsp[0].minor.yy452, OP_TYPE_LIKE); } break; case 237: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy242, NULL, OP_TYPE_LIKE); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy452, NULL, OP_TYPE_LIKE); } break; case 238: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } @@ -5271,853 +5313,865 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } break; case 241: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy242, yymsp[-1].minor.yy242, OP_TYPE_EQUAL); } +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy452, yymsp[-1].minor.yy452, OP_TYPE_EQUAL); } + break; + case 242: /* cmd ::= SHOW INDEXES FROM db_name NK_DOT table_name */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy371), createIdentifierValueNode(pCxt, &yymsp[0].minor.yy371), OP_TYPE_EQUAL); } break; - case 242: /* cmd ::= SHOW STREAMS */ + case 243: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } break; - case 243: /* cmd ::= SHOW ACCOUNTS */ + case 244: /* cmd ::= SHOW ACCOUNTS */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } break; - case 244: /* cmd ::= SHOW APPS */ + case 245: /* cmd ::= SHOW APPS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); } break; - case 245: /* cmd ::= SHOW CONNECTIONS */ + case 246: /* cmd ::= SHOW CONNECTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT); } break; - case 246: /* cmd ::= SHOW LICENCES */ - case 247: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==247); + case 247: /* cmd ::= SHOW LICENCES */ + case 248: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==248); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); } break; - case 248: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy669); } + case 249: /* cmd ::= SHOW CREATE DATABASE db_name */ +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy371); } break; - case 249: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy242); } + case 250: /* cmd ::= SHOW CREATE TABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy452); } break; - case 250: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy242); } + case 251: /* cmd ::= SHOW CREATE STABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy452); } break; - case 251: /* cmd ::= SHOW QUERIES */ + case 252: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } break; - case 252: /* cmd ::= SHOW SCORES */ + case 253: /* cmd ::= SHOW SCORES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); } break; - case 253: /* cmd ::= SHOW TOPICS */ + case 254: /* cmd ::= SHOW TOPICS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); } break; - case 254: /* cmd ::= SHOW VARIABLES */ - case 255: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==255); + case 255: /* cmd ::= SHOW VARIABLES */ + case 256: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==256); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLES_STMT); } break; - case 256: /* cmd ::= SHOW LOCAL VARIABLES */ + case 257: /* cmd ::= SHOW LOCAL VARIABLES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } break; - case 257: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy242); } + case 258: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy452); } break; - case 258: /* cmd ::= SHOW BNODES */ + case 259: /* cmd ::= SHOW BNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } break; - case 259: /* cmd ::= SHOW SNODES */ + case 260: /* cmd ::= SHOW SNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } break; - case 260: /* cmd ::= SHOW CLUSTER */ + case 261: /* cmd ::= SHOW CLUSTER */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } break; - case 261: /* cmd ::= SHOW TRANSACTIONS */ + case 262: /* cmd ::= SHOW TRANSACTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } break; - case 262: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ -{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy242); } + case 263: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ +{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy452); } break; - case 263: /* cmd ::= SHOW CONSUMERS */ + case 264: /* cmd ::= SHOW CONSUMERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } break; - case 264: /* cmd ::= SHOW SUBSCRIPTIONS */ + case 265: /* cmd ::= SHOW SUBSCRIPTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } break; - case 265: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy242, yymsp[-1].minor.yy242, OP_TYPE_EQUAL); } + case 266: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy452, yymsp[-1].minor.yy452, OP_TYPE_EQUAL); } break; - case 266: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy242, yymsp[0].minor.yy242, yymsp[-3].minor.yy174); } + case 267: /* cmd ::= SHOW TAGS FROM db_name NK_DOT table_name */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy371), createIdentifierValueNode(pCxt, &yymsp[0].minor.yy371), OP_TYPE_EQUAL); } break; - case 267: /* cmd ::= SHOW VNODES NK_INTEGER */ + case 268: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy452, yymsp[0].minor.yy452, yymsp[-3].minor.yy812); } + break; + case 269: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM db_name NK_DOT table_name */ +{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, createIdentifierValueNode(pCxt, &yymsp[0].minor.yy371), createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy371), yymsp[-4].minor.yy812); } + break; + case 270: /* cmd ::= SHOW VNODES NK_INTEGER */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); } break; - case 268: /* cmd ::= SHOW VNODES NK_STRING */ + case 271: /* cmd ::= SHOW VNODES NK_STRING */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &yymsp[0].minor.yy0)); } break; - case 269: /* cmd ::= SHOW db_name_cond_opt ALIVE */ -{ pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy242, QUERY_NODE_SHOW_DB_ALIVE_STMT); } + case 272: /* cmd ::= SHOW db_name_cond_opt ALIVE */ +{ pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy452, QUERY_NODE_SHOW_DB_ALIVE_STMT); } break; - case 270: /* cmd ::= SHOW CLUSTER ALIVE */ + case 273: /* cmd ::= SHOW CLUSTER ALIVE */ { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } break; - case 271: /* db_name_cond_opt ::= */ - case 276: /* from_db_opt ::= */ yytestcase(yyruleno==276); -{ yymsp[1].minor.yy242 = createDefaultDatabaseCondValue(pCxt); } + case 274: /* db_name_cond_opt ::= */ + case 279: /* from_db_opt ::= */ yytestcase(yyruleno==279); +{ yymsp[1].minor.yy452 = createDefaultDatabaseCondValue(pCxt); } + break; + case 275: /* db_name_cond_opt ::= db_name NK_DOT */ +{ yylhsminor.yy452 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy371); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; - case 272: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy242 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy669); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; + case 277: /* like_pattern_opt ::= LIKE NK_STRING */ +{ yymsp[-1].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; - case 274: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + case 278: /* table_name_cond ::= table_name */ +{ yylhsminor.yy452 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy371); } + yymsp[0].minor.yy452 = yylhsminor.yy452; break; - case 275: /* table_name_cond ::= table_name */ -{ yylhsminor.yy242 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy669); } - yymsp[0].minor.yy242 = yylhsminor.yy242; + case 280: /* from_db_opt ::= FROM db_name */ +{ yymsp[-1].minor.yy452 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy371); } break; - case 277: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy242 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy669); } + case 284: /* tag_item ::= TBNAME */ +{ yylhsminor.yy452 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; break; - case 281: /* tag_item ::= TBNAME */ -{ yylhsminor.yy242 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } - yymsp[0].minor.yy242 = yylhsminor.yy242; + case 287: /* tag_item ::= column_name column_alias */ +{ yylhsminor.yy452 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy371), &yymsp[0].minor.yy371); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; - case 284: /* tag_item ::= column_name column_alias */ -{ yylhsminor.yy242 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy669), &yymsp[0].minor.yy669); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; + case 288: /* tag_item ::= column_name AS column_alias */ +{ yylhsminor.yy452 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy371), &yymsp[0].minor.yy371); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 285: /* tag_item ::= column_name AS column_alias */ -{ yylhsminor.yy242 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy669), &yymsp[0].minor.yy669); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 289: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy667, yymsp[-3].minor.yy452, yymsp[-1].minor.yy452, NULL, yymsp[0].minor.yy452); } break; - case 286: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy777, yymsp[-3].minor.yy242, yymsp[-1].minor.yy242, NULL, yymsp[0].minor.yy242); } + case 290: /* cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy667, yymsp[-5].minor.yy452, yymsp[-3].minor.yy452, yymsp[-1].minor.yy812, NULL); } break; - case 287: /* cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy777, yymsp[-5].minor.yy242, yymsp[-3].minor.yy242, yymsp[-1].minor.yy174, NULL); } + case 291: /* cmd ::= DROP INDEX exists_opt full_index_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy667, yymsp[0].minor.yy452); } break; - case 288: /* cmd ::= DROP INDEX exists_opt full_index_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy777, yymsp[0].minor.yy242); } + case 292: /* full_index_name ::= index_name */ +{ yylhsminor.yy452 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy371); } + yymsp[0].minor.yy452 = yylhsminor.yy452; break; - case 289: /* full_index_name ::= index_name */ -{ yylhsminor.yy242 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy669); } - yymsp[0].minor.yy242 = yylhsminor.yy242; + case 293: /* full_index_name ::= db_name NK_DOT index_name */ +{ yylhsminor.yy452 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy371); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 290: /* full_index_name ::= db_name NK_DOT index_name */ -{ yylhsminor.yy242 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy669, &yymsp[0].minor.yy669); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 294: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-9].minor.yy452 = createIndexOption(pCxt, yymsp[-7].minor.yy812, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), NULL, yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } break; - case 291: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-9].minor.yy242 = createIndexOption(pCxt, yymsp[-7].minor.yy174, releaseRawExprNode(pCxt, yymsp[-3].minor.yy242), NULL, yymsp[-1].minor.yy242, yymsp[0].minor.yy242); } + case 295: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-11].minor.yy452 = createIndexOption(pCxt, yymsp[-9].minor.yy812, releaseRawExprNode(pCxt, yymsp[-5].minor.yy452), releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } break; - case 292: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-11].minor.yy242 = createIndexOption(pCxt, yymsp[-9].minor.yy174, releaseRawExprNode(pCxt, yymsp[-5].minor.yy242), releaseRawExprNode(pCxt, yymsp[-3].minor.yy242), yymsp[-1].minor.yy242, yymsp[0].minor.yy242); } + case 298: /* func ::= sma_func_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy452 = createFunctionNode(pCxt, &yymsp[-3].minor.yy371, yymsp[-1].minor.yy812); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; break; - case 295: /* func ::= sma_func_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy242 = createFunctionNode(pCxt, &yymsp[-3].minor.yy669, yymsp[-1].minor.yy174); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; + case 299: /* sma_func_name ::= function_name */ + case 513: /* alias_opt ::= table_alias */ yytestcase(yyruleno==513); +{ yylhsminor.yy371 = yymsp[0].minor.yy371; } + yymsp[0].minor.yy371 = yylhsminor.yy371; break; - case 296: /* sma_func_name ::= function_name */ - case 509: /* alias_opt ::= table_alias */ yytestcase(yyruleno==509); -{ yylhsminor.yy669 = yymsp[0].minor.yy669; } - yymsp[0].minor.yy669 = yylhsminor.yy669; + case 304: /* sma_stream_opt ::= */ + case 345: /* stream_options ::= */ yytestcase(yyruleno==345); +{ yymsp[1].minor.yy452 = createStreamOptions(pCxt); } break; - case 301: /* sma_stream_opt ::= */ - case 341: /* stream_options ::= */ yytestcase(yyruleno==341); -{ yymsp[1].minor.yy242 = createStreamOptions(pCxt); } + case 305: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy452)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy452); yylhsminor.yy452 = yymsp[-2].minor.yy452; } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 302: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy242)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy242); yylhsminor.yy242 = yymsp[-2].minor.yy242; } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 306: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy452)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy452); yylhsminor.yy452 = yymsp[-2].minor.yy452; } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 303: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy242)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy242); yylhsminor.yy242 = yymsp[-2].minor.yy242; } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 307: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy452)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy452); yylhsminor.yy452 = yymsp[-2].minor.yy452; } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 304: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy242)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy242); yylhsminor.yy242 = yymsp[-2].minor.yy242; } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 308: /* with_meta ::= AS */ +{ yymsp[0].minor.yy416 = 0; } break; - case 305: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ -{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy777, &yymsp[-2].minor.yy669, yymsp[0].minor.yy242); } + case 309: /* with_meta ::= WITH META AS */ +{ yymsp[-2].minor.yy416 = 1; } break; - case 306: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy777, &yymsp[-3].minor.yy669, &yymsp[0].minor.yy669, false); } + case 310: /* with_meta ::= ONLY META AS */ +{ yymsp[-2].minor.yy416 = 2; } break; - case 307: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy777, &yymsp[-5].minor.yy669, &yymsp[0].minor.yy669, true); } + case 311: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ +{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy667, &yymsp[-2].minor.yy371, yymsp[0].minor.yy452); } break; - case 308: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name where_clause_opt */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-5].minor.yy777, &yymsp[-4].minor.yy669, yymsp[-1].minor.yy242, false, yymsp[0].minor.yy242); } + case 312: /* cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy667, &yymsp[-3].minor.yy371, &yymsp[0].minor.yy371, yymsp[-2].minor.yy416); } break; - case 309: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name where_clause_opt */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-7].minor.yy777, &yymsp[-6].minor.yy669, yymsp[-1].minor.yy242, true, yymsp[0].minor.yy242); } + case 313: /* cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta STABLE full_table_name where_clause_opt */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-5].minor.yy667, &yymsp[-4].minor.yy371, yymsp[-1].minor.yy452, yymsp[-3].minor.yy416, yymsp[0].minor.yy452); } break; - case 310: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy777, &yymsp[0].minor.yy669); } + case 314: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy667, &yymsp[0].minor.yy371); } break; - case 311: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy777, &yymsp[-2].minor.yy669, &yymsp[0].minor.yy669); } + case 315: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ +{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy667, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy371); } break; - case 312: /* cmd ::= DESC full_table_name */ - case 313: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==313); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy242); } + case 316: /* cmd ::= DESC full_table_name */ + case 317: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==317); +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy452); } break; - case 314: /* cmd ::= RESET QUERY CACHE */ + case 318: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; - case 315: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - case 316: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==316); -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy777, yymsp[-1].minor.yy242, yymsp[0].minor.yy242); } + case 319: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + case 320: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==320); +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy667, yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } break; - case 319: /* explain_options ::= */ -{ yymsp[1].minor.yy242 = createDefaultExplainOptions(pCxt); } + case 323: /* explain_options ::= */ +{ yymsp[1].minor.yy452 = createDefaultExplainOptions(pCxt); } break; - case 320: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy242 = setExplainVerbose(pCxt, yymsp[-2].minor.yy242, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 324: /* explain_options ::= explain_options VERBOSE NK_BOOL */ +{ yylhsminor.yy452 = setExplainVerbose(pCxt, yymsp[-2].minor.yy452, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 321: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy242 = setExplainRatio(pCxt, yymsp[-2].minor.yy242, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 325: /* explain_options ::= explain_options RATIO NK_FLOAT */ +{ yylhsminor.yy452 = setExplainRatio(pCxt, yymsp[-2].minor.yy452, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 322: /* cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-7].minor.yy777, yymsp[-9].minor.yy777, &yymsp[-6].minor.yy669, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy794, yymsp[-1].minor.yy120, &yymsp[0].minor.yy669, yymsp[-10].minor.yy777); } + case 326: /* cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-7].minor.yy667, yymsp[-9].minor.yy667, &yymsp[-6].minor.yy371, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy310, yymsp[-1].minor.yy416, &yymsp[0].minor.yy371, yymsp[-10].minor.yy667); } break; - case 323: /* cmd ::= DROP FUNCTION exists_opt function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy777, &yymsp[0].minor.yy669); } + case 327: /* cmd ::= DROP FUNCTION exists_opt function_name */ +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy667, &yymsp[0].minor.yy371); } break; - case 328: /* language_opt ::= */ -{ yymsp[1].minor.yy669 = nil_token; } + case 332: /* language_opt ::= */ +{ yymsp[1].minor.yy371 = nil_token; } break; - case 329: /* language_opt ::= LANGUAGE NK_STRING */ -{ yymsp[-1].minor.yy669 = yymsp[0].minor.yy0; } + case 333: /* language_opt ::= LANGUAGE NK_STRING */ +{ yymsp[-1].minor.yy371 = yymsp[0].minor.yy0; } break; - case 332: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy777, &yymsp[-8].minor.yy669, yymsp[-5].minor.yy242, yymsp[-7].minor.yy242, yymsp[-3].minor.yy174, yymsp[-2].minor.yy242, yymsp[0].minor.yy242, yymsp[-4].minor.yy174); } + case 336: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy667, &yymsp[-8].minor.yy371, yymsp[-5].minor.yy452, yymsp[-7].minor.yy452, yymsp[-3].minor.yy812, yymsp[-2].minor.yy452, yymsp[0].minor.yy452, yymsp[-4].minor.yy812); } break; - case 333: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy777, &yymsp[0].minor.yy669); } + case 337: /* cmd ::= DROP STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy667, &yymsp[0].minor.yy371); } break; - case 334: /* cmd ::= PAUSE STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createPauseStreamStmt(pCxt, yymsp[-1].minor.yy777, &yymsp[0].minor.yy669); } + case 338: /* cmd ::= PAUSE STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createPauseStreamStmt(pCxt, yymsp[-1].minor.yy667, &yymsp[0].minor.yy371); } break; - case 335: /* cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ -{ pCxt->pRootNode = createResumeStreamStmt(pCxt, yymsp[-2].minor.yy777, yymsp[-1].minor.yy777, &yymsp[0].minor.yy669); } + case 339: /* cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ +{ pCxt->pRootNode = createResumeStreamStmt(pCxt, yymsp[-2].minor.yy667, yymsp[-1].minor.yy667, &yymsp[0].minor.yy371); } break; - case 342: /* stream_options ::= stream_options TRIGGER AT_ONCE */ - case 343: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ yytestcase(yyruleno==343); -{ yylhsminor.yy242 = setStreamOptions(pCxt, yymsp[-2].minor.yy242, SOPT_TRIGGER_TYPE_SET, &yymsp[0].minor.yy0, NULL); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 346: /* stream_options ::= stream_options TRIGGER AT_ONCE */ + case 347: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ yytestcase(yyruleno==347); +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-2].minor.yy452, SOPT_TRIGGER_TYPE_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 344: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ -{ yylhsminor.yy242 = setStreamOptions(pCxt, yymsp[-3].minor.yy242, SOPT_TRIGGER_TYPE_SET, &yymsp[-1].minor.yy0, releaseRawExprNode(pCxt, yymsp[0].minor.yy242)); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; + case 348: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-3].minor.yy452, SOPT_TRIGGER_TYPE_SET, &yymsp[-1].minor.yy0, releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; break; - case 345: /* stream_options ::= stream_options WATERMARK duration_literal */ -{ yylhsminor.yy242 = setStreamOptions(pCxt, yymsp[-2].minor.yy242, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy242)); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 349: /* stream_options ::= stream_options WATERMARK duration_literal */ +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-2].minor.yy452, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 346: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ -{ yylhsminor.yy242 = setStreamOptions(pCxt, yymsp[-3].minor.yy242, SOPT_IGNORE_EXPIRED_SET, &yymsp[0].minor.yy0, NULL); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; + case 350: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-3].minor.yy452, SOPT_IGNORE_EXPIRED_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; break; - case 347: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ -{ yylhsminor.yy242 = setStreamOptions(pCxt, yymsp[-2].minor.yy242, SOPT_FILL_HISTORY_SET, &yymsp[0].minor.yy0, NULL); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 351: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-2].minor.yy452, SOPT_FILL_HISTORY_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 348: /* stream_options ::= stream_options DELETE_MARK duration_literal */ -{ yylhsminor.yy242 = setStreamOptions(pCxt, yymsp[-2].minor.yy242, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy242)); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 352: /* stream_options ::= stream_options DELETE_MARK duration_literal */ +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-2].minor.yy452, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 349: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ -{ yylhsminor.yy242 = setStreamOptions(pCxt, yymsp[-3].minor.yy242, SOPT_IGNORE_UPDATE_SET, &yymsp[0].minor.yy0, NULL); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; + case 353: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ +{ yylhsminor.yy452 = setStreamOptions(pCxt, yymsp[-3].minor.yy452, SOPT_IGNORE_UPDATE_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; break; - case 351: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - case 543: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==543); - case 564: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==564); -{ yymsp[-3].minor.yy242 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy242); } + case 355: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + case 547: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==547); + case 568: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==568); +{ yymsp[-3].minor.yy452 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy452); } break; - case 354: /* cmd ::= KILL CONNECTION NK_INTEGER */ + case 358: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } break; - case 355: /* cmd ::= KILL QUERY NK_STRING */ + case 359: /* cmd ::= KILL QUERY NK_STRING */ { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 356: /* cmd ::= KILL TRANSACTION NK_INTEGER */ + case 360: /* cmd ::= KILL TRANSACTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } break; - case 357: /* cmd ::= BALANCE VGROUP */ + case 361: /* cmd ::= BALANCE VGROUP */ { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } break; - case 358: /* cmd ::= BALANCE VGROUP LEADER */ + case 362: /* cmd ::= BALANCE VGROUP LEADER */ { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt); } break; - case 359: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + case 363: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 360: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy174); } + case 364: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy812); } break; - case 361: /* cmd ::= SPLIT VGROUP NK_INTEGER */ + case 365: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 362: /* dnode_list ::= DNODE NK_INTEGER */ -{ yymsp[-1].minor.yy174 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - break; - case 364: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ -{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy242, yymsp[0].minor.yy242); } - break; - case 367: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ -{ yymsp[-6].minor.yy242 = createInsertStmt(pCxt, yymsp[-4].minor.yy242, yymsp[-2].minor.yy174, yymsp[0].minor.yy242); } - break; - case 368: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */ -{ yymsp[-3].minor.yy242 = createInsertStmt(pCxt, yymsp[-1].minor.yy242, NULL, yymsp[0].minor.yy242); } - break; - case 369: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy242 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 370: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy242 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 371: /* literal ::= NK_STRING */ -{ yylhsminor.yy242 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 372: /* literal ::= NK_BOOL */ -{ yylhsminor.yy242 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 373: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; - break; - case 374: /* literal ::= duration_literal */ - case 384: /* signed_literal ::= signed */ yytestcase(yyruleno==384); - case 405: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==405); - case 406: /* expression ::= literal */ yytestcase(yyruleno==406); - case 407: /* expression ::= pseudo_column */ yytestcase(yyruleno==407); - case 408: /* expression ::= column_reference */ yytestcase(yyruleno==408); - case 409: /* expression ::= function_expression */ yytestcase(yyruleno==409); - case 410: /* expression ::= case_when_expression */ yytestcase(yyruleno==410); - case 441: /* function_expression ::= literal_func */ yytestcase(yyruleno==441); - case 490: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==490); - case 494: /* boolean_primary ::= predicate */ yytestcase(yyruleno==494); - case 496: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==496); - case 497: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==497); - case 500: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==500); - case 502: /* table_reference ::= table_primary */ yytestcase(yyruleno==502); - case 503: /* table_reference ::= joined_table */ yytestcase(yyruleno==503); - case 507: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==507); - case 566: /* query_simple ::= query_specification */ yytestcase(yyruleno==566); - case 567: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==567); - case 570: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==570); - case 572: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==572); -{ yylhsminor.yy242 = yymsp[0].minor.yy242; } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 375: /* literal ::= NULL */ -{ yylhsminor.yy242 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 376: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy242 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 377: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy242 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 378: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 379: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - break; - case 380: /* signed ::= NK_MINUS NK_INTEGER */ + case 366: /* dnode_list ::= DNODE NK_INTEGER */ +{ yymsp[-1].minor.yy812 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + break; + case 368: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ +{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } + break; + case 371: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ +{ yymsp[-6].minor.yy452 = createInsertStmt(pCxt, yymsp[-4].minor.yy452, yymsp[-2].minor.yy812, yymsp[0].minor.yy452); } + break; + case 372: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */ +{ yymsp[-3].minor.yy452 = createInsertStmt(pCxt, yymsp[-1].minor.yy452, NULL, yymsp[0].minor.yy452); } + break; + case 373: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 374: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 375: /* literal ::= NK_STRING */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 376: /* literal ::= NK_BOOL */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 377: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 378: /* literal ::= duration_literal */ + case 388: /* signed_literal ::= signed */ yytestcase(yyruleno==388); + case 409: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==409); + case 410: /* expression ::= literal */ yytestcase(yyruleno==410); + case 411: /* expression ::= pseudo_column */ yytestcase(yyruleno==411); + case 412: /* expression ::= column_reference */ yytestcase(yyruleno==412); + case 413: /* expression ::= function_expression */ yytestcase(yyruleno==413); + case 414: /* expression ::= case_when_expression */ yytestcase(yyruleno==414); + case 445: /* function_expression ::= literal_func */ yytestcase(yyruleno==445); + case 494: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==494); + case 498: /* boolean_primary ::= predicate */ yytestcase(yyruleno==498); + case 500: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==500); + case 501: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==501); + case 504: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==504); + case 506: /* table_reference ::= table_primary */ yytestcase(yyruleno==506); + case 507: /* table_reference ::= joined_table */ yytestcase(yyruleno==507); + case 511: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==511); + case 570: /* query_simple ::= query_specification */ yytestcase(yyruleno==570); + case 571: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==571); + case 574: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==574); + case 576: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==576); +{ yylhsminor.yy452 = yymsp[0].minor.yy452; } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 379: /* literal ::= NULL */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 380: /* literal ::= NK_QUESTION */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 381: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 382: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 383: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + break; + case 384: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; - case 381: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy242 = yylhsminor.yy242; + case 385: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; break; - case 382: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + case 386: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; - case 383: /* signed ::= NK_MINUS NK_FLOAT */ + case 387: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; - break; - case 385: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 386: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 387: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } - break; - case 388: /* signed_literal ::= duration_literal */ - case 390: /* signed_literal ::= literal_func */ yytestcase(yyruleno==390); - case 461: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==461); - case 523: /* select_item ::= common_expression */ yytestcase(yyruleno==523); - case 533: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==533); - case 571: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==571); - case 573: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==573); - case 586: /* search_condition ::= common_expression */ yytestcase(yyruleno==586); -{ yylhsminor.yy242 = releaseRawExprNode(pCxt, yymsp[0].minor.yy242); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 389: /* signed_literal ::= NULL */ -{ yylhsminor.yy242 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 391: /* signed_literal ::= NK_QUESTION */ -{ yylhsminor.yy242 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 411: /* expression ::= NK_LP expression NK_RP */ - case 495: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==495); - case 585: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==585); -{ yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy242)); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; - break; - case 412: /* expression ::= NK_PLUS expr_or_subquery */ + yymsp[-1].minor.yy452 = yylhsminor.yy452; + break; + case 389: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 390: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 391: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 392: /* signed_literal ::= duration_literal */ + case 394: /* signed_literal ::= literal_func */ yytestcase(yyruleno==394); + case 465: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==465); + case 527: /* select_item ::= common_expression */ yytestcase(yyruleno==527); + case 537: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==537); + case 575: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==575); + case 577: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==577); + case 590: /* search_condition ::= common_expression */ yytestcase(yyruleno==590); +{ yylhsminor.yy452 = releaseRawExprNode(pCxt, yymsp[0].minor.yy452); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 393: /* signed_literal ::= NULL */ +{ yylhsminor.yy452 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 395: /* signed_literal ::= NK_QUESTION */ +{ yylhsminor.yy452 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 415: /* expression ::= NK_LP expression NK_RP */ + case 499: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==499); + case 589: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==589); +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy452)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 416: /* expression ::= NK_PLUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy242)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; - case 413: /* expression ::= NK_MINUS expr_or_subquery */ + case 417: /* expression ::= NK_MINUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy242), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy452), NULL)); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; - case 414: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + case 418: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy242); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 415: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + case 419: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy242); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 416: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + case 420: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy242); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 417: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + case 421: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy242); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 418: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ + case 422: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy242); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 419: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 423: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 420: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + case 424: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy242); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 421: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + case 425: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy242); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; - break; - case 424: /* column_reference ::= column_name */ -{ yylhsminor.yy242 = createRawExprNode(pCxt, &yymsp[0].minor.yy669, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy669)); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 425: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy669, &yymsp[0].minor.yy669, createColumnNode(pCxt, &yymsp[-2].minor.yy669, &yymsp[0].minor.yy669)); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; - break; - case 426: /* pseudo_column ::= ROWTS */ - case 427: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==427); - case 429: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==429); - case 430: /* pseudo_column ::= QEND */ yytestcase(yyruleno==430); - case 431: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==431); - case 432: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==432); - case 433: /* pseudo_column ::= WEND */ yytestcase(yyruleno==433); - case 434: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==434); - case 435: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==435); - case 436: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==436); - case 437: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==437); - case 443: /* literal_func ::= NOW */ yytestcase(yyruleno==443); -{ yylhsminor.yy242 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy242 = yylhsminor.yy242; - break; - case 428: /* pseudo_column ::= table_name NK_DOT TBNAME */ -{ yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy669, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy669)))); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; - break; - case 438: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 439: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==439); -{ yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy669, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy669, yymsp[-1].minor.yy174)); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; - break; - case 440: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ -{ yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy242), yymsp[-1].minor.yy794)); } - yymsp[-5].minor.yy242 = yylhsminor.yy242; - break; - case 442: /* literal_func ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy669, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy669, NULL)); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; - break; - case 457: /* star_func_para_list ::= NK_STAR */ -{ yylhsminor.yy174 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy174 = yylhsminor.yy174; - break; - case 462: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 526: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==526); -{ yylhsminor.yy242 = createColumnNode(pCxt, &yymsp[-2].minor.yy669, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; - break; - case 463: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ -{ yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy174, yymsp[-1].minor.yy242)); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; - break; - case 464: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ -{ yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy242), yymsp[-2].minor.yy174, yymsp[-1].minor.yy242)); } - yymsp[-4].minor.yy242 = yylhsminor.yy242; - break; - case 467: /* when_then_expr ::= WHEN common_expression THEN common_expression */ -{ yymsp[-3].minor.yy242 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), releaseRawExprNode(pCxt, yymsp[0].minor.yy242)); } - break; - case 469: /* case_when_else_opt ::= ELSE common_expression */ -{ yymsp[-1].minor.yy242 = releaseRawExprNode(pCxt, yymsp[0].minor.yy242); } - break; - case 470: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ - case 475: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==475); + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 428: /* column_reference ::= column_name */ +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy371, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy371)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 429: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy371, createColumnNode(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy371)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 430: /* pseudo_column ::= ROWTS */ + case 431: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==431); + case 433: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==433); + case 434: /* pseudo_column ::= QEND */ yytestcase(yyruleno==434); + case 435: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==435); + case 436: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==436); + case 437: /* pseudo_column ::= WEND */ yytestcase(yyruleno==437); + case 438: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==438); + case 439: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==439); + case 440: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==440); + case 441: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==441); + case 447: /* literal_func ::= NOW */ yytestcase(yyruleno==447); +{ yylhsminor.yy452 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy452 = yylhsminor.yy452; + break; + case 432: /* pseudo_column ::= table_name NK_DOT TBNAME */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy371)))); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 442: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 443: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==443); +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy371, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy371, yymsp[-1].minor.yy812)); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 444: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), yymsp[-1].minor.yy310)); } + yymsp[-5].minor.yy452 = yylhsminor.yy452; + break; + case 446: /* literal_func ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy371, NULL)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 461: /* star_func_para_list ::= NK_STAR */ +{ yylhsminor.yy812 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy812 = yylhsminor.yy812; + break; + case 466: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 530: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==530); +{ yylhsminor.yy452 = createColumnNode(pCxt, &yymsp[-2].minor.yy371, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; + break; + case 467: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy812, yymsp[-1].minor.yy452)); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; + break; + case 468: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), yymsp[-2].minor.yy812, yymsp[-1].minor.yy452)); } + yymsp[-4].minor.yy452 = yylhsminor.yy452; + break; + case 471: /* when_then_expr ::= WHEN common_expression THEN common_expression */ +{ yymsp[-3].minor.yy452 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452)); } + break; + case 473: /* case_when_else_opt ::= ELSE common_expression */ +{ yymsp[-1].minor.yy452 = releaseRawExprNode(pCxt, yymsp[0].minor.yy452); } + break; + case 474: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ + case 479: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==479); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy242); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy70, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy354, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 471: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + case 475: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy242); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy242), releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy452), releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } - yymsp[-4].minor.yy242 = yylhsminor.yy242; + yymsp[-4].minor.yy452 = yylhsminor.yy452; break; - case 472: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + case 476: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy242); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy242), releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy452), releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } - yymsp[-5].minor.yy242 = yylhsminor.yy242; + yymsp[-5].minor.yy452 = yylhsminor.yy452; break; - case 473: /* predicate ::= expr_or_subquery IS NULL */ + case 477: /* predicate ::= expr_or_subquery IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), NULL)); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 474: /* predicate ::= expr_or_subquery IS NOT NULL */ + case 478: /* predicate ::= expr_or_subquery IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy242), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), NULL)); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; + yymsp[-3].minor.yy452 = yylhsminor.yy452; break; - case 476: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy70 = OP_TYPE_LOWER_THAN; } + case 480: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy354 = OP_TYPE_LOWER_THAN; } break; - case 477: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy70 = OP_TYPE_GREATER_THAN; } + case 481: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy354 = OP_TYPE_GREATER_THAN; } break; - case 478: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy70 = OP_TYPE_LOWER_EQUAL; } + case 482: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy354 = OP_TYPE_LOWER_EQUAL; } break; - case 479: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy70 = OP_TYPE_GREATER_EQUAL; } + case 483: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy354 = OP_TYPE_GREATER_EQUAL; } break; - case 480: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy70 = OP_TYPE_NOT_EQUAL; } + case 484: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy354 = OP_TYPE_NOT_EQUAL; } break; - case 481: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy70 = OP_TYPE_EQUAL; } + case 485: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy354 = OP_TYPE_EQUAL; } break; - case 482: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy70 = OP_TYPE_LIKE; } + case 486: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy354 = OP_TYPE_LIKE; } break; - case 483: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy70 = OP_TYPE_NOT_LIKE; } + case 487: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy354 = OP_TYPE_NOT_LIKE; } break; - case 484: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy70 = OP_TYPE_MATCH; } + case 488: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy354 = OP_TYPE_MATCH; } break; - case 485: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy70 = OP_TYPE_NMATCH; } + case 489: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy354 = OP_TYPE_NMATCH; } break; - case 486: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy70 = OP_TYPE_JSON_CONTAINS; } + case 490: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy354 = OP_TYPE_JSON_CONTAINS; } break; - case 487: /* in_op ::= IN */ -{ yymsp[0].minor.yy70 = OP_TYPE_IN; } + case 491: /* in_op ::= IN */ +{ yymsp[0].minor.yy354 = OP_TYPE_IN; } break; - case 488: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy70 = OP_TYPE_NOT_IN; } + case 492: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy354 = OP_TYPE_NOT_IN; } break; - case 489: /* in_predicate_value ::= NK_LP literal_list NK_RP */ -{ yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy174)); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 493: /* in_predicate_value ::= NK_LP literal_list NK_RP */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy812)); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 491: /* boolean_value_expression ::= NOT boolean_primary */ + case 495: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy242), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy452), NULL)); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; - case 492: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 496: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy242); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 493: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 497: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy242); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy242); - yylhsminor.yy242 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy452); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy452); + yylhsminor.yy452 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 501: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy242 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy242, yymsp[0].minor.yy242, NULL); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 505: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy452 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy452, yymsp[0].minor.yy452, NULL); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 504: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy242 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy669, &yymsp[0].minor.yy669); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; + case 508: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy452 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy371, &yymsp[0].minor.yy371); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; - case 505: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy242 = createRealTableNode(pCxt, &yymsp[-3].minor.yy669, &yymsp[-1].minor.yy669, &yymsp[0].minor.yy669); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; + case 509: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy452 = createRealTableNode(pCxt, &yymsp[-3].minor.yy371, &yymsp[-1].minor.yy371, &yymsp[0].minor.yy371); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; break; - case 506: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy242 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy242), &yymsp[0].minor.yy669); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; + case 510: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy452 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy452), &yymsp[0].minor.yy371); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; - case 508: /* alias_opt ::= */ -{ yymsp[1].minor.yy669 = nil_token; } + case 512: /* alias_opt ::= */ +{ yymsp[1].minor.yy371 = nil_token; } break; - case 510: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy669 = yymsp[0].minor.yy669; } + case 514: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy371 = yymsp[0].minor.yy371; } break; - case 511: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 512: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==512); -{ yymsp[-2].minor.yy242 = yymsp[-1].minor.yy242; } + case 515: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 516: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==516); +{ yymsp[-2].minor.yy452 = yymsp[-1].minor.yy452; } break; - case 513: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy242 = createJoinTableNode(pCxt, yymsp[-4].minor.yy482, yymsp[-5].minor.yy242, yymsp[-2].minor.yy242, yymsp[0].minor.yy242); } - yymsp[-5].minor.yy242 = yylhsminor.yy242; + case 517: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy452 = createJoinTableNode(pCxt, yymsp[-4].minor.yy140, yymsp[-5].minor.yy452, yymsp[-2].minor.yy452, yymsp[0].minor.yy452); } + yymsp[-5].minor.yy452 = yylhsminor.yy452; break; - case 514: /* join_type ::= */ -{ yymsp[1].minor.yy482 = JOIN_TYPE_INNER; } + case 518: /* join_type ::= */ +{ yymsp[1].minor.yy140 = JOIN_TYPE_INNER; } break; - case 515: /* join_type ::= INNER */ -{ yymsp[0].minor.yy482 = JOIN_TYPE_INNER; } + case 519: /* join_type ::= INNER */ +{ yymsp[0].minor.yy140 = JOIN_TYPE_INNER; } break; - case 516: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 520: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-11].minor.yy242 = createSelectStmt(pCxt, yymsp[-10].minor.yy777, yymsp[-9].minor.yy174, yymsp[-8].minor.yy242); - yymsp[-11].minor.yy242 = addWhereClause(pCxt, yymsp[-11].minor.yy242, yymsp[-7].minor.yy242); - yymsp[-11].minor.yy242 = addPartitionByClause(pCxt, yymsp[-11].minor.yy242, yymsp[-6].minor.yy174); - yymsp[-11].minor.yy242 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy242, yymsp[-2].minor.yy242); - yymsp[-11].minor.yy242 = addGroupByClause(pCxt, yymsp[-11].minor.yy242, yymsp[-1].minor.yy174); - yymsp[-11].minor.yy242 = addHavingClause(pCxt, yymsp[-11].minor.yy242, yymsp[0].minor.yy242); - yymsp[-11].minor.yy242 = addRangeClause(pCxt, yymsp[-11].minor.yy242, yymsp[-5].minor.yy242); - yymsp[-11].minor.yy242 = addEveryClause(pCxt, yymsp[-11].minor.yy242, yymsp[-4].minor.yy242); - yymsp[-11].minor.yy242 = addFillClause(pCxt, yymsp[-11].minor.yy242, yymsp[-3].minor.yy242); + yymsp[-11].minor.yy452 = createSelectStmt(pCxt, yymsp[-10].minor.yy667, yymsp[-9].minor.yy812, yymsp[-8].minor.yy452); + yymsp[-11].minor.yy452 = addWhereClause(pCxt, yymsp[-11].minor.yy452, yymsp[-7].minor.yy452); + yymsp[-11].minor.yy452 = addPartitionByClause(pCxt, yymsp[-11].minor.yy452, yymsp[-6].minor.yy812); + yymsp[-11].minor.yy452 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy452, yymsp[-2].minor.yy452); + yymsp[-11].minor.yy452 = addGroupByClause(pCxt, yymsp[-11].minor.yy452, yymsp[-1].minor.yy812); + yymsp[-11].minor.yy452 = addHavingClause(pCxt, yymsp[-11].minor.yy452, yymsp[0].minor.yy452); + yymsp[-11].minor.yy452 = addRangeClause(pCxt, yymsp[-11].minor.yy452, yymsp[-5].minor.yy452); + yymsp[-11].minor.yy452 = addEveryClause(pCxt, yymsp[-11].minor.yy452, yymsp[-4].minor.yy452); + yymsp[-11].minor.yy452 = addFillClause(pCxt, yymsp[-11].minor.yy452, yymsp[-3].minor.yy452); } break; - case 519: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy777 = false; } + case 523: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy667 = false; } break; - case 522: /* select_item ::= NK_STAR */ -{ yylhsminor.yy242 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy242 = yylhsminor.yy242; + case 526: /* select_item ::= NK_STAR */ +{ yylhsminor.yy452 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy452 = yylhsminor.yy452; break; - case 524: /* select_item ::= common_expression column_alias */ - case 534: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==534); -{ yylhsminor.yy242 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy242), &yymsp[0].minor.yy669); } - yymsp[-1].minor.yy242 = yylhsminor.yy242; + case 528: /* select_item ::= common_expression column_alias */ + case 538: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==538); +{ yylhsminor.yy452 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy452), &yymsp[0].minor.yy371); } + yymsp[-1].minor.yy452 = yylhsminor.yy452; break; - case 525: /* select_item ::= common_expression AS column_alias */ - case 535: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==535); -{ yylhsminor.yy242 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), &yymsp[0].minor.yy669); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 529: /* select_item ::= common_expression AS column_alias */ + case 539: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==539); +{ yylhsminor.yy452 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), &yymsp[0].minor.yy371); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 530: /* partition_by_clause_opt ::= PARTITION BY partition_list */ - case 555: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==555); - case 575: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==575); -{ yymsp[-2].minor.yy174 = yymsp[0].minor.yy174; } + case 534: /* partition_by_clause_opt ::= PARTITION BY partition_list */ + case 559: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==559); + case 579: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==579); +{ yymsp[-2].minor.yy812 = yymsp[0].minor.yy812; } break; - case 537: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy242 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy242), releaseRawExprNode(pCxt, yymsp[-1].minor.yy242)); } + case 541: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy452 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), releaseRawExprNode(pCxt, yymsp[-1].minor.yy452)); } break; - case 538: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ -{ yymsp[-3].minor.yy242 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy242)); } + case 542: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ +{ yymsp[-3].minor.yy452 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy452)); } break; - case 539: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy242 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy242), NULL, yymsp[-1].minor.yy242, yymsp[0].minor.yy242); } + case 543: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy452 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), NULL, yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } break; - case 540: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy242 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy242), releaseRawExprNode(pCxt, yymsp[-3].minor.yy242), yymsp[-1].minor.yy242, yymsp[0].minor.yy242); } + case 544: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy452 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy452), releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), yymsp[-1].minor.yy452, yymsp[0].minor.yy452); } break; - case 541: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ -{ yymsp[-6].minor.yy242 = createEventWindowNode(pCxt, yymsp[-3].minor.yy242, yymsp[0].minor.yy242); } + case 545: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ +{ yymsp[-6].minor.yy452 = createEventWindowNode(pCxt, yymsp[-3].minor.yy452, yymsp[0].minor.yy452); } break; - case 545: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy242 = createFillNode(pCxt, yymsp[-1].minor.yy204, NULL); } + case 549: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy452 = createFillNode(pCxt, yymsp[-1].minor.yy844, NULL); } break; - case 546: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ -{ yymsp[-5].minor.yy242 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy174)); } + case 550: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ +{ yymsp[-5].minor.yy452 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy812)); } break; - case 547: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ -{ yymsp[-5].minor.yy242 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy174)); } + case 551: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ +{ yymsp[-5].minor.yy452 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy812)); } break; - case 548: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy204 = FILL_MODE_NONE; } + case 552: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy844 = FILL_MODE_NONE; } break; - case 549: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy204 = FILL_MODE_PREV; } + case 553: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy844 = FILL_MODE_PREV; } break; - case 550: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy204 = FILL_MODE_NULL; } + case 554: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy844 = FILL_MODE_NULL; } break; - case 551: /* fill_mode ::= NULL_F */ -{ yymsp[0].minor.yy204 = FILL_MODE_NULL_F; } + case 555: /* fill_mode ::= NULL_F */ +{ yymsp[0].minor.yy844 = FILL_MODE_NULL_F; } break; - case 552: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy204 = FILL_MODE_LINEAR; } + case 556: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy844 = FILL_MODE_LINEAR; } break; - case 553: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy204 = FILL_MODE_NEXT; } + case 557: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy844 = FILL_MODE_NEXT; } break; - case 556: /* group_by_list ::= expr_or_subquery */ -{ yylhsminor.yy174 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); } - yymsp[0].minor.yy174 = yylhsminor.yy174; + case 560: /* group_by_list ::= expr_or_subquery */ +{ yylhsminor.yy812 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } + yymsp[0].minor.yy812 = yylhsminor.yy812; break; - case 557: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ -{ yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy242))); } - yymsp[-2].minor.yy174 = yylhsminor.yy174; + case 561: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ +{ yylhsminor.yy812 = addNodeToList(pCxt, yymsp[-2].minor.yy812, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy452))); } + yymsp[-2].minor.yy812 = yylhsminor.yy812; break; - case 561: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ -{ yymsp[-5].minor.yy242 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy242), releaseRawExprNode(pCxt, yymsp[-1].minor.yy242)); } + case 565: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ +{ yymsp[-5].minor.yy452 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy452), releaseRawExprNode(pCxt, yymsp[-1].minor.yy452)); } break; - case 562: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ -{ yymsp[-3].minor.yy242 = createInterpTimePoint(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy242)); } + case 566: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ +{ yymsp[-3].minor.yy452 = createInterpTimePoint(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy452)); } break; - case 565: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 569: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy242 = addOrderByClause(pCxt, yymsp[-3].minor.yy242, yymsp[-2].minor.yy174); - yylhsminor.yy242 = addSlimitClause(pCxt, yylhsminor.yy242, yymsp[-1].minor.yy242); - yylhsminor.yy242 = addLimitClause(pCxt, yylhsminor.yy242, yymsp[0].minor.yy242); + yylhsminor.yy452 = addOrderByClause(pCxt, yymsp[-3].minor.yy452, yymsp[-2].minor.yy812); + yylhsminor.yy452 = addSlimitClause(pCxt, yylhsminor.yy452, yymsp[-1].minor.yy452); + yylhsminor.yy452 = addLimitClause(pCxt, yylhsminor.yy452, yymsp[0].minor.yy452); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; + yymsp[-3].minor.yy452 = yylhsminor.yy452; break; - case 568: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ -{ yylhsminor.yy242 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy242, yymsp[0].minor.yy242); } - yymsp[-3].minor.yy242 = yylhsminor.yy242; + case 572: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ +{ yylhsminor.yy452 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy452, yymsp[0].minor.yy452); } + yymsp[-3].minor.yy452 = yylhsminor.yy452; break; - case 569: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ -{ yylhsminor.yy242 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy242, yymsp[0].minor.yy242); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 573: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ +{ yylhsminor.yy452 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy452, yymsp[0].minor.yy452); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 577: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 581: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==581); -{ yymsp[-1].minor.yy242 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 581: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 585: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==585); +{ yymsp[-1].minor.yy452 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 578: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 582: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==582); -{ yymsp[-3].minor.yy242 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 582: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 586: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==586); +{ yymsp[-3].minor.yy452 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 579: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 583: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==583); -{ yymsp[-3].minor.yy242 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 583: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 587: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==587); +{ yymsp[-3].minor.yy452 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 584: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy242 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy242); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 588: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy452 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy452); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 589: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy242 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy242), yymsp[-1].minor.yy48, yymsp[0].minor.yy687); } - yymsp[-2].minor.yy242 = yylhsminor.yy242; + case 593: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy452 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy452), yymsp[-1].minor.yy690, yymsp[0].minor.yy399); } + yymsp[-2].minor.yy452 = yylhsminor.yy452; break; - case 590: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy48 = ORDER_ASC; } + case 594: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy690 = ORDER_ASC; } break; - case 591: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy48 = ORDER_ASC; } + case 595: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy690 = ORDER_ASC; } break; - case 592: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy48 = ORDER_DESC; } + case 596: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy690 = ORDER_DESC; } break; - case 593: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy687 = NULL_ORDER_DEFAULT; } + case 597: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy399 = NULL_ORDER_DEFAULT; } break; - case 594: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy687 = NULL_ORDER_FIRST; } + case 598: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy399 = NULL_ORDER_FIRST; } break; - case 595: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy687 = NULL_ORDER_LAST; } + case 599: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy399 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 6d27bb0d29d3525386c1b0d1cdf91e0ad8490210..856fdb48045a95e0b36a857e523b9bf86023d6d0 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -542,6 +542,18 @@ TEST_F(ParserInitialCTest, createSmaIndex) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_INDEX_STMT); SMCreateSmaReq req = {0}; + ASSERT_TRUE(pQuery->pPrevRoot); + ASSERT_EQ(QUERY_NODE_SELECT_STMT, nodeType(pQuery->pPrevRoot)); + + SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pQuery->pRoot; + SCmdMsgInfo* pCmdMsg = (SCmdMsgInfo*)taosMemoryMalloc(sizeof(SCmdMsgInfo)); + if (NULL == pCmdMsg) FAIL(); + pCmdMsg->msgType = TDMT_MND_CREATE_SMA; + pCmdMsg->msgLen = tSerializeSMCreateSmaReq(NULL, 0, pStmt->pReq); + pCmdMsg->pMsg = taosMemoryMalloc(pCmdMsg->msgLen); + if (!pCmdMsg->pMsg) FAIL(); + tSerializeSMCreateSmaReq(pCmdMsg->pMsg, pCmdMsg->msgLen, pStmt->pReq); + ((SQuery*)pQuery)->pCmdMsg = pCmdMsg; ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); ASSERT_EQ(std::string(req.name), std::string(expect.name)); diff --git a/source/libs/parser/test/parInitialDTest.cpp b/source/libs/parser/test/parInitialDTest.cpp index cddd2aa8f7399a74e51c310b25e9285462406455..937f76176eea335272dd2940267e85a1c79b7ec8 100644 --- a/source/libs/parser/test/parInitialDTest.cpp +++ b/source/libs/parser/test/parInitialDTest.cpp @@ -291,4 +291,13 @@ TEST_F(ParserInitialDTest, dropUser) { run("DROP USER wxy"); } +TEST_F(ParserInitialDTest, IntervalOnSysTable) { + login("root"); + run("SELECT count('reboot_time') FROM information_schema.ins_dnodes interval(14m) sliding(9m)", + TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED, PARSER_STAGE_TRANSLATE); + + run("SELECT count('create_time') FROM information_schema.ins_qnodes interval(14m) sliding(9m)", + TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED, PARSER_STAGE_TRANSLATE); +} + } // namespace ParserTest diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index 82abc5d1a973dbafa820c41e0614e1b2c90a92c3..092fe1741187dcf3706a64a7be64a3032835b44a 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -43,6 +43,9 @@ int32_t splitLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan); int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQueryLogicPlan** pLogicPlan); int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList); +bool isPartTableAgg(SAggLogicNode* pAgg); +bool isPartTableWinodw(SWindowLogicNode* pWindow); + #ifdef __cplusplus } #endif diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 5374fcf7a48ba52b62fc9df70a2f8ff206422d56..37c1a288631177b3f5c3690cde8fc18c3a98128e 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1027,7 +1027,6 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect return TSDB_CODE_OUT_OF_MEMORY; } - pSort->maxRows = -1; pSort->groupSort = pSelect->groupSort; pSort->node.groupAction = pSort->groupSort ? GROUP_ACTION_KEEP : GROUP_ACTION_CLEAR; pSort->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; @@ -1299,7 +1298,6 @@ static int32_t createSetOpSortLogicNode(SLogicPlanContext* pCxt, SSetOperator* p return TSDB_CODE_OUT_OF_MEMORY; } - pSort->maxRows = -1; TSWAP(pSort->node.pLimit, pSetOperator->pLimit); int32_t code = TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8575f42fa3b9950a63c8f717631595c601061cc8..5765e304a9940f288329e0bc45f85f76275e8247 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -363,6 +363,18 @@ static void scanPathOptSetScanOrder(EScanOrder scanOrder, SScanLogicNode* pScan) } } +static void scanPathOptSetGroupOrderScan(SScanLogicNode* pScan) { + if (pScan->tableType != TSDB_SUPER_TABLE) return; + + if (pScan->node.pParent && nodeType(pScan->node.pParent) == QUERY_NODE_LOGIC_PLAN_AGG) { + SAggLogicNode* pAgg = (SAggLogicNode*)pScan->node.pParent; + bool withSlimit = pAgg->node.pSlimit != NULL || (pAgg->node.pParent && pAgg->node.pParent->pSlimit); + if (withSlimit && isPartTableAgg(pAgg)) { + pScan->groupOrderScan = pAgg->node.forceCreateNonBlockingOptr = true; + } + } +} + static int32_t scanPathOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SOsdInfo info = {.scanOrder = SCAN_ORDER_ASC}; int32_t code = scanPathOptMatch(pCxt, pLogicSubplan->pNode, &info); @@ -371,6 +383,7 @@ static int32_t scanPathOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub if (!pCxt->pPlanCxt->streamQuery) { scanPathOptSetScanOrder(info.scanOrder, info.pScan); } + scanPathOptSetGroupOrderScan(info.pScan); } if (TSDB_CODE_SUCCESS == code && (NULL != info.pDsoFuncs || NULL != info.pSdrFuncs)) { info.pScan->dataRequired = scanPathOptGetDataRequired(info.pSdrFuncs); @@ -1691,6 +1704,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub if (TSDB_CODE_SUCCESS == code) { if (QUERY_NODE_LOGIC_PLAN_AGG == pNode->pParent->type) { SAggLogicNode* pParent = (SAggLogicNode*)(pNode->pParent); + scanPathOptSetGroupOrderScan(pScan); pParent->hasGroupKeyOptimized = true; } @@ -2651,11 +2665,13 @@ static bool pushDownLimitOptShouldBeOptimized(SLogicNode* pNode) { } SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pNode->pChildren, 0); + // push down to sort node if (QUERY_NODE_LOGIC_PLAN_SORT == nodeType(pChild)) { - SLimitNode* pChildLimit = (SLimitNode*)(pChild->pLimit); // if we have pushed down, we skip it - if ((*(SSortLogicNode*)pChild).maxRows != -1) return false; - } else if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pChild)) { + if (pChild->pLimit) return false; + } else if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pChild) || QUERY_NODE_LOGIC_PLAN_SORT == nodeType(pNode)) { + // push down to table scan node + // if pNode is sortNode, we skip push down limit info to table scan node return false; } return true; @@ -2670,13 +2686,10 @@ static int32_t pushDownLimitOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pNode->pChildren, 0); nodesDestroyNode(pChild->pLimit); if (QUERY_NODE_LOGIC_PLAN_SORT == nodeType(pChild)) { - SLimitNode* pLimitNode = (SLimitNode*)pNode->pLimit; - int64_t maxRows = -1; - if (pLimitNode->limit != -1) { - maxRows = pLimitNode->limit; - if (pLimitNode->offset != -1) maxRows += pLimitNode->offset; - } - ((SSortLogicNode*)pChild)->maxRows = maxRows; + pChild->pLimit = nodesCloneNode(pNode->pLimit); + SLimitNode* pLimit = (SLimitNode*)pChild->pLimit; + pLimit->limit += pLimit->offset; + pLimit->offset = 0; } else { pChild->pLimit = pNode->pLimit; pNode->pLimit = NULL; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index a349e2c0e9cddb0a7b5f4080649648586748e069..1b92dcd2e75acc6915c46ec2e509939c2189bc55 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -447,6 +447,7 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS pScanPhysiNode->uid = pScanLogicNode->tableId; pScanPhysiNode->suid = pScanLogicNode->stableId; pScanPhysiNode->tableType = pScanLogicNode->tableType; + pScanPhysiNode->groupOrderScan = pScanLogicNode->groupOrderScan; memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName)); if (NULL != pScanLogicNode->pTagCond) { pSubplan->pTagCond = nodesCloneNode(pScanLogicNode->pTagCond); @@ -880,6 +881,7 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, pAgg->mergeDataBlock = (GROUP_ACTION_KEEP == pAggLogicNode->node.groupAction ? false : true); pAgg->groupKeyOptimized = pAggLogicNode->hasGroupKeyOptimized; + pAgg->node.forceCreateNonBlockingOptr = pAggLogicNode->node.forceCreateNonBlockingOptr; SNodeList* pPrecalcExprs = NULL; SNodeList* pGroupKeys = NULL; @@ -1374,7 +1376,6 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren if (NULL == pSort) { return TSDB_CODE_OUT_OF_MEMORY; } - pSort->maxRows = pSortLogicNode->maxRows; SNodeList* pPrecalcExprs = NULL; SNodeList* pSortKeys = NULL; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index f352a2bba3089ff5ef7ef89e53a9f0b54d4eff13..84a486649efe4265794d24adf1bde7a295c779d7 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -306,54 +306,6 @@ static bool stbSplIsTableCountQuery(SLogicNode* pNode) { return QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && SCAN_TYPE_TABLE_COUNT == ((SScanLogicNode*)pChild)->scanType; } -static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - return ((SScanLogicNode*)pNode)->pGroupTags; - } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { - return ((SPartitionLogicNode*)pNode)->pPartitionKeys; - } else { - return NULL; - } -} - -static bool stbSplHasPartTbname(SNodeList* pPartKeys) { - if (NULL == pPartKeys) { - return false; - } - SNode* pPartKey = NULL; - FOREACH(pPartKey, pPartKeys) { - if (QUERY_NODE_GROUPING_SET == nodeType(pPartKey)) { - pPartKey = nodesListGetNode(((SGroupingSetNode*)pPartKey)->pParameterList, 0); - } - if ((QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType) || - (QUERY_NODE_COLUMN == nodeType(pPartKey) && COLUMN_TYPE_TBNAME == ((SColumnNode*)pPartKey)->colType)) { - return true; - } - } - return false; -} - -static bool stbSplNotSystemScan(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - return SCAN_TYPE_SYSTEM_TABLE != ((SScanLogicNode*)pNode)->scanType; - } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { - return stbSplNotSystemScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0)); - } else { - return true; - } -} - -static bool stbSplIsPartTableAgg(SAggLogicNode* pAgg) { - if (1 != LIST_LENGTH(pAgg->node.pChildren)) { - return false; - } - if (NULL != pAgg->pGroupKeys) { - return stbSplHasPartTbname(pAgg->pGroupKeys) && - stbSplNotSystemScan((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)); - } - return stbSplHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))); -} - static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: @@ -364,7 +316,7 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { return streamQuery ? false : stbSplIsMultiTbScanChild(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_AGG: return (!stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) || - stbSplIsPartTableAgg((SAggLogicNode*)pNode)) && + isPartTableAgg((SAggLogicNode*)pNode)) && stbSplHasMultiTbScan(streamQuery, pNode) && !stbSplIsTableCountQuery(pNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: return stbSplNeedSplitWindow(streamQuery, pNode); @@ -778,10 +730,6 @@ static int32_t stbSplSplitEvent(SSplitContext* pCxt, SStableSplitInfo* pInfo) { } } -static bool stbSplIsPartTableWinodw(SWindowLogicNode* pWindow) { - return stbSplHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); -} - static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { switch (((SWindowLogicNode*)pInfo->pSplitNode)->winType) { case WINDOW_TYPE_INTERVAL: @@ -834,7 +782,7 @@ static int32_t stbSplSplitWindowForPartTable(SSplitContext* pCxt, SStableSplitIn } static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - if (stbSplIsPartTableWinodw((SWindowLogicNode*)pInfo->pSplitNode)) { + if (isPartTableWinodw((SWindowLogicNode*)pInfo->pSplitNode)) { return stbSplSplitWindowForPartTable(pCxt, pInfo); } else { return stbSplSplitWindowForCrossTable(pCxt, pInfo); @@ -920,7 +868,7 @@ static int32_t stbSplSplitAggNodeForCrossTable(SSplitContext* pCxt, SStableSplit } static int32_t stbSplSplitAggNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - if (stbSplIsPartTableAgg((SAggLogicNode*)pInfo->pSplitNode)) { + if (isPartTableAgg((SAggLogicNode*)pInfo->pSplitNode)) { return stbSplSplitAggNodeForPartTable(pCxt, pInfo); } return stbSplSplitAggNodeForCrossTable(pCxt, pInfo); @@ -1018,7 +966,6 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut splSetParent((SLogicNode*)pPartSort); pPartSort->pSortKeys = pSortKeys; pPartSort->groupSort = pSort->groupSort; - pPartSort->maxRows = pSort->maxRows; code = stbSplCreateMergeKeys(pPartSort->pSortKeys, pPartSort->node.pTargets, &pMergeKeys); } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index 29e87b34ce046d166678ab54ae5095d6e9a859fe..88086cde1d0edb91e2918a26935495be0b3120ce 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -321,3 +321,57 @@ int32_t adjustLogicNodeDataRequirement(SLogicNode* pNode, EDataOrderLevel requir } return code; } + +static bool stbNotSystemScan(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + return SCAN_TYPE_SYSTEM_TABLE != ((SScanLogicNode*)pNode)->scanType; + } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { + return stbNotSystemScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0)); + } else { + return true; + } +} + +static bool stbHasPartTbname(SNodeList* pPartKeys) { + if (NULL == pPartKeys) { + return false; + } + SNode* pPartKey = NULL; + FOREACH(pPartKey, pPartKeys) { + if (QUERY_NODE_GROUPING_SET == nodeType(pPartKey)) { + pPartKey = nodesListGetNode(((SGroupingSetNode*)pPartKey)->pParameterList, 0); + } + if ((QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType) || + (QUERY_NODE_COLUMN == nodeType(pPartKey) && COLUMN_TYPE_TBNAME == ((SColumnNode*)pPartKey)->colType)) { + return true; + } + } + return false; +} + +static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + return ((SScanLogicNode*)pNode)->pGroupTags; + } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { + return ((SPartitionLogicNode*)pNode)->pPartitionKeys; + } else { + return NULL; + } +} + +bool isPartTableAgg(SAggLogicNode* pAgg) { + if (1 != LIST_LENGTH(pAgg->node.pChildren)) { + return false; + } + if (NULL != pAgg->pGroupKeys) { + return stbHasPartTbname(pAgg->pGroupKeys) && + stbNotSystemScan((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)); + } + return stbHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))); +} + +bool isPartTableWinodw(SWindowLogicNode* pWindow) { + return stbHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); +} + + diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index b9d5c85717c71cdcb73cf660ac796541896c85e0..73aca8572a0ff5c0dd916ed65a168eedf2424f95 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -12,10 +12,17 @@ IF(NOT TD_DARWIN) "${SOURCE_LIST}/../../../parser/test/mockCatalogService.cpp" ) - TARGET_LINK_LIBRARIES( - plannerTest - PUBLIC os util common nodes planner parser catalog transport gtest function qcom - ) + IF (TD_GRANT) + TARGET_LINK_LIBRARIES( + plannerTest + PUBLIC os util common nodes planner parser catalog transport gtest function qcom grant + ) + ELSE () + TARGET_LINK_LIBRARIES( + plannerTest + PUBLIC os util common nodes planner parser catalog transport gtest function qcom + ) + ENDIF() TARGET_INCLUDE_DIRECTORIES( plannerTest diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index d89e669a90230894c5a7955a2eece9b1d2cc84e3..3b432b9890e1bb5bd903b2da61ef6b044618ee48 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -441,6 +441,16 @@ class PlannerTestBaseImpl { pCxt->topicQuery = true; } else if (QUERY_NODE_CREATE_INDEX_STMT == nodeType(pQuery->pRoot)) { SMCreateSmaReq req = {0}; + SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pQuery->pRoot; + SCmdMsgInfo* pCmdMsg = (SCmdMsgInfo*)taosMemoryMalloc(sizeof(SCmdMsgInfo)); + if (NULL == pCmdMsg) FAIL(); + pCmdMsg->msgType = TDMT_MND_CREATE_SMA; + pCmdMsg->msgLen = tSerializeSMCreateSmaReq(NULL, 0, pStmt->pReq); + pCmdMsg->pMsg = taosMemoryMalloc(pCmdMsg->msgLen); + if (!pCmdMsg->pMsg) FAIL(); + tSerializeSMCreateSmaReq(pCmdMsg->pMsg, pCmdMsg->msgLen, pStmt->pReq); + ((SQuery*)pQuery)->pCmdMsg = pCmdMsg; + tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req); g_mockCatalogService->createSmaIndex(&req); nodesStringToNode(req.ast, &pCxt->pAstRoot); diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 7840fe2017a6f384e70f1553105c7293a4abb2d7..aecf3d5d916681687986ec1771d7f02e79304a8b 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -57,7 +57,7 @@ typedef enum { #define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ #define SCHEDULE_DEFAULT_MAX_NODE_NUM 20 -#define SCH_DEFAULT_TASK_TIMEOUT_USEC 60000000 +#define SCH_DEFAULT_TASK_TIMEOUT_USEC 5000000 #define SCH_MAX_TASK_TIMEOUT_USEC 300000000 #define SCH_DEFAULT_MAX_RETRY_NUM 6 #define SCH_MIN_AYSNC_EXEC_NUM 3 @@ -239,7 +239,7 @@ typedef struct SSchTask { int32_t lastMsgType; // last sent msg type int64_t timeoutUsec; // task timeout useconds before reschedule SQueryNodeAddr succeedAddr; // task executed success node address - int8_t candidateIdx; // current try condidation index + int32_t candidateIdx; // current try condidation index SArray *candidateAddrs; // condidate node addresses, element is SQueryNodeAddr SHashObj *execNodes; // all tried node for current task, element is SSchNodeInfo SSchTaskProfile profile; // task execution profile diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index adb0c111cf2ffe794fca8505d73bba94ea446ae3..9985e7d6a1260a7c6f8220022f85bbdbaa89e25e 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -745,7 +745,6 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - pTask->candidateIdx = 0; pTask->candidateAddrs = taosArrayInit(SCHEDULE_DEFAULT_MAX_NODE_NUM, sizeof(SQueryNodeAddr)); if (NULL == pTask->candidateAddrs) { SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCHEDULE_DEFAULT_MAX_NODE_NUM); @@ -770,6 +769,8 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schSetAddrsFromNodeList(pJob, pTask)); + pTask->candidateIdx = taosRand() % taosArrayGetSize(pTask->candidateAddrs); + /* for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); diff --git a/source/libs/scheduler/test/CMakeLists.txt b/source/libs/scheduler/test/CMakeLists.txt index ce928862215d3efe7dc14c1c50e581416cb18777..703bd5932b2301d0006960f94a0131f792501c43 100644 --- a/source/libs/scheduler/test/CMakeLists.txt +++ b/source/libs/scheduler/test/CMakeLists.txt @@ -7,10 +7,18 @@ IF(NOT TD_DARWIN) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(schedulerTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES( - schedulerTest - PUBLIC os util common catalog transport gtest qcom taos_static planner scheduler - ) + + IF (TD_GRANT) + TARGET_LINK_LIBRARIES( + schedulerTest + PUBLIC os util common catalog transport gtest qcom taos_static planner scheduler grant + ) + ELSE () + TARGET_LINK_LIBRARIES( + schedulerTest + PUBLIC os util common catalog transport gtest qcom taos_static planner scheduler + ) + ENDIF() TARGET_INCLUDE_DIRECTORIES( schedulerTest diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index da4e442f1a32d0cec52dcdfc749946bb1daf6a08..b6bc9c888bec1136f82e13d5f072c0acbed9d937 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -42,10 +42,11 @@ typedef struct { TdThreadMutex cfMutex; SHashObj* cfInst; int64_t defaultCfInit; -} SBackendHandle; +} SBackendWrapper; void* streamBackendInit(const char* path); void streamBackendCleanup(void* arg); +void streamBackendHandleCleanup(void* arg); SListNode* streamBackendAddCompare(void* backend, void* arg); void streamBackendDelCompare(void* backend, void* arg); diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInt.h similarity index 75% rename from source/libs/stream/inc/streamInc.h rename to source/libs/stream/inc/streamInt.h index c7ee308b6149970fc7f039cb0bc857a8031d4f73..add893c8c7dfa7d04c8bef8dfbce4df204357d23 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInt.h @@ -31,8 +31,15 @@ typedef struct { void* timer; } SStreamGlobalEnv; -static SStreamGlobalEnv streamEnv; +typedef struct { + SEpSet epset; + int32_t taskId; + SRpcMsg msg; +} SStreamContinueExecInfo; + +extern SStreamGlobalEnv streamEnv; +void streamRetryDispatchStreamBlock(SStreamTask* pTask, int64_t waitDuration); int32_t streamDispatchStreamBlock(SStreamTask* pTask); SStreamDataBlock* createStreamDataFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg); @@ -41,20 +48,23 @@ SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamT void destroyStreamDataBlock(SStreamDataBlock* pBlock); int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData); -int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* data); - int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock); int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); +int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pData); int32_t streamDispatchCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); -int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, - SEpSet* pEpSet); +int32_t streamDoDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamScanHistoryFinishReq* pReq, int32_t vgId, + SEpSet* pEpSet); SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem); +int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo, SStreamScanHistoryFinishReq* pReq); +int32_t streamNotifyUpstreamContinue(SStreamTask* pTask); + extern int32_t streamBackendId; +extern int32_t streamBackendCfWrapperId; #ifdef __cplusplus } diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 93bcd6a4d910bf3140169a3b217911e48a532cdd..f85ade591ca48f5403c4c1e0ee59f87a5760462a 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -13,14 +13,16 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" #include "ttimer.h" -#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 20480 -#define STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE (50) +#define STREAM_TASK_INPUT_QUEUE_CAPACITY 20480 +#define STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE (30) #define ONE_MB_F (1048576.0) #define QUEUE_MEM_SIZE_IN_MB(_q) (taosQueueMemorySize(_q) / ONE_MB_F) +SStreamGlobalEnv streamEnv; + int32_t streamInit() { int8_t old; while (1) { @@ -36,6 +38,7 @@ int32_t streamInit() { } atomic_store_8(&streamEnv.inited, 1); } + return 0; } @@ -52,45 +55,62 @@ void streamCleanUp() { } } -void streamSchedByTimer(void* param, void* tmrId) { +char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { + char buf[128] = {0}; + sprintf(buf, "0x%" PRIx64 "-0x%x", streamId, taskId); + return taosStrdup(buf); +} + +static void streamSchedByTimer(void* param, void* tmrId) { SStreamTask* pTask = (void*)param; + int8_t status = atomic_load_8(&pTask->triggerStatus); + qDebug("s-task:%s in scheduler, trigger status:%d, next:%dms", pTask->id.idStr, status, (int32_t)pTask->triggerParam); + if (streamTaskShouldStop(&pTask->status) || streamTaskShouldPause(&pTask->status)) { streamMetaReleaseTask(NULL, pTask); + qDebug("s-task:%s jump out of schedTimer", pTask->id.idStr); return; } - if (atomic_load_8(&pTask->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { - SStreamTrigger* trigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM, 0); - if (trigger == NULL) return; - trigger->type = STREAM_INPUT__GET_RES; - trigger->pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - if (trigger->pBlock == NULL) { - taosFreeQitem(trigger); + if (status == TASK_TRIGGER_STATUS__ACTIVE) { + SStreamTrigger* pTrigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM, 0); + if (pTrigger == NULL) { return; } - trigger->pBlock->info.type = STREAM_GET_ALL; - atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE); + pTrigger->type = STREAM_INPUT__GET_RES; + pTrigger->pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + if (pTrigger->pBlock == NULL) { + taosFreeQitem(pTrigger); + return; + } - if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)trigger) < 0) { - taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); + atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE); + pTrigger->pBlock->info.type = STREAM_GET_ALL; + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pTrigger) < 0) { + taosFreeQitem(pTrigger); + taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->schedTimer); return; } streamSchedExec(pTask); } - taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); + taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->schedTimer); } -int32_t streamSetupTrigger(SStreamTask* pTask) { - if (pTask->triggerParam != 0) { +int32_t streamSetupScheduleTrigger(SStreamTask* pTask) { + if (pTask->triggerParam != 0 && pTask->info.fillHistory == 0) { int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1); - ASSERT(ref == 2); - pTask->timer = taosTmrStart(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer); + ASSERT(ref == 2 && pTask->schedTimer == NULL); + + qDebug("s-task:%s setup scheduler trigger, delay:%"PRId64" ms", pTask->id.idStr, pTask->triggerParam); + + pTask->schedTimer = taosTmrStart(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer); pTask->triggerStatus = TASK_TRIGGER_STATUS__INACTIVE; } + return 0; } @@ -103,16 +123,20 @@ int32_t streamSchedExec(SStreamTask* pTask) { if (pRunReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + qError("failed to create msg to aunch s-task:%s, reason out of memory", pTask->id.idStr); return -1; } - pRunReq->head.vgId = pTask->nodeId; + pRunReq->head.vgId = pTask->info.nodeId; pRunReq->streamId = pTask->id.streamId; pRunReq->taskId = pTask->id.taskId; + qDebug("trigger to run s-task:%s", pTask->id.idStr); + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &msg); - qDebug("trigger to run s-task:%s", pTask->id.idStr); + } else { + qDebug("s-task:%s not launch task since sched status:%d", pTask->id.idStr, pTask->status.schedStatus); } return 0; @@ -142,7 +166,7 @@ int32_t streamTaskEnqueueBlocks(SStreamTask* pTask, const SStreamDispatchReq* pR pDispatchRsp->streamId = htobe64(pReq->streamId); pDispatchRsp->upstreamNodeId = htonl(pReq->upstreamNodeId); pDispatchRsp->upstreamTaskId = htonl(pReq->upstreamTaskId); - pDispatchRsp->downstreamNodeId = htonl(pTask->nodeId); + pDispatchRsp->downstreamNodeId = htonl(pTask->info.nodeId); pDispatchRsp->downstreamTaskId = htonl(pTask->id.taskId); pRsp->pCont = buf; @@ -158,21 +182,18 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, // enqueue if (pData != NULL) { - qDebug("s-task:%s (child %d) recv retrieve req from task:0x%x, reqId %" PRId64, pTask->id.idStr, pTask->selfChildId, - pReq->srcTaskId, pReq->reqId); + qDebug("s-task:%s (child %d) recv retrieve req from task:0x%x(vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, pTask->info.selfChildId, + pReq->srcTaskId, pReq->srcNodeId, pReq->reqId); pData->type = STREAM_INPUT__DATA_RETRIEVE; pData->srcVgId = 0; - // decode - /*pData->blocks = pReq->data;*/ - /*pBlock->sourceVer = pReq->sourceVer;*/ streamRetrieveReqToData(pReq, pData); if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pData) == 0) { status = TASK_INPUT_STATUS__NORMAL; } else { status = TASK_INPUT_STATUS__FAILED; } - } else { + } else { // todo handle oom /*streamTaskInputFail(pTask);*/ /*status = TASK_INPUT_STATUS__FAILED;*/ } @@ -187,21 +208,23 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, pRsp->pCont = buf; pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp); tmsgSendRsp(pRsp); + return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1; } // todo add log int32_t streamTaskOutputResultBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { int32_t code = 0; - if (pTask->outputType == TASK_OUTPUT__TABLE) { + int32_t type = pTask->outputInfo.type; + if (type == TASK_OUTPUT__TABLE) { pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks); destroyStreamDataBlock(pBlock); - } else if (pTask->outputType == TASK_OUTPUT__SMA) { + } else if (type == TASK_OUTPUT__SMA) { pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks); destroyStreamDataBlock(pBlock); } else { - ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH); - code = taosWriteQitem(pTask->outputQueue->queue, pBlock); + ASSERT(type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH); + code = taosWriteQitem(pTask->outputInfo.queue->queue, pBlock); if (code != 0) { // todo failed to add it into the output queue, free it. return code; } @@ -231,10 +254,28 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S return 0; } +// todo record the idle time for dispatch data int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { + if (code != TSDB_CODE_SUCCESS) { + // dispatch message failed: network error, or node not available. + // in case of the input queue is full, the code will be TSDB_CODE_SUCCESS, the and pRsp>inputStatus will be set + // flag. here we need to retry dispatch this message to downstream task immediately. handle the case the failure + // happened too fast. todo handle the shuffle dispatch failure + if (code == TSDB_CODE_STREAM_TASK_NOT_EXIST) { + qError("s-task:%s failed to dispatch msg to task:0x%x, code:%s, no-retry", pTask->id.idStr, + pRsp->downstreamTaskId, tstrerror(code)); + return code; + } else { + qError("s-task:%s failed to dispatch msg to task:0x%x, code:%s, retry cnt:%d", pTask->id.idStr, + pRsp->downstreamTaskId, tstrerror(code), ++pTask->msgInfo.retryCount); + return streamDispatchAllBlocks(pTask, pTask->msgInfo.pData); + } + } + qDebug("s-task:%s receive dispatch rsp, output status:%d code:%d", pTask->id.idStr, pRsp->inputStatus, code); - if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + // there are other dispatch message not response yet + if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); qDebug("s-task:%s is shuffle, left waiting rsp %d", pTask->id.idStr, leftRsp); if (leftRsp > 0) { @@ -242,23 +283,39 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } } - int8_t old = atomic_exchange_8(&pTask->outputStatus, pRsp->inputStatus); - ASSERT(old == TASK_OUTPUT_STATUS__WAIT); + pTask->msgInfo.retryCount = 0; + ASSERT(pTask->outputInfo.status == TASK_OUTPUT_STATUS__WAIT); + + qDebug("s-task:%s output status is set to:%d", pTask->id.idStr, pTask->outputInfo.status); - // the input queue of the (down stream) task that receive the output data is full, so the TASK_INPUT_STATUS_BLOCKED is rsp - // todo we need to send EMPTY PACKAGE to detect if the input queue is available for output of upstream task, every 50 ms. + // the input queue of the (down stream) task that receive the output data is full, + // so the TASK_INPUT_STATUS_BLOCKED is rsp + // todo blocking the output status if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { - // TODO: init recover timer - qError("s-task:%s inputQ of downstream task:0x%x is full, need to block output", pTask->id.idStr, pRsp->downstreamTaskId); + pTask->msgInfo.blockingTs = taosGetTimestampMs(); // record the blocking start time + + int32_t waitDuration = 300; // 300 ms + qError("s-task:%s inputQ of downstream task:0x%x is full, time:%" PRId64 "wait for %dms and retry dispatch data", + pTask->id.idStr, pRsp->downstreamTaskId, pTask->msgInfo.blockingTs, waitDuration); + streamRetryDispatchStreamBlock(pTask, waitDuration); + } else { // pipeline send data in output queue + // this message has been sent successfully, let's try next one. + destroyStreamDataBlock(pTask->msgInfo.pData); + pTask->msgInfo.pData = NULL; + + if (pTask->msgInfo.blockingTs != 0) { + int64_t el = taosGetTimestampMs() - pTask->msgInfo.blockingTs; + qDebug("s-task:%s resume to normal from inputQ blocking, idle time:%"PRId64"ms", pTask->id.idStr, el); + pTask->msgInfo.blockingTs = 0; + } - atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); - qError("s-task:%s ignore error, and reset task output status:%d", pTask->id.idStr, pTask->outputStatus); + // now ready for next data output + atomic_store_8(&pTask->outputInfo.status, TASK_OUTPUT_STATUS__NORMAL); - return 0; + // otherwise, continue dispatch the first block to down stream task in pipeline + streamDispatchStreamBlock(pTask); } - // otherwise, continue dispatch the first block to down stream task in pipeline - streamDispatchStreamBlock(pTask); return 0; } @@ -267,25 +324,20 @@ int32_t streamProcessRunReq(SStreamTask* pTask) { return -1; } - /*if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {*/ - /*streamDispatchStreamBlock(pTask);*/ - /*}*/ return 0; } int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) { - qDebug("s-task:%s receive retrieve req from node %d taskId:0x%x", pTask->id.idStr, pReq->srcNodeId, pReq->srcTaskId); streamTaskEnqueueRetrieve(pTask, pReq, pRsp); - - ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); + ASSERT(pTask->info.taskLevel != TASK_LEVEL__SINK); streamSchedExec(pTask); return 0; } bool tInputQueueIsFull(const SStreamTask* pTask) { - bool isFull = taosQueueItemSize((pTask->inputQueue->queue)) >= STREAM_TASK_INPUT_QUEUEU_CAPACITY; + bool isFull = taosQueueItemSize((pTask->inputQueue->queue)) >= STREAM_TASK_INPUT_QUEUE_CAPACITY; double size = QUEUE_MEM_SIZE_IN_MB(pTask->inputQueue->queue); - return (isFull || size >= STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE); + return (isFull || size >= STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE); } int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { @@ -295,15 +347,18 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { if (type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* px = (SStreamDataSubmit*)pItem; - if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && tInputQueueIsFull(pTask)) { + if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && tInputQueueIsFull(pTask)) { qError("s-task:%s input queue is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data", - pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total, + pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); streamDataSubmitDestroy(px); taosFreeQitem(pItem); return -1; } + int32_t msgLen = px->submit.msgLen; + int64_t ver = px->submit.ver; + int32_t code = taosWriteQitem(pTask->inputQueue->queue, pItem); if (code != TSDB_CODE_SUCCESS) { streamDataSubmitDestroy(px); @@ -311,19 +366,20 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { return code; } + // use the local variable to avoid the pItem be freed by other threads, since it has been put into queue already. qDebug("s-task:%s submit enqueue msgLen:%d ver:%" PRId64 ", total in queue:%d, size:%.2fMiB", pTask->id.idStr, - px->submit.msgLen, px->submit.ver, total, size + px->submit.msgLen/1048576.0); + msgLen, ver, total, size + msgLen/1048576.0); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { - if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && (tInputQueueIsFull(pTask))) { + if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && (tInputQueueIsFull(pTask))) { qError("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", - pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total, + pTask->id.idStr, STREAM_TASK_INPUT_QUEUE_CAPACITY, STREAM_TASK_INPUT_QUEUE_CAPACITY_IN_SIZE, total, size); destroyStreamDataBlock((SStreamDataBlock*) pItem); return -1; } - qDebug("s-task:%s data block enqueue, current(blocks:%d, size:%.2fMiB)", pTask->id.idStr, total, size); + qDebug("s-task:%s blockdata enqueue, total in queue:%d, size:%.2fMiB", pTask->id.idStr, total, size); int32_t code = taosWriteQitem(pTask->inputQueue->queue, pItem); if (code != TSDB_CODE_SUCCESS) { destroyStreamDataBlock((SStreamDataBlock*) pItem); @@ -339,6 +395,7 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); + qDebug("s-task:%s new data arrived, active the trigger, trigerStatus:%d", pTask->id.idStr, pTask->triggerStatus); } return 0; @@ -346,20 +403,34 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { static void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; } -void* streamQueueNextItem(SStreamQueue* queue) { - int8_t dequeueFlag = atomic_exchange_8(&queue->status, STREAM_QUEUE__PROCESSING); - if (dequeueFlag == STREAM_QUEUE__FAILED) { - ASSERT(queue->qItem != NULL); - return streamQueueCurItem(queue); +void* streamQueueNextItem(SStreamQueue* pQueue) { + int8_t flag = atomic_exchange_8(&pQueue->status, STREAM_QUEUE__PROCESSING); + + if (flag == STREAM_QUEUE__FAILED) { + ASSERT(pQueue->qItem != NULL); + return streamQueueCurItem(pQueue); } else { - queue->qItem = NULL; - taosGetQitem(queue->qall, &queue->qItem); - if (queue->qItem == NULL) { - taosReadAllQitems(queue->queue, queue->qall); - taosGetQitem(queue->qall, &queue->qItem); + pQueue->qItem = NULL; + taosGetQitem(pQueue->qall, &pQueue->qItem); + if (pQueue->qItem == NULL) { + taosReadAllQitems(pQueue->queue, pQueue->qall); + taosGetQitem(pQueue->qall, &pQueue->qItem); } - return streamQueueCurItem(queue); + + return streamQueueCurItem(pQueue); } } -void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); } \ No newline at end of file +void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); } + +SStreamChildEpInfo * streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId) { + int32_t num = taosArrayGetSize(pTask->pUpstreamEpInfoList); + for(int32_t i = 0; i < num; ++i) { + SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->pUpstreamEpInfoList, i); + if (pInfo->taskId == taskId) { + return pInfo; + } + } + + return NULL; +} \ No newline at end of file diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 242256cd347a35defe294be404c8460df8cb21f1..8534f3b0a11503101bf0a49c1f0a8aaf6811a4c7 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -16,7 +16,7 @@ #include "streamBackendRocksdb.h" #include "executor.h" #include "query.h" -#include "streamInc.h" +#include "streamInt.h" #include "tcommon.h" #include "tref.h" @@ -40,16 +40,8 @@ typedef struct { rocksdb_comparator_t** pCompares; } RocksdbCfInst; -uint32_t nextPow2(uint32_t x) { - x = x - 1; - x = x | (x >> 1); - x = x | (x >> 2); - x = x | (x >> 4); - x = x | (x >> 8); - x = x | (x >> 16); - return x + 1; -} -int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf); +uint32_t nextPow2(uint32_t x); +int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf); void destroyRocksdbCfInst(RocksdbCfInst* inst); @@ -71,7 +63,22 @@ typedef int (*BackendCmpFunc)(void* state, const char* aBuf, size_t aLen, const typedef void (*DestroyFunc)(void* state); typedef int32_t (*EncodeValueFunc)(void* value, int32_t vlen, int64_t ttl, char** dest); typedef int32_t (*DecodeValueFunc)(void* value, int32_t vlen, int64_t* ttl, char** dest); +typedef struct { + const char* key; + int32_t len; + int idx; + BackendCmpFunc cmpFunc; + EncodeFunc enFunc; + DecodeFunc deFunc; + ToStringFunc toStrFunc; + CompareName cmpName; + DestroyFunc detroyFunc; + EncodeValueFunc enValueFunc; + DecodeValueFunc deValueFunc; +} SCfInit; + +#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); const char* compareDefaultName(void* name); const char* compareStateName(void* name); const char* compareWinKeyName(void* name); @@ -80,11 +87,67 @@ const char* compareFuncKeyName(void* name); const char* compareParKeyName(void* name); const char* comparePartagKeyName(void* name); +int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +int defaultKeyEncode(void* k, char* buf); +int defaultKeyDecode(void* k, char* buf); +int defaultKeyToString(void* k, char* buf); + +int stateKeyDBComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +int stateKeyEncode(void* k, char* buf); +int stateKeyDecode(void* k, char* buf); +int stateKeyToString(void* k, char* buf); + +int stateSessionKeyDBComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +int stateSessionKeyEncode(void* ses, char* buf); +int stateSessionKeyDecode(void* ses, char* buf); +int stateSessionKeyToString(void* k, char* buf); + +int winKeyDBComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +int winKeyEncode(void* k, char* buf); +int winKeyDecode(void* k, char* buf); +int winKeyToString(void* k, char* buf); + +int tupleKeyDBComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +int tupleKeyEncode(void* k, char* buf); +int tupleKeyDecode(void* k, char* buf); +int tupleKeyToString(void* k, char* buf); + +int parKeyDBComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen); +int parKeyEncode(void* k, char* buf); +int parKeyDecode(void* k, char* buf); +int parKeyToString(void* k, char* buf); + +int stremaValueEncode(void* k, char* buf); +int streamValueDecode(void* k, char* buf); +int32_t streamValueToString(void* k, char* buf); +int32_t streaValueIsStale(void* k, int64_t ts); +void destroyFunc(void* arg); + +int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest); +int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest); + +SCfInit ginitDict[] = { + {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName, + destroyFunc, encodeValueFunc, decodeValueFunc}, + {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyFunc, + encodeValueFunc, decodeValueFunc}, + {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc}, + {"sess", 4, 3, stateSessionKeyDBComp, stateSessionKeyEncode, stateSessionKeyDecode, stateSessionKeyToString, + compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc}, + {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc}, + {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc}, + {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyFunc, + encodeValueFunc, decodeValueFunc}, +}; + void* streamBackendInit(const char* path) { uint32_t dbMemLimit = nextPow2(tsMaxStreamBackendCache) << 20; qDebug("start to init stream backend at %s", path); - SBackendHandle* pHandle = taosMemoryCalloc(1, sizeof(SBackendHandle)); + SBackendWrapper* pHandle = taosMemoryCalloc(1, sizeof(SBackendWrapper)); pHandle->list = tdListNew(sizeof(SCfComparator)); taosThreadMutexInit(&pHandle->mutex, NULL); taosThreadMutexInit(&pHandle->cfMutex, NULL); @@ -154,8 +217,8 @@ _EXIT: return NULL; } void streamBackendCleanup(void* arg) { - SBackendHandle* pHandle = (SBackendHandle*)arg; - RocksdbCfInst** pIter = (RocksdbCfInst**)taosHashIterate(pHandle->cfInst, NULL); + SBackendWrapper* pHandle = (SBackendWrapper*)arg; + RocksdbCfInst** pIter = (RocksdbCfInst**)taosHashIterate(pHandle->cfInst, NULL); while (pIter != NULL) { RocksdbCfInst* inst = *pIter; destroyRocksdbCfInst(inst); @@ -194,17 +257,79 @@ void streamBackendCleanup(void* arg) { taosMemoryFree(pHandle); return; } +void streamBackendHandleCleanup(void* arg) { + SBackendCfWrapper* wrapper = arg; + bool remove = wrapper->remove; + qDebug("start to do-close backendwrapper %p, %s", wrapper, wrapper->idstr); + if (wrapper->rocksdb == NULL) { + return; + } + + int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); + + char* err = NULL; + if (remove) { + for (int i = 0; i < cfLen; i++) { + if (wrapper->pHandle[i] != NULL) + rocksdb_drop_column_family(wrapper->rocksdb, ((rocksdb_column_family_handle_t**)wrapper->pHandle)[i], &err); + if (err != NULL) { + // qError("failed to create cf:%s_%s, reason:%s", wrapper->idstr, ginitDict[i].key, err); + taosMemoryFreeClear(err); + } + } + } else { + rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); + for (int i = 0; i < cfLen; i++) { + if (wrapper->pHandle[i] != NULL) rocksdb_flush_cf(wrapper->rocksdb, flushOpt, wrapper->pHandle[i], &err); + if (err != NULL) { + qError("failed to create cf:%s_%s, reason:%s", wrapper->idstr, ginitDict[i].key, err); + taosMemoryFreeClear(err); + } + } + rocksdb_flushoptions_destroy(flushOpt); + } + + for (int i = 0; i < cfLen; i++) { + if (wrapper->pHandle[i] != NULL) { + rocksdb_column_family_handle_destroy(wrapper->pHandle[i]); + } + } + taosMemoryFreeClear(wrapper->pHandle); + for (int i = 0; i < cfLen; i++) { + rocksdb_options_destroy(wrapper->cfOpts[i]); + rocksdb_block_based_options_destroy(((RocksdbCfParam*)wrapper->param)[i].tableOpt); + } + + if (remove) { + streamBackendDelCompare(wrapper->pBackend, wrapper->pComparNode); + } + rocksdb_writeoptions_destroy(wrapper->writeOpts); + wrapper->writeOpts = NULL; + + rocksdb_readoptions_destroy(wrapper->readOpts); + wrapper->readOpts = NULL; + taosMemoryFreeClear(wrapper->cfOpts); + taosMemoryFreeClear(wrapper->param); + + taosThreadRwlockDestroy(&wrapper->rwLock); + wrapper->rocksdb = NULL; + taosReleaseRef(streamBackendId, wrapper->backendId); + + qDebug("end to do-close backendwrapper %p, %s", wrapper, wrapper->idstr); + taosMemoryFree(wrapper); + return; +} SListNode* streamBackendAddCompare(void* backend, void* arg) { - SBackendHandle* pHandle = (SBackendHandle*)backend; - SListNode* node = NULL; + SBackendWrapper* pHandle = (SBackendWrapper*)backend; + SListNode* node = NULL; taosThreadMutexLock(&pHandle->mutex); node = tdListAdd(pHandle->list, arg); taosThreadMutexUnlock(&pHandle->mutex); return node; } void streamBackendDelCompare(void* backend, void* arg) { - SBackendHandle* pHandle = (SBackendHandle*)backend; - SListNode* node = NULL; + SBackendWrapper* pHandle = (SBackendWrapper*)backend; + SListNode* node = NULL; taosThreadMutexLock(&pHandle->mutex); node = tdListPopNode(pHandle->list, arg); taosThreadMutexUnlock(&pHandle->mutex); @@ -542,23 +667,6 @@ void destroyFunc(void* arg) { return; } -typedef struct { - const char* key; - int32_t len; - int idx; - BackendCmpFunc cmpFunc; - EncodeFunc enFunc; - DecodeFunc deFunc; - ToStringFunc toStrFunc; - CompareName cmpName; - DestroyFunc detroyFunc; - EncodeValueFunc enValueFunc; - DecodeValueFunc deValueFunc; - -} SCfInit; - -#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); - int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest) { SStreamValue key = {.unixTimestamp = ttl, .len = vlen, .data = (char*)(value)}; int32_t len = 0; @@ -613,22 +721,6 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { } return key.len; } -SCfInit ginitDict[] = { - {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName, - destroyFunc, encodeValueFunc, decodeValueFunc}, - {"state", 5, 1, stateKeyDBComp, stateKeyEncode, stateKeyDecode, stateKeyToString, compareStateName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"fill", 4, 2, winKeyDBComp, winKeyEncode, winKeyDecode, winKeyToString, compareWinKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"sess", 4, 3, stateSessionKeyDBComp, stateSessionKeyEncode, stateSessionKeyDecode, stateSessionKeyToString, - compareSessionKeyName, destroyFunc, encodeValueFunc, decodeValueFunc}, - {"func", 4, 4, tupleKeyDBComp, tupleKeyEncode, tupleKeyDecode, tupleKeyToString, compareFuncKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"parname", 7, 5, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, compareParKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, - {"partag", 6, 6, parKeyDBComp, parKeyEncode, parKeyDecode, parKeyToString, comparePartagKeyName, destroyFunc, - encodeValueFunc, decodeValueFunc}, -}; const char* compareDefaultName(void* arg) { (void)arg; @@ -697,11 +789,11 @@ void destroyRocksdbCfInst(RocksdbCfInst* inst) { } int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf) { - SBackendHandle* handle = backend; - char* err = NULL; - int64_t streamId; - int32_t taskId, dummy = 0; - char suffix[64] = {0}; + SBackendWrapper* handle = backend; + char* err = NULL; + int64_t streamId; + int32_t taskId, dummy = 0; + char suffix[64] = {0}; rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); RocksdbCfParam* params = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam)); @@ -821,23 +913,30 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t int streamStateOpenBackend(void* backend, SStreamState* pState) { qInfo("start to open state %p on backend %p 0x%" PRIx64 "-%d", pState, backend, pState->streamId, pState->taskId); taosAcquireRef(streamBackendId, pState->streamBackendRid); - SBackendHandle* handle = backend; - - sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-%d", pState->streamId, pState->taskId); + SBackendWrapper* handle = backend; + SBackendCfWrapper* pBackendCfWrapper = taosMemoryCalloc(1, sizeof(SBackendCfWrapper)); taosThreadMutexLock(&handle->cfMutex); + RocksdbCfInst** ppInst = taosHashGet(handle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); if (ppInst != NULL && *ppInst != NULL) { RocksdbCfInst* inst = *ppInst; - pState->pTdbState->rocksdb = inst->db; - pState->pTdbState->pHandle = (void**)inst->pHandle; - pState->pTdbState->writeOpts = inst->wOpt; - pState->pTdbState->readOpts = inst->rOpt; - pState->pTdbState->cfOpts = (void**)(inst->cfOpt); - pState->pTdbState->dbOpt = handle->dbOpt; - pState->pTdbState->param = inst->param; - pState->pTdbState->pBackend = handle; - pState->pTdbState->pComparNode = inst->pCompareNode; + pBackendCfWrapper->rocksdb = inst->db; + pBackendCfWrapper->pHandle = (void**)inst->pHandle; + pBackendCfWrapper->writeOpts = inst->wOpt; + pBackendCfWrapper->readOpts = inst->rOpt; + pBackendCfWrapper->cfOpts = (void**)(inst->cfOpt); + pBackendCfWrapper->dbOpt = handle->dbOpt; + pBackendCfWrapper->param = inst->param; + pBackendCfWrapper->pBackend = handle; + pBackendCfWrapper->pComparNode = inst->pCompareNode; taosThreadMutexUnlock(&handle->cfMutex); + pBackendCfWrapper->backendId = pState->streamBackendRid; + memcpy(pBackendCfWrapper->idstr, pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr)); + + int64_t id = taosAddRef(streamBackendCfWrapperId, pBackendCfWrapper); + pState->pTdbState->backendCfWrapperId = id; + pState->pTdbState->pBackendCfWrapper = pBackendCfWrapper; + qInfo("succ to open state %p on backendWrapper, %p, %s", pState, pBackendCfWrapper, pBackendCfWrapper->idstr); return 0; } taosThreadMutexUnlock(&handle->cfMutex); @@ -870,27 +969,33 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { pCompare[i] = compare; } rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(cfLen, sizeof(rocksdb_column_family_handle_t*)); - pState->pTdbState->rocksdb = handle->db; - pState->pTdbState->pHandle = (void**)cfHandle; - pState->pTdbState->writeOpts = rocksdb_writeoptions_create(); - pState->pTdbState->readOpts = rocksdb_readoptions_create(); - pState->pTdbState->cfOpts = (void**)cfOpt; - pState->pTdbState->dbOpt = handle->dbOpt; - pState->pTdbState->param = param; - pState->pTdbState->pBackend = handle; - - taosThreadRwlockInit(&pState->pTdbState->rwLock, NULL); + pBackendCfWrapper->rocksdb = handle->db; + pBackendCfWrapper->pHandle = (void**)cfHandle; + pBackendCfWrapper->writeOpts = rocksdb_writeoptions_create(); + pBackendCfWrapper->readOpts = rocksdb_readoptions_create(); + pBackendCfWrapper->cfOpts = (void**)cfOpt; + pBackendCfWrapper->dbOpt = handle->dbOpt; + pBackendCfWrapper->param = param; + pBackendCfWrapper->pBackend = handle; + pBackendCfWrapper->backendId = pState->streamBackendRid; + taosThreadRwlockInit(&pBackendCfWrapper->rwLock, NULL); SCfComparator compare = {.comp = pCompare, .numOfComp = cfLen}; - pState->pTdbState->pComparNode = streamBackendAddCompare(handle, &compare); - rocksdb_writeoptions_disable_WAL(pState->pTdbState->writeOpts, 1); - qInfo("succ to open state %p on backend, %p, 0x%" PRIx64 "-%d", pState, handle, pState->streamId, pState->taskId); + pBackendCfWrapper->pComparNode = streamBackendAddCompare(handle, &compare); + rocksdb_writeoptions_disable_WAL(pBackendCfWrapper->writeOpts, 1); + memcpy(pBackendCfWrapper->idstr, pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr)); + + int64_t id = taosAddRef(streamBackendCfWrapperId, pBackendCfWrapper); + pState->pTdbState->backendCfWrapperId = id; + pState->pTdbState->pBackendCfWrapper = pBackendCfWrapper; + qInfo("succ to open state %p on backendWrapper %p %s", pState, pBackendCfWrapper, pBackendCfWrapper->idstr); return 0; } void streamStateCloseBackend(SStreamState* pState, bool remove) { - SBackendHandle* pHandle = pState->pTdbState->pBackend; + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SBackendWrapper* pHandle = wrapper->pBackend; taosThreadMutexLock(&pHandle->cfMutex); - RocksdbCfInst** ppInst = taosHashGet(pHandle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); + RocksdbCfInst** ppInst = taosHashGet(pHandle->cfInst, wrapper->idstr, strlen(pState->pTdbState->idstr) + 1); if (ppInst != NULL && *ppInst != NULL) { RocksdbCfInst* inst = *ppInst; taosMemoryFree(inst); @@ -899,63 +1004,10 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) { taosThreadMutexUnlock(&pHandle->cfMutex); char* status[] = {"close", "drop"}; - qInfo("start to close %s state %p on backend %p 0x%" PRIx64 "-%d", status[remove == false ? 0 : 1], pState, pHandle, - pState->streamId, pState->taskId); - if (pState->pTdbState->rocksdb == NULL) { - return; - } - - int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); - - char* err = NULL; - if (remove) { - for (int i = 0; i < cfLen; i++) { - if (pState->pTdbState->pHandle[i] != NULL) - rocksdb_drop_column_family(pState->pTdbState->rocksdb, - ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[i], &err); - if (err != NULL) { - qError("failed to create cf:%s_%s, reason:%s", pState->pTdbState->idstr, ginitDict[i].key, err); - taosMemoryFreeClear(err); - } - } - } else { - rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); - for (int i = 0; i < cfLen; i++) { - if (pState->pTdbState->pHandle[i] != NULL) - rocksdb_flush_cf(pState->pTdbState->rocksdb, flushOpt, pState->pTdbState->pHandle[i], &err); - if (err != NULL) { - qError("failed to create cf:%s_%s, reason:%s", pState->pTdbState->idstr, ginitDict[i].key, err); - taosMemoryFreeClear(err); - } - } - rocksdb_flushoptions_destroy(flushOpt); - } - - for (int i = 0; i < cfLen; i++) { - if (pState->pTdbState->pHandle[i] != NULL) { - rocksdb_column_family_handle_destroy(pState->pTdbState->pHandle[i]); - } - } - taosMemoryFreeClear(pState->pTdbState->pHandle); - for (int i = 0; i < cfLen; i++) { - rocksdb_options_destroy(pState->pTdbState->cfOpts[i]); - rocksdb_block_based_options_destroy(((RocksdbCfParam*)pState->pTdbState->param)[i].tableOpt); - } - - if (remove) { - streamBackendDelCompare(pState->pTdbState->pBackend, pState->pTdbState->pComparNode); - } - rocksdb_writeoptions_destroy(pState->pTdbState->writeOpts); - pState->pTdbState->writeOpts = NULL; - - rocksdb_readoptions_destroy(pState->pTdbState->readOpts); - pState->pTdbState->readOpts = NULL; - taosMemoryFreeClear(pState->pTdbState->cfOpts); - taosMemoryFreeClear(pState->pTdbState->param); - - taosThreadRwlockDestroy(&pState->pTdbState->rwLock); - pState->pTdbState->rocksdb = NULL; - taosReleaseRef(streamBackendId, pState->streamBackendRid); + qInfo("start to close %s state %p on backendWrapper %p %s", status[remove == false ? 0 : 1], pState, wrapper, + wrapper->idstr); + wrapper->remove |= remove; // update by other pState + taosReleaseRef(streamBackendCfWrapperId, pState->pTdbState->backendCfWrapperId); } void streamStateDestroyCompar(void* arg) { SCfComparator* comp = (SCfComparator*)arg; @@ -975,26 +1027,26 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) { } } if (pState != NULL && idx != -1) { + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; rocksdb_column_family_handle_t* cf = NULL; - taosThreadRwlockRdlock(&pState->pTdbState->rwLock); - cf = pState->pTdbState->pHandle[idx]; - taosThreadRwlockUnlock(&pState->pTdbState->rwLock); + taosThreadRwlockRdlock(&wrapper->rwLock); + cf = wrapper->pHandle[idx]; + taosThreadRwlockUnlock(&wrapper->rwLock); if (cf == NULL) { char buf[128] = {0}; - GEN_COLUMN_FAMILY_NAME(buf, pState->pTdbState->idstr, ginitDict[idx].key); + GEN_COLUMN_FAMILY_NAME(buf, wrapper->idstr, ginitDict[idx].key); char* err = NULL; - taosThreadRwlockWrlock(&pState->pTdbState->rwLock); - cf = rocksdb_create_column_family(pState->pTdbState->rocksdb, pState->pTdbState->cfOpts[idx], buf, &err); + taosThreadRwlockWrlock(&wrapper->rwLock); + cf = rocksdb_create_column_family(wrapper->rocksdb, wrapper->cfOpts[idx], buf, &err); if (err != NULL) { idx = -1; - qError("failed to to open cf, %p 0x%" PRIx64 "-%d_%s, reason:%s", pState, pState->streamId, pState->taskId, - funcName, err); + qError("failed to to open cf, %p %s_%s, reason:%s", pState, wrapper->idstr, funcName, err); taosMemoryFree(err); } else { - pState->pTdbState->pHandle[idx] = cf; + wrapper->pHandle[idx] = cf; } - taosThreadRwlockUnlock(&pState->pTdbState->rwLock); + taosThreadRwlockUnlock(&wrapper->rwLock); } } @@ -1012,18 +1064,19 @@ bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len } rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt) { - int idx = streamStateGetCfIdx(pState, cfName); + int idx = streamStateGetCfIdx(pState, cfName); + rocksdb_readoptions_t* rOpt = rocksdb_readoptions_create(); *readOpt = rOpt; + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; if (snapshot != NULL) { - *snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(pState->pTdbState->rocksdb); + *snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(wrapper->rocksdb); rocksdb_readoptions_set_snapshot(rOpt, *snapshot); rocksdb_readoptions_set_fill_cache(rOpt, 0); } - return rocksdb_create_iterator_cf(pState->pTdbState->rocksdb, rOpt, - ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[idx]); + return rocksdb_create_iterator_cf(wrapper->rocksdb, rOpt, ((rocksdb_column_family_handle_t**)wrapper->pHandle)[idx]); } #define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \ @@ -1037,15 +1090,15 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa code = -1; \ break; \ } \ - char toString[128] = {0}; \ + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; \ + char toString[128] = {0}; \ if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = \ - ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[ginitDict[i].idx]; \ - rocksdb_t* db = pState->pTdbState->rocksdb; \ - rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \ - char* ttlV = NULL; \ - int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pHandle)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->rocksdb; \ + rocksdb_writeoptions_t* opts = wrapper->writeOpts; \ + char* ttlV = NULL; \ + int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ if (err != NULL) { \ qError("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ @@ -1057,81 +1110,76 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa taosMemoryFree(ttlV); \ } while (0); -#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamStateGetCfIdx(pState, funcname); \ - if (i < 0) { \ - qWarn("streamState failed to get cf name: %s", funcname); \ - code = -1; \ - break; \ - } \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = \ - ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[ginitDict[i].idx]; \ - rocksdb_t* db = pState->pTdbState->rocksdb; \ - rocksdb_readoptions_t* opts = pState->pTdbState->readOpts; \ - size_t len = 0; \ - char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ - if (val == NULL || len == 0) { \ - if (err == NULL) { \ - qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, pState->pTdbState->idstr, \ - funcname); \ - } else { \ - qError("streamState str: %s failed to read from %s_%s, err: %s", toString, pState->pTdbState->idstr, funcname, \ - err); \ - taosMemoryFreeClear(err); \ - } \ - code = -1; \ - } else { \ - char* p = NULL; \ - int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ - if (tlen <= 0) { \ - qError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, pState->pTdbState->idstr, \ - funcname); \ - code = -1; \ - } else { \ - qTrace("streamState str: %s succ to read from %s_%s, valLen:%d", toString, pState->pTdbState->idstr, funcname, \ - tlen); \ - } \ - taosMemoryFree(val); \ - if (vLen != NULL) *vLen = tlen; \ - } \ - if (code == 0) \ - qDebug("streamState str: %s succ to read from %s_%s", toString, pState->pTdbState->idstr, funcname); \ +#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamStateGetCfIdx(pState, funcname); \ + if (i < 0) { \ + qWarn("streamState failed to get cf name: %s", funcname); \ + code = -1; \ + break; \ + } \ + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; \ + char toString[128] = {0}; \ + if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pHandle)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->rocksdb; \ + rocksdb_readoptions_t* opts = wrapper->readOpts; \ + size_t len = 0; \ + char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ + if (val == NULL || len == 0) { \ + if (err == NULL) { \ + qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, wrapper->idstr, funcname); \ + } else { \ + qError("streamState str: %s failed to read from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ + taosMemoryFreeClear(err); \ + } \ + code = -1; \ + } else { \ + char* p = NULL; \ + int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ + if (tlen <= 0) { \ + qError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, wrapper->idstr, \ + funcname); \ + code = -1; \ + } else { \ + qTrace("streamState str: %s succ to read from %s_%s, valLen:%d", toString, wrapper->idstr, funcname, tlen); \ + } \ + taosMemoryFree(val); \ + if (vLen != NULL) *vLen = tlen; \ + } \ + if (code == 0) qDebug("streamState str: %s succ to read from %s_%s", toString, wrapper->idstr, funcname); \ } while (0); -#define STREAM_STATE_DEL_ROCKSDB(pState, funcname, key) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamStateGetCfIdx(pState, funcname); \ - if (i < 0) { \ - qWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ - code = -1; \ - break; \ - } \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = \ - ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[ginitDict[i].idx]; \ - rocksdb_t* db = pState->pTdbState->rocksdb; \ - rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \ - rocksdb_delete_cf(db, opts, pHandle, (const char*)buf, klen, &err); \ - if (err != NULL) { \ - qError("streamState str: %s failed to del from %s_%s, err: %s", toString, pState->pTdbState->idstr, funcname, \ - err); \ - taosMemoryFree(err); \ - code = -1; \ - } else { \ - qTrace("streamState str: %s succ to del from %s_%s", toString, pState->pTdbState->idstr, funcname); \ - } \ +#define STREAM_STATE_DEL_ROCKSDB(pState, funcname, key) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamStateGetCfIdx(pState, funcname); \ + if (i < 0) { \ + qWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ + code = -1; \ + break; \ + } \ + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; \ + char toString[128] = {0}; \ + if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = ((rocksdb_column_family_handle_t**)wrapper->pHandle)[ginitDict[i].idx]; \ + rocksdb_t* db = wrapper->rocksdb; \ + rocksdb_writeoptions_t* opts = wrapper->writeOpts; \ + rocksdb_delete_cf(db, opts, pHandle, (const char*)buf, klen, &err); \ + if (err != NULL) { \ + qError("streamState str: %s failed to del from %s_%s, err: %s", toString, wrapper->idstr, funcname, err); \ + taosMemoryFree(err); \ + code = -1; \ + } else { \ + qTrace("streamState str: %s succ to del from %s_%s", toString, wrapper->idstr, funcname); \ + } \ } while (0); // state cf @@ -1157,18 +1205,19 @@ int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key) { int32_t streamStateClear_rocksdb(SStreamState* pState) { qDebug("streamStateClear_rocksdb"); - char sKeyStr[128] = {0}; - char eKeyStr[128] = {0}; - SStateKey sKey = {.key = {.ts = 0, .groupId = 0}, .opNum = pState->number}; - SStateKey eKey = {.key = {.ts = INT64_MAX, .groupId = UINT64_MAX}, .opNum = pState->number}; + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + char sKeyStr[128] = {0}; + char eKeyStr[128] = {0}; + SStateKey sKey = {.key = {.ts = 0, .groupId = 0}, .opNum = pState->number}; + SStateKey eKey = {.key = {.ts = INT64_MAX, .groupId = UINT64_MAX}, .opNum = pState->number}; int sLen = stateKeyEncode(&sKey, sKeyStr); int eLen = stateKeyEncode(&eKey, eKeyStr); - if (pState->pTdbState->pHandle[1] != NULL) { + if (wrapper->pHandle[1] != NULL) { char* err = NULL; - rocksdb_delete_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->writeOpts, pState->pTdbState->pHandle[1], - sKeyStr, sLen, eKeyStr, eLen, &err); + rocksdb_delete_range_cf(wrapper->rocksdb, wrapper->writeOpts, wrapper->pHandle[1], sKeyStr, sLen, eKeyStr, eLen, + &err); if (err != NULL) { char toStringStart[128] = {0}; char toStringEnd[128] = {0}; @@ -1178,7 +1227,7 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) { qWarn("failed to delete range cf(state) start: %s, end:%s, reason:%s", toStringStart, toStringEnd, err); taosMemoryFree(err); } else { - rocksdb_compact_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[1], sKeyStr, sLen, eKeyStr, eLen); + rocksdb_compact_range_cf(wrapper->rocksdb, wrapper->pHandle[1], sKeyStr, sLen, eKeyStr, eLen); } } @@ -1214,6 +1263,8 @@ int32_t streamStateGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, if (pKey->groupId == groupId) { return 0; } + taosMemoryFree((void*)*pVal); + *pVal = NULL; } return -1; } @@ -1272,8 +1323,9 @@ SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWin if (pCur == NULL) { return NULL; } + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; pCur->number = pState->number; - pCur->db = pState->pTdbState->rocksdb; + pCur->db = wrapper->rocksdb; pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -1306,15 +1358,16 @@ SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWin SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateGetCur_rocksdb"); - int32_t code = 0; + int32_t code = 0; + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + const SStateKey maxStateKey = {.key = {.groupId = UINT64_MAX, .ts = INT64_MAX}, .opNum = INT64_MAX}; STREAM_STATE_PUT_ROCKSDB(pState, "state", &maxStateKey, "", 0); - char buf[128] = {0}; - int32_t klen = stateKeyEncode((void*)&maxStateKey, buf); - + char buf[128] = {0}; + int32_t klen = stateKeyEncode((void*)&maxStateKey, buf); SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) return NULL; - pCur->db = pState->pTdbState->rocksdb; + pCur->db = wrapper->rocksdb; pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); rocksdb_iter_seek(pCur->iter, buf, (size_t)klen); @@ -1334,10 +1387,11 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinK SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateGetCur_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) return NULL; - pCur->db = pState->pTdbState->rocksdb; + pCur->db = wrapper->rocksdb; pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -1425,12 +1479,14 @@ int32_t streamStateSessionDel_rocksdb(SStreamState* pState, const SSessionKey* k } SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pState, const SSessionKey* key) { qDebug("streamStateSessionSeekKeyCurrentPrev_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } pCur->number = pState->number; - pCur->db = pState->pTdbState->rocksdb; + pCur->db = wrapper->rocksdb; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -1466,11 +1522,12 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta } SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pState, SSessionKey* key) { qDebug("streamStateSessionSeekKeyCurrentNext_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } - pCur->db = pState->pTdbState->rocksdb; + pCur->db = wrapper->rocksdb; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -1503,11 +1560,12 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, const SSessionKey* key) { qDebug("streamStateSessionSeekKeyNext_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } - pCur->db = pState->pTdbState->rocksdb; + pCur->db = wrapper->rocksdb; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); pCur->number = pState->number; @@ -1596,11 +1654,12 @@ int32_t streamStateFillDel_rocksdb(SStreamState* pState, const SWinKey* key) { SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillGetCur_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; if (pCur == NULL) return NULL; - pCur->db = pState->pTdbState->rocksdb; + pCur->db = wrapper->rocksdb; pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -1655,12 +1714,13 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillSeekKeyNext_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (!pCur) { return NULL; } - pCur->db = pState->pTdbState->rocksdb; + pCur->db = wrapper->rocksdb; pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -1691,12 +1751,13 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const } SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const SWinKey* key) { qDebug("streamStateFillSeekKeyPrev_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; } - pCur->db = pState->pTdbState->rocksdb; + pCur->db = wrapper->rocksdb; pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -1727,12 +1788,13 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const } int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { qDebug("streamStateSessionGetKeyByRange_rocksdb"); - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return -1; } pCur->number = pState->number; - pCur->db = pState->pTdbState->rocksdb; + pCur->db = wrapper->rocksdb; pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); @@ -1804,7 +1866,6 @@ int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKe if (sessionRangeKeyCmpr(&searchKey, key) == 0) { memcpy(tmp, *pVal, valSize); taosMemoryFreeClear(*pVal); - streamStateSessionDel_rocksdb(pState, key); goto _end; } taosMemoryFreeClear(*pVal); @@ -1820,7 +1881,6 @@ int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKe if (code == 0) { if (sessionRangeKeyCmpr(&searchKey, key) == 0) { memcpy(tmp, *pVal, valSize); - streamStateSessionDel_rocksdb(pState, key); goto _end; } } @@ -1878,14 +1938,12 @@ int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* if (code == 0) { if (key->win.skey <= tmpKey.win.skey && tmpKey.win.ekey <= key->win.ekey) { memcpy(tmp, *pVal, valSize); - streamStateSessionDel_rocksdb(pState, key); goto _end; } void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); if (fn(pKeyData, stateKey) == true) { memcpy(tmp, *pVal, valSize); - streamStateSessionDel_rocksdb(pState, key); goto _end; } @@ -1901,7 +1959,6 @@ int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); if (fn(pKeyData, stateKey) == true) { memcpy(tmp, *pVal, valSize); - streamStateSessionDel_rocksdb(pState, key); goto _end; } } @@ -1963,6 +2020,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co int code = 0; char* err = NULL; + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; rocksdb_snapshot_t* snapshot = NULL; rocksdb_readoptions_t* readopts = NULL; rocksdb_iterator_t* pIter = streamStateIterCreate(pState, "default", &snapshot, &readopts); @@ -1995,15 +2053,16 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co } rocksdb_iter_next(pIter); } - rocksdb_release_snapshot(pState->pTdbState->rocksdb, snapshot); + rocksdb_release_snapshot(wrapper->rocksdb, snapshot); rocksdb_readoptions_destroy(readopts); rocksdb_iter_destroy(pIter); return code; } void* streamDefaultIterCreate_rocksdb(SStreamState* pState) { - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; - pCur->db = pState->pTdbState->rocksdb; + pCur->db = wrapper->rocksdb; pCur->iter = streamStateIterCreate(pState, "default", (rocksdb_snapshot_t**)&pCur->snapshot, (rocksdb_readoptions_t**)&pCur->readOpt); return pCur; @@ -2050,7 +2109,8 @@ void streamStateClearBatch(void* pBatch) { rocksdb_writebatch_clear((rocksdb_ void streamStateDestroyBatch(void* pBatch) { rocksdb_writebatch_destroy((rocksdb_writebatch_t*)pBatch); } int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_writebatch_t* pBatch, void* key, void* val, int32_t vlen, int64_t ttl) { - int i = streamStateGetCfIdx(pState, cfName); + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + int i = streamStateGetCfIdx(pState, cfName); if (i < 0) { qError("streamState failed to put to cf name:%s", cfName); @@ -2061,7 +2121,7 @@ int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_wr char* ttlV = NULL; int32_t ttlVLen = ginitDict[i].enValueFunc(val, vlen, ttl, &ttlV); - rocksdb_column_family_handle_t* pCf = pState->pTdbState->pHandle[ginitDict[i].idx]; + rocksdb_column_family_handle_t* pCf = wrapper->pHandle[ginitDict[i].idx]; rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); taosMemoryFree(ttlV); return 0; @@ -2073,7 +2133,9 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb char* ttlV = tmpBuf; int32_t ttlVLen = ginitDict[cfIdx].enValueFunc(val, vlen, ttl, &ttlV); - rocksdb_column_family_handle_t* pCf = pState->pTdbState->pHandle[ginitDict[cfIdx].idx]; + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + + rocksdb_column_family_handle_t* pCf = wrapper->pHandle[ginitDict[cfIdx].idx]; rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); if (tmpBuf == NULL) { @@ -2082,8 +2144,9 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb return 0; } int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch) { - char* err = NULL; - rocksdb_write(pState->pTdbState->rocksdb, pState->pTdbState->writeOpts, (rocksdb_writebatch_t*)pBatch, &err); + char* err = NULL; + SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; + rocksdb_write(wrapper->rocksdb, wrapper->writeOpts, (rocksdb_writebatch_t*)pBatch, &err); if (err != NULL) { qError("streamState failed to write batch, err:%s", err); taosMemoryFree(err); @@ -2091,3 +2154,13 @@ int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch) { } return 0; } + +uint32_t nextPow2(uint32_t x) { + x = x - 1; + x = x | (x >> 1); + x = x | (x >> 2); + x = x | (x >> 4); + x = x | (x >> 8); + x = x | (x >> 16); + return x + 1; +} \ No newline at end of file diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 670cfbead1e180061fe0f972290351125eb9852c..722c557b8f1e2b6b44bf851454f60ed6ca14ad23 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -123,7 +123,7 @@ int32_t tDecodeSStreamCheckpointRsp(SDecoder* pDecoder, SStreamCheckpointRsp* pR static int32_t streamAlignCheckpoint(SStreamTask* pTask, int64_t checkpointId, int32_t childId) { if (pTask->checkpointingId == 0) { pTask->checkpointingId = checkpointId; - pTask->checkpointAlignCnt = taosArrayGetSize(pTask->childEpInfo); + pTask->checkpointAlignCnt = taosArrayGetSize(pTask->pUpstreamEpInfoList); } ASSERT(pTask->checkpointingId == checkpointId); @@ -165,7 +165,7 @@ int32_t streamProcessCheckpointReq(SStreamMeta* pMeta, SStreamTask* pTask, SStre int64_t checkpointId = pReq->checkpointId; int32_t childId = pReq->childId; - if (taosArrayGetSize(pTask->childEpInfo) > 0) { + if (taosArrayGetSize(pTask->pUpstreamEpInfoList) > 0) { code = streamAlignCheckpoint(pTask, checkpointId, childId); if (code > 0) { return 0; diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index baf2f73f905a15a145d0ad76966b1ce696c4b9e6..bb4b842787a640435f561d6e75074869da8885af 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" SStreamDataBlock* createStreamDataFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg) { SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, pReq->totalLen); @@ -65,11 +65,11 @@ SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamT if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)pItem; - pStreamBlocks->childId = pTask->selfChildId; + pStreamBlocks->childId = pTask->info.selfChildId; pStreamBlocks->sourceVer = pSubmit->ver; } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)pItem; - pStreamBlocks->childId = pTask->selfChildId; + pStreamBlocks->childId = pTask->info.selfChildId; pStreamBlocks->sourceVer = pMerged->ver; } @@ -165,27 +165,9 @@ int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubm return 0; } -static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) { - atomic_add_fetch_32(pDataSubmit->dataRef, 1); -} - -SStreamDataSubmit* streamSubmitBlockClone(SStreamDataSubmit* pSubmit) { - int32_t len = 0; - if (pSubmit->type == STREAM_INPUT__DATA_SUBMIT) { - len = pSubmit->submit.msgLen; - } - - SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, len); - if (pSubmitClone == NULL) { - return NULL; - } - - streamDataSubmitRefInc(pSubmit); - memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit)); - return pSubmitClone; -} - SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem) { + terrno = 0; + if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)pElem; @@ -201,7 +183,10 @@ SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* return dst; } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamMergedSubmit* pMerged = streamMergedSubmitNew(); - // todo handle error + if (pMerged == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } streamMergeSubmit(pMerged, (SStreamDataSubmit*)dst); streamMergeSubmit(pMerged, (SStreamDataSubmit*)pElem); @@ -209,6 +194,7 @@ SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* taosFreeQitem(pElem); return (SStreamQueueItem*)pMerged; } else { + qDebug("block type:%d not merged with existed blocks list, type:%d", pElem->type, dst->type); return NULL; } } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 7af3219f85f16bc0e87af16e66b2ef576ae2951c..8334ea1c88b05f6516acf136f4da69424fc7f1c7 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -13,16 +13,25 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" +#include "ttimer.h" -#define MAX_BLOCK_NAME_NUM 1024 +#define MAX_BLOCK_NAME_NUM 1024 +#define DISPATCH_RETRY_INTERVAL_MS 300 +#define MAX_CONTINUE_RETRY_COUNT 5 typedef struct SBlockName { uint32_t hashValue; char parTbName[TSDB_TABLE_NAME_LEN]; } SBlockName; -int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* pReq) { +static void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen) { + pMsg->msgType = msgType; + pMsg->pCont = pCont; + pMsg->contLen = contLen; +} + +static int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; @@ -44,6 +53,37 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p return pEncoder->pos; } +static int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { + int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + void* buf = taosMemoryCalloc(1, dataStrLen); + if (buf == NULL) return -1; + + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; + pRetrieve->useconds = 0; + pRetrieve->precision = TSDB_DEFAULT_PRECISION; + pRetrieve->compressed = 0; + pRetrieve->completed = 1; + pRetrieve->streamBlockType = pBlock->info.type; + pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); + pRetrieve->skey = htobe64(pBlock->info.window.skey); + pRetrieve->ekey = htobe64(pBlock->info.window.ekey); + pRetrieve->version = htobe64(pBlock->info.version); + pRetrieve->watermark = htobe64(pBlock->info.watermark); + memcpy(pRetrieve->parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); + + int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); + pRetrieve->numOfCols = htonl(numOfCols); + + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); + actualLen += sizeof(SRetrieveTableRsp); + ASSERT(actualLen <= dataStrLen); + taosArrayPush(pReq->dataLen, &actualLen); + taosArrayPush(pReq->data, &buf); + + pReq->totalLen += dataStrLen; + return 0; +} + int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; @@ -72,6 +112,27 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { return 0; } +int32_t tInitStreamDispatchReq(SStreamDispatchReq* pReq, const SStreamTask* pTask, int32_t vgId, int32_t numOfBlocks, + int64_t dstTaskId) { + pReq->streamId = pTask->id.streamId; + pReq->dataSrcVgId = vgId; + pReq->upstreamTaskId = pTask->id.taskId; + pReq->upstreamChildId = pTask->info.selfChildId; + pReq->upstreamNodeId = pTask->info.nodeId; + pReq->blockNum = numOfBlocks; + pReq->taskId = dstTaskId; + + pReq->data = taosArrayInit(numOfBlocks, POINTER_BYTES); + pReq->dataLen = taosArrayInit(numOfBlocks, sizeof(int32_t)); + if (pReq->data == NULL || pReq->dataLen == NULL) { + taosArrayDestroyP(pReq->data, taosMemoryFree); + taosArrayDestroy(pReq->dataLen); + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + void tDeleteStreamDispatchReq(SStreamDispatchReq* pReq) { taosArrayDestroyP(pReq->data, taosMemoryFree); taosArrayDestroy(pReq->dataLen); @@ -132,17 +193,17 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) SStreamRetrieveReq req = { .streamId = pTask->id.streamId, - .srcNodeId = pTask->nodeId, + .srcNodeId = pTask->info.nodeId, .srcTaskId = pTask->id.taskId, .pRetrieve = pRetrieve, .retrieveLen = dataStrLen, }; - int32_t sz = taosArrayGetSize(pTask->childEpInfo); + int32_t sz = taosArrayGetSize(pTask->pUpstreamEpInfoList); ASSERT(sz > 0); for (int32_t i = 0; i < sz; i++) { req.reqId = tGenIdPI64(); - SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i); + SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->pUpstreamEpInfoList, i); req.dstNodeId = pEpInfo->nodeId; req.dstTaskId = pEpInfo->taskId; int32_t len; @@ -171,8 +232,8 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) } buf = NULL; - qDebug("s-task:%s (child %d) send retrieve req to task %d at node %d, reqId:0x%" PRIx64, pTask->id.idStr, - pTask->selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId); + qDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, + pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId); } code = 0; @@ -182,44 +243,13 @@ CLEAR: return code; } -static int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { - int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); - void* buf = taosMemoryCalloc(1, dataStrLen); - if (buf == NULL) return -1; - - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; - pRetrieve->useconds = 0; - pRetrieve->precision = TSDB_DEFAULT_PRECISION; - pRetrieve->compressed = 0; - pRetrieve->completed = 1; - pRetrieve->streamBlockType = pBlock->info.type; - pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); - pRetrieve->skey = htobe64(pBlock->info.window.skey); - pRetrieve->ekey = htobe64(pBlock->info.window.ekey); - pRetrieve->version = htobe64(pBlock->info.version); - pRetrieve->watermark = htobe64(pBlock->info.watermark); - memcpy(pRetrieve->parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); - - int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); - pRetrieve->numOfCols = htonl(numOfCols); - - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); - actualLen += sizeof(SRetrieveTableRsp); - ASSERT(actualLen <= dataStrLen); - taosArrayPush(pReq->dataLen, &actualLen); - taosArrayPush(pReq->data, &buf); - - pReq->totalLen += dataStrLen; - return 0; -} - int32_t streamDispatchCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet) { void* buf = NULL; int32_t code = -1; SRpcMsg msg = {0}; int32_t tlen; - tEncodeSize(tEncodeSStreamTaskCheckReq, pReq, tlen, code); + tEncodeSize(tEncodeStreamTaskCheckReq, pReq, tlen, code); if (code < 0) { return -1; } @@ -234,7 +264,7 @@ int32_t streamDispatchCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pR SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); - if ((code = tEncodeSStreamTaskCheckReq(&encoder, pReq)) < 0) { + if ((code = tEncodeStreamTaskCheckReq(&encoder, pReq)) < 0) { rpcFreeCont(buf); return code; } @@ -245,21 +275,21 @@ int32_t streamDispatchCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pR msg.pCont = buf; msg.msgType = TDMT_STREAM_TASK_CHECK; - qDebug("s-task:%s dispatch check msg to downstream s-task:%" PRIx64 ":%d node %d: check msg", pTask->id.idStr, - pReq->streamId, pReq->downstreamTaskId, nodeId); + qDebug("s-task:%s (level:%d) dispatch check msg to s-task:%" PRIx64 ":0x%x (vgId:%d)", pTask->id.idStr, + pTask->info.taskLevel, pReq->streamId, pReq->downstreamTaskId, nodeId); tmsgSendReq(pEpSet, &msg); return 0; } -int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, - SEpSet* pEpSet) { +int32_t streamDoDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamScanHistoryFinishReq* pReq, int32_t vgId, + SEpSet* pEpSet) { void* buf = NULL; int32_t code = -1; SRpcMsg msg = {0}; int32_t tlen; - tEncodeSize(tEncodeSStreamRecoverFinishReq, pReq, tlen, code); + tEncodeSize(tEncodeStreamScanHistoryFinishReq, pReq, tlen, code); if (code < 0) { return -1; } @@ -275,7 +305,7 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); - if ((code = tEncodeSStreamRecoverFinishReq(&encoder, pReq)) < 0) { + if ((code = tEncodeStreamScanHistoryFinishReq(&encoder, pReq)) < 0) { if (buf) { rpcFreeCont(buf); } @@ -286,17 +316,17 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov msg.contLen = tlen + sizeof(SMsgHead); msg.pCont = buf; - msg.msgType = TDMT_STREAM_RECOVER_FINISH; - msg.info.noResp = 1; + msg.msgType = TDMT_STREAM_SCAN_HISTORY_FINISH; tmsgSendReq(pEpSet, &msg); - qDebug("s-task:%s dispatch recover finish msg to downstream taskId:0x%x node %d: recover finish msg", pTask->id.idStr, - pReq->taskId, vgId); + const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); + qDebug("s-task:%s status:%s dispatch scan-history finish msg to taskId:0x%x (vgId:%d)", pTask->id.idStr, pStatus, + pReq->downstreamTaskId, vgId); return 0; } -int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet) { +static int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet) { void* buf = NULL; int32_t code = -1; SRpcMsg msg = {0}; @@ -304,7 +334,10 @@ int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, in // serialize int32_t tlen; tEncodeSize(tEncodeStreamDispatchReq, pReq, tlen, code); - if (code < 0) goto FAIL; + if (code < 0) { + goto FAIL; + } + code = -1; buf = rpcMallocCont(sizeof(SMsgHead) + tlen); if (buf == NULL) { @@ -323,16 +356,16 @@ int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, in msg.contLen = tlen + sizeof(SMsgHead); msg.pCont = buf; - msg.msgType = pTask->dispatchMsgType; + msg.msgType = pTask->msgInfo.msgType; qDebug("s-task:%s dispatch msg to taskId:0x%x vgId:%d data msg", pTask->id.idStr, pReq->taskId, vgId); - tmsgSendReq(pEpSet, &msg); - - code = 0; - return 0; + return tmsgSendReq(pEpSet, &msg); FAIL: - if (buf) rpcFreeCont(buf); + if (buf) { + rpcFreeCont(buf); + } + return code; } @@ -365,8 +398,6 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->shuffleDispatcher.dbInfo.db, pDataBlock->info.parTbName); } - SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - /*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/ SUseDbRsp* pDbInfo = &pTask->shuffleDispatcher.dbInfo; hashValue = @@ -386,13 +417,16 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S for (j = 0; j < vgSz; j++) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); ASSERT(pVgInfo->vgId > 0); + if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) { return -1; } + if (pReqs[j].blockNum == 0) { atomic_add_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); } + pReqs[j].blockNum++; found = true; break; @@ -404,25 +438,17 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pData) { int32_t code = 0; + int32_t numOfBlocks = taosArrayGetSize(pData->blocks); ASSERT(numOfBlocks != 0); - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - SStreamDispatchReq req = { - .streamId = pTask->id.streamId, - .dataSrcVgId = pData->srcVgId, - .upstreamTaskId = pTask->id.taskId, - .upstreamChildId = pTask->selfChildId, - .upstreamNodeId = pTask->nodeId, - .blockNum = numOfBlocks, - }; - - req.data = taosArrayInit(numOfBlocks, sizeof(void*)); - req.dataLen = taosArrayInit(numOfBlocks, sizeof(int32_t)); - if (req.data == NULL || req.dataLen == NULL) { - taosArrayDestroyP(req.data, taosMemoryFree); - taosArrayDestroy(req.dataLen); - return TSDB_CODE_OUT_OF_MEMORY; + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + SStreamDispatchReq req = {0}; + + int32_t downstreamTaskId = pTask->fixedEpDispatcher.taskId; + code = tInitStreamDispatchReq(&req, pTask, pData->srcVgId, numOfBlocks, downstreamTaskId); + if (code != TSDB_CODE_SUCCESS) { + return code; } for (int32_t i = 0; i < numOfBlocks; i++) { @@ -438,23 +464,21 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat int32_t vgId = pTask->fixedEpDispatcher.nodeId; SEpSet* pEpSet = &pTask->fixedEpDispatcher.epSet; - int32_t downstreamTaskId = pTask->fixedEpDispatcher.taskId; - req.taskId = downstreamTaskId; - - qDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to down stream s-task:0x%x in vgId:%d", pTask->id.idStr, - pTask->selfChildId, numOfBlocks, downstreamTaskId, vgId); + qDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d)", pTask->id.idStr, + pTask->info.selfChildId, numOfBlocks, downstreamTaskId, vgId); code = doSendDispatchMsg(pTask, &req, vgId, pEpSet); taosArrayDestroyP(req.data, taosMemoryFree); taosArrayDestroy(req.dataLen); return code; - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t rspCnt = atomic_load_32(&pTask->shuffleDispatcher.waitingRspCnt); ASSERT(rspCnt == 0); - SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t vgSz = taosArrayGetSize(vgInfo); + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t vgSz = taosArrayGetSize(vgInfo); + SStreamDispatchReq* pReqs = taosMemoryCalloc(vgSz, sizeof(SStreamDispatchReq)); if (pReqs == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -462,20 +486,11 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat } for (int32_t i = 0; i < vgSz; i++) { - pReqs[i].streamId = pTask->id.streamId; - pReqs[i].dataSrcVgId = pData->srcVgId; - pReqs[i].upstreamTaskId = pTask->id.taskId; - pReqs[i].upstreamChildId = pTask->selfChildId; - pReqs[i].upstreamNodeId = pTask->nodeId; - pReqs[i].blockNum = 0; - pReqs[i].data = taosArrayInit(0, sizeof(void*)); - pReqs[i].dataLen = taosArrayInit(0, sizeof(int32_t)); - if (pReqs[i].data == NULL || pReqs[i].dataLen == NULL) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); + code = tInitStreamDispatchReq(&pReqs[i], pTask, pData->srcVgId, 0, pVgInfo->taskId); + if (code != TSDB_CODE_SUCCESS) { goto FAIL_SHUFFLE_DISPATCH; } - - SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - pReqs[i].taskId = pVgInfo->taskId; } for (int32_t i = 0; i < numOfBlocks; i++) { @@ -483,15 +498,18 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat // TODO: do not use broadcast if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + for (int32_t j = 0; j < vgSz; j++) { if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) { goto FAIL_SHUFFLE_DISPATCH; } + if (pReqs[j].blockNum == 0) { atomic_add_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); } pReqs[j].blockNum++; } + continue; } @@ -500,16 +518,17 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat } } - qDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to %d vgroups", pTask->id.idStr, pTask->selfChildId, + qDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to %d vgroups", pTask->id.idStr, pTask->info.selfChildId, numOfBlocks, vgSz); for (int32_t i = 0; i < vgSz; i++) { if (pReqs[i].blockNum > 0) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - qDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to vgId:%d", pTask->id.idStr, pTask->selfChildId, + qDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to vgId:%d", pTask->id.idStr, pTask->info.selfChildId, pReqs[i].blockNum, pVgInfo->vgId); - if (doSendDispatchMsg(pTask, &pReqs[i], pVgInfo->vgId, &pVgInfo->epSet) < 0) { + code = doSendDispatchMsg(pTask, &pReqs[i], pVgInfo->vgId, &pVgInfo->epSet); + if (code < 0) { goto FAIL_SHUFFLE_DISPATCH; } } @@ -522,46 +541,177 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat taosArrayDestroyP(pReqs[i].data, taosMemoryFree); taosArrayDestroy(pReqs[i].dataLen); } + taosMemoryFree(pReqs); } + return code; } +static void doRetryDispatchData(void* param, void* tmrId) { + SStreamTask* pTask = param; + ASSERT(pTask->outputInfo.status == TASK_OUTPUT_STATUS__WAIT); + + int32_t code = streamDispatchAllBlocks(pTask, pTask->msgInfo.pData); + if (code != TSDB_CODE_SUCCESS) { + qDebug("s-task:%s reset the waitRspCnt to be 0 before launch retry dispatch", pTask->id.idStr); + atomic_store_32(&pTask->shuffleDispatcher.waitingRspCnt, 0); + streamRetryDispatchStreamBlock(pTask, DISPATCH_RETRY_INTERVAL_MS); + } +} + +void streamRetryDispatchStreamBlock(SStreamTask* pTask, int64_t waitDuration) { + qError("s-task:%s dispatch data in %"PRId64"ms", pTask->id.idStr, waitDuration); + taosTmrReset(doRetryDispatchData, waitDuration, pTask, streamEnv.timer, &pTask->launchTaskTimer); +} + int32_t streamDispatchStreamBlock(SStreamTask* pTask) { - ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH); - int32_t numOfElems = taosQueueItemSize(pTask->outputQueue->queue); + STaskOutputInfo* pInfo = &pTask->outputInfo; + ASSERT((pInfo->type == TASK_OUTPUT__FIXED_DISPATCH || pInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH)); + + int32_t numOfElems = taosQueueItemSize(pInfo->queue->queue); if (numOfElems > 0) { qDebug("s-task:%s try to dispatch intermediate result block to downstream, elem in outputQ:%d", pTask->id.idStr, numOfElems); } - int8_t old = - atomic_val_compare_exchange_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL, TASK_OUTPUT_STATUS__WAIT); + // to make sure only one dispatch is running + int8_t old = atomic_val_compare_exchange_8(&pInfo->status, TASK_OUTPUT_STATUS__NORMAL, TASK_OUTPUT_STATUS__WAIT); if (old != TASK_OUTPUT_STATUS__NORMAL) { - qDebug("s-task:%s task wait for dispatch rsp, not dispatch now, output status:%d", pTask->id.idStr, old); + qDebug("s-task:%s wait for dispatch rsp, not dispatch now, output status:%d", pTask->id.idStr, old); return 0; } - qDebug("s-task:%s start to dispatch msg, set output status:%d", pTask->id.idStr, pTask->outputStatus); + ASSERT(pTask->msgInfo.pData == NULL); + qDebug("s-task:%s start to dispatch msg, set output status:%d", pTask->id.idStr, pInfo->status); - SStreamDataBlock* pDispatchedBlock = streamQueueNextItem(pTask->outputQueue); - if (pDispatchedBlock == NULL) { - atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); - qDebug("s-task:%s stop dispatching since no output in output queue, output status:%d", pTask->id.idStr, - pTask->outputStatus); + SStreamDataBlock* pBlock = streamQueueNextItem(pInfo->queue); + if (pBlock == NULL) { + atomic_store_8(&pInfo->status, TASK_OUTPUT_STATUS__NORMAL); + qDebug("s-task:%s not dispatch since no elems in outputQ, output status:%d", pTask->id.idStr, pInfo->status); return 0; } - ASSERT(pDispatchedBlock->type == STREAM_INPUT__DATA_BLOCK); + pTask->msgInfo.pData = pBlock; + ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - int32_t code = streamDispatchAllBlocks(pTask, pDispatchedBlock); - if (code != TSDB_CODE_SUCCESS) { - streamQueueProcessFail(pTask->outputQueue); - atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); - qDebug("s-task:%s failed to dispatch msg to downstream, output status:%d", pTask->id.idStr, pTask->outputStatus); + int32_t retryCount = 0; + + while (1) { + int32_t code = streamDispatchAllBlocks(pTask, pBlock); + if (code == TSDB_CODE_SUCCESS) { + break; + } + + qDebug("s-task:%s failed to dispatch msg to downstream, code:%s, output status:%d, retry cnt:%d", pTask->id.idStr, + tstrerror(terrno), pInfo->status, retryCount); + + // todo deal with only partially success dispatch case + atomic_store_32(&pTask->shuffleDispatcher.waitingRspCnt, 0); + if (terrno == TSDB_CODE_APP_IS_STOPPING) { // in case of this error, do not retry anymore + destroyStreamDataBlock(pTask->msgInfo.pData); + pTask->msgInfo.pData = NULL; + return code; + } + + if (++retryCount > MAX_CONTINUE_RETRY_COUNT) { // add to timer to retry + qDebug("s-task:%s failed to dispatch msg to downstream for %d times, code:%s, add timer to retry in %dms", + pTask->id.idStr, retryCount, tstrerror(terrno), DISPATCH_RETRY_INTERVAL_MS); + streamRetryDispatchStreamBlock(pTask, DISPATCH_RETRY_INTERVAL_MS); + break; + } } - // this block can be freed only when it has been pushed to down stream. - destroyStreamDataBlock(pDispatchedBlock); - return code; + // this block can not be deleted until it has been sent to downstream task successfully. + return TSDB_CODE_SUCCESS; +} + +int32_t tEncodeCompleteHistoryDataMsg(SEncoder* pEncoder, const SStreamCompleteHistoryMsg* pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->downstreamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->downstreamNode) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeCompleteHistoryDataMsg(SDecoder* pDecoder, SStreamCompleteHistoryMsg* pRsp) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->downstreamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->downstreamNode) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->upstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->upstreamNodeId) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo, SStreamScanHistoryFinishReq* pReq) { + int32_t len = 0; + int32_t code = 0; + SEncoder encoder; + + SStreamCompleteHistoryMsg msg = { + .streamId = pReq->streamId, + .upstreamTaskId = pReq->upstreamTaskId, + .upstreamNodeId = pReq->upstreamNodeId, + .downstreamId = pReq->downstreamTaskId, + .downstreamNode = pTask->pMeta->vgId, + }; + + tEncodeSize(tEncodeCompleteHistoryDataMsg, &msg, len, code); + if (code < 0) { + return code; + } + + void* pBuf = rpcMallocCont(sizeof(SMsgHead) + len); + if (pBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + ((SMsgHead*)pBuf)->vgId = htonl(pReq->upstreamNodeId); + + void* abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); + + tEncoderInit(&encoder, (uint8_t*)abuf, len); + tEncodeCompleteHistoryDataMsg(&encoder, &msg); + tEncoderClear(&encoder); + + SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId); + + SStreamContinueExecInfo info = {.taskId = pReq->upstreamTaskId, .epset = pInfo->epSet}; + initRpcMsg(&info.msg, 0, pBuf, sizeof(SMsgHead) + len); + info.msg.info = *pRpcInfo; + + taosThreadMutexLock(&pTask->lock); + if (pTask->pRspMsgList == NULL) { + pTask->pRspMsgList = taosArrayInit(4, sizeof(SStreamContinueExecInfo)); + } + taosArrayPush(pTask->pRspMsgList, &info); + taosThreadMutexUnlock(&pTask->lock); + + int32_t num = taosArrayGetSize(pTask->pRspMsgList); + qDebug("s-task:%s add scan history finish rsp msg for task:0x%x, total:%d", pTask->id.idStr, pReq->upstreamTaskId, + num); + return TSDB_CODE_SUCCESS; +} + +int32_t streamNotifyUpstreamContinue(SStreamTask* pTask) { + ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG || pTask->info.taskLevel == TASK_LEVEL__SINK); + + int32_t num = taosArrayGetSize(pTask->pRspMsgList); + for (int32_t i = 0; i < num; ++i) { + SStreamContinueExecInfo* pInfo = taosArrayGet(pTask->pRspMsgList, i); + tmsgSendRsp(&pInfo->msg); + + qDebug("s-task:%s level:%d notify upstream:0x%x to continue process data from WAL", pTask->id.idStr, pTask->info.taskLevel, + pInfo->taskId); + } + + taosArrayClear(pTask->pRspMsgList); + qDebug("s-task:%s level:%d checkpoint ready msg sent to all %d upstreams", pTask->id.idStr, pTask->info.taskLevel, + num); + return 0; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 298d585481d77ecc903dbf4392f38c5f67e42e8c..4ef7d6084d9a0da77c6e58254fb2c1d3e00e6eb6 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" // maximum allowed processed block batches. One block may include several submit blocks #define MAX_STREAM_EXEC_BATCH_NUM 32 @@ -29,7 +29,7 @@ bool streamTaskShouldStop(const SStreamStatus* pStatus) { bool streamTaskShouldPause(const SStreamStatus* pStatus) { int32_t status = atomic_load_8((int8_t*)&pStatus->taskStatus); - return (status == TASK_STATUS__PAUSE); + return (status == TASK_STATUS__PAUSE || status == TASK_STATUS__HALT); } static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* pRes, int32_t size, int64_t* totalSize, @@ -108,10 +108,11 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); block.info.type = STREAM_PULL_OVER; - block.info.childId = pTask->selfChildId; + block.info.childId = pTask->info.selfChildId; taosArrayPush(pRes, &block); numOfBlocks += 1; - qDebug("s-task:%s(child %d) processed retrieve, reqId:0x%" PRIx64, pTask->id.idStr, pTask->selfChildId, + + qDebug("s-task:%s(child %d) retrieve process completed, reqId:0x%" PRIx64" dump results", pTask->id.idStr, pTask->info.selfChildId, pRetrieveBlock->reqId); } @@ -127,7 +128,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i SSDataBlock block = {0}; assignOneDataBlock(&block, output); - block.info.childId = pTask->selfChildId; + block.info.childId = pTask->info.selfChildId; size += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block); numOfBlocks += 1; @@ -135,7 +136,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i taosArrayPush(pRes, &block); qDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr, - pTask->selfChildId, numOfBlocks, size / 1048576.0); + pTask->info.selfChildId, numOfBlocks, size / 1048576.0); // current output should be dispatched to down stream nodes if (numOfBlocks >= MAX_STREAM_RESULT_DUMP_THRESHOLD) { @@ -164,13 +165,19 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { int32_t code = 0; - ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); + ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); void* exec = pTask->exec.pExecutor; qSetStreamOpOpen(exec); bool finished = false; while (1) { + if (streamTaskShouldPause(&pTask->status)) { + double el = (taosGetTimestampMs() - pTask->tsInfo.step1Start) / 1000.0; + qDebug("s-task:%s paused from the scan-history task, elapsed time:%.2fsec", pTask->id.idStr, el); + return 0; + } + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -179,7 +186,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { int32_t batchCnt = 0; while (1) { - if (streamTaskShouldStop(&pTask->status) || streamTaskShouldPause(&pTask->status)) { + if (streamTaskShouldStop(&pTask->status)) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); return 0; } @@ -189,18 +196,37 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { if (qExecTask(exec, &output, &ts) < 0) { continue; } + if (output == NULL) { if (qStreamRecoverScanFinished(exec)) { finished = true; } else { qSetStreamOpOpen(exec); + if (streamTaskShouldPause(&pTask->status)) { + SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); + if (qRes == NULL) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + qRes->type = STREAM_INPUT__DATA_BLOCK; + qRes->blocks = pRes; + code = streamTaskOutputResultBlock(pTask, qRes); + if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + taosFreeQitem(qRes); + return code; + } + return 0; + } } break; } SSDataBlock block = {0}; assignOneDataBlock(&block, output); - block.info.childId = pTask->selfChildId; + block.info.childId = pTask->info.selfChildId; taosArrayPush(pRes, &block); batchCnt++; @@ -238,11 +264,6 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { taosFreeQitem(qRes); return code; } -// -// if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { -// qDebug("s-task:%s scan exec dispatch blocks:%d", pTask->id.idStr, batchCnt); -// streamDispatchStreamBlock(pTask); -// } if (finished) { break; @@ -276,7 +297,7 @@ int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) { return -1; } - if (pTask->taskLevel == TASK_LEVEL__SINK) { + if (pTask->info.taskLevel == TASK_LEVEL__SINK) { ASSERT(((SStreamQueueItem*)pItem)->type == STREAM_INPUT__DATA_BLOCK); streamTaskOutputResultBlock(pTask, (SStreamDataBlock*)pItem); } @@ -318,6 +339,104 @@ int32_t updateCheckPointInfo(SStreamTask* pTask) { return TSDB_CODE_SUCCESS; } +static void waitForTaskIdle(SStreamTask* pTask, SStreamTask* pStreamTask) { + // wait for the stream task to be idle + int64_t st = taosGetTimestampMs(); + + while (!streamTaskIsIdle(pStreamTask)) { + qDebug("s-task:%s level:%d wait for stream task:%s to be idle, check again in 100ms", pTask->id.idStr, + pTask->info.taskLevel, pStreamTask->id.idStr); + taosMsleep(100); + } + + double el = (taosGetTimestampMs() - st) / 1000.0; + if (el > 0) { + qDebug("s-task:%s wait for stream task:%s for %.2fs to be idle", pTask->id.idStr, + pStreamTask->id.idStr, el); + } +} + +static int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { + SStreamMeta* pMeta = pTask->pMeta; + + SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.taskId); + if (pStreamTask == NULL) { + // todo: destroy this task here + qError("s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed", pTask->id.idStr, + pTask->streamTaskId.taskId); + return TSDB_CODE_STREAM_TASK_NOT_EXIST; + } else { + qDebug("s-task:%s fill-history task end, update related stream task:%s info, transfer exec state", pTask->id.idStr, + pStreamTask->id.idStr); + } + + ASSERT(pStreamTask->historyTaskId.taskId == pTask->id.taskId && pTask->status.transferState == true); + + STimeWindow* pTimeWindow = &pStreamTask->dataRange.window; + + // It must be halted for a source stream task, since when the related scan-history-data task start scan the history + // for the step 2. For a agg task + int8_t status = pStreamTask->status.taskStatus; + if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) { + ASSERT(status == TASK_STATUS__HALT); + } else { + ASSERT(status == TASK_STATUS__SCAN_HISTORY); + pStreamTask->status.taskStatus = TASK_STATUS__HALT; + qDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, pTask->id.idStr); + } + + // wait for the stream task to handle all in the inputQ, and to be idle + waitForTaskIdle(pTask, pStreamTask); + + // In case of sink tasks, no need to halt them. + // In case of source tasks and agg tasks, we should HALT them, and wait for them to be idle. And then, it's safe to + // start the task state transfer procedure. + // When a task is idle with halt status, all data in inputQ are consumed. + if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) { + // update the scan data range for source task. + qDebug("s-task:%s level:%d stream task window %" PRId64 " - %" PRId64 " update to %" PRId64 " - %" PRId64 + ", status:%s, sched-status:%d", + pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN, + pTimeWindow->ekey, streamGetTaskStatusStr(TASK_STATUS__NORMAL), pStreamTask->status.schedStatus); + } else { + qDebug("s-task:%s no need to update time window for non-source task", pStreamTask->id.idStr); + } + + // expand the query time window for stream scanner + pTimeWindow->skey = INT64_MIN; + qResetStreamInfoTimeWindow(pStreamTask->exec.pExecutor); + + // transfer the ownership of executor state + streamTaskReleaseState(pTask); + streamTaskReloadState(pStreamTask); + + // clear the link between fill-history task and stream task info + pStreamTask->historyTaskId.taskId = 0; + streamTaskResumeFromHalt(pStreamTask); + + qDebug("s-task:%s fill-history task set status to be dropping, save the state into disk", pTask->id.idStr); + int32_t taskId = pTask->id.taskId; + + // free it and remove it from disk meta-store + streamMetaUnregisterTask(pMeta, taskId); + + // save to disk + taosWLockLatch(&pMeta->lock); + + streamMetaSaveTask(pMeta, pStreamTask); + if (streamMetaCommit(pMeta) < 0) { + // persist to disk + } + taosWUnLockLatch(&pMeta->lock); + + // pause allowed + streamTaskEnablePause(pStreamTask); + + streamSchedExec(pStreamTask); + streamMetaReleaseTask(pMeta, pStreamTask); + return TSDB_CODE_SUCCESS; +} + static int32_t extractMsgFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks, const char* id) { int32_t retryTimes = 0; @@ -331,7 +450,7 @@ static int32_t extractMsgFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); if (qItem == NULL) { - if (pTask->taskLevel == TASK_LEVEL__SOURCE && (++retryTimes) < MAX_RETRY_TIMES) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE && (++retryTimes) < MAX_RETRY_TIMES) { taosMsleep(10); qDebug("===stream===try again batchSize:%d, retry:%d", *numOfBlocks, retryTimes); continue; @@ -342,7 +461,7 @@ static int32_t extractMsgFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu } // do not merge blocks for sink node - if (pTask->taskLevel == TASK_LEVEL__SINK) { + if (pTask->info.taskLevel == TASK_LEVEL__SINK) { *numOfBlocks = 1; *pInput = qItem; return TSDB_CODE_SUCCESS; @@ -355,7 +474,12 @@ static int32_t extractMsgFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu // todo we need to sort the data block, instead of just appending into the array list. void* newRet = streamMergeQueueItem(*pInput, qItem); if (newRet == NULL) { - qError("s-task:%s failed to merge blocks from inputQ, numOfBlocks:%d", id, *numOfBlocks); + if (terrno == 0) { + qDebug("s-task:%s failed to merge blocks from inputQ, numOfBlocks:%d", id, *numOfBlocks); + } else { + qDebug("s-task:%s failed to merge blocks from inputQ, numOfBlocks:%d, code:%s", id, *numOfBlocks, + tstrerror(terrno)); + } streamQueueProcessFail(pTask->inputQueue); return TSDB_CODE_SUCCESS; } @@ -383,6 +507,10 @@ int32_t streamExecForAll(SStreamTask* pTask) { while (1) { int32_t batchSize = 0; SStreamQueueItem* pInput = NULL; + if (streamTaskShouldStop(&pTask->status)) { + qDebug("s-task:%s stream task stopped, abort", id); + break; + } // merge multiple input data if possible in the input queue. qDebug("s-task:%s start to extract data block from inputQ", id); @@ -393,33 +521,13 @@ int32_t streamExecForAll(SStreamTask* pTask) { break; } - if (pTask->taskLevel == TASK_LEVEL__SINK) { + if (pTask->info.taskLevel == TASK_LEVEL__SINK) { ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK); qDebug("s-task:%s sink task start to sink %d blocks", id, batchSize); streamTaskOutputResultBlock(pTask, (SStreamDataBlock*)pInput); continue; } - // wait for the task to be ready to go - while (pTask->taskLevel == TASK_LEVEL__SOURCE) { - int8_t status = atomic_load_8(&pTask->status.taskStatus); - if (status == TASK_STATUS__DROPPING) { - if (pInput != NULL) { - streamFreeQitem(pInput); - } - - qError("s-task:%s task is dropped, abort exec", id); - return TSDB_CODE_SUCCESS; - } - - if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__PAUSE && status != TASK_STATUS__STOP) { - qError("stream task wait for the end of fill history, s-task:%s, status:%d", id, status); - taosMsleep(100); - } else { - break; - } - } - int64_t st = taosGetTimestampMs(); qDebug("s-task:%s start to process batch of blocks, num:%d", id, batchSize); @@ -432,7 +540,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { const SStreamTrigger* pTrigger = (const SStreamTrigger*)pInput; qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { - ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); + ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit, @@ -473,27 +581,102 @@ int32_t streamExecForAll(SStreamTask* pTask) { return 0; } +bool streamTaskIsIdle(const SStreamTask* pTask) { + return (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE); +} + +int32_t streamTaskEndScanWAL(SStreamTask* pTask) { + const char* id = pTask->id.idStr; + double el = (taosGetTimestampMs() - pTask->tsInfo.step2Start) / 1000.0; + qDebug("s-task:%s scan-history from WAL stage(step 2) ended, elapsed time:%.2fs", id, el); + + // 3. notify downstream tasks to transfer executor state after handle all history blocks. + pTask->status.transferState = true; + + int32_t code = streamDispatchTransferStateMsg(pTask); + if (code != TSDB_CODE_SUCCESS) { + // todo handle error + } + + // the last execution of fill-history task, in order to transfer task operator states. + code = streamTransferStateToStreamTask(pTask); + if (code != TSDB_CODE_SUCCESS) { // todo handle this + return code; + } + + return TSDB_CODE_SUCCESS; +} + int32_t streamTryExec(SStreamTask* pTask) { // this function may be executed by multi-threads, so status check is required. int8_t schedStatus = atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE); + const char* id = pTask->id.idStr; + if (schedStatus == TASK_SCHED_STATUS__WAITING) { int32_t code = streamExecForAll(pTask); - if (code < 0) { + if (code < 0) { // todo this status shoudl be removed atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__FAILED); return -1; } // todo the task should be commit here - atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); - qDebug("s-task:%s exec completed", pTask->id.idStr); + if (taosQueueEmpty(pTask->inputQueue->queue)) { + // fill-history WAL scan has completed + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE && pTask->status.transferState == true) { + streamTaskRecoverSetAllStepFinished(pTask); + streamTaskEndScanWAL(pTask); + } else { + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + qDebug("s-task:%s exec completed, status:%s, sched-status:%d", id, streamGetTaskStatusStr(pTask->status.taskStatus), + pTask->status.schedStatus); + } + } else { + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + qDebug("s-task:%s exec completed, status:%s, sched-status:%d", id, streamGetTaskStatusStr(pTask->status.taskStatus), + pTask->status.schedStatus); - if (!taosQueueEmpty(pTask->inputQueue->queue) && (!streamTaskShouldStop(&pTask->status)) && - (!streamTaskShouldPause(&pTask->status))) { - streamSchedExec(pTask); + if ((!streamTaskShouldStop(&pTask->status)) && (!streamTaskShouldPause(&pTask->status))) { + streamSchedExec(pTask); + } } + } else { + qDebug("s-task:%s already started to exec by other thread, status:%s, sched-status:%d", id, + streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus); } return 0; } + +int32_t streamTaskReleaseState(SStreamTask* pTask) { + qDebug("s-task:%s release exec state", pTask->id.idStr); + void* pExecutor = pTask->exec.pExecutor; + if (pExecutor != NULL) { + int32_t code = qStreamOperatorReleaseState(pExecutor); + return code; + } else { + return TSDB_CODE_SUCCESS; + } +} + +int32_t streamTaskReloadState(SStreamTask* pTask) { + qDebug("s-task:%s reload exec state", pTask->id.idStr); + void* pExecutor = pTask->exec.pExecutor; + if (pExecutor != NULL) { + int32_t code = qStreamOperatorReloadState(pExecutor); + return code; + } else { + return TSDB_CODE_SUCCESS; + } +} + +int32_t streamAlignTransferState(SStreamTask* pTask) { + int32_t numOfUpstream = taosArrayGetSize(pTask->pUpstreamEpInfoList); + int32_t old = atomic_val_compare_exchange_32(&pTask->transferStateAlignCnt, 0, numOfUpstream); + if (old == 0) { + qDebug("s-task:%s set the transfer state aligncnt %d", pTask->id.idStr, numOfUpstream); + } + + return atomic_sub_fetch_32(&pTask->transferStateAlignCnt, 1); +} diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 5c31b1dd602595a264693aa965fdba9b7448ae13..ae077388685c0c732d956a466016ea2660db63c2 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -15,16 +15,24 @@ #include "executor.h" #include "streamBackendRocksdb.h" -#include "streamInc.h" +#include "streamInt.h" #include "tref.h" #include "ttimer.h" static TdThreadOnce streamMetaModuleInit = PTHREAD_ONCE_INIT; int32_t streamBackendId = 0; -static void streamMetaEnvInit() { streamBackendId = taosOpenRef(20, streamBackendCleanup); } +int32_t streamBackendCfWrapperId = 0; + +static void streamMetaEnvInit() { + streamBackendId = taosOpenRef(64, streamBackendCleanup); + streamBackendCfWrapperId = taosOpenRef(64, streamBackendHandleCleanup); +} void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } -void streamMetaCleanup() { taosCloseRef(streamBackendId); } +void streamMetaCleanup() { + taosCloseRef(streamBackendId); + taosCloseRef(streamBackendCfWrapperId); +} SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId) { int32_t code = -1; @@ -93,10 +101,14 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); + pMeta->pTaskBackendUnique = + taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); taosMemoryFree(streamPath); taosInitRWLatch(&pMeta->lock); + taosThreadMutexInit(&pMeta->backendMutex, NULL); + return pMeta; _err: @@ -127,9 +139,14 @@ void streamMetaClose(SStreamMeta* pMeta) { } SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->timer) { - taosTmrStop(pTask->timer); - pTask->timer = NULL; + if (pTask->schedTimer) { + taosTmrStop(pTask->schedTimer); + pTask->schedTimer = NULL; + } + + if (pTask->launchTaskTimer) { + taosTmrStop(pTask->launchTaskTimer); + pTask->launchTaskTimer = NULL; } tFreeStreamTask(pTask); @@ -139,6 +156,8 @@ void streamMetaClose(SStreamMeta* pMeta) { taosRemoveRef(streamBackendId, pMeta->streamBackendRid); pMeta->pTaskList = taosArrayDestroy(pMeta->pTaskList); taosMemoryFree(pMeta->path); + taosThreadMutexDestroy(&pMeta->backendMutex); + taosHashCleanup(pMeta->pTaskBackendUnique); taosMemoryFree(pMeta); } @@ -198,6 +217,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { tEncoderClear(&encoder); if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) { + qError("s-task:%s save to disk failed, code:%s", pTask->id.idStr, tstrerror(terrno)); return -1; } @@ -205,8 +225,21 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { return 0; } +int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { + int32_t code = tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(taskId), pMeta->txn); + if (code != 0) { + qError("vgId:%d failed to remove task:0x%x from metastore, code:%s", pMeta->vgId, taskId, tstrerror(terrno)); + } else { + qDebug("vgId:%d remove task:0x%x from metastore", pMeta->vgId, taskId); + } + + return code; +} + // add to the ready tasks hash map, not the restored tasks hash map -int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { +int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded) { + *pAdded = false; + void* p = taosHashGet(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId)); if (p == NULL) { if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { @@ -214,6 +247,8 @@ int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* return -1; } + taosArrayPush(pMeta->pTaskList, &pTask->id.taskId); + if (streamMetaSaveTask(pMeta, pTask) < 0) { tFreeStreamTask(pTask); return -1; @@ -223,19 +258,18 @@ int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* tFreeStreamTask(pTask); return -1; } - taosArrayPush(pMeta->pTaskList, &pTask->id.taskId); } else { return 0; } taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES); + *pAdded = true; return 0; } -int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta) { +int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta) { size_t size = taosHashGetSize(pMeta->pTasks); ASSERT(taosArrayGetSize(pMeta->pTaskList) == taosHashGetSize(pMeta->pTasks)); - return (int32_t)size; } @@ -245,8 +279,9 @@ SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) { SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); if (ppTask != NULL) { if (!streamTaskShouldStop(&(*ppTask)->status)) { - atomic_add_fetch_32(&(*ppTask)->refCnt, 1); + int32_t ref = atomic_add_fetch_32(&(*ppTask)->refCnt, 1); taosRUnLockLatch(&pMeta->lock); + qTrace("s-task:%s acquire task, ref:%d", (*ppTask)->id.idStr, ref); return *ppTask; } } @@ -256,43 +291,90 @@ SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) { } void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) { - int32_t left = atomic_sub_fetch_32(&pTask->refCnt, 1); - if (left < 0) { - qError("task ref is invalid, ref:%d, %s", left, pTask->id.idStr); - } else if (left == 0) { + int32_t ref = atomic_sub_fetch_32(&pTask->refCnt, 1); + if (ref > 0) { + qTrace("s-task:%s release task, ref:%d", pTask->id.idStr, ref); + } else if (ref == 0) { ASSERT(streamTaskShouldStop(&pTask->status)); + qTrace("s-task:%s all refs are gone, free it", pTask->id.idStr); tFreeStreamTask(pTask); + } else if (ref < 0) { + qError("task ref is invalid, ref:%d, %s", ref, pTask->id.idStr); } } -void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { - taosWLockLatch(&pMeta->lock); +static void doRemoveIdFromList(SStreamMeta* pMeta, int32_t num, int32_t taskId) { + for (int32_t i = 0; i < num; ++i) { + int32_t* pTaskId = taosArrayGet(pMeta->pTaskList, i); + if (*pTaskId == taskId) { + taosArrayRemove(pMeta->pTaskList, i); + break; + } + } +} + +int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int32_t taskId) { + SStreamTask* pTask = NULL; + // pre-delete operation + taosWLockLatch(&pMeta->lock); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); if (ppTask) { - SStreamTask* pTask = *ppTask; + pTask = *ppTask; + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING); + } else { + qDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId); + taosWUnLockLatch(&pMeta->lock); + return 0; + } + taosWUnLockLatch(&pMeta->lock); - taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t)); - tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn); + qDebug("s-task:0x%x set task status:%s", taskId, streamGetTaskStatusStr(TASK_STATUS__DROPPING)); - atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING); - int32_t num = taosArrayGetSize(pMeta->pTaskList); + while(1) { + taosRLockLatch(&pMeta->lock); + ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); - qDebug("s-task:%s set the drop task flag, remain running s-task:%d", pTask->id.idStr, num - 1); - for (int32_t i = 0; i < num; ++i) { - int32_t* pTaskId = taosArrayGet(pMeta->pTaskList, i); - if (*pTaskId == taskId) { - taosArrayRemove(pMeta->pTaskList, i); + if (ppTask) { + if ((*ppTask)->status.timerActive == 0) { + taosRUnLockLatch(&pMeta->lock); break; } + + taosMsleep(10); + qDebug("s-task:%s wait for quit from timer", (*ppTask)->id.idStr); + taosRUnLockLatch(&pMeta->lock); + } else { + taosRUnLockLatch(&pMeta->lock); + break; } + } + // let's do delete of stream task + taosWLockLatch(&pMeta->lock); + ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); + if (ppTask) { + taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t)); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING); + + ASSERT(pTask->status.timerActive == 0); + + int32_t num = taosArrayGetSize(pMeta->pTaskList); + doRemoveIdFromList(pMeta, num, pTask->id.taskId); + + // remove the ref by timer + if (pTask->triggerParam != 0) { + taosTmrStop(pTask->schedTimer); + } + + streamMetaRemoveTask(pMeta, taskId); streamMetaReleaseTask(pMeta, pTask); } else { - qDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId); + qDebug("vgId:%d failed to find the task:0x%x, it may have been dropped already", pMeta->vgId, taskId); } taosWUnLockLatch(&pMeta->lock); + return 0; } int32_t streamMetaBegin(SStreamMeta* pMeta) { @@ -337,7 +419,9 @@ int32_t streamMetaAbort(SStreamMeta* pMeta) { int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { TBC* pCur = NULL; + if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { + qError("vgId:%d failed to open stream meta, code:%s", pMeta->vgId, tstrerror(terrno)); return -1; } @@ -346,6 +430,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { void* pVal = NULL; int32_t vLen = 0; SDecoder decoder; + SArray* pRecycleList = taosArrayInit(4, sizeof(int32_t)); tdbTbcMoveToFirst(pCur); @@ -355,6 +440,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); + taosArrayDestroy(pRecycleList); return -1; } @@ -362,16 +448,29 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { tDecodeStreamTask(&decoder, pTask); tDecoderClear(&decoder); - // remove duplicate + if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { + int32_t taskId = pTask->id.taskId; + tFreeStreamTask(pTask); + + taosArrayPush(pRecycleList, &taskId); + + int32_t total = taosArrayGetSize(pRecycleList); + qDebug("s-task:0x%x is already dropped, add into recycle list, total:%d", taskId, total); + continue; + } + + // do duplicate task check. void* p = taosHashGet(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId)); if (p == NULL) { if (pMeta->expandFunc(pMeta->ahandle, pTask, pTask->chkInfo.version) < 0) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); - taosMemoryFree(pTask); + tFreeStreamTask(pTask); + taosArrayDestroy(pRecycleList); return -1; } + taosArrayPush(pMeta->pTaskList, &pTask->id.taskId); } else { tdbFree(pKey); @@ -380,25 +479,34 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { taosMemoryFree(pTask); continue; } + if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, sizeof(void*)) < 0) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); - taosMemoryFree(pTask); + tFreeStreamTask(pTask); + taosArrayDestroy(pRecycleList); return -1; } - if (pTask->fillHistory) { - ASSERT(pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM); - streamTaskCheckDownstream(pTask, ver); - } + ASSERT(pTask->status.downstreamReady == 0); } tdbFree(pKey); tdbFree(pVal); if (tdbTbcClose(pCur) < 0) { + taosArrayDestroy(pRecycleList); return -1; } + if (taosArrayGetSize(pRecycleList) > 0) { + for(int32_t i = 0; i < taosArrayGetSize(pRecycleList); ++i) { + int32_t taskId = *(int32_t*) taosArrayGet(pRecycleList, i); + streamMetaRemoveTask(pMeta, taskId); + } + } + + qDebug("vgId:%d load %d task from disk", pMeta->vgId, (int32_t) taosArrayGetSize(pMeta->pTaskList)); + taosArrayDestroy(pRecycleList); return 0; } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 4cfeedab57fb289d12643e996f0b4cdd3f65c531..aaf9fdec724cf6765232cb8a77adfb36754ce6d9 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" SStreamQueue* streamQueueOpen(int64_t cap) { SStreamQueue* pQueue = taosMemoryCalloc(1, sizeof(SStreamQueue)); diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index eb2535782ea6810fd82440f814012279a54bd64a..df45ff2759e512b799b2f8b1203e47d3823573f5 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -13,94 +13,146 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" +#include "ttimer.h" +#include "wal.h" -int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) { - qDebug("s-task:%s at node %d launch recover", pTask->id.idStr, pTask->nodeId); +static void launchFillHistoryTask(SStreamTask* pTask); +static void streamTaskSetRangeStreamCalc(SStreamTask* pTask); - if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__RECOVER_PREPARE); - qDebug("s-task:%s set task status:%d and start to recover", pTask->id.idStr, pTask->status.taskStatus); +static void streamTaskSetForReady(SStreamTask* pTask, int32_t numOfReqs) { + ASSERT(pTask->status.downstreamReady == 0); + pTask->status.downstreamReady = 1; + int64_t el = (taosGetTimestampMs() - pTask->tsInfo.init); - streamSetParamForRecover(pTask); - streamSourceRecoverPrepareStep1(pTask, version); + qDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%dms, task status:%s", + pTask->id.idStr, numOfReqs, (int32_t) el, streamGetTaskStatusStr(pTask->status.taskStatus)); +} - SStreamRecoverStep1Req req; - streamBuildSourceRecover1Req(pTask, &req); - int32_t len = sizeof(SStreamRecoverStep1Req); +int32_t streamStartRecoverTask(SStreamTask* pTask, int8_t igUntreated) { + SStreamScanHistoryReq req; + streamBuildSourceRecover1Req(pTask, &req, igUntreated); + int32_t len = sizeof(SStreamScanHistoryReq); - void* serializedReq = rpcMallocCont(len); - if (serializedReq == NULL) { - return -1; - } + void* serializedReq = rpcMallocCont(len); + if (serializedReq == NULL) { + return -1; + } - memcpy(serializedReq, &req, len); + memcpy(serializedReq, &req, len); - SRpcMsg rpcMsg = { .contLen = len, .pCont = serializedReq, .msgType = TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE }; - if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) { - /*ASSERT(0);*/ - } + SRpcMsg rpcMsg = {.contLen = len, .pCont = serializedReq, .msgType = TDMT_VND_STREAM_SCAN_HISTORY}; + if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) { + /*ASSERT(0);*/ + } - } else if (pTask->taskLevel == TASK_LEVEL__AGG) { - atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); - streamSetParamForRecover(pTask); - streamAggRecoverPrepare(pTask); - } else if (pTask->taskLevel == TASK_LEVEL__SINK) { - // sink nodes has no specified operation for fill history - atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); + return 0; +} + +const char* streamGetTaskStatusStr(int32_t status) { + switch(status) { + case TASK_STATUS__NORMAL: return "normal"; + case TASK_STATUS__SCAN_HISTORY: return "scan-history"; + case TASK_STATUS__HALT: return "halt"; + case TASK_STATUS__PAUSE: return "paused"; + case TASK_STATUS__DROPPING: return "dropping"; + default:return ""; } +} + +static int32_t doLaunchScanHistoryTask(SStreamTask* pTask) { + SVersionRange* pRange = &pTask->dataRange.range; + if (pTask->info.fillHistory) { + streamSetParamForScanHistory(pTask); + } + streamSetParamForStreamScannerStep1(pTask, pRange, &pTask->dataRange.window); + + int32_t code = streamStartRecoverTask(pTask, 0); + return code; +} +int32_t streamTaskLaunchScanHistory(SStreamTask* pTask) { + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + if (pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) { + return doLaunchScanHistoryTask(pTask); + } else { + ASSERT(pTask->status.taskStatus == TASK_STATUS__NORMAL); + qDebug("s-task:%s no need to scan-history-data, status:%s, sched-status:%d, ver:%" PRId64, pTask->id.idStr, + streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus, + walReaderGetCurrentVer(pTask->exec.pWalReader)); + } + } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) { + if (pTask->info.fillHistory) { + streamSetParamForScanHistory(pTask); + } + streamTaskEnablePause(pTask); + streamTaskScanHistoryPrepare(pTask); + } else if (pTask->info.taskLevel == TASK_LEVEL__SINK) { + qDebug("s-task:%s sink task do nothing to handle scan-history", pTask->id.idStr); + streamTaskScanHistoryPrepare(pTask); + } return 0; } -// checkstatus -int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { - qDebug("s-task:%s in fill history stage, ver:%"PRId64, pTask->id.idStr, version); +// check status +int32_t streamTaskDoCheckDownstreamTasks(SStreamTask* pTask) { + SHistDataRange* pRange = &pTask->dataRange; + STimeWindow* pWindow = &pRange->window; SStreamTaskCheckReq req = { .streamId = pTask->id.streamId, .upstreamTaskId = pTask->id.taskId, - .upstreamNodeId = pTask->nodeId, - .childId = pTask->selfChildId, + .upstreamNodeId = pTask->info.nodeId, + .childId = pTask->info.selfChildId, }; // serialize - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { req.reqId = tGenIdPI64(); req.downstreamNodeId = pTask->fixedEpDispatcher.nodeId; req.downstreamTaskId = pTask->fixedEpDispatcher.taskId; pTask->checkReqId = req.reqId; - qDebug("s-task:%s at node %d check downstream task:0x%x at node %d", pTask->id.idStr, pTask->nodeId, req.downstreamTaskId, - req.downstreamNodeId); + qDebug("s-task:%s check single downstream task:0x%x(vgId:%d) ver:%" PRId64 "-%" PRId64 " window:%" PRId64 + "-%" PRId64 ", req:0x%" PRIx64, + pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer, pRange->range.maxVer, + pWindow->skey, pWindow->ekey, req.reqId); + streamDispatchCheckMsg(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; int32_t numOfVgs = taosArrayGetSize(vgInfo); - pTask->recoverTryingDownstream = numOfVgs; + pTask->notReadyTasks = numOfVgs; pTask->checkReqIds = taosArrayInit(numOfVgs, sizeof(int64_t)); + qDebug("s-task:%s check %d downstream tasks, ver:%" PRId64 "-%" PRId64 " window:%" PRId64 "-%" PRId64, + pTask->id.idStr, numOfVgs, pRange->range.minVer, pRange->range.maxVer, pWindow->skey, pWindow->ekey); + for (int32_t i = 0; i < numOfVgs; i++) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); req.reqId = tGenIdPI64(); taosArrayPush(pTask->checkReqIds, &req.reqId); req.downstreamNodeId = pVgInfo->vgId; req.downstreamTaskId = pVgInfo->taskId; - qDebug("s-task:%s at node %d check downstream task:0x%x at node %d (shuffle)", pTask->id.idStr, pTask->nodeId, - req.downstreamTaskId, req.downstreamNodeId); + qDebug("s-task:%s (vgId:%d) check downstream task:0x%x (vgId:%d) (shuffle), idx:%d", pTask->id.idStr, pTask->info.nodeId, + req.downstreamTaskId, req.downstreamNodeId, i); streamDispatchCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); } } else { - qDebug("s-task:%s at node %d direct launch recover since no downstream", pTask->id.idStr, pTask->nodeId); - streamTaskLaunchRecover(pTask, version); + qDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", pTask->id.idStr, pTask->info.nodeId); + + streamTaskSetForReady(pTask, 0); + streamTaskSetRangeStreamCalc(pTask); + streamTaskLaunchScanHistory(pTask); + + launchFillHistoryTask(pTask); } return 0; } -int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) { +int32_t streamRecheckDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) { SStreamTaskCheckReq req = { .reqId = pRsp->reqId, .streamId = pRsp->streamId, @@ -111,12 +163,12 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp .childId = pRsp->childId, }; - qDebug("s-task:%s at node %d check downstream task:0x%x at node %d (recheck)", pTask->id.idStr, pTask->nodeId, + qDebug("s-task:%s (vgId:%d) check downstream task:0x%x (vgId:%d) (recheck)", pTask->id.idStr, pTask->info.nodeId, req.downstreamTaskId, req.downstreamNodeId); - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { streamDispatchCheckMsg(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet); - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; int32_t numOfVgs = taosArrayGetSize(vgInfo); @@ -132,17 +184,36 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp } int32_t streamTaskCheckStatus(SStreamTask* pTask) { - return atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL? 1:0; + return (pTask->status.downstreamReady == 1)? 1:0; } -int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version) { - ASSERT(pTask->id.taskId == pRsp->upstreamTaskId); +static void doProcessDownstreamReadyRsp(SStreamTask* pTask, int32_t numOfReqs) { + streamTaskSetForReady(pTask, numOfReqs); + const char* id = pTask->id.idStr; + + int8_t status = pTask->status.taskStatus; + const char* str = streamGetTaskStatusStr(status); - qDebug("s-task:%s at node %d recv check rsp from task:0x%x at node %d: status %d", pTask->id.idStr, - pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status); + ASSERT(status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__NORMAL); + streamTaskSetRangeStreamCalc(pTask); + + if (status == TASK_STATUS__SCAN_HISTORY) { + qDebug("s-task:%s enter into scan-history data stage, status:%s", id, str); + streamTaskLaunchScanHistory(pTask); + } else { + qDebug("s-task:%s downstream tasks are ready, now ready for data from wal, status:%s", id, str); + } + + // when current stream task is ready, check the related fill history task. + launchFillHistoryTask(pTask); +} + +int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) { + ASSERT(pTask->id.taskId == pRsp->upstreamTaskId); + const char* id = pTask->id.idStr; if (pRsp->status == 1) { - if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { bool found = false; int32_t numOfReqs = taosArrayGetSize(pTask->checkReqIds); @@ -158,146 +229,463 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* return -1; } - int32_t left = atomic_sub_fetch_32(&pTask->recoverTryingDownstream, 1); + int32_t left = atomic_sub_fetch_32(&pTask->notReadyTasks, 1); ASSERT(left >= 0); if (left == 0) { taosArrayDestroy(pTask->checkReqIds); pTask->checkReqIds = NULL; - qDebug("s-task:%s all %d downstream tasks are ready, now enter into recover stage", pTask->id.idStr, numOfReqs); - streamTaskLaunchRecover(pTask, version); + doProcessDownstreamReadyRsp(pTask, numOfReqs); + } else { + int32_t total = taosArrayGetSize(pTask->shuffleDispatcher.dbInfo.pVgroupInfos); + qDebug("s-task:%s (vgId:%d) recv check rsp from task:0x%x (vgId:%d) status:%d, total:%d not ready:%d", id, + pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status, total, left); } - } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + } else { + ASSERT(pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH); if (pRsp->reqId != pTask->checkReqId) { return -1; } - streamTaskLaunchRecover(pTask, version); - } else { - ASSERT(0); + doProcessDownstreamReadyRsp(pTask, 1); } - } else { // not ready, wait for 100ms and retry - qDebug("s-task:%s downstream taskId:0x%x (vgId:%d) not ready, wait for 100ms and retry", pTask->id.idStr, - pRsp->downstreamTaskId, pRsp->downstreamNodeId); + } else { // not ready, wait for 100ms and retry + qDebug("s-task:%s downstream taskId:0x%x (vgId:%d) not ready, wait for 100ms and retry", id, pRsp->downstreamTaskId, + pRsp->downstreamNodeId); taosMsleep(100); - streamRecheckOneDownstream(pTask, pRsp); + streamRecheckDownstream(pTask, pRsp); } return 0; } +int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp, + SRpcHandleInfo *pRpcInfo, int32_t taskId) { + SEncoder encoder; + int32_t code; + int32_t len; + + tEncodeSize(tEncodeStreamTaskCheckRsp, pRsp, len, code); + if (code < 0) { + qError("vgId:%d failed to encode task check rsp, s-task:0x%x", pMeta->vgId, taskId); + return -1; + } + + void* buf = rpcMallocCont(sizeof(SMsgHead) + len); + ((SMsgHead*)buf)->vgId = htonl(pReq->upstreamNodeId); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncoderInit(&encoder, (uint8_t*)abuf, len); + tEncodeStreamTaskCheckRsp(&encoder, pRsp); + tEncoderClear(&encoder); + + SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo}; + + tmsgSendRsp(&rspMsg); + return 0; +} + // common -int32_t streamSetParamForRecover(SStreamTask* pTask) { - void* exec = pTask->exec.pExecutor; - return qStreamSetParamForRecover(exec); +int32_t streamSetParamForScanHistory(SStreamTask* pTask) { + qDebug("s-task:%s set operator option for scan-history data", pTask->id.idStr); + return qSetStreamOperatorOptionForScanHistory(pTask->exec.pExecutor); } + int32_t streamRestoreParam(SStreamTask* pTask) { - void* exec = pTask->exec.pExecutor; - return qStreamRestoreParam(exec); + qDebug("s-task:%s restore operator param after scan-history", pTask->id.idStr); + return qRestoreStreamOperatorOption(pTask->exec.pExecutor); } int32_t streamSetStatusNormal(SStreamTask* pTask) { - atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); - return 0; + int32_t status = atomic_load_8(&pTask->status.taskStatus); + if (status == TASK_STATUS__DROPPING) { + qError("s-task:%s cannot be set normal, since in dropping state", pTask->id.idStr); + return -1; + } else { + qDebug("s-task:%s set task status to be normal, prev:%s", pTask->id.idStr, streamGetTaskStatusStr(status)); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); + return 0; + } } // source -int32_t streamSourceRecoverPrepareStep1(SStreamTask* pTask, int64_t ver) { - void* exec = pTask->exec.pExecutor; - return qStreamSourceRecoverStep1(exec, ver); +int32_t streamSetParamForStreamScannerStep1(SStreamTask* pTask, SVersionRange *pVerRange, STimeWindow* pWindow) { + return qStreamSourceScanParamForHistoryScanStep1(pTask->exec.pExecutor, pVerRange, pWindow); } -int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamRecoverStep1Req* pReq) { - pReq->msgHead.vgId = pTask->nodeId; +int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange *pVerRange, STimeWindow* pWindow) { + return qStreamSourceScanParamForHistoryScanStep2(pTask->exec.pExecutor, pVerRange, pWindow); +} + +int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated) { + pReq->msgHead.vgId = pTask->info.nodeId; pReq->streamId = pTask->id.streamId; pReq->taskId = pTask->id.taskId; + pReq->igUntreated = igUntreated; return 0; } -int32_t streamSourceRecoverScanStep1(SStreamTask* pTask) { +int32_t streamSourceScanHistoryData(SStreamTask* pTask) { return streamScanExec(pTask, 100); } -int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* pReq) { - pReq->msgHead.vgId = pTask->nodeId; - pReq->streamId = pTask->id.streamId; - pReq->taskId = pTask->id.taskId; +int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask) { + SStreamScanHistoryFinishReq req = { + .streamId = pTask->id.streamId, + .childId = pTask->info.selfChildId, + .upstreamTaskId = pTask->id.taskId, + .upstreamNodeId = pTask->pMeta->vgId, + }; + + // serialize + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + req.downstreamTaskId = pTask->fixedEpDispatcher.taskId; + pTask->notReadyTasks = 1; + streamDoDispatchScanHistoryFinishMsg(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t numOfVgs = taosArrayGetSize(vgInfo); + pTask->notReadyTasks = numOfVgs; + + qDebug("s-task:%s send scan-history data complete msg to downstream (shuffle-dispatch) %d tasks, status:%s", pTask->id.idStr, + numOfVgs, streamGetTaskStatusStr(pTask->status.taskStatus)); + for (int32_t i = 0; i < numOfVgs; i++) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); + req.downstreamTaskId = pVgInfo->taskId; + streamDoDispatchScanHistoryFinishMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + } + } else { + qDebug("s-task:%s no downstream tasks, invoke scan-history finish rsp directly", pTask->id.idStr); + streamProcessScanHistoryFinishRsp(pTask); + } + return 0; } -int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) { - void* exec = pTask->exec.pExecutor; - const char* id = pTask->id.idStr; +static int32_t doDispatchTransferMsg(SStreamTask* pTask, const SStreamTransferReq* pReq, int32_t vgId, SEpSet* pEpSet) { + void* buf = NULL; + int32_t code = -1; + SRpcMsg msg = {0}; - int64_t st = taosGetTimestampMs(); - qDebug("s-task:%s recover step2(blocking stage) started", id); - if (qStreamSourceRecoverStep2(exec, ver) < 0) { + int32_t tlen; + tEncodeSize(tEncodeStreamScanHistoryFinishReq, pReq, tlen, code); + if (code < 0) { + return -1; } - int32_t code = streamScanExec(pTask, 100); + buf = rpcMallocCont(sizeof(SMsgHead) + tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } - double el = (taosGetTimestampMs() - st) / 1000.0; - qDebug("s-task:%s recover step2(blocking stage) ended, elapsed time:%.2fs", id, el); + ((SMsgHead*)buf)->vgId = htonl(vgId); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - return code; + SEncoder encoder; + tEncoderInit(&encoder, abuf, tlen); + if ((code = tEncodeStreamScanHistoryFinishReq(&encoder, pReq)) < 0) { + if (buf) { + rpcFreeCont(buf); + } + return code; + } + + tEncoderClear(&encoder); + + msg.contLen = tlen + sizeof(SMsgHead); + msg.pCont = buf; + msg.msgType = TDMT_STREAM_TRANSFER_STATE; + msg.info.noResp = 1; + + tmsgSendReq(pEpSet, &msg); + qDebug("s-task:%s level:%d, status:%s dispatch transfer state msg to taskId:0x%x (vgId:%d)", pTask->id.idStr, + pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus), pReq->downstreamTaskId, vgId); + + return 0; } -int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { - SStreamRecoverFinishReq req = { .streamId = pTask->id.streamId, .childId = pTask->selfChildId }; +int32_t streamDispatchTransferStateMsg(SStreamTask* pTask) { + SStreamTransferReq req = { .streamId = pTask->id.streamId, .childId = pTask->info.selfChildId }; // serialize - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - qDebug("s-task:%s send recover finish msg to downstream (fix-dispatch) to taskId:%d, status:%d", pTask->id.idStr, - pTask->fixedEpDispatcher.taskId, pTask->status.taskStatus); - - req.taskId = pTask->fixedEpDispatcher.taskId; - streamDispatchOneRecoverFinishReq(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + req.downstreamTaskId = pTask->fixedEpDispatcher.taskId; + doDispatchTransferMsg(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t vgSz = taosArrayGetSize(vgInfo); - for (int32_t i = 0; i < vgSz; i++) { + + int32_t numOfVgs = taosArrayGetSize(vgInfo); + for (int32_t i = 0; i < numOfVgs; i++) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - req.taskId = pVgInfo->taskId; - streamDispatchOneRecoverFinishReq(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + req.downstreamTaskId = pVgInfo->taskId; + doDispatchTransferMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); } } + return 0; } // agg -int32_t streamAggRecoverPrepare(SStreamTask* pTask) { - pTask->recoverWaitingUpstream = taosArrayGetSize(pTask->childEpInfo); - qDebug("s-task:%s wait for %d upstreams", pTask->id.idStr, pTask->recoverWaitingUpstream); +int32_t streamTaskScanHistoryPrepare(SStreamTask* pTask) { + pTask->numOfWaitingUpstream = taosArrayGetSize(pTask->pUpstreamEpInfoList); + qDebug("s-task:%s level:%d task wait for %d upstream tasks complete scan-history procedure, status:%s", + pTask->id.idStr, pTask->info.taskLevel, pTask->numOfWaitingUpstream, + streamGetTaskStatusStr(pTask->status.taskStatus)); return 0; } -int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) { +int32_t streamAggUpstreamScanHistoryFinish(SStreamTask* pTask) { void* exec = pTask->exec.pExecutor; - if (qStreamRestoreParam(exec) < 0) { + if (pTask->info.fillHistory && qRestoreStreamOperatorOption(exec) < 0) { return -1; } + if (qStreamRecoverFinish(exec) < 0) { return -1; } - streamSetStatusNormal(pTask); return 0; } -int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) { - if (pTask->taskLevel == TASK_LEVEL__AGG) { - int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingUpstream, 1); - qDebug("s-task:%s remain unfinished child tasks:%d", pTask->id.idStr, left); - ASSERT(left >= 0); - if (left == 0) { - streamAggChildrenRecoverFinish(pTask); +int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, + SRpcHandleInfo* pRpcInfo) { + int32_t taskLevel = pTask->info.taskLevel; + ASSERT(taskLevel == TASK_LEVEL__AGG || taskLevel == TASK_LEVEL__SINK); + + // sink node do not send end of scan history msg to its upstream, which is agg task. + streamAddEndScanHistoryMsg(pTask, pRpcInfo, pReq); + + int32_t left = atomic_sub_fetch_32(&pTask->numOfWaitingUpstream, 1); + ASSERT(left >= 0); + + if (left == 0) { + int32_t numOfTasks = taosArrayGetSize(pTask->pUpstreamEpInfoList); + qDebug( + "s-task:%s all %d upstream tasks finish scan-history data, set param for agg task for stream data and send " + "rsp to all upstream tasks", + pTask->id.idStr, numOfTasks); + + if (pTask->info.taskLevel == TASK_LEVEL__AGG) { + streamAggUpstreamScanHistoryFinish(pTask); + } + + streamNotifyUpstreamContinue(pTask); + + // sink node does not receive the pause msg from mnode, so does not need enable it + if (pTask->info.taskLevel == TASK_LEVEL__AGG) { + streamTaskEnablePause(pTask); + } + } else { + qDebug("s-task:%s receive scan-history data finish msg from upstream:0x%x(index:%d), unfinished:%d", + pTask->id.idStr, pReq->upstreamTaskId, pReq->childId, left); + } + + return 0; +} + +int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask) { + ASSERT(pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY); + SStreamMeta* pMeta = pTask->pMeta; + + // execute in the scan history complete call back msg, ready to process data from inputQ + streamSetStatusNormal(pTask); + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + + taosWLockLatch(&pMeta->lock); + streamMetaSaveTask(pMeta, pTask); + taosWUnLockLatch(&pMeta->lock); + + // history data scan in the stream time window finished, now let's enable the pause + streamTaskEnablePause(pTask); + + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + streamSchedExec(pTask); + } + + return TSDB_CODE_SUCCESS; +} + +static void doCheckDownstreamStatus(SStreamTask* pTask, SStreamTask* pHTask) { + pHTask->dataRange.range.minVer = 0; + pHTask->dataRange.range.maxVer = pTask->chkInfo.currentVer; + + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + qDebug("s-task:%s set the launch condition for fill-history s-task:%s, window:%" PRId64 " - %" PRId64 + " ver range:%" PRId64 " - %" PRId64, + pTask->id.idStr, pHTask->id.idStr, pHTask->dataRange.window.skey, pHTask->dataRange.window.ekey, + pHTask->dataRange.range.minVer, pHTask->dataRange.range.maxVer); + } else { + qDebug("s-task:%s no fill history condition for non-source task:%s", pTask->id.idStr, pHTask->id.idStr); + } + + // check if downstream tasks have been ready + streamTaskDoCheckDownstreamTasks(pHTask); +} + +typedef struct SStreamTaskRetryInfo { + SStreamMeta* pMeta; + int32_t taskId; +} SStreamTaskRetryInfo; + +static void tryLaunchHistoryTask(void* param, void* tmrId) { + SStreamTaskRetryInfo* pInfo = param; + SStreamMeta* pMeta = pInfo->pMeta; + + qDebug("s-task:0x%x in timer to launch related history task", pInfo->taskId); + + taosWLockLatch(&pMeta->lock); + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &pInfo->taskId, sizeof(int32_t)); + if (ppTask) { + ASSERT((*ppTask)->status.timerActive == 1); + + if (streamTaskShouldStop(&(*ppTask)->status)) { + const char* pStatus = streamGetTaskStatusStr((*ppTask)->status.taskStatus); + qDebug("s-task:%s status:%s quit timer task", (*ppTask)->id.idStr, pStatus); + + taosMemoryFree(pInfo); + (*ppTask)->status.timerActive = 0; + taosWUnLockLatch(&pMeta->lock); + return; + } + } + taosWUnLockLatch(&pMeta->lock); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pInfo->taskId); + if (pTask != NULL) { + ASSERT(pTask->status.timerActive == 1); + + // abort the timer if intend to stop task + SStreamTask* pHTask = streamMetaAcquireTask(pMeta, pTask->historyTaskId.taskId); + if (pHTask == NULL && (!streamTaskShouldStop(&pTask->status))) { + const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); + qWarn( + "s-task:%s vgId:%d status:%s failed to launch history task:0x%x, since it may not be built, or may have been " + "destroyed, or should stop", + pTask->id.idStr, pMeta->vgId, pStatus, pTask->historyTaskId.taskId); + + taosTmrReset(tryLaunchHistoryTask, 100, pInfo, streamEnv.timer, &pTask->launchTaskTimer); + streamMetaReleaseTask(pMeta, pTask); + return; + } + + if (pHTask != NULL) { + doCheckDownstreamStatus(pTask, pHTask); + streamMetaReleaseTask(pMeta, pHTask); + } + + // not in timer anymore + pTask->status.timerActive = 0; + streamMetaReleaseTask(pMeta, pTask); + } else { + qError("s-task:0x%x failed to load task, it may have been destroyed", pInfo->taskId); + } + + taosMemoryFree(pInfo); +} + +// todo fix the bug: 2. race condition +// an fill history task needs to be started. +int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { + SStreamMeta* pMeta = pTask->pMeta; + int32_t hTaskId = pTask->historyTaskId.taskId; + + // Set the execute conditions, including the query time window and the version range + SStreamTask** pHTask = taosHashGet(pMeta->pTasks, &hTaskId, sizeof(hTaskId)); + if (pHTask == NULL) { + qWarn("s-task:%s vgId:%d failed to launch history task:0x%x, since it is not built yet", pTask->id.idStr, + pMeta->vgId, hTaskId); + + SStreamTaskRetryInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamTaskRetryInfo)); + pInfo->taskId = pTask->id.taskId; + pInfo->pMeta = pTask->pMeta; + + if (pTask->launchTaskTimer == NULL) { + pTask->launchTaskTimer = taosTmrStart(tryLaunchHistoryTask, 100, pInfo, streamEnv.timer); + if (pTask->launchTaskTimer == NULL) { + // todo failed to create timer + taosMemoryFree(pInfo); + } else { + pTask->status.timerActive = 1; // timer is active + qDebug("s-task:%s set timer active flag", pTask->id.idStr); + } + } else { // timer exists + pTask->status.timerActive = 1; + qDebug("s-task:%s set timer active flag, task timer not null", pTask->id.idStr); + taosTmrReset(tryLaunchHistoryTask, 100, pInfo, streamEnv.timer, &pTask->launchTaskTimer); + } + + // try again in 500ms + return TSDB_CODE_SUCCESS; + } + + doCheckDownstreamStatus(pTask, *pHTask); + return TSDB_CODE_SUCCESS; +} + +int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { + return 0; + } + + // restore param + int32_t code = 0; + if (pTask->info.fillHistory) { + code = streamRestoreParam(pTask); + if (code < 0) { + return -1; } } + + // dispatch recover finish req to all related downstream task + code = streamDispatchScanHistoryFinishMsg(pTask); + if (code < 0) { + return -1; + } + return 0; } -int32_t tEncodeSStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) { +bool streamTaskRecoverScanStep1Finished(SStreamTask* pTask) { + void* exec = pTask->exec.pExecutor; + return qStreamRecoverScanStep1Finished(exec); +} + +bool streamTaskRecoverScanStep2Finished(SStreamTask* pTask) { + void* exec = pTask->exec.pExecutor; + return qStreamRecoverScanStep2Finished(exec); +} + +int32_t streamTaskRecoverSetAllStepFinished(SStreamTask* pTask) { + void* exec = pTask->exec.pExecutor; + return qStreamRecoverSetAllStepFinished(exec); +} + +bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer) { + SVersionRange* pRange = &pTask->dataRange.range; + ASSERT(latestVer >= pRange->maxVer); + + int64_t nextStartVer = pRange->maxVer + 1; + if (nextStartVer > latestVer - 1) { + // no input data yet. no need to execute the secondardy scan while stream task halt + streamTaskRecoverSetAllStepFinished(pTask); + qDebug( + "s-task:%s no need to perform secondary scan-history data(step 2), since no data ingest during step1 scan, " + "related stream task currentVer:%" PRId64, + pTask->id.idStr, latestVer); + return true; + } else { + // 2. do secondary scan of the history data, the time window remain, and the version range is updated to + // [pTask->dataRange.range.maxVer, ver1] + pRange->minVer = nextStartVer; + pRange->maxVer = latestVer - 1; + return false; + } +} + + +int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->reqId) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; @@ -310,7 +698,7 @@ int32_t tEncodeSStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq return pEncoder->pos; } -int32_t tDecodeSStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq) { +int32_t tDecodeStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->reqId) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; @@ -323,7 +711,7 @@ int32_t tDecodeSStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq return 0; } -int32_t tEncodeSStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp) { +int32_t tEncodeStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pRsp->reqId) < 0) return -1; if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; @@ -337,7 +725,7 @@ int32_t tEncodeSStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp return pEncoder->pos; } -int32_t tDecodeSStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) { +int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pRsp->reqId) < 0) return -1; if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1; @@ -351,19 +739,203 @@ int32_t tDecodeSStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp return 0; } -int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq) { +int32_t tEncodeStreamScanHistoryFinishReq(SEncoder* pEncoder, const SStreamScanHistoryFinishReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->downstreamTaskId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->childId) < 0) return -1; tEndEncode(pEncoder); return pEncoder->pos; } -int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq) { + +int32_t tDecodeStreamScanHistoryFinishReq(SDecoder* pDecoder, SStreamScanHistoryFinishReq* pReq) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->downstreamTaskId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->childId) < 0) return -1; tEndDecode(pDecoder); return 0; } + +void streamTaskSetRangeStreamCalc(SStreamTask* pTask) { + if (pTask->historyTaskId.taskId == 0) { + SHistDataRange* pRange = &pTask->dataRange; + if (pTask->info.fillHistory == 1) { + qDebug("s-task:%s fill-history task, time window:%" PRId64 "-%" PRId64 ", verRange:%" PRId64 + "-%" PRId64, + pTask->id.idStr, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, pRange->range.maxVer); + } else { + qDebug("s-task:%s no related fill-history task, stream time window:%" PRId64 "-%" PRId64 ", verRange:%" PRId64 + "-%" PRId64, + pTask->id.idStr, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, pRange->range.maxVer); + } + } else { + SHistDataRange* pRange = &pTask->dataRange; + + int64_t ekey = 0; + if (pRange->window.ekey < INT64_MAX) { + ekey = pRange->window.ekey + 1; + } else { + ekey = pRange->window.ekey; + } + + int64_t ver = pRange->range.minVer; + + pRange->window.skey = ekey; + pRange->window.ekey = INT64_MAX; + pRange->range.minVer = 0; + pRange->range.maxVer = ver; + + qDebug("s-task:%s level:%d related fill-history task exists, update stream calc time window:%" PRId64 " - %" PRId64 + ", verRang:%" PRId64 " - %" PRId64, + pTask->id.idStr, pTask->info.taskLevel, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, + pRange->range.maxVer); + } +} + +void launchFillHistoryTask(SStreamTask* pTask) { + int32_t tId = pTask->historyTaskId.taskId; + if (tId == 0) { + return; + } + + ASSERT(pTask->status.downstreamReady == 1); + qDebug("s-task:%s start to launch related fill-history task:0x%x", pTask->id.idStr, tId); + + // launch associated fill history task + streamLaunchFillHistoryTask(pTask); +} + +void streamTaskCheckDownstreamTasks(SStreamTask* pTask) { + if (pTask->info.fillHistory) { + qDebug("s-task:%s fill history task, wait for being launched", pTask->id.idStr); + return; + } + + ASSERT(pTask->status.downstreamReady == 0); + + // check downstream tasks for itself + streamTaskDoCheckDownstreamTasks(pTask); +} + +// normal -> pause, pause/stop/dropping -> pause, halt -> pause, scan-history -> pause +void streamTaskPause(SStreamTask* pTask) { + SStreamMeta* pMeta = pTask->pMeta; + + int64_t st = taosGetTimestampMs(); + + int8_t status = pTask->status.taskStatus; + if (status == TASK_STATUS__DROPPING) { + qDebug("vgId:%d s-task:%s task already dropped, do nothing", pMeta->vgId, pTask->id.idStr); + return; + } + + const char* str = streamGetTaskStatusStr(status); + if (status == TASK_STATUS__STOP || status == TASK_STATUS__PAUSE) { + qDebug("vgId:%d s-task:%s task already stopped/paused, status:%s, do nothing", pMeta->vgId, pTask->id.idStr, str); + return; + } + + while (!pTask->status.pauseAllowed || (pTask->status.taskStatus == TASK_STATUS__HALT)) { + status = pTask->status.taskStatus; + if (status == TASK_STATUS__DROPPING) { + qDebug("vgId:%d s-task:%s task already dropped, do nothing", pMeta->vgId, pTask->id.idStr); + return; + } + + if (status == TASK_STATUS__STOP || status == TASK_STATUS__PAUSE) { + qDebug("vgId:%d s-task:%s task already stopped/paused, status:%s, do nothing", pMeta->vgId, pTask->id.idStr, str); + return; + } + + const char* pStatus = streamGetTaskStatusStr(status); + qDebug("s-task:%s wait for the task can be paused, status:%s, vgId:%d", pTask->id.idStr, pStatus, pMeta->vgId); + taosMsleep(100); + } + + // todo: use the lock of the task. + taosWLockLatch(&pMeta->lock); + + status = pTask->status.taskStatus; + if (status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP) { + taosWUnLockLatch(&pMeta->lock); + qDebug("vgId:%d s-task:%s task already dropped/stopped/paused, do nothing", pMeta->vgId, pTask->id.idStr); + return; + } + + atomic_store_8(&pTask->status.keepTaskStatus, pTask->status.taskStatus); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__PAUSE); + taosWUnLockLatch(&pMeta->lock); + + int64_t el = taosGetTimestampMs() - st; + qDebug("vgId:%d s-task:%s set pause flag, prev:%s, elapsed time:%dms", pMeta->vgId, pTask->id.idStr, + streamGetTaskStatusStr(pTask->status.keepTaskStatus), (int32_t)el); +} + +void streamTaskResume(SStreamTask* pTask) { + int8_t status = pTask->status.taskStatus; + if (status == TASK_STATUS__PAUSE) { + pTask->status.taskStatus = pTask->status.keepTaskStatus; + pTask->status.keepTaskStatus = TASK_STATUS__NORMAL; + qDebug("s-task:%s resume from pause", pTask->id.idStr); + } else { + qError("s-task:%s not in pause, failed to resume, status:%s", pTask->id.idStr, streamGetTaskStatusStr(status)); + } +} + +// todo fix race condition +void streamTaskDisablePause(SStreamTask* pTask) { + // pre-condition check + const char* id = pTask->id.idStr; + while (pTask->status.taskStatus == TASK_STATUS__PAUSE) { + qDebug("s-task:%s already in pause, wait for pause being cancelled, and set pause disabled, recheck in 100ms", id); + taosMsleep(100); + } + + qDebug("s-task:%s disable task pause", id); + pTask->status.pauseAllowed = 0; +} + +void streamTaskEnablePause(SStreamTask* pTask) { + qDebug("s-task:%s enable task pause", pTask->id.idStr); + pTask->status.pauseAllowed = 1; +} + +void streamTaskHalt(SStreamTask* pTask) { + int8_t status = pTask->status.taskStatus; + if (status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP) { + return; + } + + if (status == TASK_STATUS__HALT) { + return; + } + + // upgrade to halt status + if (status == TASK_STATUS__PAUSE) { + qDebug("s-task:%s upgrade status to %s from %s", pTask->id.idStr, streamGetTaskStatusStr(TASK_STATUS__HALT), + streamGetTaskStatusStr(TASK_STATUS__PAUSE)); + } else { + qDebug("s-task:%s halt task", pTask->id.idStr); + } + + pTask->status.keepTaskStatus = status; + pTask->status.taskStatus = TASK_STATUS__HALT; +} + +void streamTaskResumeFromHalt(SStreamTask* pTask) { + const char* id = pTask->id.idStr; + int8_t status = pTask->status.taskStatus; + if (status != TASK_STATUS__HALT) { + qError("s-task:%s not in halt status, status:%s", id, streamGetTaskStatusStr(status)); + return; + } + + pTask->status.taskStatus = pTask->status.keepTaskStatus; + pTask->status.keepTaskStatus = TASK_STATUS__NORMAL; + qDebug("s-task:%s resume from halt, current status:%s", id, streamGetTaskStatusStr(pTask->status.taskStatus)); +} diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 967c7733c9160b49cabab54b504ad47f520d1d2c..0a4f73a67c8afb319aa4da90b5f0253368d85dee 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -18,7 +18,7 @@ #include "osMemory.h" #include "rocksdb/c.h" #include "streamBackendRocksdb.h" -#include "streamInc.h" +#include "streamInt.h" #include "tcoding.h" #include "tcommon.h" #include "tcompare.h" @@ -116,16 +116,33 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz pState->taskId = pStreamTask->id.taskId; pState->streamId = pStreamTask->id.streamId; + sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-%d", pState->streamId, pState->taskId); #ifdef USE_ROCKSDB SStreamMeta* pMeta = pStreamTask->pMeta; pState->streamBackendRid = pMeta->streamBackendRid; - int code = streamStateOpenBackend(pMeta->streamBackend, pState); - if (code == -1) { - taosReleaseRef(streamBackendId, pMeta->streamBackendRid); - taosMemoryFree(pState); - pState = NULL; + // taosWLockLatch(&pMeta->lock); + taosThreadMutexLock(&pMeta->backendMutex); + void* uniqueId = + taosHashGet(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); + if (uniqueId == NULL) { + int code = streamStateOpenBackend(pMeta->streamBackend, pState); + if (code == -1) { + taosReleaseRef(streamBackendId, pState->streamBackendRid); + taosThreadMutexUnlock(&pMeta->backendMutex); + taosMemoryFree(pState); + return NULL; + } + taosHashPut(pMeta->pTaskBackendUnique, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1, + &pState->pTdbState->backendCfWrapperId, sizeof(pState->pTdbState->backendCfWrapperId)); + } else { + int64_t id = *(int64_t*)uniqueId; + pState->pTdbState->backendCfWrapperId = id; + pState->pTdbState->pBackendCfWrapper = taosAcquireRef(streamBackendCfWrapperId, id); + + taosAcquireRef(streamBackendId, pState->streamBackendRid); } + taosThreadMutexUnlock(&pMeta->backendMutex); pState->pTdbState->pOwner = pTask; pState->pFileState = NULL; @@ -385,8 +402,8 @@ int32_t streamStateClear(SStreamState* pState) { streamStatePut(pState, &key, NULL, 0); while (1) { SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &key); - SWinKey delKey = {0}; - int32_t code = streamStateGetKVByCur(pCur, &delKey, NULL, 0); + SWinKey delKey = {0}; + int32_t code = streamStateGetKVByCur(pCur, &delKey, NULL, 0); streamStateFreeCur(pCur); if (code == 0) { streamStateDel(pState, &delKey); @@ -498,7 +515,7 @@ int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** return -1; } const SStateKey* pKTmp = NULL; - int32_t kLen; + int32_t kLen; if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { return -1; } @@ -518,7 +535,7 @@ int32_t streamStateFillGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const vo return -1; } const SWinKey* pKTmp = NULL; - int32_t kLen; + int32_t kLen; if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { return -1; } @@ -535,7 +552,7 @@ int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const v return -1; } uint64_t groupId = pKey->groupId; - int32_t code = streamStateFillGetKVByCur(pCur, pKey, pVal, pVLen); + int32_t code = streamStateFillGetKVByCur(pCur, pKey, pVal, pVLen); if (code == 0) { if (pKey->groupId == groupId) { return 0; @@ -553,7 +570,7 @@ int32_t streamStateGetFirst(SStreamState* pState, SWinKey* key) { SWinKey tmp = {.ts = 0, .groupId = 0}; streamStatePut(pState, &tmp, NULL, 0); SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &tmp); - int32_t code = streamStateGetKVByCur(pCur, key, NULL, 0); + int32_t code = streamStateGetKVByCur(pCur, key, NULL, 0); streamStateFreeCur(pCur); streamStateDel(pState, &tmp); return code; @@ -593,7 +610,7 @@ SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key } SStateKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c) < 0) { streamStateFreeCur(pCur); return NULL; @@ -726,9 +743,9 @@ int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVa #else SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pState, key); - SSessionKey resKey = *key; - void* tmp = NULL; - int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, &tmp, pVLen); + SSessionKey resKey = *key; + void* tmp = NULL; + int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, &tmp, pVLen); if (code == 0) { if (key->win.skey != resKey.win.skey) { code = -1; @@ -745,6 +762,7 @@ int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVa int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { #ifdef USE_ROCKSDB + qDebug("===stream===delete skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64, key->win.skey,key->win.ekey, key->groupId); return streamStateSessionDel_rocksdb(pState, key); #else SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; @@ -767,7 +785,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, cons } SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { streamStateFreeCur(pCur); return NULL; @@ -798,7 +816,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, cons } SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { streamStateFreeCur(pCur); return NULL; @@ -830,7 +848,7 @@ SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSess } SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { streamStateFreeCur(pCur); return NULL; @@ -854,7 +872,7 @@ int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, v return -1; } SStateSessionKey* pKTmp = NULL; - int32_t kLen; + int32_t kLen; if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, (const void**)pVal, pVLen) < 0) { return -1; } @@ -873,13 +891,13 @@ int32_t streamStateSessionClear(SStreamState* pState) { #ifdef USE_ROCKSDB return streamStateSessionClear_rocksdb(pState); #else - SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; + SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pState, &key); while (1) { SSessionKey delKey = {0}; - void* buf = NULL; - int32_t size = 0; - int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, &buf, &size); + void* buf = NULL; + int32_t size = 0; + int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, &buf, &size); if (code == 0 && size > 0) { memset(buf, 0, size); streamStateSessionPut(pState, &delKey, buf, size); @@ -908,14 +926,14 @@ int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* } SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { streamStateFreeCur(pCur); return -1; } SSessionKey resKey = *key; - int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); + int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { *curKey = resKey; streamStateFreeCur(pCur); @@ -951,19 +969,19 @@ int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, return streamStateSessionAddIfNotExist_rocksdb(pState, key, gap, pVal, pVLen); #else // todo refactor - int32_t res = 0; + int32_t res = 0; SSessionKey originKey = *key; SSessionKey searchKey = *key; searchKey.win.skey = key->win.skey - gap; searchKey.win.ekey = key->win.ekey + gap; int32_t valSize = *pVLen; - void* tmp = tdbRealloc(NULL, valSize); + void* tmp = tdbRealloc(NULL, valSize); if (!tmp) { return -1; } SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pState, key); - int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); + int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); if (code == 0) { if (sessionRangeKeyCmpr(&searchKey, key) == 0) { memcpy(tmp, *pVal, valSize); @@ -1006,16 +1024,16 @@ int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, ch #ifdef USE_ROCKSDB return streamStateStateAddIfNotExist_rocksdb(pState, key, pKeyData, keyDataLen, fn, pVal, pVLen); #else - int32_t res = 0; + int32_t res = 0; SSessionKey tmpKey = *key; - int32_t valSize = *pVLen; - void* tmp = tdbRealloc(NULL, valSize); + int32_t valSize = *pVLen; + void* tmp = tdbRealloc(NULL, valSize); if (!tmp) { return -1; } SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pState, key); - int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); + int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); if (code == 0) { if (key->win.skey <= tmpKey.win.skey && tmpKey.win.ekey <= key->win.ekey) { memcpy(tmp, *pVal, valSize); @@ -1113,6 +1131,8 @@ int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark) { #endif } +void streamStateReloadInfo(SStreamState* pState, TSKEY ts) { streamFileStateReloadInfo(pState->pFileState, ts); } + #if 0 char* streamStateSessionDump(SStreamState* pState) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 284d1ecab63423a7b52e80c23d197639d36b6844..1eb8d119168d23b96a851e0946e968c4e1b1fef8 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -13,13 +13,15 @@ * along with this program. If not, see . */ +#include +#include #include "executor.h" #include "tstream.h" #include "wal.h" -static int32_t mndAddToTaskset(SArray* pArray, SStreamTask* pTask) { +static int32_t addToTaskset(SArray* pArray, SStreamTask* pTask) { int32_t childId = taosArrayGetSize(pArray); - pTask->selfChildId = childId; + pTask->info.selfChildId = childId; taosArrayPush(pArray, &pTask); return 0; } @@ -33,8 +35,8 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHisto pTask->id.taskId = tGenIdPI32(); pTask->id.streamId = streamId; - pTask->taskLevel = taskLevel; - pTask->fillHistory = fillHistory; + pTask->info.taskLevel = taskLevel; + pTask->info.fillHistory = fillHistory; pTask->triggerParam = triggerParam; char buf[128] = {0}; @@ -42,10 +44,11 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHisto pTask->id.idStr = taosStrdup(buf); pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; + pTask->status.taskStatus = TASK_STATUS__SCAN_HISTORY; pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; - pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; + pTask->outputInfo.status = TASK_OUTPUT_STATUS__NORMAL; - mndAddToTaskset(pTaskList, pTask); + addToTaskset(pTaskList, pTask); return pTask; } @@ -71,46 +74,56 @@ int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pTask->id.streamId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->id.taskId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->totalLevel) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->taskLevel) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->outputType) < 0) return -1; - if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->info.totalLevel) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->info.taskLevel) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->outputInfo.type) < 0) return -1; + if (tEncodeI16(pEncoder, pTask->msgInfo.msgType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->status.taskStatus) < 0) return -1; if (tEncodeI8(pEncoder, pTask->status.schedStatus) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; - if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->info.selfChildId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->info.nodeId) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pTask->info.epSet) < 0) return -1; if (tEncodeI64(pEncoder, pTask->chkInfo.id) < 0) return -1; if (tEncodeI64(pEncoder, pTask->chkInfo.version) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->fillHistory) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->info.fillHistory) < 0) return -1; - int32_t epSz = taosArrayGetSize(pTask->childEpInfo); + if (tEncodeI64(pEncoder, pTask->historyTaskId.streamId)) return -1; + if (tEncodeI32(pEncoder, pTask->historyTaskId.taskId)) return -1; + if (tEncodeI64(pEncoder, pTask->streamTaskId.streamId)) return -1; + if (tEncodeI32(pEncoder, pTask->streamTaskId.taskId)) return -1; + + if (tEncodeU64(pEncoder, pTask->dataRange.range.minVer)) return -1; + if (tEncodeU64(pEncoder, pTask->dataRange.range.maxVer)) return -1; + if (tEncodeI64(pEncoder, pTask->dataRange.window.skey)) return -1; + if (tEncodeI64(pEncoder, pTask->dataRange.window.ekey)) return -1; + + int32_t epSz = taosArrayGetSize(pTask->pUpstreamEpInfoList); if (tEncodeI32(pEncoder, epSz) < 0) return -1; for (int32_t i = 0; i < epSz; i++) { - SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->childEpInfo, i); + SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->pUpstreamEpInfoList, i); if (tEncodeStreamEpInfo(pEncoder, pInfo) < 0) return -1; } - if (pTask->taskLevel != TASK_LEVEL__SINK) { + if (pTask->info.taskLevel != TASK_LEVEL__SINK) { if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1; } - if (pTask->outputType == TASK_OUTPUT__TABLE) { + if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { if (tEncodeI64(pEncoder, pTask->tbSink.stbUid) < 0) return -1; if (tEncodeCStr(pEncoder, pTask->tbSink.stbFullName) < 0) return -1; if (tEncodeSSchemaWrapper(pEncoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__SMA) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { if (tEncodeI64(pEncoder, pTask->smaSink.smaId) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__FETCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) { if (tEncodeI8(pEncoder, pTask->fetchSink.reserved) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { if (tSerializeSUseDbRspImp(pEncoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1; if (tEncodeCStr(pEncoder, pTask->shuffleDispatcher.stbFullName) < 0) return -1; } @@ -124,25 +137,36 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pTask->id.streamId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->id.taskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->totalLevel) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->taskLevel) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->outputType) < 0) return -1; - if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->info.totalLevel) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->info.taskLevel) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->outputInfo.type) < 0) return -1; + if (tDecodeI16(pDecoder, &pTask->msgInfo.msgType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->status.taskStatus) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->status.schedStatus) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; - if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->info.selfChildId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->info.nodeId) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &pTask->info.epSet) < 0) return -1; if (tDecodeI64(pDecoder, &pTask->chkInfo.id) < 0) return -1; if (tDecodeI64(pDecoder, &pTask->chkInfo.version) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->fillHistory) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->info.fillHistory) < 0) return -1; + + if (tDecodeI64(pDecoder, &pTask->historyTaskId.streamId)) return -1; + if (tDecodeI32(pDecoder, &pTask->historyTaskId.taskId)) return -1; + if (tDecodeI64(pDecoder, &pTask->streamTaskId.streamId)) return -1; + if (tDecodeI32(pDecoder, &pTask->streamTaskId.taskId)) return -1; + + if (tDecodeU64(pDecoder, &pTask->dataRange.range.minVer)) return -1; + if (tDecodeU64(pDecoder, &pTask->dataRange.range.maxVer)) return -1; + if (tDecodeI64(pDecoder, &pTask->dataRange.window.skey)) return -1; + if (tDecodeI64(pDecoder, &pTask->dataRange.window.ekey)) return -1; int32_t epSz; if (tDecodeI32(pDecoder, &epSz) < 0) return -1; - pTask->childEpInfo = taosArrayInit(epSz, sizeof(void*)); + + pTask->pUpstreamEpInfoList = taosArrayInit(epSz, POINTER_BYTES); for (int32_t i = 0; i < epSz; i++) { SStreamChildEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamChildEpInfo)); if (pInfo == NULL) return -1; @@ -150,28 +174,28 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { taosMemoryFreeClear(pInfo); return -1; } - taosArrayPush(pTask->childEpInfo, &pInfo); + taosArrayPush(pTask->pUpstreamEpInfoList, &pInfo); } - if (pTask->taskLevel != TASK_LEVEL__SINK) { + if (pTask->info.taskLevel != TASK_LEVEL__SINK) { if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1; } - if (pTask->outputType == TASK_OUTPUT__TABLE) { + if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { if (tDecodeI64(pDecoder, &pTask->tbSink.stbUid) < 0) return -1; if (tDecodeCStrTo(pDecoder, pTask->tbSink.stbFullName) < 0) return -1; pTask->tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); if (pTask->tbSink.pSchemaWrapper == NULL) return -1; if (tDecodeSSchemaWrapper(pDecoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__SMA) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { if (tDecodeI64(pDecoder, &pTask->smaSink.smaId) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__FETCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__FETCH) { if (tDecodeI8(pDecoder, &pTask->fetchSink.reserved) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { if (tDeserializeSUseDbRspImp(pDecoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1; if (tDecodeCStrTo(pDecoder, pTask->shuffleDispatcher.stbFullName) < 0) return -1; } @@ -181,15 +205,23 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { return 0; } +static void freeItem(void* p) { + SStreamContinueExecInfo* pInfo = p; + rpcFreeCont(pInfo->msg.pCont); +} + void tFreeStreamTask(SStreamTask* pTask) { - qDebug("free s-task:%s", pTask->id.idStr); + qDebug("free s-task:%s, %p", pTask->id.idStr, pTask); + int32_t status = atomic_load_8((int8_t*)&(pTask->status.taskStatus)); if (pTask->inputQueue) { streamQueueClose(pTask->inputQueue); } - if (pTask->outputQueue) { - streamQueueClose(pTask->outputQueue); + + if (pTask->outputInfo.queue) { + streamQueueClose(pTask->outputInfo.queue); } + if (pTask->exec.qmsg) { taosMemoryFree(pTask->exec.qmsg); } @@ -203,14 +235,12 @@ void tFreeStreamTask(SStreamTask* pTask) { walCloseReader(pTask->exec.pWalReader); } - taosArrayDestroyP(pTask->childEpInfo, taosMemoryFree); - if (pTask->outputType == TASK_OUTPUT__TABLE) { + taosArrayDestroyP(pTask->pUpstreamEpInfoList, taosMemoryFree); + if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { tDeleteSchemaWrapper(pTask->tbSink.pSchemaWrapper); taosMemoryFree(pTask->tbSink.pTSchema); tSimpleHashCleanup(pTask->tbSink.pTblInfo); - } - - if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { taosArrayDestroy(pTask->shuffleDispatcher.dbInfo.pVgroupInfos); taosArrayDestroy(pTask->checkReqIds); pTask->checkReqIds = NULL; @@ -228,5 +258,11 @@ void tFreeStreamTask(SStreamTask* pTask) { tSimpleHashCleanup(pTask->pNameMap); } + if (pTask->pRspMsgList != NULL) { + taosArrayDestroyEx(pTask->pRspMsgList, freeItem); + pTask->pRspMsgList = NULL; + } + + taosThreadMutexDestroy(&pTask->lock); taosMemoryFree(pTask); } diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index c10ef1e5572d692ae430088d4d7884a2ba5be490..dd857141c1288da621b8ed2b58af8373d28fbe31 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -43,12 +43,13 @@ struct SStreamFileState { uint64_t maxRowCount; uint64_t curRowCount; GetTsFun getTs; + char* id; }; typedef SRowBuffPos SRowBuffInfo; SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, - GetTsFun fp, void* pFile, TSKEY delMark) { + GetTsFun fp, void* pFile, TSKEY delMark, const char* idstr) { if (memSize <= 0) { memSize = DEFAULT_MAX_STREAM_BUFFER_SIZE; } @@ -70,6 +71,7 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ if (!pFileState->usedBuffs || !pFileState->freeBuffs || !pFileState->rowBuffMap) { goto _error; } + pFileState->keyLen = keySize; pFileState->rowSize = rowSize; pFileState->selectivityRowSize = selectRowSize; @@ -81,6 +83,8 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ pFileState->deleteMark = delMark; pFileState->flushMark = INT64_MIN; pFileState->maxTs = INT64_MIN; + pFileState->id = taosStrdup(idstr); + recoverSnapshot(pFileState); return pFileState; @@ -124,6 +128,8 @@ void streamFileStateDestroy(SStreamFileState* pFileState) { if (!pFileState) { return; } + + taosMemoryFree(pFileState->id); tdListFreeP(pFileState->usedBuffs, destroyRowBuffAllPosPtr); tdListFreeP(pFileState->freeBuffs, destroyRowBuff); tSimpleHashCleanup(pFileState->rowBuffMap); @@ -177,7 +183,8 @@ void popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uin i++; } } - qInfo("do stream state flush %d rows to disck. is used: %d", listNEles(pFlushList), used); + + qInfo("stream state flush %d rows to disk. is used:%d", listNEles(pFlushList), used); } int32_t flushRowBuff(SStreamFileState* pFileState) { @@ -185,13 +192,17 @@ int32_t flushRowBuff(SStreamFileState* pFileState) { if (!pFlushList) { return TSDB_CODE_OUT_OF_MEMORY; } + uint64_t num = (uint64_t)(pFileState->curRowCount * FLUSH_RATIO); num = TMAX(num, FLUSH_NUM); popUsedBuffs(pFileState, pFlushList, num, false); + if (isListEmpty(pFlushList)) { popUsedBuffs(pFileState, pFlushList, num, true); } + flushSnapshot(pFileState, pFlushList, false); + SListIter fIter = {0}; tdListInitIter(pFlushList, &fIter, TD_LIST_FORWARD); SListNode* pNode = NULL; @@ -201,6 +212,7 @@ int32_t flushRowBuff(SStreamFileState* pFileState) { tdListAppend(pFileState->freeBuffs, &pPos->pRowBuff); pPos->pRowBuff = NULL; } + tdListFreeP(pFlushList, destroyRowBuffPosPtr); return TSDB_CODE_SUCCESS; } @@ -269,13 +281,13 @@ int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, voi TSKEY ts = pFileState->getTs(pKey); if (ts > pFileState->maxTs - pFileState->deleteMark && ts < pFileState->flushMark) { int32_t len = 0; - void* pVal = NULL; - int32_t code = streamStateGet_rocksdb(pFileState->pFileStore, pKey, &pVal, &len); + void* p = NULL; + int32_t code = streamStateGet_rocksdb(pFileState->pFileStore, pKey, &p, &len); qDebug("===stream===get %" PRId64 " from disc, res %d", ts, code); if (code == TSDB_CODE_SUCCESS) { - memcpy(pNewPos->pRowBuff, pVal, len); + memcpy(pNewPos->pRowBuff, p, len); } - taosMemoryFree(pVal); + taosMemoryFree(p); } tSimpleHashPut(pFileState->rowBuffMap, pKey, keyLen, &pNewPos, POINTER_BYTES); @@ -348,7 +360,10 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, tdListInitIter(pSnapshot, &iter, TD_LIST_FORWARD); const int32_t BATCH_LIMIT = 256; - SListNode* pNode = NULL; + + int64_t st = taosGetTimestampMs(); + int32_t numOfElems = listNEles(pSnapshot); + SListNode* pNode = NULL; int idx = streamStateGetCfIdx(pFileState->pFileStore, "state"); @@ -359,6 +374,7 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, while ((pNode = tdListNext(&iter)) != NULL && code == TSDB_CODE_SUCCESS) { SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data; ASSERT(pPos->pRowBuff && pFileState->rowSize > 0); + if (streamStateGetBatchSize(batch) >= BATCH_LIMIT) { streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); streamStateClearBatch(batch); @@ -367,16 +383,22 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, SStateKey sKey = {.key = *((SWinKey*)pPos->pKey), .opNum = ((SStreamState*)pFileState->pFileStore)->number}; code = streamStatePutBatchOptimize(pFileState->pFileStore, idx, batch, &sKey, pPos->pRowBuff, pFileState->rowSize, 0, buf); + // todo handle failure memset(buf, 0, len); - qDebug("===stream===put %" PRId64 " to disc, res %d", sKey.key.ts, code); +// qDebug("===stream===put %" PRId64 " to disc, res %d", sKey.key.ts, code); } taosMemoryFree(buf); if (streamStateGetBatchSize(batch) > 0) { streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); } + streamStateClearBatch(batch); + int64_t elapsed = taosGetTimestampMs() - st; + qDebug("%s flush to disk in batch model completed, rows:%d, batch size:%d, elapsed time:%"PRId64"ms", pFileState->id, numOfElems, + BATCH_LIMIT, elapsed); + if (flushState) { const char* taskKey = "streamFileState"; { @@ -398,8 +420,8 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, } streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); } - streamStateDestroyBatch(batch); + streamStateDestroyBatch(batch); return code; } @@ -502,3 +524,8 @@ int32_t recoverSnapshot(SStreamFileState* pFileState) { } int32_t streamFileStateGeSelectRowSize(SStreamFileState* pFileState) { return pFileState->selectivityRowSize; } + +void streamFileStateReloadInfo(SStreamFileState* pFileState, TSKEY ts) { + pFileState->flushMark = TMAX(pFileState->flushMark, ts); + pFileState->maxTs = TMAX(pFileState->maxTs, ts); +} diff --git a/source/libs/stream/test/tstreamUpdateTest.cpp b/source/libs/stream/test/tstreamUpdateTest.cpp index 18c60aff284414e5ba5044d50000a9bd45718965..0e84d6b8bdad0e40e7257a0d60880439d1f3ba37 100644 --- a/source/libs/stream/test/tstreamUpdateTest.cpp +++ b/source/libs/stream/test/tstreamUpdateTest.cpp @@ -158,7 +158,7 @@ TEST(TD_STREAM_UPDATE_TEST, update) { // void *buf = taosMemoryCalloc(1, bufLen); // int32_t resSize = updateInfoSerialize(buf, bufLen, pSU7); - // SUpdateInfo *pSU6 = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); + // SUpdateInfo *pSU6 = taosMemoryCalloc(1, sizeof(SUpdateInfo)); // int32_t desSize = updateInfoDeserialize(buf, bufLen, pSU6); // GTEST_ASSERT_EQ(desSize, 0); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 7d336c83135a20449f09431c2d9427a6dd6cc0c5..f74e43f47f4b3d81ec13a7e99578f7224b05bf5d 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -213,7 +213,7 @@ typedef struct SSyncNode { int64_t minMatchIndex; int64_t startTime; - int64_t leaderTime; + int64_t roleTimeMs; int64_t lastReplicateTime; int32_t electNum; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f8e21af2c3adc06c428eede38a2dac00cbfb74cc..f1e3c35a49228b9805b12711e8ba6d41f5a44dd9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -508,12 +508,14 @@ SSyncState syncGetState(int64_t rid) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode != NULL) { state.state = pSyncNode->state; + state.roleTimeMs = pSyncNode->roleTimeMs; state.restored = pSyncNode->restoreFinish; if (pSyncNode->vgId != 1) { state.canRead = syncNodeIsReadyForRead(pSyncNode); } else { state.canRead = state.restored; } + state.term = raftStoreGetTerm(pSyncNode); syncNodeRelease(pSyncNode); } @@ -898,6 +900,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init TLA+ server vars pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; + pSyncNode->roleTimeMs = taosGetTimestampMs(); if (raftStoreOpen(pSyncNode) != 0) { sError("vgId:%d, failed to open raft store at path %s", pSyncNode->vgId, pSyncNode->raftStorePath); goto _error; @@ -1035,7 +1038,6 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { int64_t timeNow = taosGetTimestampMs(); pSyncNode->startTime = timeNow; - pSyncNode->leaderTime = timeNow; pSyncNode->lastReplicateTime = timeNow; // snapshotting @@ -1131,6 +1133,7 @@ int32_t syncNodeStart(SSyncNode* pSyncNode) { int32_t syncNodeStartStandBy(SSyncNode* pSyncNode) { // state change pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; + pSyncNode->roleTimeMs = taosGetTimestampMs(); syncNodeStopHeartbeatTimer(pSyncNode); // reset elect timer, long enough @@ -1667,6 +1670,7 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // state change pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; + pSyncNode->roleTimeMs = taosGetTimestampMs(); syncNodeStopHeartbeatTimer(pSyncNode); // trace log @@ -1695,6 +1699,7 @@ void syncNodeBecomeLearner(SSyncNode* pSyncNode, const char* debugStr) { // state change pSyncNode->state = TAOS_SYNC_STATE_LEARNER; + pSyncNode->roleTimeMs = taosGetTimestampMs(); // trace log sNTrace(pSyncNode, "become learner %s", debugStr); @@ -1730,8 +1735,6 @@ void syncNodeBecomeLearner(SSyncNode* pSyncNode, const char* debugStr) { // /\ UNCHANGED <> // void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { - pSyncNode->leaderTime = taosGetTimestampMs(); - pSyncNode->becomeLeaderNum++; pSyncNode->hbrSlowNum = 0; @@ -1740,6 +1743,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // state change pSyncNode->state = TAOS_SYNC_STATE_LEADER; + pSyncNode->roleTimeMs = taosGetTimestampMs(); // set leader cache pSyncNode->leaderCache = pSyncNode->myRaftId; @@ -1839,6 +1843,7 @@ int32_t syncNodePeerStateInit(SSyncNode* pSyncNode) { void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { ASSERT(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER); pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; + pSyncNode->roleTimeMs = taosGetTimestampMs(); SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); sInfo("vgId:%d, become candidate from follower. term:%" PRId64 ", commit index:%" PRId64 ", last index:%" PRId64, pSyncNode->vgId, raftStoreGetTerm(pSyncNode), pSyncNode->commitIndex, lastIndex); diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 5ee67da9ab8d2cef07b7d85ef275199e8cf2ef0e..37166805cee2746313b8511776928ccb74255878 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -87,22 +87,6 @@ static int32_t syncNodeTimerRoutine(SSyncNode* ths) { } } - if (atomic_load_64(&ths->snapshottingIndex) != SYNC_INDEX_INVALID) { - // end timeout wal snapshot - if (timeNow - ths->snapshottingTime > SYNC_DEL_WAL_MS && - atomic_load_64(&ths->snapshottingIndex) != SYNC_INDEX_INVALID) { - SSyncLogStoreData* pData = ths->pLogStore->data; - int32_t code = walEndSnapshot(pData->pWal); - if (code != 0) { - sNError(ths, "timer wal snapshot end error since:%s", terrstr()); - return -1; - } else { - sNTrace(ths, "wal snapshot end, index:%" PRId64, atomic_load_64(&ths->snapshottingIndex)); - atomic_store_64(&ths->snapshottingIndex, SYNC_INDEX_INVALID); - } - } - } - if (!syncNodeIsMnode(ths)) { syncRespClean(ths->pSyncRespMgr); } diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index db05f3e28712504e7b1addec72b3874304d08152..612179b2053ded041941a01e9f2aca905da3f72b 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -233,7 +233,11 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { int ret; tdbBtcOpen(&btc, pBt, pTxn); - + /* + btc.coder.ofps = taosArrayInit(8, sizeof(SPage *)); + // btc.coder.ofps = taosArrayInit(8, sizeof(SPgno)); + //pBtc->coder.ofps = taosArrayInit(8, sizeof(SPage *)); + */ tdbTrace("tdb delete, btc: %p, pTxn: %p", &btc, pTxn); // move the cursor @@ -254,7 +258,18 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { tdbBtcClose(&btc); return -1; } + /* + SArray *ofps = btc.coder.ofps; + if (ofps) { + for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { + SPage *ofp = *(SPage **)taosArrayGet(ofps, i); + tdbPagerInsertFreePage(btc.pBt->pPager, ofp, btc.pTxn); + } + taosArrayDestroy(ofps); + btc.coder.ofps = NULL; + } + */ tdbBtcClose(&btc); return 0; } @@ -563,6 +578,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } } // copy the parent key out if child pages are not leaf page + // childNotLeaf = !(TDB_BTREE_PAGE_IS_LEAF(pOlds[0]) || TDB_BTREE_PAGE_IS_OVFL(pOlds[0])); childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(pOlds[0]); if (childNotLeaf) { for (int i = 0; i < nOlds; i++) { @@ -592,7 +608,30 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx for (int i = 0; i < nOlds; i++) { nCells = TDB_PAGE_TOTAL_CELLS(pParent); if (sIdx < nCells) { + bool destroyOfps = false; + if (!childNotLeaf) { + if (!pParent->pPager->ofps) { + pParent->pPager->ofps = taosArrayInit(8, sizeof(SPage *)); + destroyOfps = true; + } + } + tdbPageDropCell(pParent, sIdx, pTxn, pBt); + + if (!childNotLeaf) { + SArray *ofps = pParent->pPager->ofps; + if (ofps) { + for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { + SPage *ofp = *(SPage **)taosArrayGet(ofps, i); + tdbPagerInsertFreePage(pParent->pPager, ofp, pTxn); + } + + if (destroyOfps) { + taosArrayDestroy(ofps); + pParent->pPager->ofps = NULL; + } + } + } } else { ((SIntHdr *)pParent->pData)->pgno = 0; } @@ -861,6 +900,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx if (!TDB_BTREE_PAGE_IS_LEAF(pNews[0])) { ((SIntHdr *)(pParent->pData))->pgno = ((SIntHdr *)(pNews[0]->pData))->pgno; } + + tdbPagerInsertFreePage(pBt->pPager, pNews[0], pTxn); } for (int i = 0; i < 3; i++) { @@ -870,6 +911,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } for (pageIdx = 0; pageIdx < nOlds; ++pageIdx) { + if (pageIdx >= nNews) { + tdbPagerInsertFreePage(pBt->pPager, pOlds[pageIdx], pTxn); + } tdbPagerReturnPage(pBt->pPager, pOlds[pageIdx], pTxn); } for (; pageIdx < nNews; ++pageIdx) { @@ -1311,7 +1355,11 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, if (ret < 0) { return -1; } - + /* + if (pDecoder->ofps) { + taosArrayPush(pDecoder->ofps, &ofp); + } + */ ofpCell = tdbPageGetCell(ofp, 0); if (nLeft <= ofp->maxLocal - sizeof(SPgno)) { @@ -1346,11 +1394,17 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, int lastKeyPageSpace = 0; // load left key & val to ovpages while (pgno != 0) { + tdbTrace("tdb decode-ofp, pTxn: %p, pgno:%u by cell:%p", pTxn, pgno, pCell); + // printf("tdb decode-ofp, pTxn: %p, pgno:%u by cell:%p\n", pTxn, pgno, pCell); ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt); if (ret < 0) { return -1; } - + /* + if (pDecoder->ofps) { + taosArrayPush(pDecoder->ofps, &ofp); + } + */ ofpCell = tdbPageGetCell(ofp, 0); int lastKeyPage = 0; @@ -1518,8 +1572,8 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * if (pPage->vLen == TDB_VARIANT_LEN) { if (!leaf) { - tdbError("tdb/btree-cell-size: not a leaf page."); - return -1; + tdbError("tdb/btree-cell-size: not a leaf page:%p, pgno:%" PRIu32 ".", pPage, TDB_PAGE_PGNO(pPage)); + // return -1; } nHeader += tdbGetVarInt(pCell + nHeader, &vLen); } else if (leaf) { @@ -1559,8 +1613,27 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * bytes = ofp->maxLocal - sizeof(SPgno); } + // SPgno origPgno = pgno; memcpy(&pgno, ofpCell + bytes, sizeof(pgno)); + ret = tdbPagerWrite(pBt->pPager, ofp); + if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); + return -1; + } + /* + tdbPageDropCell(ofp, 0, pTxn, pBt); + */ + // SIntHdr *pIntHdr = (SIntHdr *)(ofp->pData); + // pIntHdr->flags = TDB_FLAG_ADD(0, TDB_BTREE_OVFL); + // pIntHdr->pgno = 0; + // ofp->pPager = NULL; + + SArray *ofps = pPage->pPager->ofps; + if (ofps) { + taosArrayPush(ofps, &ofp); + } + tdbPagerReturnPage(pPage->pPager, ofp, pTxn); nLeft -= bytes; @@ -1980,6 +2053,11 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { return -1; } + if (TDB_BTREE_PAGE_IS_OVFL(pBtc->pPage)) { + tdbError("tdb/btc-move-downward: should not be a ovfl page here."); + return -1; + } + if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); pgno = ((SPgno *)pCell)[0]; @@ -2068,8 +2146,27 @@ int tdbBtcDelete(SBTC *pBtc) { return -1; } + bool destroyOfps = false; + if (!pBtc->pPage->pPager->ofps) { + pBtc->pPage->pPager->ofps = taosArrayInit(8, sizeof(SPage *)); + destroyOfps = true; + } + tdbPageDropCell(pBtc->pPage, idx, pBtc->pTxn, pBtc->pBt); + SArray *ofps = pBtc->pPage->pPager->ofps; + if (ofps) { + for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { + SPage *ofp = *(SPage **)taosArrayGet(ofps, i); + tdbPagerInsertFreePage(pBtc->pPage->pPager, ofp, pBtc->pTxn); + } + + if (destroyOfps) { + taosArrayDestroy(ofps); + pBtc->pPage->pPager->ofps = NULL; + } + } + // update interior page or do balance if (idx == nCells - 1) { if (idx) { @@ -2113,6 +2210,8 @@ int tdbBtcDelete(SBTC *pBtc) { return -1; } + // printf("tdb/btc-delete: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pBtc->pPage)); + ret = tdbBtreeBalance(pBtc); if (ret < 0) { tdbError("tdb/btc-delete: btree balance failed with ret: %d.", ret); @@ -2181,7 +2280,13 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int tdbError("tdb/btc-upsert: page insert/update cell failed with ret: %d.", ret); return -1; } - + /* + bool destroyOfps = false; + if (!pBtc->pPage->pPager->ofps) { + pBtc->pPage->pPager->ofps = taosArrayInit(8, sizeof(SPage *)); + destroyOfps = true; + } + */ // check balance if (pBtc->pPage->nOverflow > 0) { ret = tdbBtreeBalance(pBtc); @@ -2190,7 +2295,20 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int return -1; } } + /* + SArray *ofps = pBtc->pPage->pPager->ofps; + if (ofps) { + for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { + SPage *ofp = *(SPage **)taosArrayGet(ofps, i); + tdbPagerInsertFreePage(pBtc->pPage->pPager, ofp, pBtc->pTxn); + } + if (destroyOfps) { + taosArrayDestroy(ofps); + pBtc->pPage->pPager->ofps = NULL; + } + } + */ return 0; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 0e9f1922261b226cb4362535ea9161a2b90015be..4f595d8d4ae83f4003082bd1b1b3ffb6816d473e 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -73,6 +73,11 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i if (ret < 0) { return -1; } + + ret = tdbTbOpen(TDB_FREEDB_NAME, sizeof(SPgno), 0, NULL, pDb, &pDb->pFreeDb, rollback); + if (ret < 0) { + return -1; + } #endif *ppDb = pDb; @@ -85,6 +90,7 @@ int tdbClose(TDB *pDb) { if (pDb) { #ifdef USE_MAINDB if (pDb->pMainDb) tdbTbClose(pDb->pMainDb); + if (pDb->pFreeDb) tdbTbClose(pDb->pFreeDb); #endif for (pPager = pDb->pgrList; pPager; pPager = pDb->pgrList) { diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 5144a94a2594b2e38dbc32ff356591a2c177b3aa..474e5d227071de65cd91811ac27c018131da818d 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -292,7 +292,23 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { */ return 0; } +/* +int tdbPagerCancelDirty(SPager *pPager, SPage *pPage, TXN *pTxn) { + SRBTreeNode *pNode = tRBTreeGet(&pPager->rbt, (SRBTreeNode *)pPage); + if (pNode) { + pPage->isDirty = 0; + + tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); + if (pTxn->jPageSet) { + hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); + } + + tdbPCacheRelease(pPager->pCache, pPage, pTxn); + } + return 0; +} +*/ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { SPage *pPage; int ret; @@ -338,10 +354,13 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { if (pTxn->jPageSet) { hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); } + + tdbTrace("tdb/pager-commit: remove page: %p %d from dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt); + tdbPCacheRelease(pPager->pCache, pPage, pTxn); } - tdbTrace("pager/commit reset dirty tree: %p", &pPager->rbt); + tdbTrace("tdb/pager-commit reset dirty tree: %p", &pPager->rbt); tRBTreeCreate(&pPager->rbt, pageCmpFn); // sync the db file @@ -629,6 +648,8 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) { return 0; } +static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno, TXN *pTxn); + int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, TXN *pTxn) { SPage *pPage; @@ -643,7 +664,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa // alloc new page if (pgno == 0) { loadPage = 0; - ret = tdbPagerAllocPage(pPager, &pgno); + ret = tdbPagerAllocPage(pPager, &pgno, pTxn); if (ret < 0) { tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); return -1; @@ -695,23 +716,114 @@ void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn) { // TDB_PAGE_PGNO(pPage), pPage); } -static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) { - // TODO: Allocate a page from the free list +int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) { + int code = 0; + SPgno pgno = TDB_PAGE_PGNO(pPage); + + if (pPager->frps) { + taosArrayPush(pPager->frps, &pgno); + pPage->pPager = NULL; + return code; + } + + pPager->frps = taosArrayInit(8, sizeof(SPgno)); + // memset(pPage->pData, 0, pPage->pageSize); + tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno); + // printf("tdb/insert-free-page: tbc recycle page: %d.\n", pgno); + code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn); + if (code < 0) { + tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); + taosArrayDestroy(pPager->frps); + pPager->frps = NULL; + return -1; + } + + while (TARRAY_SIZE(pPager->frps) > 0) { + pgno = *(SPgno *)taosArrayPop(pPager->frps); + + code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn); + if (code < 0) { + tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); + taosArrayDestroy(pPager->frps); + pPager->frps = NULL; + return -1; + } + } + + taosArrayDestroy(pPager->frps); + pPager->frps = NULL; + + pPage->pPager = NULL; + + return code; +} + +static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) { + int code = 0; + TBC *pCur; + + if (!pPager->pEnv->pFreeDb) { + return code; + } + + if (pPager->frps) { + return code; + } + + code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, pTxn); + if (code < 0) { + return 0; + } + + code = tdbTbcMoveToFirst(pCur); + if (code) { + tdbError("tdb/remove-free-page: moveto first failed with ret: %d.", code); + tdbTbcClose(pCur); + return 0; + } + + void *pKey = NULL; + int nKey = 0; + + code = tdbTbcGet(pCur, (const void **)&pKey, &nKey, NULL, NULL); + if (code < 0) { + // tdbError("tdb/remove-free-page: tbc get failed with ret: %d.", code); + tdbTbcClose(pCur); + return 0; + } + + *pPgno = *(SPgno *)pKey; + tdbTrace("tdb/remove-free-page: tbc get page: %d.", *pPgno); + // printf("tdb/remove-free-page: tbc get page: %d.\n", *pPgno); + + code = tdbTbcDelete(pCur); + if (code < 0) { + tdbError("tdb/remove-free-page: tbc delete failed with ret: %d.", code); + tdbTbcClose(pCur); + return 0; + } + tdbTbcClose(pCur); return 0; } +static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno, TXN *pTxn) { + // Allocate a page from the free list + return tdbPagerRemoveFreePage(pPager, ppgno, pTxn); +} + static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) { *ppgno = ++pPager->dbFileSize; + // tdbError("tdb/alloc-new-page: %d.", *ppgno); return 0; } -int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) { +static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno, TXN *pTxn) { int ret; *ppgno = 0; // Try to allocate from the free list of the pager - ret = tdbPagerAllocFreePage(pPager, ppgno); + ret = tdbPagerAllocFreePage(pPager, ppgno, pTxn); if (ret < 0) { return -1; } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 7a0bcc00a42a95d1f27b23384e494416e110b2e1..8ce294a3c6565ccffb3efa214fe75a147768143b 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -131,13 +131,14 @@ typedef struct SBtInfo { #define TDB_CELLDECODER_FREE_VAL(pCellDecoder) ((pCellDecoder)->freeKV & TDB_CELLD_F_VAL) typedef struct { - int kLen; - u8 *pKey; - int vLen; - u8 *pVal; - SPgno pgno; - u8 *pBuf; - u8 freeKV; + int kLen; + u8 *pKey; + int vLen; + u8 *pVal; + SPgno pgno; + u8 *pBuf; + u8 freeKV; + SArray *ofps; } SCellDecoder; struct SBTC { @@ -198,9 +199,10 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn); int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, TXN *pTxn); void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); -int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); -int tdbPagerRestoreJournals(SPager *pPager); -int tdbPagerRollback(SPager *pPager); +int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn); +// int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); +int tdbPagerRestoreJournals(SPager *pPager); +int tdbPagerRollback(SPager *pPager); // tdbPCache.c ==================================== #define TDB_PCACHE_PAGE \ @@ -373,6 +375,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { #ifdef USE_MAINDB #define TDB_MAINDB_NAME "main.tdb" +#define TDB_FREEDB_NAME "_free.db" #endif struct STDB { @@ -386,6 +389,7 @@ struct STDB { SPager **pgrHash; #ifdef USE_MAINDB TTB *pMainDb; + TTB *pFreeDb; #endif int64_t txnId; }; @@ -403,6 +407,8 @@ struct SPager { SRBTree rbt; // u8 inTran; TXN *pActiveTxn; + SArray *ofps; + SArray *frps; SPager *pNext; // used by TDB SPager *pHashNext; // used by TDB #ifdef USE_MAINDB diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index fd4d7c101d06ce73de37678e858716326e83feee..4715ccbd4157e65eb8665677a94788e2c0ed234b 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -14,3 +14,7 @@ target_link_libraries(tdbExOVFLTest tdb gtest gtest_main) add_executable(tdbPageDefragmentTest "tdbPageDefragmentTest.cpp") target_link_libraries(tdbPageDefragmentTest tdb gtest gtest_main) +# page recycling testing +add_executable(tdbPageRecycleTest "tdbPageRecycleTest.cpp") +target_link_libraries(tdbPageRecycleTest tdb gtest gtest_main) + diff --git a/source/libs/tdb/test/tdbExOVFLTest.cpp b/source/libs/tdb/test/tdbExOVFLTest.cpp index b16bc643d3687e00b04cfdcdc9f544e56cd0d0c4..325703c946842308249dcd0ec7170447cdb2bd92 100644 --- a/source/libs/tdb/test/tdbExOVFLTest.cpp +++ b/source/libs/tdb/test/tdbExOVFLTest.cpp @@ -190,6 +190,15 @@ static void insertOfp(void) { // commit current transaction tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); } // TEST(TdbOVFLPagesTest, DISABLED_TbInsertTest) { @@ -233,6 +242,13 @@ TEST(TdbOVFLPagesTest, TbGetTest) { tdbFree(pVal); } + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); } // TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) { @@ -334,6 +350,15 @@ tdbBegin(pEnv, &txn); // commit current transaction tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); } // TEST(tdb_test, DISABLED_simple_insert1) { @@ -407,6 +432,8 @@ TEST(tdb_test, simple_insert1) { tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); + closePool(pPool); + { // Query the data void *pVal = NULL; int vLen; diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4d7b314917a822fde5d4c2ad091ba6130ed47b22 --- /dev/null +++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp @@ -0,0 +1,835 @@ +#include + +#define ALLOW_FORBID_FUNC +#include "os.h" +#include "tdb.h" + +#include +#include +#include +#include +#include "tlog.h" + +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void clearPool(SPoolMem *pPool) { + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); + } while (1); + + assert(pPool->size == 0); +} + +static void closePool(SPoolMem *pPool) { + clearPool(pPool); + taosMemoryFree(pPool); +} + +static void *poolMalloc(void *arg, size_t size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); +} + +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + int k1, k2; + + std::string s1((char *)pKey1 + 3, kLen1 - 3); + std::string s2((char *)pKey2 + 3, kLen2 - 3); + k1 = stoi(s1); + k2 = stoi(s2); + + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } else { + return 0; + } +} + +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; +} + +static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { + TDB *pEnv = NULL; + + int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0); + if (ret) { + pEnv = NULL; + } + + return pEnv; +} + +static void generateBigVal(char *val, int valLen) { + for (int i = 0; i < valLen; ++i) { + char c = char(i & 0xff); + if (c == 0) { + c = 1; + } + val[i] = c; + } +} + +static void insertOfp(void) { + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + char const *key = "key123456789"; + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); +} + +static void clearDb(char const *db) { taosRemoveDir(db); } + +TEST(TdbPageRecycleTest, DISABLED_TbInsertTest) { + // TEST(TdbPageRecycleTest, TbInsertTest) { + // ofp inserting + clearDb("tdb"); + insertOfp(); +} + +TEST(TdbPageRecycleTest, DISABLED_TbGetTest) { + // TEST(TdbPageRecycleTest, TbGetTest) { + clearDb("tdb"); + insertOfp(); + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // int ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + int ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + { // Query the data + void *pVal = NULL; + int vLen; + + // char const *key = "key1"; + char const *key = "key123456789"; + ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, valLen); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + + tdbFree(pVal); + } +} + +TEST(TdbPageRecycleTest, DISABLED_TbDeleteTest) { + // TEST(TdbPageRecycleTest, TbDeleteTest) { + int ret = 0; + + taosRemoveDir("tdb"); + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[((4083 - 4 - 3 - 2) + 1) * 2]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + { // insert the generated big data + ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + { // query the data + void *pVal = NULL; + int vLen; + + ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, valLen); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + + tdbFree(pVal); + } + /* open to debug committed file +tdbCommit(pEnv, &txn); +tdbTxnClose(&txn); + +++txnid; +tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); +tdbBegin(pEnv, &txn); + */ + { // upsert the data + ret = tdbTbUpsert(pDb, "key1", strlen("key1"), "value1", strlen("value1"), txn); + GTEST_ASSERT_EQ(ret, 0); + } + + { // query the upserted data + void *pVal = NULL; + int vLen; + + ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, strlen("value1")); + GTEST_ASSERT_EQ(memcmp("value1", pVal, vLen), 0); + + tdbFree(pVal); + } + + { // delete the data + ret = tdbTbDelete(pDb, "key1", strlen("key1"), txn); + GTEST_ASSERT_EQ(ret, 0); + } + + { // query the deleted data + void *pVal = NULL; + int vLen = -1; + + ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); + ASSERT(ret == -1); + GTEST_ASSERT_EQ(ret, -1); + + GTEST_ASSERT_EQ(vLen, -1); + GTEST_ASSERT_EQ(pVal, nullptr); + + tdbFree(pVal); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); +} + +TEST(TdbPageRecycleTest, DISABLED_simple_insert1) { + // TEST(TdbPageRecycleTest, simple_insert1) { + int ret; + TDB *pEnv; + TTB *pDb; + tdb_cmpr_fn_t compFunc; + int nData = 1; + TXN *txn; + int const pageSize = 4096; + + taosRemoveDir("tdb"); + + // Open Env + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tKeyCmpr; + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + { + char key[64]; + // char val[(4083 - 4 - 3 - 2)]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int iData = 1; iData <= nData; iData++) { + sprintf(key, "key0"); + sprintf(val, "value%d", iData); + + // ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + // GTEST_ASSERT_EQ(ret, 0); + + // generate value payload + int valLen = sizeof(val) / sizeof(val[0]); + for (int i = 6; i < valLen; ++i) { + char c = char(i & 0xff); + if (c == 0) { + c = 1; + } + val[i] = c; + } + + ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + + // commit the transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + { // Query the data + void *pVal = NULL; + int vLen; + + for (int i = 1; i <= nData; i++) { + sprintf(key, "key%d", i); + // sprintf(val, "value%d", i); + + ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, sizeof(val) / sizeof(val[0])); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + } + + tdbFree(pVal); + } + + { // Iterate to query the DB data + TBC *pDBC; + void *pKey = NULL; + void *pVal = NULL; + int vLen, kLen; + int count = 0; + + ret = tdbTbcOpen(pDb, &pDBC, NULL); + GTEST_ASSERT_EQ(ret, 0); + + tdbTbcMoveToFirst(pDBC); + + for (;;) { + ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + + // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + // std::cout << std::endl; + + count++; + } + + GTEST_ASSERT_EQ(count, nData); + + tdbTbcClose(pDBC); + + tdbFree(pKey); + tdbFree(pVal); + } + } + + ret = tdbTbDrop(pDb); + GTEST_ASSERT_EQ(ret, 0); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); +} + +static void insertDb(int nData) { + int ret = 0; + TDB *pEnv = NULL; + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc; + TXN *txn = NULL; + int const pageSize = 4 * 1024; + + // Open Env + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tKeyCmpr; + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // 1, insert nData kv + { + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int iData = 0; iData < nData; ++iData) { + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); + GTEST_ASSERT_EQ(ret, 0); + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + + // commit the transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // 2, delete nData/2 records + + closePool(pPool); + } + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} + +static void deleteDb(int nData) { + int ret = 0; + TDB *pEnv = NULL; + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc; + TXN *txn = NULL; + int const pageSize = 4 * 1024; + + // Open Env + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tKeyCmpr; + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // 2, delete nData/2 records + { + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int iData = 0; iData < nData; iData++) { + // if (iData % 2 == 0) continue; + + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + { // delete the data + ret = tdbTbDelete(pDb, key, strlen(key), txn); + GTEST_ASSERT_EQ(ret, 0); + } + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + + // commit the transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + } + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} + +static const int nDataConst = 256 * 19; + +// TEST(TdbPageRecycleTest, DISABLED_seq_insert) { +TEST(TdbPageRecycleTest, seq_insert) { + clearDb("tdb"); + insertDb(nDataConst); +} + +// TEST(TdbPageRecycleTest, DISABLED_seq_delete) { +TEST(TdbPageRecycleTest, seq_delete) { deleteDb(nDataConst); } + +// TEST(TdbPageRecycleTest, DISABLED_recycly_insert) { +TEST(TdbPageRecycleTest, recycly_insert) { insertDb(nDataConst); } + +// TEST(TdbPageRecycleTest, DISABLED_recycly_seq_insert_ofp) { +TEST(TdbPageRecycleTest, recycly_seq_insert_ofp) { + clearDb("tdb"); + insertOfp(); + system("ls -l ./tdb"); +} + +static void deleteOfp(void) { + // open Env + int ret = 0; + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + { // delete the data + char const *key = "key123456789"; + ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + ret = tdbTbDrop(pDb); + GTEST_ASSERT_EQ(ret, 0); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); +} + +// TEST(TdbPageRecycleTest, DISABLED_seq_delete_ofp) { +TEST(TdbPageRecycleTest, seq_delete_ofp) { + deleteOfp(); + system("ls -l ./tdb"); +} + +// TEST(TdbPageRecycleTest, DISABLED_recycly_seq_insert_ofp_again) { +TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_again) { + insertOfp(); + system("ls -l ./tdb"); +} + +// TEST(TdbPageRecycleTest, DISABLED_recycly_seq_insert_ofp_nocommit) { +TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_nocommit) { + clearDb("tdb"); + insertOfp(); + system("ls -l ./tdb"); + + // open Env + int ret = 0; + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + { // delete the data + char const *key = "key123456789"; + ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // 1, insert nData kv + { + int nData = nDataConst; + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + + for (int iData = 0; iData < nData; ++iData) { + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); + GTEST_ASSERT_EQ(ret, 0); + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} + +// TEST(TdbPageRecycleTest, DISABLED_recycly_delete_interior_ofp_nocommit) { +TEST(TdbPageRecycleTest, recycly_delete_interior_ofp_nocommit) { + clearDb("tdb"); + + // open Env + int ret = 0; + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = NULL; // tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + char key[1024] = {0}; + int count = sizeof(key) / sizeof(key[0]); + for (int i = 0; i < count - 1; ++i) { + key[i] = 'a'; + } + + // insert n ofp keys to form 2-layer btree + { + for (int i = 0; i < 7; ++i) { + // sprintf(&key[count - 2], "%c", i); + key[count - 2] = '0' + i; + + ret = tdbTbInsert(pDb, key, count, NULL, NULL, txn); + GTEST_ASSERT_EQ(ret, 0); + } + } + /* + // delete one interior key + { + sprintf(&key[count - 2], "%c", 2); + key[count - 2] = '0' + 2; + + ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + GTEST_ASSERT_EQ(ret, 0); + } + */ + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index bedd14353fa94379416d2c1d0f44421e3c3b450c..8adaab91a186c021da4137a1a7dcd528a69a21a1 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -113,6 +113,15 @@ SDiskSize tfsGetSize(STfs *pTfs) { return size; } +int32_t tfsGetDisksAtLevel(STfs *pTfs, int32_t level) { + if (level < 0 || level >= pTfs->nlevel) { + return 0; + } + + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + return pTier->ndisk; +} + int32_t tfsGetLevel(STfs *pTfs) { return pTfs->nlevel; } int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) { @@ -227,6 +236,9 @@ int32_t tfsMkdirAt(STfs *pTfs, const char *rname, SDiskID diskId) { STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); char aname[TMPNAME_LEN]; + if (pDisk == NULL) { + return -1; + } snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname); if (taosMkDir(aname) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -269,6 +281,20 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { return 0; } +int32_t tfsMkdirRecur(STfs *pTfs, const char *rname) { + for (int32_t level = 0; level < pTfs->nlevel; level++) { + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + for (int32_t id = 0; id < pTier->ndisk; id++) { + SDiskID did = {.id = id, .level = level}; + if (tfsMkdirRecurAt(pTfs, rname, did) < 0) { + return -1; + } + } + } + + return 0; +} + int32_t tfsMkdir(STfs *pTfs, const char *rname) { for (int32_t level = 0; level < pTfs->nlevel; level++) { STfsTier *pTier = TFS_TIER_AT(pTfs, level); @@ -311,25 +337,60 @@ int32_t tfsRmdir(STfs *pTfs, const char *rname) { return 0; } -int32_t tfsRename(STfs *pTfs, const char *orname, const char *nrname) { +static int32_t tfsRenameAt(STfs *pTfs, SDiskID diskId, const char *orname, const char *nrname) { char oaname[TMPNAME_LEN] = "\0"; char naname[TMPNAME_LEN] = "\0"; - for (int32_t level = 0; level < pTfs->nlevel; level++) { + int32_t level = diskId.level; + int32_t id = diskId.id; + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + STfsDisk *pDisk = pTier->disks[id]; + snprintf(oaname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, orname); + snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname); + + if (taosRenameFile(oaname, naname) != 0 && errno != ENOENT) { + terrno = TAOS_SYSTEM_ERROR(errno); + fError("failed to rename %s to %s since %s", oaname, naname, terrstr()); + return -1; + } + + return 0; +} + +int32_t tfsRename(STfs *pTfs, int32_t diskPrimary, const char *orname, const char *nrname) { + for (int32_t level = pTfs->nlevel - 1; level >= 0; level--) { STfsTier *pTier = TFS_TIER_AT(pTfs, level); - for (int32_t id = 0; id < pTier->ndisk; id++) { - STfsDisk *pDisk = pTier->disks[id]; - snprintf(oaname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, orname); - snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname); - if (taosRenameFile(oaname, naname) != 0 && errno != ENOENT) { - terrno = TAOS_SYSTEM_ERROR(errno); - fError("failed to rename %s to %s since %s", oaname, naname, terrstr()); + for (int32_t id = pTier->ndisk - 1; id >= 0; id--) { + if (level == 0 && id == diskPrimary) { + continue; + } + + SDiskID diskId = {.level = level, .id = id}; + if (tfsRenameAt(pTfs, diskId, orname, nrname)) { return -1; } } } - return 0; + SDiskID diskId = {.level = 0, .id = diskPrimary}; + return tfsRenameAt(pTfs, diskId, orname, nrname); +} + +int32_t tfsSearch(STfs *pTfs, int32_t level, const char *fname) { + if (level < 0 || level >= pTfs->nlevel) { + return -1; + } + char path[TMPNAME_LEN] = {0}; + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + + for (int32_t id = 0; id < pTier->ndisk; id++) { + STfsDisk *pDisk = pTier->disks[id]; + snprintf(path, TMPNAME_LEN - 1, "%s%s%s", pDisk->path, TD_DIRSEP, fname); + if (taosCheckExistFile(path)) { + return id; + } + } + return -1; } STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index df37630fd784e0e561b2675202a9a714370865cc..9bbf6bc729f38141b4ef38ba60af90bc87402654 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -156,7 +156,7 @@ TEST_F(TfsTest, 03_Dir) { EXPECT_NE(taosDirExist(ap4), 1); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); EXPECT_EQ(taosDirExist(ap4), 1); - EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); + EXPECT_EQ(tfsRename(pTfs, 0, p44, p45), 0); EXPECT_EQ(tfsRmdir(pTfs, p4), 0); EXPECT_NE(taosDirExist(ap4), 1); @@ -609,7 +609,7 @@ TEST_F(TfsTest, 05_MultiDisk) { EXPECT_NE(taosDirExist(_ap22), 1); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); EXPECT_EQ(taosDirExist(_ap22), 1); - EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); + EXPECT_EQ(tfsRename(pTfs, 0, p44, p45), 0); EXPECT_EQ(tfsRmdir(pTfs, p4), 0); EXPECT_NE(taosDirExist(_ap22), 1); } @@ -721,4 +721,4 @@ TEST_F(TfsTest, 05_MultiDisk) { } tfsClose(pTfs); -} \ No newline at end of file +} diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 3d457c9b5f179256359c3a3c1ea46ecd5c868c86..01d23a7e96a8313e0e111ee0a1fe3f361bf9b219 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -47,9 +47,7 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { } static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { - int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - terrno = TSDB_CODE_SUCCESS; - + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); @@ -74,13 +72,12 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { int64_t capacity = 0; int64_t readSize = 0; char* buf = NULL; - bool firstTrial = pFileInfo->fileSize < fileSize; int64_t offset = TMIN(pFileInfo->fileSize, fileSize); - int64_t offsetForward = offset - stepSize + walCkHeadSz - 1; - int64_t offsetBackward = offset; int64_t retVer = -1; int64_t lastEntryBeginOffset = 0; int64_t lastEntryEndOffset = 0; + int64_t recordLen = 0; + bool forwardStage = false; // check recover size if (2 * tsWalFsyncDataSizeLimit + offset < end) { @@ -91,14 +88,8 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { // search for the valid last WAL entry, e.g. block by block while (1) { - offset = (firstTrial) ? TMIN(fileSize, offsetForward + stepSize - walCkHeadSz + 1) - : TMAX(0, offsetBackward - stepSize + walCkHeadSz - 1); + offset = (lastEntryEndOffset > 0) ? offset : TMAX(0, offset - stepSize + walCkHeadSz - 1); end = TMIN(offset + stepSize, fileSize); - if (firstTrial) { - offsetForward = offset; - } else { - offsetBackward = offset; - } readSize = end - offset; capacity = readSize + sizeof(magic); @@ -129,7 +120,16 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { int64_t pos = 0; SWalCkHead* logContent = NULL; - while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) { + while (true) { + forwardStage = (lastEntryEndOffset > 0 || offset == 0); + terrno = TSDB_CODE_SUCCESS; + if (forwardStage) { + candidate = (readSize - (haystack - buf)) > 0 ? haystack : NULL; + } else { + candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic)); + } + + if (candidate == NULL) break; pos = candidate - buf; // validate head @@ -137,13 +137,14 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { if (len < walCkHeadSz) { break; } + logContent = (SWalCkHead*)(buf + pos); if (walValidHeadCksum(logContent) != 0) { terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; wWarn("vgId:%d, failed to validate checksum of wal entry header. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, offset + pos, fnameStr); haystack = buf + pos + 1; - if (firstTrial) { + if (forwardStage) { break; } else { continue; @@ -151,9 +152,9 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { } // validate body - int64_t size = walCkHeadSz + logContent->head.bodyLen; - if (len < size) { - int64_t extraSize = size - len; + recordLen = walCkHeadSz + logContent->head.bodyLen; + if (len < recordLen) { + int64_t extraSize = recordLen - len; if (capacity < readSize + extraSize + sizeof(magic)) { capacity += extraSize; void* ptr = taosMemoryRealloc(buf, capacity); @@ -184,7 +185,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, offset + pos, fnameStr); haystack = buf + pos + 1; - if (firstTrial) { + if (forwardStage) { break; } else { continue; @@ -194,21 +195,14 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { // found one retVer = logContent->head.version; lastEntryBeginOffset = offset + pos; - lastEntryEndOffset = offset + pos + sizeof(SWalCkHead) + logContent->head.bodyLen; + lastEntryEndOffset = offset + pos + recordLen; // try next - haystack = buf + pos + 1; + haystack = buf + pos + recordLen; } - if (end == fileSize) firstTrial = false; - if (firstTrial) { - if (terrno == TSDB_CODE_SUCCESS) { - continue; - } else { - firstTrial = false; - } - } - if (retVer >= 0 || offset == 0) break; + offset = (lastEntryEndOffset > 0) ? lastEntryEndOffset : offset; + if (forwardStage && (terrno != TSDB_CODE_SUCCESS || end == fileSize)) break; } if (retVer < 0) { diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 84004ed3c18d1bb4baa218d450e7c825d25cd385..5f73251e3b9dff37395198709269dc75c599f8a9 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -327,17 +327,19 @@ bool getWinVersionReleaseName(char *releaseName, int32_t maxLen) { } #endif -int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { +int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t maxLen) { #ifdef WINDOWS if (!getWinVersionReleaseName(releaseName, maxLen)) { snprintf(releaseName, maxLen, "Windows"); } + if(sName) snprintf(sName, maxLen, "Windows"); return 0; #elif defined(_TD_DARWIN_64) char osversion[32]; size_t osversion_len = sizeof(osversion) - 1; int osversion_name[] = { CTL_KERN, KERN_OSRELEASE }; + if(sName) snprintf(sName, maxLen, "macOS"); if (sysctl(osversion_name, 2, osversion, &osversion_len, NULL, 0) == -1) { return -1; } @@ -357,24 +359,35 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { return 0; #else char line[1024]; + char *dest = NULL; size_t size = 0; int32_t code = -1; + int32_t cnt = 0; TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) return false; + if (pFile == NULL) return code; while ((size = taosGetsFile(pFile, sizeof(line), line)) != -1) { line[size - 1] = '\0'; - if (strncmp(line, "PRETTY_NAME", 11) == 0) { - const char *p = strchr(line, '=') + 1; - if (*p == '"') { - p++; - line[size - 2] = 0; - } - tstrncpy(releaseName, p, maxLen); + if (strncmp(line, "NAME", 4) == 0) { + dest = sName; + } else if (strncmp(line, "PRETTY_NAME", 11) == 0) { + dest = releaseName; code = 0; - break; + } else if (strncmp(line, "VERSION_ID", 10) == 0) { + dest = ver; + } else { + continue; } + if (!dest) continue; + const char *p = strchr(line, '=') + 1; + if (*p == '"') { + p++; + line[size - 2] = 0; + } + tstrncpy(dest, p, maxLen); + + if (++cnt >= 3) break; } taosCloseFile(&pFile); @@ -948,6 +961,18 @@ char *taosGetCmdlineByPID(int pid) { #endif } +int64_t taosGetOsUptime() { +#ifdef WINDOWS +#elif defined(_TD_DARWIN_64) +#else + struct sysinfo info; + if (0 == sysinfo(&info)) { + return (int64_t)info.uptime * 1000; + } +#endif + return 0; +} + void taosSetCoreDump(bool enable) { if (!enable) return; #ifdef WINDOWS diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index 1d6542e78cd987e5228c94609cf0dd865a6d6396..a2ccc4de021f90bc83279960d2dacb8959aca445 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -37,7 +37,7 @@ TEST(osTest, osSystem) { const int sysLen = 64; char osSysName[sysLen]; - int ret = taosGetOsReleaseName(osSysName, sysLen); + int ret = taosGetOsReleaseName(osSysName, NULL, NULL, sysLen); printf("os systeme name:%s\n", osSysName); ASSERT_EQ(ret, 0); } diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 8906391a9a9de3244e434137675f9ca5eff30405..f5e15e7436240e612e4dae3263acaa09363a38d1 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -191,7 +191,7 @@ void* taosArrayGet(const SArray* pArray, size_t index) { } if (index >= pArray->size) { - uError("index is out of range, current:%"PRIzu" max:%d", index, pArray->capacity); + uError("index is out of range, current:%" PRIzu " max:%d", index, pArray->capacity); return NULL; } @@ -221,7 +221,7 @@ size_t taosArrayGetSize(const SArray* pArray) { return TARRAY_SIZE(pArray); } -void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { +void* taosArrayInsert(SArray* pArray, size_t index, const void* pData) { if (pArray == NULL || pData == NULL) { return NULL; } @@ -492,7 +492,7 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t // order array void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { taosqsort(pArray->pData, pArray->size, pArray->elemSize, param, fn); -// taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); + // taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); } void taosArraySwap(SArray* a, SArray* b) { diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 288ea6052b580909198788f528871633048e19f1..3681c7a423c82cbd343d3660cdd6432614a6c919 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -380,43 +380,43 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { return 0; } -int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, bool tsc) { - SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .tsc = tsc}; +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope) { + SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .scope = scope}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, bool tsc) { +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval, .tsc = tsc}; + SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval, .scope = scope}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, bool tsc) { +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval, .tsc = tsc}; + SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval, .scope = scope}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, bool tsc) { +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, int8_t scope) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval, .tsc = tsc}; + SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval, .scope = scope}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc) { - SConfigItem item = {.dtype = CFG_DTYPE_STRING, .tsc = tsc}; +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { + SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope}; item.str = taosStrdup(defaultVal); if (item.str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -425,8 +425,8 @@ int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bo return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc) { - SConfigItem item = {.dtype = CFG_DTYPE_DIR, .tsc = tsc}; +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { + SConfigItem item = {.dtype = CFG_DTYPE_DIR, .scope = scope}; if (cfgCheckAndSetDir(&item, defaultVal) != 0) { return -1; } @@ -434,8 +434,8 @@ int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, bool return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal) { - SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .tsc = 1}; +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { + SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .scope = scope}; if (cfgCheckAndSetLocale(&item, defaultVal) != 0) { return -1; } @@ -443,8 +443,8 @@ int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal) { return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal) { - SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .tsc = 1}; +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { + SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .scope = scope}; if (cfgCheckAndSetCharset(&item, defaultVal) != 0) { return -1; } @@ -452,8 +452,8 @@ int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal) { return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal) { - SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .tsc = 1}; +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { + SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .scope = scope}; if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) { return -1; } @@ -543,6 +543,27 @@ void cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p *pLen = len; } +void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen) { + int32_t len = 0; + switch (pItem->scope) { + case CFG_SCOPE_SERVER: + len = snprintf(buf, bufSize, "server"); + break; + case CFG_SCOPE_CLIENT: + len = snprintf(buf, bufSize, "client"); + break; + case CFG_SCOPE_BOTH: + len = snprintf(buf, bufSize, "both"); + break; + } + + if (len > bufSize) { + len = bufSize; + } + + *pLen = len; +} + void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { if (dump) { printf(" global config"); @@ -560,7 +581,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { int32_t size = taosArrayGetSize(pCfg->array); for (int32_t i = 0; i < size; ++i) { SConfigItem *pItem = taosArrayGet(pCfg->array, i); - if (tsc && !pItem->tsc) continue; + if (tsc && pItem->scope == CFG_SCOPE_SERVER) continue; if (dump && strcmp(pItem->name, "scriptDir") == 0) continue; if (dump && strcmp(pItem->name, "simDebugFlag") == 0) continue; tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 83a50f70515accfe9ae642de104ae3f82efa138e..e330a659d5ea545c69b17e65c17f00a2a3bc2533 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -329,6 +329,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE, "Please use this comma // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INIT_FAILED, "Vnode init failure") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_EXIST, "Vnode not exist") TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_EXIST, "Vnode already exist") @@ -568,6 +569,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_SELECTED_EXPR, "Invalid SELECTed ex TAOS_DEFINE_ERROR(TSDB_CODE_PAR_GET_META_ERROR, "Fail to get table info") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_UNIQUE_TABLE_ALIAS, "Not unique table/alias") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC, "System table not allowed") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED, "System table not allowed") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error") //planner @@ -618,9 +620,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_QTASKINFO_CREATE, "Rsma qtaskinfo crea TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_SCHEMA, "Rsma invalid schema") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_STREAM_STATE_OPEN, "Rsma stream state open") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_STREAM_STATE_COMMIT, "Rsma stream state commit") -TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_REF, "Rsma fs ref error") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_SYNC, "Rsma fs sync error") -TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_UPDATE, "Rsma fs update error") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_RESULT, "Rsma result error") //index TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Index is rebuilding") diff --git a/source/util/src/theap.c b/source/util/src/theap.c index d60606008fbacfb958d8e6ba695464da5e903838..315ddf93676831f41b820e82ca70fb2ac993b644 100644 --- a/source/util/src/theap.c +++ b/source/util/src/theap.c @@ -230,7 +230,7 @@ static void pqSwapPQNode(PriorityQueueNode* a, PriorityQueueNode* b) { size_t taosPQSize(PriorityQueue* pq) { return pqContainerSize(pq); } -static void pqHeapify(PriorityQueue* pq, size_t from, size_t last) { +static PriorityQueueNode* pqHeapify(PriorityQueue* pq, size_t from, size_t last) { size_t largest = from; do { from = largest; @@ -246,6 +246,7 @@ static void pqHeapify(PriorityQueue* pq, size_t from, size_t last) { pqSwapPQNode(pqContainerGetEle(pq, from), pqContainerGetEle(pq, largest)); } } while (largest != from); + return pqContainerGetEle(pq, largest); } static void pqBuildHeap(PriorityQueue* pq) { @@ -257,12 +258,13 @@ static void pqBuildHeap(PriorityQueue* pq) { } } -static void pqReverseHeapify(PriorityQueue* pq, size_t i) { +static PriorityQueueNode* pqReverseHeapify(PriorityQueue* pq, size_t i) { while (i > 0 && !pq->fn(pqContainerGetEle(pq, i)->data, pqContainerGetEle(pq, pqParent(i))->data, pq->param)) { size_t parentIdx = pqParent(i); pqSwapPQNode(pqContainerGetEle(pq, i), pqContainerGetEle(pq, parentIdx)); i = parentIdx; } + return pqContainerGetEle(pq, i); } static void pqUpdate(PriorityQueue* pq, size_t i) { @@ -290,9 +292,9 @@ PriorityQueueNode* taosPQTop(PriorityQueue* pq) { return pqContainerGetEle(pq, 0); } -void taosPQPush(PriorityQueue* pq, const PriorityQueueNode* node) { +PriorityQueueNode* taosPQPush(PriorityQueue* pq, const PriorityQueueNode* node) { taosArrayPush(pq->container, node); - pqReverseHeapify(pq, pqContainerSize(pq) - 1); + return pqReverseHeapify(pq, pqContainerSize(pq) - 1); } void taosPQPop(PriorityQueue* pq) { @@ -324,16 +326,20 @@ void destroyBoundedQueue(BoundedQueue* q) { taosMemoryFree(q); } -void taosBQPush(BoundedQueue* q, PriorityQueueNode* n) { +PriorityQueueNode* taosBQPush(BoundedQueue* q, PriorityQueueNode* n) { if (pqContainerSize(q->queue) == q->maxSize + 1) { PriorityQueueNode* top = pqContainerGetEle(q->queue, 0); - void *p = top->data; - top->data = n->data; - n->data = p; - if (q->queue->deleteFn) q->queue->deleteFn(n->data); - pqHeapify(q->queue, 0, taosBQSize(q)); + if (q->queue->fn(top->data, n->data, q->queue->param)) { + return NULL; + } else { + void* p = top->data; + top->data = n->data; + n->data = p; + if (q->queue->deleteFn) q->queue->deleteFn(n->data); + } + return pqHeapify(q->queue, 0, taosBQSize(q)); } else { - taosPQPush(q->queue, n); + return taosPQPush(q->queue, n); } } diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index e7386d5912dd83c5a76af3c902bcd910b5ffef87..e1000f7bc153176d76c676f81601f2a49f7d0213 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -105,7 +105,7 @@ static void tRBTreeTransplant(SRBTree *pTree, SRBTreeNode *u, SRBTreeNode *v) { v->parent = u->parent; } -static SRBTreeNode *tRBTreeSuccessor(SRBTree *pTree, SRBTreeNode *pNode) { +static SRBTreeNode *tRBTreeSuccessor(const SRBTree *pTree, SRBTreeNode *pNode) { if (pNode->right != pTree->NIL) { pNode = pNode->right; while (pNode->left != pTree->NIL) { @@ -125,7 +125,7 @@ static SRBTreeNode *tRBTreeSuccessor(SRBTree *pTree, SRBTreeNode *pNode) { return pNode; } -static SRBTreeNode *tRBTreePredecessor(SRBTree *pTree, SRBTreeNode *pNode) { +static SRBTreeNode *tRBTreePredecessor(const SRBTree *pTree, SRBTreeNode *pNode) { if (pNode->left != pTree->NIL) { pNode = pNode->left; while (pNode->right != pTree->NIL) { @@ -443,7 +443,7 @@ SRBTreeNode *tRBTreeDropMax(SRBTree *pTree) { return pNode; } -SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode) { +SRBTreeNode *tRBTreeGet(const SRBTree *pTree, const SRBTreeNode *pKeyNode) { SRBTreeNode *pNode = pTree->root; while (pNode != pTree->NIL) { diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 6d95660103e2c78203e7531af927e8d59ae4c358..6b6878ec83e9da35957c9c6aa5bbc6b37b1a404d 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -351,10 +351,10 @@ int32_t titoa(uint64_t val, size_t radix, char str[]) { int32_t i = 0; uint64_t v = val; - while(v > 0) { + do { buf[i++] = s[v % radix]; v /= radix; - } + } while (v > 0); // reverse order for(int32_t j = 0; j < i; ++j) { diff --git a/tests/develop-test/2-query/show_create_db.py b/tests/develop-test/2-query/show_create_db.py index 5574a59ec2e328f10a215adb3061e5626e28b100..af31d7f03aa2aef4f533e3701b5f4bc7200c15ad 100644 --- a/tests/develop-test/2-query/show_create_db.py +++ b/tests/develop-test/2-query/show_create_db.py @@ -1,4 +1,4 @@ -import sys +import sys from util.log import * from util.cases import * from util.sql import * @@ -8,15 +8,15 @@ from math import inf class TDTestCase: def caseDescription(self): ''' - case1: [TD-11204]Difference improvement that can ignore negative - ''' + case1: [TD-11204]Difference improvement that can ignore negative + ''' return - + def init(self, conn, logSql, replicaVer=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), False) self._conn = conn - + def restartTaosd(self, index=1, dbname="db"): tdDnodes.stop(index) tdDnodes.startWithoutSleep(index) @@ -42,17 +42,17 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 1 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0") self.restartTaosd(1, dbname='scd') @@ -60,17 +60,17 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 1 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0") tdSql.execute('drop database scd') diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index f6126642bbe93515c9edee1d63c3d63d86486567..1ec5102d9b281511d157c83e6f92496b88fb9081 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -213,6 +213,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionUS.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionNS.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -R @@ -449,7 +450,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 #,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 ,,n,system-test,python3 ./test.py -f 6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py -N 6 -M 3 -,,n,system-test,python ./test.py -f 6-cluster/5dnode3mnodeRoll.py -N 3 -C 1 +#,,n,system-test,python ./test.py -f 6-cluster/5dnode3mnodeRoll.py -N 3 -C 1 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 #,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 7da8da09bfcf3810b692ff25f37333c93ae87497..5b1773e66418e96902c887f455032083bc0ddd2c 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -118,7 +118,7 @@ echo "statusInterval 1" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG -echo "tmrDebugFlag 131" >> $TAOS_CFG +echo "tmrDebugFlag 143" >> $TAOS_CFG echo "uDebugFlag 143" >> $TAOS_CFG echo "rpcDebugFlag 143" >> $TAOS_CFG echo "jniDebugFlag 143" >> $TAOS_CFG diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index f20f861bd0d40df47ab9c6e1792651410678f515..6c98d43794d5bb9a692e9a52e6a341e7e53df9f9 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -111,18 +111,12 @@ endi if $data21_db != 1000 then # wal_level fsyncperiod return -1 endi -if $data22_db != 0 then # wal_retention_period +if $data22_db != 3600 then # wal_retention_period return -1 endi if $data23_db != 0 then # wal_retention_size return -1 endi -if $data24_db != 0 then # wal_roll_period - return -1 -endi -if $data25_db != 0 then # wal_segment_size - return -1 -endi #sql show db.vgroups #if $data[0][4] == leader then diff --git a/tests/script/tsim/parser/fill.sim b/tests/script/tsim/parser/fill.sim index 0510f80419413d893c5172e34b441dd25de0fb9b..0534aa5d5b762c24bb50bba9c52b6733a2d7b429 100644 --- a/tests/script/tsim/parser/fill.sim +++ b/tests/script/tsim/parser/fill.sim @@ -1143,4 +1143,185 @@ if $rows != 20026 then return -1 endi +print ===================== TD-25209 test fill prev/next/linear after data range +sql use $db + +sql select _wstart,_wend,count(*) from tm0 where ts >= '2020-01-01 01:03:06.000' and ts <= '2020-01-01 01:03:10.000' interval(1s) fill(prev); + +if $rows != 5 then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +if $data12 != 1 then + return -1 +endi + +if $data22 != 1 then + return -1 +endi + +if $data32 != 1 then + return -1 +endi + +if $data42 != 1 then + return -1 +endi + +sql select _wstart,_wend,count(*) from tm0 where ts >= '2020-01-01 01:03:06.000' and ts <= '2020-01-01 01:03:10.000' interval(1s) fill(next); + +if $rows != 5 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +if $data12 != 1 then + return -1 +endi + +if $data22 != 1 then + return -1 +endi + +if $data32 != 1 then + return -1 +endi + +if $data42 != NULL then + return -1 +endi + +sql select _wstart,_wend,count(*) from tm0 where ts >= '2020-01-01 01:03:06.000' and ts <= '2020-01-01 01:03:10.000' interval(1s) fill(linear); + +if $rows != 5 then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +if $data12 != 1 then + return -1 +endi + +if $data22 != 1 then + return -1 +endi + +if $data32 != 1 then + return -1 +endi + +if $data42 != NULL then + return -1 +endi + +print ===================== TD-3625 test fill value NULL +sql use $db + +sql select _wstart,_wend,count(*) from tm0 where ts >= '2020-01-01 01:03:06.000' and ts <= '2020-01-01 01:03:10.000' interval(1s) fill(value, NULL); + +if $rows != 5 then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +if $data12 != 1 then + return -1 +endi + +if $data22 != 1 then + return -1 +endi + +if $data32 != 1 then + return -1 +endi + +if $data42 != NULL then + return -1 +endi + +sql select _wstart,_wend,count(*),sum(k),avg(k) from tm0 where ts >= '2020-01-01 01:03:06.000' and ts <= '2020-01-01 01:03:10.000' interval(1s) fill(value, 1, NULL, 1); + +if $rows != 5 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +if $data12 != 1 then + return -1 +endi + +if $data22 != 1 then + return -1 +endi + +if $data32 != 1 then + return -1 +endi + +if $data42 != 1 then + return -1 +endi + + +if $data03 != NULL then + return -1 +endi + +if $data13 != 7 then + return -1 +endi + +if $data23 != 8 then + return -1 +endi + +if $data33 != 9 then + return -1 +endi + +if $data43 != NULL then + return -1 +endi + + +if $data04 != 1.000000000 then + return -1 +endi + +if $data14 != 7.000000000 then + return -1 +endi + +if $data24 != 8.000000000 then + return -1 +endi + +if $data34 != 9.000000000 then + return -1 +endi + +if $data44 != 1.000000000 then + return -1 +endi + + system sh/exec.sh -n dnode1 -s stop -x SIGINT + + diff --git a/tests/script/tsim/sma/drop_sma.sim b/tests/script/tsim/sma/drop_sma.sim index 0d2712f8db9f65ae051466d00f5f39ab9d27093f..8fd8ebdcfd28849d64a85fe23df0727d232287ab 100644 --- a/tests/script/tsim/sma/drop_sma.sim +++ b/tests/script/tsim/sma/drop_sma.sim @@ -129,6 +129,7 @@ sql DROP INDEX sma_index_3 ; print ========== step8 sql drop database if exists db; +sleep 2000 sql create database db duration 300; sql use db; sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); diff --git a/tests/script/tsim/sma/rsmaCreateInsertQuery.sim b/tests/script/tsim/sma/rsmaCreateInsertQuery.sim index 7932cb68ac3251e78ea356796e20d6bfd1cffbce..b3144e4e0dd217319a0d58bf3222360fcd5fa355 100644 --- a/tests/script/tsim/sma/rsmaCreateInsertQuery.sim +++ b/tests/script/tsim/sma/rsmaCreateInsertQuery.sim @@ -4,9 +4,6 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect -#todo xukaili sma should use rocksdb. -return 1 - print =============== create database with retentions sql create database d0 retentions 5s:7d,10s:21d,15s:365d; sql use d0 diff --git a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim index 75969b1d0eddf9f122655d5483ec9c827b5ab550..0b3938d77342d1d572196ceabf10cc39b7bdd558 100644 --- a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim +++ b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim @@ -4,7 +4,7 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect -#todo xukaili sma should use rocksdb. +#todo wait for streamState checkpoint return 1 print =============== create database with retentions @@ -13,17 +13,18 @@ sql use d0 print =============== create super table and register rsma sql create table if not exists stb (ts timestamp, c1 int, c2 float) tags (city binary(20),district binary(20)) rollup(max) max_delay 5s,5s watermark 2s,3s; +sql create table if not exists stb1 (ts timestamp, c1 int, c2 float) tags (city binary(20),district binary(20)) rollup(max) max_delay 5s,5s watermark 2s,3s; sql show stables -if $rows != 1 then +if $rows != 2 then return -1 endi print =============== create child table -sql create table ct1 using stb tags("BeiJing", "ChaoYang"); +sql create table ct1 using stb tags("BeiJing", "ChaoYang") ct_1 using stb1 tags("BeiJing", "ChaoYang"); sql show tables -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -31,6 +32,9 @@ print =============== insert data and trigger rollup sql insert into ct1 values(now, 10, 10.0); sql insert into ct1 values(now+1s, 1, 1.0); sql insert into ct1 values(now+2s, 100, 100.0); +sql insert into ct_1 values(now, 10, 10.0); +sql insert into ct_1 values(now+1s, 1, 1.0); +sql insert into ct_1 values(now+2s, 100, 100.0); print =============== wait maxdelay 5+2 seconds for results sleep 7000 @@ -44,6 +48,20 @@ if $rows > 2 then return -1 endi +if $data01 != 100 then + if $data01 != 10 then + print retention level 2 file result $data01 != 100 or 10 + return -1 + endi +endi + +sql select * from ct_1; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 2 file rows $rows > 2 + return -1 +endi if $data01 != 100 then if $data01 != 10 then @@ -68,6 +86,21 @@ if $data01 != 100 then endi endi +sql select * from ct_1 where ts > now-8d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 1 file rows $rows > 2 + return -1 +endi + +if $data01 != 100 then + if $data01 != 10 then + print retention level 1 file result $data01 != 100 or 10 + return -1 + endi +endi + print =============== select * from retention level 0 from memory sql select * from ct1 where ts > now-3d; print $data00 $data01 $data02 @@ -84,6 +117,21 @@ if $data01 != 10 then return -1 endi +sql select * from ct_1 where ts > now-3d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 + +if $rows < 1 then + print retention level 0 file rows $rows < 1 + return -1 +endi + +if $data01 != 10 then + print retention level 0 file result $data01 != 10 + return -1 +endi + #=================================================================== system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start @@ -100,6 +148,22 @@ if $rows > 2 then endi +if $data01 != 100 then + if $data01 != 10 then + print retention level 2 file result $data01 != 100 or 10 + return -1 + endi +endi + +sql select * from ct_1; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 2 file rows $rows > 2 + return -1 +endi + + if $data01 != 100 then if $data01 != 10 then print retention level 2 file result $data01 != 100 or 10 @@ -123,6 +187,21 @@ if $data01 != 100 then endi endi +sql select * from ct_1 where ts > now-8d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 1 file rows $rows > 2 + return -1 +endi + +if $data01 != 100 then + if $data01 != 10 then + print retention level 1 file result $data01 != 100 or 10 + return -1 + endi +endi + print =============== select * from retention level 0 from memory after reboot sql select * from ct1 where ts > now-3d; print $data00 $data01 $data02 @@ -139,6 +218,21 @@ if $data01 != 10 then return -1 endi +sql select * from ct_1 where ts > now-3d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 + +if $rows < 1 then + print retention level 0 file rows $rows < 1 + return -1 +endi + +if $data01 != 10 then + print retention level 0 file result $data01 != 10 + return -1 +endi + #==================== flush database to trigger commit data to file sql flush database d0; @@ -161,6 +255,21 @@ if $data01 != 100 then endi endi +sql select * from ct_1; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 2 file rows $rows > 2 + return -1 +endi + +if $data01 != 100 then + if $data01 != 10 then + print retention level 2 file result $data01 != 100 or 10 + return -1 + endi +endi + print =============== select * from retention level 1 from file sql select * from ct1 where ts > now-8d; print $data00 $data01 $data02 @@ -177,6 +286,21 @@ if $data01 != 100 then endi endi +sql select * from ct_1 where ts > now-8d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 1 file rows $rows > 2 + return -1 +endi + +if $data01 != 100 then + if $data01 != 10 then + print retention level 1 file result $data01 != 100 or 10 + return -1 + endi +endi + print =============== select * from retention level 0 from file sql select * from ct1 where ts > now-3d; print $data00 $data01 $data02 @@ -192,9 +316,25 @@ if $data01 != 10 then return -1 endi +sql select * from ct_1 where ts > now-3d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 +if $rows < 1 then + print retention level 0 file rows $rows < 1 + return -1 +endi + +if $data01 != 10 then + print retention level 0 file result $data01 != 10 + return -1 +endi + print =============== insert after rsma qtaskinfo recovery sql insert into ct1 values(now, 50, 500.0); sql insert into ct1 values(now+1s, 40, 40.0); +sql insert into ct_1 values(now, 50, 500.0); +sql insert into ct_1 values(now+1s, 40, 40.0); print =============== wait maxdelay 5+2 seconds for results sleep 7000 @@ -217,11 +357,37 @@ endi if $data02 != 500.00000 then if $data02 != 100.00000 then - print retention level 1 file/mem result $data02 != 500.00000 or 100.00000 + if $data02 != 10.00000 then + print retention level 1 file/mem result $data02 != 500.00000 or 100.00000 or 10.00000 + return -1 + endi + endi +endi + +sql select * from ct_1; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 2 file/mem rows $rows > 2 + return -1 +endi + +if $data01 != 100 then + if $data01 != 10 then + print retention level 2 file/mem result $data01 != 100 or 10 return -1 endi endi +if $data02 != 500.00000 then + if $data02 != 100.00000 then + if $data02 != 10.00000 then + print retention level 1 file/mem result $data02 != 500.00000 or 100.00000 or 10.00000 + return -1 + endi + endi +endi + print =============== select * from retention level 1 from file and memory after rsma qtaskinfo recovery sql select * from ct1 where ts > now-8d; print $data00 $data01 $data02 @@ -240,11 +406,37 @@ endi if $data02 != 500.00000 then if $data02 != 100.00000 then - print retention level 1 file/mem result $data02 != 500.00000 or 100.00000 + if $data02 != 10.00000 then + print retention level 1 file/mem result $data02 != 500.00000 or 100.00000 or 10.00000 + return -1 + endi + endi +endi + +sql select * from ct_1 where ts > now-8d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 1 file/mem rows $rows > 2 + return -1 +endi + +if $data01 != 100 then + if $data01 != 10 then + print retention level 1 file/mem result $data01 != 100 or 10 return -1 endi endi +if $data02 != 500.00000 then + if $data02 != 100.00000 then + if $data02 != 10.00000 then + print retention level 1 file/mem result $data02 != 500.00000 or 100.00000 or 10.00000 + return -1 + endi + endi +endi + print =============== select * from retention level 0 from file and memory after rsma qtaskinfo recovery sql select * from ct1 where ts > now-3d; @@ -295,6 +487,61 @@ if $data42 != 40.00000 then return -1 endi +sql select * from ct_1 where ts > now-3d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 +print $data40 $data41 $data42 + +if $rows < 1 then + print retention level 0 file/mem rows $rows < 1 + return -1 +endi + +if $data01 != 10 then + print retention level 0 file/mem result $data01 != 10 + return -1 +endi + +if $data11 != 1 then + print retention level 0 file/mem result $data11 != 1 + return -1 +endi + +if $data21 != 100 then + print retention level 0 file/mem result $data21 != 100 + return -1 +endi + +if $data31 != 50 then + print retention level 0 file/mem result $data31 != 50 + return -1 +endi + +if $data32 != 500.00000 then + print retention level 0 file/mem result $data32 != 500.00000 + return -1 +endi + + +if $data41 != 40 then + print retention level 0 file/mem result $data41 != 40 + return -1 +endi + +if $data42 != 40.00000 then + print retention level 0 file/mem result $data42 != 40.00000 + return -1 +endi + +print =============== drop stb1 +sql drop table stb1; +sql flush database d0; +print =============== select * from retention level 0 from file and memory after rsma qtaskinfo recovery +sql_error select * from ct_1 where ts > now-3d; +sql_error select * from ct_1 where ts > now-8d; +sql_error select * from ct_1; system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/basic3.sim b/tests/script/tsim/stream/basic3.sim index 2df33541b461b1f2fe1c772cb80550e421fe56aa..f18061a6df012b5093beef6c8f1588a61b69f5b8 100644 --- a/tests/script/tsim/stream/basic3.sim +++ b/tests/script/tsim/stream/basic3.sim @@ -1,11 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c debugflag -v 131 system sh/cfg.sh -n dnode1 -c keepColumnName -v 1 system sh/exec.sh -n dnode1 -s start -sleep 5000 - +sleep 1000 sql connect print ========== interval\session\state window @@ -32,7 +30,6 @@ sql create stream streamd6 into streamt6 as select ca, _wstart,_wend, count(*), sql alter local 'keepColumnName' '1' - sql CREATE STABLE `meters_test_data` (`ts` TIMESTAMP, `close` FLOAT, `parttime` TIMESTAMP, `parttime_str` VARCHAR(32)) TAGS (`id` VARCHAR(32)); sql_error create stream realtime_meters fill_history 1 into realtime_meters as select last(parttime),first(close),last(close) from meters_test_data partition by tbname state_window(parttime_str); @@ -58,17 +55,13 @@ sql_error create stream streamd11 into streamd11 as select _wstart, _wend, count sql alter local 'keepColumnName' '0' sql create stream realtime_meters fill_history 1 into realtime_meters as select last(parttime),first(close),last(close) from meters_test_data partition by tbname state_window(parttime_str); - sql desc realtime_meters; - if $rows == 0 then return -1 endi -sql create stream streamd7 into streamt7 as select _wstart, _wend, count(*), first(ca), last(ca) from t1 interval(10s); - +sql create stream streamd7 into streamt7 as select _wstart t1, _wend t2, count(*), first(ca), last(ca) from t1 interval(10s); sql desc streamt7; - if $rows == 0 then return -1 endi @@ -76,12 +69,11 @@ endi sql create stream streamd71 into streamt71 as select _wstart, _wend, count(*) as ca, first(ca), last(ca) as c2 from t1 interval(10s); sql desc streamt71; - if $rows == 0 then return -1 endi -sleep 3000 +sleep 1000 sql drop stream if exists streamd1; sql drop stream if exists streamd2; @@ -93,23 +85,19 @@ sql drop stream if exists streamd6; sql create stream streamd10 into streamd10 as select _wstart, _wend, count(*), first(ca), last(cb) as c2 from t1 interval(10s); sql desc streamd10; - if $rows == 0 then return -1 endi sql_error create stream streamd11 into streamd11 as select _wstart, _wend, count(*), last(ca), last(ca) from t1 interval(10s); - sql create stream streamd12 into streamd12 as select _wstart, _wend, count(*), last(ca), last(cb) as c2 from t1 interval(10s); - sql desc streamd12; if $rows == 0 then return -1 endi - _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check diff --git a/tests/script/tsim/stream/fillHistoryTransform.sim b/tests/script/tsim/stream/fillHistoryTransform.sim new file mode 100644 index 0000000000000000000000000000000000000000..fe58b76b78d6a2bd9614d05022b4f2694bf23db4 --- /dev/null +++ b/tests/script/tsim/stream/fillHistoryTransform.sim @@ -0,0 +1,405 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database test vgroups 1; +sql select * from information_schema.ins_databases +if $rows != 3 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use test; + +print =====step1 + +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql insert into t1 values(1648791213000,10,2,3,1.0); + +sql create stream stream0 trigger at_once fill_history 1 IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, sum(a) from t1 interval(10s); + +$loop_count = 0 +loop00: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop00 +endi + +if $data01 != 10 then + print =====data01=$data01 + goto loop00 +endi + +sql insert into t1 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 +loop0: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop0 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop0 +endi + +sql insert into t1 values(1648791213001,2,2,3,1.0); + +$loop_count = 0 +loop1: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop1 +endi + +if $data01 != 3 then + print ======$data01 + goto loop1 +endi + + +sql insert into t1 values(1648791223001,3,2,3,1.0); + +sql insert into t1 values(1648791223002,4,2,3,1.0); + +$loop_count = 0 +loop2: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt; + +if $rows != 2 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop2 +endi + +if $data01 != 3 then + print ======$data01 + goto loop2 +endi + +if $data11 != 7 then + print ======$data01 + goto loop2 +endi + +print =====step1 over + +print =====step2 + +sql create database test1 vgroups 4; + +sql use test1; + +sql create stable st(ts timestamp,a int,b int,c int,d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql insert into t1 values(1648791213000,10,2,3,1.0); + +sql create stream stream1 trigger at_once fill_history 1 IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, sum(a) from st interval(10s); + +$loop_count = 0 +loop00: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt1; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop00 +endi + +if $data01 != 10 then + print =====data01=$data01 + goto loop00 +endi + +sql insert into t1 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 +loop0: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt1; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop0 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop0 +endi + +sql insert into t1 values(1648791213001,2,2,3,1.0); + +$loop_count = 0 +loop1: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt1; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop1 +endi + +if $data01 != 3 then + print ======$data01 + goto loop1 +endi + + +sql insert into t1 values(1648791223001,3,2,3,1.0); + +sql insert into t1 values(1648791223002,4,2,3,1.0); + +$loop_count = 0 +loop2: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt1; + +if $rows != 2 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop2 +endi + +if $data01 != 3 then + print ======$data01 + goto loop2 +endi + +if $data11 != 7 then + print ======$data01 + goto loop2 +endi + +print =====step2 over + +print =====step3 + +sql create database test2 vgroups 4; + +sql use test2; + +sql create stable st(ts timestamp,a int,b int,c int,d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql insert into t1 values(1648791213000,10,2,3,1.0); + +sql create stream stream2 trigger at_once fill_history 1 IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, sum(a) from st partition by ta interval(10s); + +$loop_count = 0 +loop00: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt2; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop00 +endi + +if $data01 != 10 then + print =====data01=$data01 + goto loop00 +endi + +sql insert into t1 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 +loop0: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt2; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop0 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop0 +endi + +sql insert into t1 values(1648791213001,2,2,3,1.0); + +$loop_count = 0 +loop1: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt2; + +if $rows != 1 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop1 +endi + +if $data01 != 3 then + print ======$data01 + goto loop1 +endi + + +sql insert into t1 values(1648791223001,3,2,3,1.0); + +sql insert into t1 values(1648791223002,4,2,3,1.0); + +$loop_count = 0 +loop2: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt2; + +if $rows != 2 then + print ======$rows + print data00,data01, data02 + print data10,data11, data12 + print data20,data21, data22 + goto loop2 +endi + +if $data01 != 3 then + print ======$data01 + goto loop2 +endi + +if $data11 != 7 then + print ======$data01 + goto loop2 +endi + +print =====step3 over + +print =====over + + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/partitionby.sim b/tests/script/tsim/stream/partitionby.sim index df4b60314fc8b0b52fa2f7075262cea2cd0106ce..9a660741e7eab7242564225377ba2fc77691604a 100644 --- a/tests/script/tsim/stream/partitionby.sim +++ b/tests/script/tsim/stream/partitionby.sim @@ -14,6 +14,7 @@ sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test0.streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by ta,tb,tc interval(10s); +sleep 500 sql insert into ts1 values(1648791213001,1,12,3,1.0); sql insert into ts2 values(1648791213001,1,12,3,1.0); diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index 05eb7dacba92254c947289332a5cbcce8410271a..18893245fa6b735168f388563566537818d4c88f 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -575,8 +575,6 @@ endi $loop_count = 0 print step 7 - - sql create database test3 vgroups 6; sql use test3; sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); diff --git a/tests/script/tsim/table/hash.sim b/tests/script/tsim/table/hash.sim index 664f86713720e1ae1969027508e2931d23397f08..45ce689b5ac8dabf29066dd0b6ac98d9d78e8155 100644 --- a/tests/script/tsim/table/hash.sim +++ b/tests/script/tsim/table/hash.sim @@ -7,11 +7,11 @@ sql connect #sql create database d1 vgroups 2 sql create database d1 vgroups 2 table_prefix 3 table_suffix 2 sql select * from information_schema.ins_databases -print $data(d1)[27] $data(d1)[28] -if $data(d1)[27] != 3 then +print $data(d1)[25] $data(d1)[26] +if $data(d1)[25] != 3 then return -1 endi -if $data(d1)[28] != 2 then +if $data(d1)[26] != 2 then return -1 endi diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index 4551228f2f67e6da9beed93f984e93113a77055f..fe6ec04a205313f084b120bf7d7bdf0d6096d916 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -87,7 +86,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -162,7 +160,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -236,7 +233,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic1Of2Cons.sim b/tests/script/tsim/tmq/basic1Of2Cons.sim index 51d39e8d110f0be00e3a16cf7cdd47e6be226a78..c12351cbe81538ad87fdde313bd0e4f4c074875b 100644 --- a/tests/script/tsim/tmq/basic1Of2Cons.sim +++ b/tests/script/tsim/tmq/basic1Of2Cons.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -87,7 +86,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for stb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -193,7 +191,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -298,7 +295,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic2.sim b/tests/script/tsim/tmq/basic2.sim index 8356a60b672c54feb37663d5c3b8f3391c99a456..5c7528ea5dc749da644b83f0928f7b3cc35f8c7a 100644 --- a/tests/script/tsim/tmq/basic2.sim +++ b/tests/script/tsim/tmq/basic2.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -122,7 +121,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -182,7 +180,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic2Of2Cons.sim b/tests/script/tsim/tmq/basic2Of2Cons.sim index 63e7e2dcf4e600e340f6c3767ab91dfd19fc5338..23598c17a4bce516d2b0c888cb55c6405766f2d8 100644 --- a/tests/script/tsim/tmq/basic2Of2Cons.sim +++ b/tests/script/tsim/tmq/basic2Of2Cons.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -151,7 +150,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -241,7 +239,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index cfdae059dc862d4177d12891ffd9499900ba3b41..1223a94fa7e666540d4e790440c2a9039ab2feab 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -172,7 +171,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -266,7 +264,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic3.sim b/tests/script/tsim/tmq/basic3.sim index a64dd6924d7c30dec12e315924439b37019af500..8bb34cefa2382fb89856cf477b9a1bd82daf800e 100644 --- a/tests/script/tsim/tmq/basic3.sim +++ b/tests/script/tsim/tmq/basic3.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -87,7 +86,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -161,7 +159,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -235,7 +232,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic3Of2Cons.sim b/tests/script/tsim/tmq/basic3Of2Cons.sim index 4e47e3dbf9a0eeaf4380e43ceb6caef81f3c8770..75d762c44b6b572f2c9dfd5633b77eb5ff53ec39 100644 --- a/tests/script/tsim/tmq/basic3Of2Cons.sim +++ b/tests/script/tsim/tmq/basic3Of2Cons.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -86,7 +85,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -204,7 +202,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -309,7 +306,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic4.sim b/tests/script/tsim/tmq/basic4.sim index 6b35342ad119431d1eece232486069c9792015d9..c72d8ff412cc8b8687f4388ed471951eb7c09a47 100644 --- a/tests/script/tsim/tmq/basic4.sim +++ b/tests/script/tsim/tmq/basic4.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -119,7 +118,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -179,7 +177,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/basic4Of2Cons.sim b/tests/script/tsim/tmq/basic4Of2Cons.sim index 122a91af3627fb12d00be61a503942e32dca604d..bb006a354c0f0af476d0365de4cd28f0dea2a7de 100644 --- a/tests/script/tsim/tmq/basic4Of2Cons.sim +++ b/tests/script/tsim/tmq/basic4Of2Cons.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -160,7 +159,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -251,7 +249,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/snapshot.sim b/tests/script/tsim/tmq/snapshot.sim index 81fff3522499ff99ccfa9a92be5483ea42d775ba..fbdaba7d28d266526bd02aaa86a5de3c85f41338 100644 --- a/tests/script/tsim/tmq/snapshot.sim +++ b/tests/script/tsim/tmq/snapshot.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -87,7 +86,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -159,7 +157,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -233,7 +230,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/snapshot1.sim b/tests/script/tsim/tmq/snapshot1.sim index c79892ae1d943a071c58fe2236eb90ef9a5e10d4..5349981cc719e41d6d91a9a0a6e18555a7b8fd86 100644 --- a/tests/script/tsim/tmq/snapshot1.sim +++ b/tests/script/tsim/tmq/snapshot1.sim @@ -35,7 +35,6 @@ sql connect sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb @@ -151,7 +150,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) @@ -231,7 +229,6 @@ sleep 500 sql use $cdbName print == alter database -sql alter database $cdbName wal_retention_period 3600 print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim index 78c4c561af9ae55d203635cdccc562eb8bec869d..8d4b506b2492b4abbb178f3f8b99f25c1e88e9d9 100644 --- a/tests/script/tsim/tmq/topic.sim +++ b/tests/script/tsim/tmq/topic.sim @@ -40,7 +40,6 @@ endi sql use $dbName print == alter database -sql alter database $dbName wal_retention_period 3600 print == create super table sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) diff --git a/tests/system-test/0-others/backquote_check.py b/tests/system-test/0-others/backquote_check.py index be8590f913f110e0156d71fa81560fc99fa39132..7c91fd9e8cb9ef732b51efc32452df5601510555 100644 --- a/tests/system-test/0-others/backquote_check.py +++ b/tests/system-test/0-others/backquote_check.py @@ -22,7 +22,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) self.dbname = 'db' self.setsql = TDSetSql() self.stbname = 'stb' diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 016f0a6b1a40c64c23dc54ee654e7987271465f6..98a0fbe18d2ebadb253cd003f563811476141a7d 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -138,9 +138,9 @@ class TDTestCase: tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{BASEVERSION}") tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") - os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ") - os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ') - os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ') + # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ") + # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ') + # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ') os.system(f"sed -i 's/\/etc\/taos/{cPath}/' 0-others/tmqBasic.json ") # os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/tmqBasic.json -y ") os.system('LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists tmq_test_topic as select current,voltage,phase from test.meters where voltage <= 106 and current <= 5;" ') @@ -193,7 +193,6 @@ class TDTestCase: tdsql.execute("drop database if exists db") tdsql.execute("create database db") tdsql.execute("use db") - tdsql.execute("alter database db wal_retention_period 3600") tdsql.execute("create stable db.stb1 (ts timestamp, c1 int) tags (t1 int);") tdsql.execute("insert into db.ct1 using db.stb1 TAGS(1) values(now(),11);") tdsql.error(" insert into `db.ct2` using db.stb1 TAGS(9) values(now(),11);") @@ -224,7 +223,7 @@ class TDTestCase: args = (caller.filename, caller.lineno) tdLog.exit("%s(%d) failed" % args) tdsql.query("show streams;") - tdsql.checkRows(2) + tdsql.checkRows(0) tdsql.query("select *,tbname from d0.almlog where mcid='m0103';") tdsql.checkRows(6) expectList = [0,3003,20031,20032,20033,30031] diff --git a/tests/system-test/0-others/performance_schema.py b/tests/system-test/0-others/performance_schema.py index 9d2a362254714a2daefc98787a3c1ec236c3e80c..b4a16794bb0f36b907c21ff4dd657b9b920321a1 100755 --- a/tests/system-test/0-others/performance_schema.py +++ b/tests/system-test/0-others/performance_schema.py @@ -75,7 +75,6 @@ class TDTestCase: def prepare_data(self): tdSql.execute(f"create database if not exists {self.dbname} vgroups 2") #1 query tdSql.execute(f'use {self.dbname}') #1 query - tdsql.execute(f"alter database {self.dbname} wal_retention_period 3600") tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) #1 query for i in range(self.tbnum): #self.tbnum query diff --git a/tests/system-test/0-others/show.py b/tests/system-test/0-others/show.py index b284605a0e8dc619c5ab004d4a65d022b0eb8946..9d26b3a2aeb679195246e0989605b48f12b91a77 100644 --- a/tests/system-test/0-others/show.py +++ b/tests/system-test/0-others/show.py @@ -45,8 +45,6 @@ class TDTestCase: "replica":1, "wal_level":1, "wal_fsync_period":6000, - "wal_roll_period":0, - "wal_segment_size":1024, "vgroups":self.vgroups, "stt_trigger":1, "tsdb_pagesize":16 @@ -83,12 +81,37 @@ class TDTestCase: tag_sql += f"{k} {v}, " create_stb_sql = f'create stable {stbname} ({column_sql[:-2]}) tags ({tag_sql[:-2]})' return create_stb_sql - + def set_create_database_sql(self,sql_dict): create_sql = 'create' for key,value in sql_dict.items(): create_sql += f' {key} {value}' return create_sql + + def show_create_sysdb_sql(self): + sysdb_list = {'information_schema', 'performance_schema'} + for db in sysdb_list: + tdSql.query(f'show create database {db}') + tdSql.checkEqual(f'{db}',tdSql.queryResult[0][0]) + tdSql.checkEqual(f'CREATE DATABASE `{db}`',tdSql.queryResult[0][1]) + + def show_create_systb_sql(self): + for param in self.ins_param_list: + tdSql.query(f'show create table information_schema.ins_{param}') + tdSql.checkEqual(f'ins_{param}',tdSql.queryResult[0][0]) + + tdSql.execute(f'use information_schema') + tdSql.query(f'show create table ins_{param}') + tdSql.checkEqual(f'ins_{param}',tdSql.queryResult[0][0]) + + for param in self.perf_param_list: + tdSql.query(f'show create table performance_schema.perf_{param}') + tdSql.checkEqual(f'perf_{param}',tdSql.queryResult[0][0]) + + tdSql.execute(f'use performance_schema') + tdSql.query(f'show create table perf_{param}') + tdSql.checkEqual(f'perf_{param}',tdSql.queryResult[0][0]) + def show_create_sql(self): create_db_sql = self.set_create_database_sql(self.db_param) print(create_db_sql) @@ -108,7 +131,7 @@ class TDTestCase: tdSql.query('show vnodes 1') tdSql.checkRows(self.vgroups) tdSql.execute(f'use {self.dbname}') - + column_dict = { '`ts`': 'timestamp', '`col1`': 'tinyint', @@ -124,7 +147,7 @@ class TDTestCase: '`col11`': 'bool', '`col12`': 'varchar(20)', '`col13`': 'nchar(20)' - + } tag_dict = { '`t1`': 'tinyint', @@ -141,7 +164,7 @@ class TDTestCase: '`t12`': 'varchar(20)', '`t13`': 'nchar(20)', '`t14`': 'timestamp' - + } create_table_sql = self.set_stb_sql(self.stbname,column_dict,tag_dict) tdSql.execute(create_table_sql) @@ -152,7 +175,7 @@ class TDTestCase: tag_sql = '(' for tag_keys in tag_dict.keys(): tag_sql += f'{tag_keys}, ' - tags = f'{tag_sql[:-2]})' + tags = f'{tag_sql[:-2]})' sql = f'create table {self.tbname} using {self.stbname} {tags} tags (1, 1, 1, 1, 1, 1, 1, 1, 1.000000e+00, 1.000000e+00, true, "abc", "abc123", 0)' tdSql.query(f'show create table {self.tbname}') query_result = tdSql.queryResult @@ -175,7 +198,7 @@ class TDTestCase: taosd_info = os.popen('taosd -V').read() taosd_gitinfo = re.findall("^gitinfo.*",taosd_info,re.M) tdSql.checkEqual(taosd_gitinfo_sql,taosd_gitinfo[0]) - + def show_base(self): for sql in ['dnodes','mnodes','cluster']: tdSql.query(f'show {sql}') @@ -193,6 +216,8 @@ class TDTestCase: self.ins_check() self.perf_check() self.show_create_sql() + self.show_create_sysdb_sql() + self.show_create_systb_sql() def stop(self): tdSql.close() diff --git a/tests/system-test/0-others/show_tag_index.py b/tests/system-test/0-others/show_tag_index.py new file mode 100644 index 0000000000000000000000000000000000000000..6c19dbce0df4424e53a5e9a7bf62619270c92f38 --- /dev/null +++ b/tests/system-test/0-others/show_tag_index.py @@ -0,0 +1,184 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import re +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def check_tags(self): + tdSql.checkRows(2) + tdSql.checkCols(6) + tdSql.checkData(0, 0, 'ctb1') + tdSql.checkData(0, 1, 'db') + tdSql.checkData(0, 2, 'stb') + tdSql.checkData(0, 3, 't0') + tdSql.checkData(0, 4, 'INT') + tdSql.checkData(0, 5, 1) + tdSql.checkData(1, 0, 'ctb1') + tdSql.checkData(1, 1, 'db') + tdSql.checkData(1, 2, 'stb') + tdSql.checkData(1, 3, 't1') + tdSql.checkData(1, 4, 'INT') + tdSql.checkData(1, 5, 1) + + def check_table_tags(self, is_super_table): + + if is_super_table == False: + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 'ctb1') + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + else: + tdSql.checkRows(2) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 'ctb1') + tdSql.checkData(1, 0, 'ctb2') + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 2, 2) + + def check_indexes(self): + tdSql.checkRows(1) + tdSql.checkCols(7) + tdSql.checkData(0, 0, 'idx1') + tdSql.checkData(0, 1, 'db') + tdSql.checkData(0, 2, 'stb') + tdSql.checkData(0, 3, -1) + tdSql.checkData(0, 5, 't1') + tdSql.checkData(0, 6, 'tag_index') + + def run(self): + tdSql.execute(f'create database db') + tdSql.execute(f'use db') + tdSql.execute(f'create table stb (ts timestamp, c0 int) tags (t0 int, t1 int)') + tdSql.execute(f'create table ctb1 using stb tags (1, 1)') + tdSql.execute(f'create table ctb2 using stb tags (2, 2)') + tdSql.execute(f'insert into ctb1 values (now, 1)') + tdSql.execute(f'insert into ctb2 values (now, 2)') + + # show tags + tdSql.query(f'show tags from stb') + tdSql.checkRows(0) + tdSql.query(f'show tags from stb') + tdSql.checkRows(0); + tdSql.query(f'show tags from `stb`') + tdSql.checkRows(0); + tdSql.query(f'show tags from stb from db') + tdSql.checkRows(0); + tdSql.query(f'show tags from `stb` from `db`') + tdSql.checkRows(0); + tdSql.query(f'show tags from db.stb') + tdSql.checkRows(0); + tdSql.query(f'show tags from `db`.`stb`') + tdSql.checkRows(0); + tdSql.query(f'show tags from ctb1') + self.check_tags(); + tdSql.query(f'show tags from `ctb1`') + self.check_tags(); + tdSql.query(f'show tags from ctb1 from db') + self.check_tags(); + tdSql.query(f'show tags from `ctb1` from `db`') + self.check_tags(); + tdSql.query(f'show tags from db.ctb1') + self.check_tags(); + tdSql.query(f'show tags from `db`.`ctb1`') + self.check_tags(); + + tdSql.error(f'show tags from db.stb from db') + tdSql.error(f'show tags from `db`.`stb` from db') + tdSql.error(f'show tags from db.ctb1 from db') + tdSql.error(f'show tags from `db`.`ctb1` from db') + + # show table tags + tdSql.query(f'show table tags from stb') + self.check_table_tags(True); + tdSql.query(f'show table tags from `stb`') + self.check_table_tags(True); + tdSql.query(f'show table tags from stb from db') + self.check_table_tags(True); + tdSql.query(f'show table tags from `stb` from `db`') + self.check_table_tags(True); + tdSql.query(f'show table tags from db.stb') + self.check_table_tags(True); + tdSql.query(f'show table tags from `db`.`stb`') + self.check_table_tags(True); + + tdSql.query(f'show table tags from ctb1') + self.check_table_tags(False); + tdSql.query(f'show table tags from `ctb1`') + self.check_table_tags(False); + tdSql.query(f'show table tags from ctb1 from db') + self.check_table_tags(False); + tdSql.query(f'show table tags from `ctb1` from `db`') + self.check_table_tags(False); + tdSql.query(f'show table tags from db.ctb1') + self.check_table_tags(False); + tdSql.query(f'show table tags from `db`.`ctb1`') + self.check_table_tags(False); + + tdSql.error(f'show table tags from db.stb from db') + tdSql.error(f'show table tags from `db`.`stb` from db') + tdSql.error(f'show table tags from db.ctb1 from db') + tdSql.error(f'show table tags from `db`.`ctb1` from db') + + # show indexes + tdSql.execute(f'create index idx1 on stb (t1)') + + tdSql.query(f'show indexes from stb') + self.check_indexes(); + tdSql.query(f'show indexes from `stb`') + self.check_indexes(); + tdSql.query(f'show indexes from stb from db') + self.check_indexes(); + tdSql.query(f'show indexes from `stb` from `db`') + self.check_indexes(); + tdSql.query(f'show indexes from db.stb') + self.check_indexes(); + tdSql.query(f'show indexes from `db`.`stb`') + self.check_indexes(); + + tdSql.query(f'show indexes from ctb1') + tdSql.checkRows(0) + tdSql.query(f'show indexes from `ctb1`') + tdSql.checkRows(0) + tdSql.query(f'show indexes from ctb1 from db') + tdSql.checkRows(0) + tdSql.query(f'show indexes from `ctb1` from `db`') + tdSql.checkRows(0) + tdSql.query(f'show indexes from db.ctb1') + tdSql.checkRows(0) + tdSql.query(f'show indexes from `db`.`ctb1`') + tdSql.checkRows(0) + + tdSql.error(f'show indexes from db.stb from db') + tdSql.error(f'show indexes from `db`.`stb` from db') + tdSql.error(f'show indexes from db.ctb1 from db') + tdSql.error(f'show indexes from `db`.`ctb1` from db') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/splitVGroup.py b/tests/system-test/0-others/splitVGroup.py index 450996106608aa60cec781392bec8f9463ff2171..9fd00892e45ade46da43e4fb4a38d532109e1fb4 100644 --- a/tests/system-test/0-others/splitVGroup.py +++ b/tests/system-test/0-others/splitVGroup.py @@ -283,14 +283,14 @@ class TDTestCase: # normal table - # all rows - sql = "select * from @db_name.ta" - self.queryDouble(sql) - # count sql = "select count(*) from @db_name.ta" self.queryDouble(sql) + # all rows + sql = "select * from @db_name.ta" + self.queryDouble(sql) + # sum sql = "select sum(c1) from @db_name.ta" self.queryDouble(sql) @@ -316,7 +316,8 @@ class TDTestCase: tdSql.execute(sql) # wait end - for i in range(100): + seconds = 300 + for i in range(seconds): sql ="show transactions;" rows = tdSql.query(sql) if rows == 0: @@ -325,7 +326,7 @@ class TDTestCase: #tdLog.info(f"i={i} wait split vgroup ...") time.sleep(1) - tdLog.exit("split vgroup transaction is not finished after executing 50s") + tdLog.exit(f"split vgroup transaction is not finished after executing {seconds}s") return False # split error @@ -382,6 +383,14 @@ class TDTestCase: self.expectSplitError("topicdb") tdSql.execute("drop topic toa;") self.expectSplitOk("topicdb") + + # compact and check db2 + def compactAndCheck(self): + tdLog.info("compact db2 and check result ...") + # compact + tdSql.execute(f"compact database {self.db2};") + # check result + self.checkResult() # run def run(self): @@ -390,12 +399,15 @@ class TDTestCase: for i in range(5): # split vgroup on db2 + start = time.time() self.splitVGroup(self.db2) + end = time.time() self.vgroups2 += 1 - + # check two db query result same self.checkResult() - tdLog.info(f"split vgroup i={i} passed.") + spend = "%.3f"%(end-start) + tdLog.info(f"split vgroup i={i} passed. spend = {spend}s") # split empty db self.splitEmptyDB() @@ -403,6 +415,9 @@ class TDTestCase: # check topic and stream forib self.checkForbid() + # compact database + self.compactAndCheck() + # stop def stop(self): tdSql.close() diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 6c21eb8daa2138b10b5405e1d68b3599da04a39b..a07d7f411e2528033711972addf300fbcd909de4 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -185,6 +185,19 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): if "total" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["total"] <= 0: tdLog.exit("total is null!") + # log_infos ==================================== + + if "log_infos" not in infoDict or infoDict["log_infos"]== None: + tdLog.exit("log_infos is null!") + + if "summary" not in infoDict["log_infos"] or len(infoDict["log_infos"]["summary"])!= 4: + tdLog.exit("summary is null!") + + if "total" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["total"] < 0 : + tdLog.exit("total is null!") + + if "level" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: + tdLog.exit("level is null!") def do_GET(self): """ diff --git a/tests/system-test/0-others/user_manage.py b/tests/system-test/0-others/user_manage.py index 1e33d4bb1c628bd43569b68d8f7fc07e68f2bec8..6f90a2873afa4e08c1ec4b72457b6cba9f232d48 100644 --- a/tests/system-test/0-others/user_manage.py +++ b/tests/system-test/0-others/user_manage.py @@ -115,7 +115,6 @@ class TDTestCase: jiacy0_read_conn = taos.connect(user='jiacy0_read', password='123') jiacy0_write_conn = taos.connect(user='jiacy0_write', password='123') jiacy0_none_conn = taos.connect(user='jiacy0_none', password='123') - tdSql.execute('alter database db wal_retention_period 3600') tdSql.execute('create topic root_db as select * from db.stb') for user in [jiacy1_all_conn, jiacy1_read_conn, jiacy0_all_conn, jiacy0_read_conn]: user.execute(f'create topic db_jiacy as select * from db.stb') diff --git a/tests/system-test/1-insert/db_tb_name_check.py b/tests/system-test/1-insert/db_tb_name_check.py index 23bb53962038a416590a28b99bdd2763bf52a954..fa43603e258f67b8cead63f30130424fdbc60461 100644 --- a/tests/system-test/1-insert/db_tb_name_check.py +++ b/tests/system-test/1-insert/db_tb_name_check.py @@ -44,7 +44,7 @@ class TDTestCase: new_dbname = list(dbname) new_dbname.insert(i,j) dbname_1 = ''.join(new_dbname) - tdSql.execute(f'create database if not exists `{dbname_1}`') + tdSql.execute(f'create database if not exists `{dbname_1}` vgroups 1 replica 1') tdSql.query('select * from information_schema.ins_databases') tdSql.checkEqual(tdSql.queryResult[2][0],str(dbname_1)) tdSql.execute(f'drop database `{dbname_1}`') @@ -56,7 +56,7 @@ class TDTestCase: def tb_name_check(self): dbname = tdCom.getLongName(10) - tdSql.execute(f'create database if not exists `{dbname}`') + tdSql.execute(f'create database if not exists `{dbname}` vgroups 1 replica 1') tdSql.execute(f'use `{dbname}`') tbname = tdCom.getLongName(5) for i in self.special_name: diff --git a/tests/system-test/1-insert/delete_stable.py b/tests/system-test/1-insert/delete_stable.py index 8ebe7b6692e31bba12bdc0a3cbc885112eb96562..67561c51e506056d34eff5a0a7f1ee3eb7367fdb 100644 --- a/tests/system-test/1-insert/delete_stable.py +++ b/tests/system-test/1-insert/delete_stable.py @@ -24,10 +24,11 @@ from util.common import * from util.sqlset import TDSetSql class TDTestCase: + updatecfgDict = {'tsdbdebugFlag': 143} def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) self.dbname = 'db_test' self.ns_dbname = 'ns_test' self.us_dbname = 'us_test' diff --git a/tests/system-test/2-query/columnLenUpdated.py b/tests/system-test/2-query/columnLenUpdated.py index e43b32a716017702382ce0384bb377d692e64684..93d9a492f946db423b78f7891b533af8c9a6da9f 100644 --- a/tests/system-test/2-query/columnLenUpdated.py +++ b/tests/system-test/2-query/columnLenUpdated.py @@ -202,7 +202,7 @@ class TDTestCase: if retCode != "TAOS_OK": tdLog.exit("taos -s fail") - tdSql.query("select count(*) from stb group by tg1") + tdSql.query("select count(*) from stb group by tg1 order by count(*) desc") tdSql.checkData(0, 0, 2) tdSql.checkData(1, 0, 1) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index b6cefbe36fda9954188d59f813db9be4069a1af8..986c63839b1c6e9cfc8c8c6b857a668146d03237 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -147,6 +147,57 @@ class TDTestCase: tdSql.checkData(11, 0, 15) tdSql.checkData(12, 0, 1) + for col in col_list: + tdSql.query(f"select interp({col}) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1.0)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 1) + tdSql.checkData(9, 0, 1) + tdSql.checkData(10, 0, 1) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, 1) + + for col in col_list: + tdSql.query(f"select interp({col}) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, true)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 1) + tdSql.checkData(9, 0, 1) + tdSql.checkData(10, 0, 1) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, 1) + + for col in col_list: + tdSql.query(f"select interp({col}) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, NULL)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, None) + tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") tdSql.checkRows(13) tdSql.checkData(0, 0, 1.0) @@ -163,6 +214,54 @@ class TDTestCase: tdSql.checkData(11, 0, 15.0) tdSql.checkData(12, 0, 1.0) + tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1.0)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, true)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c4) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, NULL)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, None) + tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") tdSql.checkRows(13) tdSql.checkData(0, 0, 1.0) @@ -179,6 +278,54 @@ class TDTestCase: tdSql.checkData(11, 0, 15.0) tdSql.checkData(12, 0, 1.0) + tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1.0)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, true)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1.0) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, 1.0) + tdSql.checkData(3, 0, 1.0) + tdSql.checkData(4, 0, 1.0) + tdSql.checkData(5, 0, 1.0) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, 1.0) + tdSql.checkData(8, 0, 1.0) + tdSql.checkData(9, 0, 1.0) + tdSql.checkData(10, 0, 1.0) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, 1.0) + + tdSql.query(f"select interp(c5) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, NULL)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 5.0) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, 10.0) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, 15.0) + tdSql.checkData(12, 0, None) + tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") tdSql.checkRows(13) tdSql.checkData(0, 0, True) @@ -195,6 +342,54 @@ class TDTestCase: tdSql.checkData(11, 0, True) tdSql.checkData(12, 0, True) + tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1.0)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, True) + tdSql.checkData(1, 0, True) + tdSql.checkData(2, 0, True) + tdSql.checkData(3, 0, True) + tdSql.checkData(4, 0, True) + tdSql.checkData(5, 0, True) + tdSql.checkData(6, 0, True) + tdSql.checkData(7, 0, True) + tdSql.checkData(8, 0, True) + tdSql.checkData(9, 0, True) + tdSql.checkData(10, 0, True) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, True) + + tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, true)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, True) + tdSql.checkData(1, 0, True) + tdSql.checkData(2, 0, True) + tdSql.checkData(3, 0, True) + tdSql.checkData(4, 0, True) + tdSql.checkData(5, 0, True) + tdSql.checkData(6, 0, True) + tdSql.checkData(7, 0, True) + tdSql.checkData(8, 0, True) + tdSql.checkData(9, 0, True) + tdSql.checkData(10, 0, True) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, True) + + tdSql.query(f"select interp(c6) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, NULL)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, True) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, True) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, None) + ## {} ... tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(value, 1)") tdSql.checkRows(4) @@ -2587,25 +2782,25 @@ class TDTestCase: tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') - tdSql.checkData(0, 2, False) + tdSql.checkData(0, 2, None) tdSql.checkData(1, 2, False) - tdSql.checkData(2, 2, False) + tdSql.checkData(2, 2, None) tdSql.checkData(3, 2, True) - tdSql.checkData(4, 2, False) + tdSql.checkData(4, 2, None) tdSql.checkData(5, 2, False) - tdSql.checkData(6, 2, False) + tdSql.checkData(6, 2, None) tdSql.checkData(7, 2, True) - tdSql.checkData(8, 2, False) + tdSql.checkData(8, 2, None) tdSql.checkData(9, 2, True) - tdSql.checkData(10, 2, False) + tdSql.checkData(10, 2, None) tdSql.checkData(11, 2, False) - tdSql.checkData(12, 2, False) + tdSql.checkData(12, 2, None) tdSql.checkData(13, 2, False) - tdSql.checkData(14, 2, False) + tdSql.checkData(14, 2, None) tdSql.checkData(15, 2, None) - tdSql.checkData(16, 2, False) + tdSql.checkData(16, 2, None) tdSql.checkData(17, 2, None) - tdSql.checkData(18, 2, False) + tdSql.checkData(18, 2, None) tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') diff --git a/tests/system-test/2-query/smaBasic.py b/tests/system-test/2-query/smaBasic.py index 43c379ee53889aa0af5410332bf9d02cfb1ca291..c221a70605064a3d23210c7a064c36657d5a9b5a 100644 --- a/tests/system-test/2-query/smaBasic.py +++ b/tests/system-test/2-query/smaBasic.py @@ -127,7 +127,7 @@ class TDTestCase: self.c2Sum = None # create database db - sql = f"create database db vgroups 5 replica 3" + sql = f"create database db vgroups 5 replica 3 stt_trigger 1" tdLog.info(sql) tdSql.execute(sql) sql = f"use db" diff --git a/tests/system-test/5-taos-tools/taosbenchmark/insertMix.py b/tests/system-test/5-taos-tools/taosbenchmark/insertMix.py index 60daa8cdc27f5e683239d6722e4f28ae1b8d90d1..b4046b8c98b311d8f73fc811cf4e70a2d2510e38 100644 --- a/tests/system-test/5-taos-tools/taosbenchmark/insertMix.py +++ b/tests/system-test/5-taos-tools/taosbenchmark/insertMix.py @@ -79,6 +79,11 @@ class TDTestCase: tdSql.query("select count(*) from (select * from meters order by ts desc)") tdSql.checkData(0, 0, allCnt) + rowCnt = tdSql.query("select tbname, count(*) from meters partition by tbname slimit 11") + if rowCnt != 10: + tdLog.exit("partition by tbname should return 10 rows of table data which is " + str(rowCnt)) + return + def run(self): binPath = self.getPath() diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index 080b431ffe54f43e725e22d84a3f365f8bfc4cea..a16f2c348fdd925e8eadd1558ef12de926e64f92 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -149,7 +149,6 @@ class TDTestCase: topicFromStb = 'topic_stb_column' topicFromCtb = 'topic_ctb_column' - tdSql.execute("alter database %s wal_retention_period 3600" % (parameterDict['dbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName'])) diff --git a/tests/system-test/7-tmq/create_wrong_topic.py b/tests/system-test/7-tmq/create_wrong_topic.py index 77d43149cd611225d2ee8f7b017013b0fc784bfb..66814d22ea05578408793132a9ba3cd3df16a4db 100644 --- a/tests/system-test/7-tmq/create_wrong_topic.py +++ b/tests/system-test/7-tmq/create_wrong_topic.py @@ -44,7 +44,6 @@ class TDTestCase: def wrong_topic(self): tdSql.prepare() tdSql.execute('use db') - tdSql.execute('alter database db wal_retention_period 3600') stbname = f'db.{tdCom.getLongName(5, "letters")}' tag_dict = { 't0':'int' diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py index 44f58279be2df9cc97ee504babebbbfda1793f44..808a4935e30ccec4d06863e54e5e2ff3e1ba635d 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py @@ -67,7 +67,6 @@ class TDTestCase: tdLog.info("flush db to let data falls into the disk") tdSql.query("flush database %s"%(paraDict['dbName'])) - tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal.py b/tests/system-test/7-tmq/dataFromTsdbNWal.py index 0f4f1694c17701d9d4361c788980215ae418db5e..8386c22cd0430128a1b873f3f5150ef51704b245 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal.py @@ -67,7 +67,6 @@ class TDTestCase: tdLog.info("flush db to let data falls into the disk") tdSql.query("flush database %s"%(paraDict['dbName'])) - tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py index 7d11684ed81fd79584253e032f67cae1bd833de8..e25fb412af44231e0a0a5deb672ed44a15b1e879 100644 --- a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py +++ b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py @@ -106,7 +106,6 @@ class TDTestCase: paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl - tdSql.execute("alter database dbt wal_retention_period 3600") tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha' "%(paraDict['dbName'], paraDict['stbName']) diff --git a/tests/system-test/7-tmq/stbFilter.py b/tests/system-test/7-tmq/stbFilter.py index 3f862ae0472f6a1312ac5f2ea680adbd4b95f172..6b48a6d57076b4baf54d7c08db7cd8a625fab29c 100644 --- a/tests/system-test/7-tmq/stbFilter.py +++ b/tests/system-test/7-tmq/stbFilter.py @@ -45,7 +45,6 @@ class TDTestCase: tmqCom.initConsumerTable() tmqCom.create_database(tsql=tdSql, dbName=paraDict["dbName"],dropFlag=paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=paraDict['replica']) - tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict["dbName"])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/stbFilterWhere.py b/tests/system-test/7-tmq/stbFilterWhere.py index 8d8d046cef14dae6e4e01828b050201fcb575185..9b3e8b77109c7064f8920cbc2b964e17e6ed39dd 100644 --- a/tests/system-test/7-tmq/stbFilterWhere.py +++ b/tests/system-test/7-tmq/stbFilterWhere.py @@ -45,7 +45,6 @@ class TDTestCase: tmqCom.initConsumerTable() tmqCom.create_database(tsql=tdSql, dbName=paraDict["dbName"],dropFlag=paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=paraDict['replica']) - tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict["dbName"])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/stbTagFilter-1ctb.py b/tests/system-test/7-tmq/stbTagFilter-1ctb.py index 1867dc54cb3744077f477f976f88edf59f744dc3..1e6011d5db14f98b8bf29b60812a0c3eb880cb62 100644 --- a/tests/system-test/7-tmq/stbTagFilter-1ctb.py +++ b/tests/system-test/7-tmq/stbTagFilter-1ctb.py @@ -106,7 +106,6 @@ class TDTestCase: # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - tdSql.execute("alter database dbt wal_retention_period 3600") tdLog.info("create topics from stb1") topicFromStb1 = 'topic_UpperCase_stb1' # queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) diff --git a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py index 67cc60d196bfd31f75e71978145f58ac40b698c7..5043c46f00dfda378d750357659504f2ecc542b6 100644 --- a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py +++ b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index 764362c70882e0851d3cd5eb80ac4f0096a71113..c14d3b27b11fa8e36bd216f23a2371018358aaf0 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -65,7 +65,6 @@ class TDTestCase: tmqCom.initConsumerTable(self.cdbName) tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) - tdSql.execute("alter database %s wal_retention_period 3600" % (self.paraDict['dbName'])) self.paraDict["stbName"] = 'stb1' tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index 0740830696987a6009db020de14bdc768d56c69a..8c5dc5e693a43fd6d7a4ea724db5bdfa0fd7b8a4 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -172,7 +172,6 @@ class TDTestCase: tdLog.info("async insert data") pThread = tmqCom.asyncInsertData(paraDict) - tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create topics from stb with filter") # queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) diff --git a/tests/system-test/7-tmq/tmqAlterSchema.py b/tests/system-test/7-tmq/tmqAlterSchema.py index 1a8b0693b86cae8d12eaeb11f3af91f587a23a6f..a70678219f15ba94b7f9ceb1f81b9aa3881660d4 100644 --- a/tests/system-test/7-tmq/tmqAlterSchema.py +++ b/tests/system-test/7-tmq/tmqAlterSchema.py @@ -65,7 +65,6 @@ class TDTestCase: queryStringList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") @@ -176,7 +175,6 @@ class TDTestCase: queryStringList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ntb") diff --git a/tests/system-test/7-tmq/tmqCheckData.py b/tests/system-test/7-tmq/tmqCheckData.py index cb5a40642aab7ba2053780b898fa498e2c8b49a3..4d5edf87f1bb6b88ed9fe3dd8402f94e6c915744 100644 --- a/tests/system-test/7-tmq/tmqCheckData.py +++ b/tests/system-test/7-tmq/tmqCheckData.py @@ -53,7 +53,6 @@ class TDTestCase: tdLog.info("insert data") tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create topics from stb with filter") queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topicNameList[0], queryString) diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py index b4fec94dcc3a2a266a35400dc789ee13aa463caf..1209c2812c536f9b3b6d23c8ad64ac1bbad033bc 100644 --- a/tests/system-test/7-tmq/tmqCheckData1.py +++ b/tests/system-test/7-tmq/tmqCheckData1.py @@ -45,7 +45,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqClientConsLog.py b/tests/system-test/7-tmq/tmqClientConsLog.py index a56bdecb5895481ca5e6460a376e1157747f3798..7f755726cee55a18400e58f9c5b0b58837cdb3bb 100644 --- a/tests/system-test/7-tmq/tmqClientConsLog.py +++ b/tests/system-test/7-tmq/tmqClientConsLog.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py index 3ad1d097e11b85f04f1d0cc9b2cd75a9ea1bdb0a..f372a2b7428a77b02b8dcdb5c4c546756ef7e1f7 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py index fdd683d08daba5625017029a3363b379d2a206b1..c7f95f6f41cbf72e8d64c3a9bd8218dfd7a1c6d5 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py index f05f0abeff3f3f6dabde18ac3d3543d9df032f4e..26dacf514d4f66273d36f6ac3fe49ee17602747a 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py index 75b49a34fc583217e5ee0bb81324abc8421ed31a..d6f100041b15c205e8294cca964353d880b9ecc4 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index 26f7a9fb4d1300907807651a5e2fd9dd80579725..11fc7dbcc0587b20fd65bc71047ba04c1adb3f91 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -56,7 +56,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index cca29c178db281b86d145b8623f72a2dd8df1366..8ed4a6df973b57f7302d5a2c193debffbf7286a1 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py index 00d2491c97b0a178fbfd289cf44e2617107fdfe9..6a03f0f75192dda0b8b8394b1733bdec02222bda 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py index 3b1dbae443f0e8d6f71dc3c8127975790c23401a..c11159c6e551d60b90aab16bb7c90e4b26021ac2 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py index a799fa57195fe84065c7bb0b295dc7c0f14cdd09..37946d0c22dae353ac62cb8e564c1361115bf8f9 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py index f0bedbb187b9987e8f367f1afed7b69fb4c41001..439845aa54a6d29c1cd7633e9949da3eaa40cfee 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py index a63927dd8bdee0483cc943ececaac1855248ef57..53ff020b08abf2b758a6de00984b0cb799528d45 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py index 8fcc991d4e3904da04480c7bbc28cac49cf5c068..4bb6cf463f59519565f43e04e18d7d7902b73108 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py index f05f600f27b1d1c3194c85a03edfbcac1aeee3b7..e64d8552340ba9c348ee637e1f676556fd6beabb 100644 --- a/tests/system-test/7-tmq/tmqConsumerGroup.py +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -45,7 +45,6 @@ class TDTestCase: queryRowsList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index 648d629e5cc2e5cbcc8d60cf1599ba93c37d080a..74aba317264cea673cba4ed459165c20caec1cf3 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -187,7 +187,6 @@ class TDTestCase: tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) # tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py index 7a9c1bbb8c808fe62e8fede76640ac78e19f2f30..c8bcdd6235782f4a29cc18a4f243e8e53ed5ffe5 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py @@ -57,7 +57,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") @@ -144,7 +143,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py index 0b9cb7e66a2d481f6162f1e5e92d8c1f68dcb300..3fc5a2fdc70b64ec95324c370625f5397287ae7d 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py @@ -57,7 +57,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") @@ -144,7 +143,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") diff --git a/tests/system-test/7-tmq/tmqDropStb.py b/tests/system-test/7-tmq/tmqDropStb.py index 0b252a733495041464a3504a3bc8d94e54fcf48c..00affabafcb8c729e2f3ca9f988a0f7930eb030b 100644 --- a/tests/system-test/7-tmq/tmqDropStb.py +++ b/tests/system-test/7-tmq/tmqDropStb.py @@ -64,7 +64,6 @@ class TDTestCase: tmqCom.initConsumerTable(self.cdbName) tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) - tdSql.execute("alter database %s wal_retention_period 3600" % (self.paraDict['dbName'])) self.paraDict["stbName"] = 'stb1' tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) diff --git a/tests/system-test/7-tmq/tmqDropStbCtb.py b/tests/system-test/7-tmq/tmqDropStbCtb.py index 587baf12aab02a3ae3e7d4d9783cafab6ac0ceca..c9e34136cc1e59af173205a8e0cb2796acb1a14a 100644 --- a/tests/system-test/7-tmq/tmqDropStbCtb.py +++ b/tests/system-test/7-tmq/tmqDropStbCtb.py @@ -54,7 +54,6 @@ class TDTestCase: # tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqMaxTopic.py b/tests/system-test/7-tmq/tmqMaxTopic.py index 1bf20dfbd8dd4ab27c6e721abbb1af53e2547d71..62bc9ccb4ed7c6f74d05a7ccc2d81db21945ed0b 100644 --- a/tests/system-test/7-tmq/tmqMaxTopic.py +++ b/tests/system-test/7-tmq/tmqMaxTopic.py @@ -71,7 +71,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqParamsTest.py b/tests/system-test/7-tmq/tmqParamsTest.py index 6c8253c54286d6a8074e1361acba67e8c156a5ad..0fc7a6cdd98613e684776f6abc3b9077149b6254 100644 --- a/tests/system-test/7-tmq/tmqParamsTest.py +++ b/tests/system-test/7-tmq/tmqParamsTest.py @@ -63,7 +63,7 @@ class TDTestCase: queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topic_name, queryString) tdSql.query(f'select * from information_schema.ins_databases') - db_wal_retention_period_list = list(map(lambda x:x[-8] if x[0] == paraDict['dbName'] else None, tdSql.queryResult)) + db_wal_retention_period_list = list(map(lambda x:x[-6] if x[0] == paraDict['dbName'] else None, tdSql.queryResult)) for i in range(len(db_wal_retention_period_list)): if db_wal_retention_period_list[0] is None or db_wal_retention_period_list[-1] is None: db_wal_retention_period_list.remove(None) diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py index e9234f6c7aea99cf339be76ed799b34d3ec44721..31ddc1b0f831df5d6e10f4337536488ddeaa892e 100644 --- a/tests/system-test/7-tmq/tmqShow.py +++ b/tests/system-test/7-tmq/tmqShow.py @@ -51,7 +51,6 @@ class TDTestCase: consumerIdList = [0, 1, 2, 3] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py index 7f322dc2583201bce47f40a9c1883daabadf8e42..85222a941bf466631feedd1ead0387ef5f984096 100644 --- a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py +++ b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py @@ -94,7 +94,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replica) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py index bee38ca8ee192c542ebc2c09718d4e4224904c1e..5f9fb42c453ed49d16461ec7ec597bff69c3ed46 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py @@ -88,7 +88,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -136,7 +135,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -239,7 +237,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py index d3b64d2b2150521c1a3bc98fe41734390609084f..6278527c64061806013e64367c65bc685d76386b 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py @@ -88,7 +88,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -136,7 +135,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -239,7 +237,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUdf.py b/tests/system-test/7-tmq/tmqUdf.py index 5da1625cb125091dddc6ddbe3e635dc404352bbb..8af4406f780916087cbf5c6a8fffe549d90b42af 100644 --- a/tests/system-test/7-tmq/tmqUdf.py +++ b/tests/system-test/7-tmq/tmqUdf.py @@ -88,7 +88,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -136,7 +135,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -240,7 +238,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUpdate-1ctb.py b/tests/system-test/7-tmq/tmqUpdate-1ctb.py index 8fdf7748a3425b1efe7966949a1098e5867b6ee2..920e8e77e466eaf27c99fd9ac462bbd27a18ed94 100644 --- a/tests/system-test/7-tmq/tmqUpdate-1ctb.py +++ b/tests/system-test/7-tmq/tmqUpdate-1ctb.py @@ -54,7 +54,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py index 8b67f6f8252e5e3682284c5c2d5b4295308db26c..2f1d3e263175e2adedc6fad6f4693a0fcda0d4bb 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py @@ -55,7 +55,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py index 5a35c4f5ee84c950ee7a611e5be93fe13371b031..6b8c10de271b28726639baad8648e2586c02f9a1 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py @@ -55,7 +55,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py index 84617efae4d78cf57cf6d522a7b3fbf50321e5f2..3975013e747f09d8cc0548b5e32f9f01c63fb9a4 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py @@ -55,7 +55,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index c3ec4875ceeeb25a0ee3b9f921a8e7d5c40fdc07..86c40fdc727fbba7d0167209c14dbb8714a318b4 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -220,6 +220,17 @@ class TDTestCase: return + def checkWal1VgroupOnlyMeta(self): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + cmdStr = '%s/build/bin/tmq_taosx_ci -c %s -sv 1 -dv 1 -d -onlymeta'%(buildPath, cfgPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + self.checkJson(cfgPath, "tmq_taosx_tmp") + + return + def checkWal1VgroupTable(self): buildPath = tdCom.getBuildPath() cfgPath = tdCom.getClientCfgPath() @@ -301,6 +312,8 @@ class TDTestCase: def run(self): tdSql.prepare() + self.checkWal1VgroupOnlyMeta() + self.checkWal1Vgroup() self.checkSnapshot1Vgroup() diff --git a/tests/system-test/99-TDcase/TD-16821.py b/tests/system-test/99-TDcase/TD-16821.py index 2e23002059c5d157b3e1f6edf2f21a5d291739bc..26b41e6afc9a7e90b8993ef9ca1b8cf33eb27e05 100644 --- a/tests/system-test/99-TDcase/TD-16821.py +++ b/tests/system-test/99-TDcase/TD-16821.py @@ -45,7 +45,6 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/99-TDcase/TD-17255.py b/tests/system-test/99-TDcase/TD-17255.py index 5f68a5b7389d3ea095a8886e2801dd82fb6af8d9..0f83468754783ee5adca78113c7a8df5c78bbe99 100644 --- a/tests/system-test/99-TDcase/TD-17255.py +++ b/tests/system-test/99-TDcase/TD-17255.py @@ -53,7 +53,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -98,7 +97,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -183,7 +181,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -270,7 +267,6 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("insert data by auto create ctb") diff --git a/tests/system-test/99-TDcase/TD-17699.py b/tests/system-test/99-TDcase/TD-17699.py index 6956e88aec7d0c8adb84c8af9aff31cb6d80fa1f..2862f4a78d5391098f0493afa35e482c5a032817 100644 --- a/tests/system-test/99-TDcase/TD-17699.py +++ b/tests/system-test/99-TDcase/TD-17699.py @@ -65,7 +65,6 @@ class TDTestCase: tmqCom.initConsumerTable(self.cdbName) tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) - tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) self.paraDict["stbName"] = 'stb1' tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) diff --git a/tests/system-test/eco-system/manager/cmul.py b/tests/system-test/eco-system/manager/cmul.py new file mode 100644 index 0000000000000000000000000000000000000000..ac2fa5e4f28743513e7c78c3a9c3ecbc5c7592e9 --- /dev/null +++ b/tests/system-test/eco-system/manager/cmul.py @@ -0,0 +1,104 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# The option for wal_retetion_period and wal_retention_size is work well +# + +import taos +from taos.tmq import Consumer + +import os +import sys +import threading +import json +import time +import random +from datetime import date +from datetime import datetime +from datetime import timedelta +from os import path + + +topicName = "topic" +topicNum = 100 + +# consume topic +def consume_topic(topic_name, group,consume_cnt, index, wait): + consumer = Consumer( + { + "group.id": group, + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "enable.auto.commit": "true", + } + ) + + print(f"start consumer topic:{topic_name} group={group} index={index} ...") + consumer.subscribe([topic_name]) + cnt = 0 + try: + while True and cnt < consume_cnt: + res = consumer.poll(1) + if not res: + if wait: + continue + else: + break + err = res.error() + if err is not None: + raise err + val = res.value() + cnt += 1 + print(f" consume {cnt} ") + for block in val: + datas = block.fetchall() + data = datas[0][:50] + + print(f" {topic_name}_{group}_{index} {cnt} {data}") + + finally: + consumer.unsubscribe() + consumer.close() + +def consumerThread(index): + global topicName, topicNum + print(f' thread {index} start...') + while True: + idx = random.randint(0, topicNum - 1) + name = f"{topicName}{idx}" + group = f"group_{index}_{idx}" + consume_topic(name, group, 100, index, True) + + + +if __name__ == "__main__": + print(sys.argv) + threadCnt = 10 + + if len(sys.argv) == 1: + threadCnt = int(sys.argv[1]) + + + threads = [] + print(f'consumer with {threadCnt} threads...') + for i in range(threadCnt): + x = threading.Thread(target=consumerThread, args=(i,)) + x.start() + threads.append(x) + + # wait + for i, thread in enumerate(threads): + thread.join() + print(f'join thread {i} end.') + diff --git a/tests/system-test/eco-system/manager/mul.py b/tests/system-test/eco-system/manager/mul.py new file mode 100644 index 0000000000000000000000000000000000000000..d78b63d386209d06a78ff9f77aa8552d08f1a181 --- /dev/null +++ b/tests/system-test/eco-system/manager/mul.py @@ -0,0 +1,114 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import sys +import random +import time + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + + # prepareEnv + def prepareEnv(self): + self.dbName = "mullevel" + self.stbName = "meters" + self.topicName = "topic" + self.topicNum = 100 + self.loop = 50000 + + sql = f"use {self.dbName}" + tdSql.execute(sql) + + # generate topic sql + self.sqls = [ + f"select * from {self.stbName}", + f"select * from {self.stbName} where ui < 200", + f"select * from {self.stbName} where fc > 20.1", + f"select * from {self.stbName} where nch like '%%a%%'", + f"select * from {self.stbName} where fc > 20.1", + f"select lower(bin) from {self.stbName} where length(bin) < 10;", + f"select upper(bin) from {self.stbName} where length(nch) > 10;", + f"select upper(bin) from {self.stbName} where ti > 10 or ic < 40;", + f"select * from {self.stbName} where ic < 100 " + ] + + + + # prepareEnv + def createTopics(self): + for i in range(self.topicNum): + topicName = f"{self.topicName}{i}" + sql = random.choice(self.sqls) + createSql = f"create topic if not exists {topicName} as {sql}" + try: + tdSql.execute(createSql, 3, True) + except: + tdLog.info(f" create topic {topicName} failed.") + + + # random del topic + def managerTopics(self): + + for i in range(self.loop): + tdLog.info(f"start modify loop={i}") + idx = random.randint(0, self.topicNum - 1) + # delete + topicName = f"{self.topicName}{idx}" + sql = f"drop topic if exist {topicName}" + try: + tdSql.execute(sql, 3, True) + except: + tdLog.info(f" drop topic {topicName} failed.") + + + # create topic + sql = random.choice(self.sqls) + createSql = f"create topic if not exists {topicName} as {sql}" + try: + tdSql.execute(createSql, 3, True) + except: + tdLog.info(f" create topic {topicName} failed.") + + seconds = [0.1, 0.5, 3, 2.5, 1.5, 0.4, 5.2, 2.6, 0.4, 0.2] + time.sleep(random.choice(seconds)) + + + # run + def run(self): + # prepare env + self.prepareEnv() + + # create topic + self.createTopics() + + # modify topic + self.managerTopics() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index c4becdd38196727b0ac3417c028fb4f9d698ae80..5d4d73c4484092cab49fdb52e9a3479d471b7d9e 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -27,6 +27,7 @@ typedef struct { bool snapShot; bool dropTable; bool subTable; + int meta; int srcVgroups; int dstVgroups; char dir[64]; @@ -511,14 +512,18 @@ int32_t create_topic() { taos_free_result(pRes); if (g_conf.subTable) { - pRes = taos_query(pConn, "create topic meters_summary_t1 with meta as stable meters_summary"); + char topic[128] = {0}; + sprintf(topic, "create topic meters_summary_t1 %s as stable meters_summary", g_conf.meta == 0 ? "with meta" : "only meta"); + pRes = taos_query(pConn, topic); if (taos_errno(pRes) != 0) { printf("failed to create topic meters_summary_t1, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); } else { - pRes = taos_query(pConn, "create topic topic_db with meta as database abc1"); + char topic[128] = {0}; + sprintf(topic, "create topic topic_db %s as database abc1", g_conf.meta == 0 ? "with meta" : "only meta"); + pRes = taos_query(pConn, topic); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_db, reason:%s\n", taos_errstr(pRes)); return -1; @@ -692,95 +697,185 @@ void initLogFile() { } } } else { - if (g_conf.subTable) { - char* result[] = { - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"meters_summary\",\"columns\":[{\"name\":\"_" - "wstart\",\"type\":9},{\"name\":\"current\",\"type\":6},{\"name\":\"groupid\",\"type\":4},{\"name\":" - "\"location\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"group_id\",\"type\":14}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"t_d2a450ee819dcf7576f0282d9ac22dbc\",\"using\":" - "\"meters_summary\",\"tagNum\":1,\"tags\":[{\"name\":\"group_id\",\"type\":14,\"value\":1.313555008277358e+" - "19}],\"createList\":[]}"}; - - for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { - taosFprintfFile(pFile2, result[i]); - taosFprintfFile(pFile2, "\n"); + if (g_conf.meta) { + if (g_conf.subTable){ + + }else{ + char* result[] = { + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":" + "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," + "\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\"," + "\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\"," + "\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"alterType\":4,\"colName\":\"t1\"," + "\"colValue\":\"5000\",\"colValueNull\":false}", + "{\"type\":\"drop\",\"tableNameList\":[\"ct3\",\"ct1\"]}", + "{\"type\":\"drop\",\"tableType\":\"super\",\"tableName\":\"st1\"}", + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\"," + "\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\"," + "\"colType\":10,\"colLength\":8}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":10,\"colName\":\"c3\"," + "\"colNewName\":\"cc3\"}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":9}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":6,\"colName\":\"c1\"}", + "{\"type\":\"drop\",\"tableNameList\":[\"n1\"]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9}," + "{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{" + "\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]" + ",\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9}," + "{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", + "{\"type\":\"drop\",\"tableType\":\"super\",\"tableName\":\"st1\"}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":" + "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt1\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb1\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt2\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb2\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt3\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb3\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt4\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb4\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}"}; + + for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } } } else { - char* result[] = { - "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":" - "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" - ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," - "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" - "1}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" - "{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{" - "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" - "{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" - "],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" - "{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}", - "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," - "\"colType\":5}", - "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\"," - "\"colType\":8,\"colLength\":64}", - "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\"," - "\"colType\":8,\"colLength\":64}", - "{\"type\":\"alter\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"alterType\":4,\"colName\":\"t1\"," - "\"colValue\":\"5000\",\"colValueNull\":false}", - "{\"type\":\"delete\",\"sql\":\"delete from `ct3` where `ts` >= 1626006833600 and `ts` <= 1626006833605\"}", - "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" - ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", - "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\"," - "\"colType\":5}", - "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\"," - "\"colType\":10,\"colLength\":8}", - "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":10,\"colName\":\"c3\"," - "\"colNewName\":\"cc3\"}", - "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":9}", - "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":6,\"colName\":\"c1\"}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9}," - "{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{" - "\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]" - ",\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9}" - ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," - "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" - "1}]}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":" - "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," - "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" - "1}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" - "[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{" - "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt1\",\"using\":\"stt\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb1\",\"using\":\"sttb\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt2\",\"using\":\"stt\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb2\",\"using\":\"sttb\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" - "[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{" - "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt3\",\"using\":\"stt\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb3\",\"using\":\"sttb\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"sttb3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt4\",\"using\":\"stt\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"stt4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb4\",\"using\":\"sttb\"," - "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":" - "\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}"}; - - for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { - taosFprintfFile(pFile2, result[i]); - taosFprintfFile(pFile2, "\n"); + if (g_conf.subTable) { + char* result[] = { + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"meters_summary\",\"columns\":[{\"name\":\"_" + "wstart\",\"type\":9},{\"name\":\"current\",\"type\":6},{\"name\":\"groupid\",\"type\":4},{\"name\":" + "\"location\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"group_id\",\"type\":14}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"t_d2a450ee819dcf7576f0282d9ac22dbc\",\"using\":" + "\"meters_summary\",\"tagNum\":1,\"tags\":[{\"name\":\"group_id\",\"type\":14,\"value\":1.313555008277358e+" + "19}],\"createList\":[]}"}; + + for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } + } + else { + char* result[] = { + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":" + "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," + "\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\"," + "\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\"," + "\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"alterType\":4,\"colName\":\"t1\"," + "\"colValue\":\"5000\",\"colValueNull\":false}", + "{\"type\":\"delete\",\"sql\":\"delete from `ct3` where `ts` >= 1626006833600 and `ts` <= 1626006833605\"}", + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\"," + "\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\"," + "\"colType\":10,\"colLength\":8}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":10,\"colName\":\"c3\"," + "\"colNewName\":\"cc3\"}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":9}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":6,\"colName\":\"c1\"}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9}," + "{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{" + "\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]" + ",\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":" + "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt1\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb1\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt2\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb2\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt3\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb3\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt4\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb4\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}"}; + + for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } } } } @@ -802,6 +897,8 @@ int main(int argc, char* argv[]) { g_conf.dstVgroups = atol(argv[++i]); } else if (strcmp(argv[i], "-t") == 0) { g_conf.subTable = true; + } else if (strcmp(argv[i], "-onlymeta") == 0) { + g_conf.meta = 1; } }