diff --git a/cmake/cmake.version b/cmake/cmake.version index 5150ee3b75e2ac429961850995d06b5b2fd40cc9..232e86d891afebc524c1a6b2df78fac6a9137211 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.3.2") + SET(TD_VER_NUMBER "3.0.4.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index b2f335e1f7b8a30a46713a322c2ef95b5605c9d9..ba937b40c14305dbf45ccc81db9bb79ed9d89cb5 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG cb1e89c + GIT_TAG e02ddb2 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/compile_flags.txt b/compile_flags.txt new file mode 100644 index 0000000000000000000000000000000000000000..c61f9701abb7750b5e36c10eac480f61886fd5ca --- /dev/null +++ b/compile_flags.txt @@ -0,0 +1,9 @@ +-DLINUX +-DWEBSOCKET +-I/usr/include +-Iinclude +-Iinclude/os +-Iinclude/common +-Iinclude/util +-Iinclude/libs/transport +-Itools/shell/inc diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 8dfa9c2851c16ac16d43559555e492991b44dd15..5c1a833e05c927e91363a7405339750204761ce0 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -459,12 +459,17 @@ TO_JSON(str_literal) #### TO_UNIXTIMESTAMP ```sql -TO_UNIXTIMESTAMP(expr) +TO_UNIXTIMESTAMP(expr [, return_timestamp]) + +return_timestamp: { + 0 + | 1 +} ``` **Description**: UNIX timestamp converted from a string of date/time format -**Return value type**: BIGINT +**Return value type**: BIGINT, TIMESTAMP **Applicable column types**: VARCHAR and NCHAR @@ -476,6 +481,7 @@ TO_UNIXTIMESTAMP(expr) - The input string must be compatible with ISO8601/RFC3339 standard, NULL will be returned if the string can't be converted - The precision of the returned timestamp is same as the precision set for the current data base in use +- return_timestamp indicates whether the returned value type is TIMESTAMP or not. If this parameter set to 1, function will return TIMESTAMP type. Otherwise function will return BIGINT type. If parameter is omitted, default return value type is BIGINT. ### Time and Date Functions diff --git a/docs/en/12-taos-sql/23-perf.md b/docs/en/12-taos-sql/23-perf.md index fc369ec663cf9130bd7d1af650f1f47a1054fad6..43ff8e3091309e4f55ca1e7b59d1edca09424e3d 100644 --- a/docs/en/12-taos-sql/23-perf.md +++ b/docs/en/12-taos-sql/23-perf.md @@ -69,7 +69,7 @@ Provides information about SQL queries currently running. Similar to SHOW QUERIE | 1 | consumer_id | BIGINT | Consumer ID | | 2 | consumer_group | BINARY(192) | Consumer group | | 3 | client_id | BINARY(192) | Client ID (user-defined) | -| 4 | status | BINARY(20) | Consumer status | +| 4 | status | BINARY(20) | Consumer status. All possible status include: ready(consumer is in normal state), lost(the connection between consumer and mnode is broken), rebalance(the redistribution of vgroups that belongs to current consumer is now in progress), unknown(consumer is in invalid state) | 5 | topics | BINARY(204) | Subscribed topic. Returns one row for each topic. | | 6 | up_time | TIMESTAMP | Time of first connection to TDengine Server | | 7 | subscribe_time | TIMESTAMP | Time of first subscription | diff --git a/docs/en/12-taos-sql/29-changes.md b/docs/en/12-taos-sql/29-changes.md index a695a2cae18f28e090816ec98a978674a028df30..f4606f263fbda42e8e221d067912b884a3be4a40 100644 --- a/docs/en/12-taos-sql/29-changes.md +++ b/docs/en/12-taos-sql/29-changes.md @@ -27,7 +27,7 @@ The following data types can be used in the schema for standard tables. | - | :------- | :-------- | :------- | | 1 | ALTER ACCOUNT | Deprecated| This Enterprise Edition-only statement has been removed. It returns the error "This statement is no longer supported." | 2 | ALTER ALL DNODES | Added | Modifies the configuration of all dnodes. -| 3 | ALTER DATABASE | Modified | Deprecated +| 3 | ALTER DATABASE | Modified | Deprecated | 4 | ALTER STABLE | Modified | Deprecated | 5 | ALTER TABLE | Modified | Deprecated | 6 | ALTER USER | Modified | Deprecated diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index bfbdd929c263a59df42e1f310b267ba1521032c2..cc5c8f4e69e208c98c773e94c6aa2ca1ace334e7 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -459,6 +459,56 @@ For a more detailed description of the `sql()` method, please refer to [RestClie +### Schemaless Insert + +Connector support schemaless insert. + + + + +Simple insert + +```python +{{#include docs/examples/python/schemaless_insert.py}} +``` + +Insert with ttl argument + +```python +{{#include docs/examples/python/schemaless_insert_ttl.py}} +``` + +Insert with req_id argument + +```python +{{#include docs/examples/python/schemaless_insert_req_id.py}} +``` + + + + + +Simple insert + +```python +{{#include docs/examples/python/schemaless_insert_raw.py}} +``` + +Insert with ttl argument + +```python +{{#include docs/examples/python/schemaless_insert_raw_ttl.py}} +``` + +Insert with req_id argument + +```python +{{#include docs/examples/python/schemaless_insert_raw_req_id.py}} +``` + + + + ### Other sample programs | Example program links | Example program content | diff --git a/docs/en/14-reference/03-connector/_category_.yml b/docs/en/14-reference/03-connector/_category_.yml index e470f64aa013b137f05f03db112641faf2956297..6a766e96574844db87a70ff7eb5f0005cb6acbfb 100644 --- a/docs/en/14-reference/03-connector/_category_.yml +++ b/docs/en/14-reference/03-connector/_category_.yml @@ -1 +1 @@ -label: "connector" \ No newline at end of file +label: "Connector" diff --git a/docs/en/14-reference/04-taosadapter.md b/docs/en/14-reference/04-taosadapter.md index 7ab894a1c72651e3568f3effa351b044c397b2dd..6bc49768c692a47ea2796fb006d9ee942ef818ba 100644 --- a/docs/en/14-reference/04-taosadapter.md +++ b/docs/en/14-reference/04-taosadapter.md @@ -54,94 +54,91 @@ Command-line arguments take precedence over environment variables over configura ```shell Usage of taosAdapter: - --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") - --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) - --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") - --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) - --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" - --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") - --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) - -c, --config string config path default /etc/taos/taosadapter.toml - --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) - --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" - --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" - --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" - --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" - --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" - --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) - --help Print this help message and exit - --httpCodeServerError Use a non-200 http status code when taosd returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" - --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) - --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" - --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") - --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) - --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") - --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) - --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) - --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") - --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) - --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") - --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) - --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" - --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" - --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" - --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" - --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") - --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) - --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) - --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") - --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) - --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" - --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" - --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" - --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") - --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" - --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) - --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" - --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" - --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" - --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) - --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" - --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") - --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) - --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" - --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) - --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") - --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) - --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) - --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) - --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" - --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" - --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) - --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") - --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) - --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" - --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" - --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") - --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" - --pool.maxConnect int max connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" - --pool.maxIdle int max idle connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_IDLE" - -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) - --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) - --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) - --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) - --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") - --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) - --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) - --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) - --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) - --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) - --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) - --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) - --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") - --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) - --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") - --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" - --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" - --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") - --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) - --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" - --version Print the version and exit + --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") + --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) + --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") + --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" + --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") + --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + -c, --config string config path default /etc/taos/taosadapter.toml + --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) + --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" + --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" + --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" + --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" + --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) + --help Print this help message and exit + --httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" + --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) + --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" + --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) + --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") + --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) + --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) + --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") + --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) + --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") + --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) + --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" + --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" + --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" + --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" + --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") + --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) + --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) + --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) + --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" + --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" + --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" + --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") + --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" + --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) + --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" + --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" + --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" + --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) + --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" + --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") + --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" + --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) + --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) + --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) + --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) + --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" + --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" + --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) + --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") + --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) + --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" + --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" + --pool.maxConnect int max connections to server. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" + --pool.maxIdle int max idle connections to server. Env "TAOS_ADAPTER_POOL_MAX_IDLE" + -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) + --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) + --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) + --smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB" + --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) + --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) + --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) + --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) + --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) + --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) + --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) + --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) + --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") + --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) + --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") + --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" + --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") + --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" + --tmq.releaseIntervalMultiplierForAutocommit int When set to autocommit, the interval for message release is a multiple of the autocommit interval, with a default value of 2 and a minimum value of 1 and a maximum value of 10. Env "TAOS_ADAPTER_TMQ_RELEASE_INTERVAL_MULTIPLIER_FOR_AUTOCOMMIT" (default 2) + --version Print the version and exit ``` Note: @@ -332,6 +329,10 @@ This parameter controls the number of results returned by the following interfac taosAdapter uses the parameter `httpCodeServerError` to set whether to return a non-200 http status code http status code other than when the C interface returns an error. When set to true, different http status codes will be returned according to the error code returned by C. For details, see [RESTful API](https://docs.tdengine.com/reference/rest-api/) HTTP Response Code chapter. +## Configure whether schemaless writes automatically create DBs + +Starting from version 3.0.4.0, the taosAdapter provides the parameter "smlAutoCreateDB" to control whether to automatically create DBs when writing with the schemaless protocol. The default value is false, which means that the DB will not be automatically created and the user needs to manually create the DB before performing schemaless writing. + ## Troubleshooting You can check the taosAdapter running status with the `systemctl status taosadapter` command. diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 430487a3af9f8729e199251b9cb6c913ee059769..2e6f5ec1e27b7cbba035ad745e4d1f6ec3633f22 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -628,6 +628,16 @@ The charset that takes effect is UTF-8. | Default Value | 1 | | Note | The core file is generated under root directory `systemctl start taosd`/`launchctl start com.tdengine.taosd` is used to start, or under the working directory if `taosd` is started directly on Linux/macOS Shell. | +### enableScience + +| Attribute | Description | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| Applicable | Only taos-CLI client | +| Meaning | Whether to show float and double with the scientific notation | +| Value Range | 0: false, 1: true | +| Default Value | 0 | + + ### udf | Attribute | Description | diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index b160058d02c22dc824b0e960ad61cbe596999a70..acfbf6a0baa63d830aed57e7f08c370ea04722c0 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://w import Release from "/components/ReleaseV3"; +## 3.0.4.0 + + + ## 3.0.3.2 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index 17581b780a8cba5e0e73ef7d0ff8da8e54f0de56..e8f7a54566128260937cce3a31a5b82f22f1d1b6 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -10,6 +10,10 @@ For other historical version installers, please visit [here](https://www.taosdat import Release from "/components/ReleaseV3"; +## 2.4.12 + + + ## 2.4.11 diff --git a/docs/examples/python/schemaless_insert.py b/docs/examples/python/schemaless_insert.py new file mode 100644 index 0000000000000000000000000000000000000000..334a4b728f5fb3914bdd346083ba76607ea78ce2 --- /dev/null +++ b/docs/examples/python/schemaless_insert.py @@ -0,0 +1,21 @@ +import taos + +conn = taos.connect() +dbname = "pytest_line" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', +] +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED) +print("inserted") + +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED) + +result = conn.query("show tables") +for row in result: + print(row) + +conn.execute("drop database if exists %s" % dbname) diff --git a/docs/examples/python/schemaless_insert_raw.py b/docs/examples/python/schemaless_insert_raw.py new file mode 100644 index 0000000000000000000000000000000000000000..0fda7dc505e8f1ff95912282e85938eef8d8963c --- /dev/null +++ b/docs/examples/python/schemaless_insert_raw.py @@ -0,0 +1,74 @@ +import taos +from taos import utils +from taos import TaosConnection +from taos.cinterface import * +from taos.error import OperationalError, SchemalessError + +conn = taos.connect() +dbname = "taos_schemaless_insert" +try: + conn.execute("drop database if exists %s" % dbname) + + if taos.IS_V3: + conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + else: + conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + + conn.select_db(dbname) + + lines = '''st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000 + st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin, abc",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000 + stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + + res = conn.schemaless_insert_raw(lines, 1, 0) + print("affected rows: ", res) + assert (res == 3) + + lines = '''stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + res = conn.schemaless_insert_raw(lines, 1, 0) + print("affected rows: ", res) + assert (res == 1) + + result = conn.query("select * from st") + dict2 = result.fetch_all_into_dict() + print(dict2) + print(result.row_count) + + all = result.rows_iter() + for row in all: + print(row) + result.close() + assert (result.row_count == 2) + + # error test + lines = ''',t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000''' + try: + res = conn.schemaless_insert_raw(lines, 1, 0) + print(res) + # assert(False) + except SchemalessError as err: + print('**** error: ', err) + # assert (err.msg == 'Invalid data format') + + result = conn.query("select * from st") + print(result.row_count) + all = result.rows_iter() + for row in all: + print(row) + result.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() +except InterfaceError as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) +except SchemalessError as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) +except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) + raise err diff --git a/docs/examples/python/schemaless_insert_raw_req_id.py b/docs/examples/python/schemaless_insert_raw_req_id.py new file mode 100644 index 0000000000000000000000000000000000000000..606e51098665a611d54379368fb3c099f9b6e33b --- /dev/null +++ b/docs/examples/python/schemaless_insert_raw_req_id.py @@ -0,0 +1,76 @@ +import taos +from taos import utils +from taos import TaosConnection +from taos.cinterface import * +from taos.error import OperationalError, SchemalessError + +conn = taos.connect() +dbname = "taos_schemaless_insert" +try: + conn.execute("drop database if exists %s" % dbname) + + if taos.IS_V3: + conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + else: + conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + + conn.select_db(dbname) + + lines = '''st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000 + st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin, abc",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000 + stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + + ttl = 1000 + req_id = utils.gen_req_id() + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl, req_id=req_id) + print("affected rows: ", res) + assert (res == 3) + + lines = '''stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + ttl = 1000 + req_id = utils.gen_req_id() + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl, req_id=req_id) + print("affected rows: ", res) + assert (res == 1) + + result = conn.query("select * from st") + dict2 = result.fetch_all_into_dict() + print(dict2) + print(result.row_count) + + all = result.rows_iter() + for row in all: + print(row) + result.close() + assert (result.row_count == 2) + + # error test + lines = ''',t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000''' + try: + ttl = 1000 + req_id = utils.gen_req_id() + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl, req_id=req_id) + print(res) + # assert(False) + except SchemalessError as err: + print('**** error: ', err) + # assert (err.msg == 'Invalid data format') + + result = conn.query("select * from st") + print(result.row_count) + all = result.rows_iter() + for row in all: + print(row) + result.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() +except InterfaceError as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) +except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) + raise err diff --git a/docs/examples/python/schemaless_insert_raw_ttl.py b/docs/examples/python/schemaless_insert_raw_ttl.py new file mode 100644 index 0000000000000000000000000000000000000000..cf57792534cce73317dbdac3a9915adfa73a389c --- /dev/null +++ b/docs/examples/python/schemaless_insert_raw_ttl.py @@ -0,0 +1,73 @@ +import taos +from taos import utils +from taos import TaosConnection +from taos.cinterface import * +from taos.error import OperationalError, SchemalessError + +conn = taos.connect() +dbname = "taos_schemaless_insert" +try: + conn.execute("drop database if exists %s" % dbname) + + if taos.IS_V3: + conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + else: + conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + + conn.select_db(dbname) + + lines = '''st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000 + st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin, abc",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000 + stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + + ttl = 1000 + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl) + print("affected rows: ", res) + assert (res == 3) + + lines = '''stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + ttl = 1000 + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl) + print("affected rows: ", res) + assert (res == 1) + + result = conn.query("select * from st") + dict2 = result.fetch_all_into_dict() + print(dict2) + print(result.row_count) + + all = result.rows_iter() + for row in all: + print(row) + result.close() + assert (result.row_count == 2) + + # error test + lines = ''',t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000''' + try: + ttl = 1000 + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl) + print(res) + # assert(False) + except SchemalessError as err: + print('**** error: ', err) + # assert (err.msg == 'Invalid data format') + + result = conn.query("select * from st") + print(result.row_count) + all = result.rows_iter() + for row in all: + print(row) + result.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() +except InterfaceError as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) +except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) + raise err diff --git a/docs/examples/python/schemaless_insert_req_id.py b/docs/examples/python/schemaless_insert_req_id.py new file mode 100644 index 0000000000000000000000000000000000000000..ee1472db6939474ccf34c591fe4dcf4891f0d2ab --- /dev/null +++ b/docs/examples/python/schemaless_insert_req_id.py @@ -0,0 +1,22 @@ +import taos +from taos import SmlProtocol, SmlPrecision + +conn = taos.connect() +dbname = "pytest_line" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', +] +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED, req_id=1) +print("inserted") + +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED, req_id=2) + +result = conn.query("show tables") +for row in result: + print(row) + +conn.execute("drop database if exists %s" % dbname) diff --git a/docs/examples/python/schemaless_insert_ttl.py b/docs/examples/python/schemaless_insert_ttl.py new file mode 100644 index 0000000000000000000000000000000000000000..85050439f2a451b9d12f7fe20ebbae5297adb5a4 --- /dev/null +++ b/docs/examples/python/schemaless_insert_ttl.py @@ -0,0 +1,22 @@ +import taos +from taos import SmlProtocol, SmlPrecision + +conn = taos.connect() +dbname = "pytest_line" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', +] +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED, ttl=1000) +print("inserted") + +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED, ttl=1000) + +result = conn.query("show tables") +for row in result: + print(row) + +conn.execute("drop database if exists %s" % dbname) diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index 5395610df3ce5535293f8017f5e14c403e9a8279..1cff142e11d39e6afe86fab187697d222f37a9dd 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -484,6 +484,56 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 +### 无模式写入 + +连接器支持无模式写入功能。 + + + + +简单写入 + +```python +{{#include docs/examples/python/schemaless_insert.py}} +``` + +带有 ttl 参数的写入 + +```python +{{#include docs/examples/python/schemaless_insert_ttl.py}} +``` + +带有 req_id 参数的写入 + +```python +{{#include docs/examples/python/schemaless_insert_req_id.py}} +``` + + + + + +简单写入 + +```python +{{#include docs/examples/python/schemaless_insert_raw.py}} +``` + +带有 ttl 参数的写入 + +```python +{{#include docs/examples/python/schemaless_insert_raw_ttl.py}} +``` + +带有 req_id 参数的写入 + +```python +{{#include docs/examples/python/schemaless_insert_raw_req_id.py}} +``` + + + + ### 其它示例程序 | 示例程序链接 | 示例程序内容 | diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 94f805205183aca6a7a08d9a836b0df33c7ed1f0..50e82e6b902d0beddbc205a7fde5867389c1cbc3 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -459,12 +459,17 @@ TO_JSON(str_literal) #### TO_UNIXTIMESTAMP ```sql -TO_UNIXTIMESTAMP(expr) +TO_UNIXTIMESTAMP(expr [, return_timestamp]) + +return_timestamp: { + 0 + | 1 +} ``` **功能说明**：将日期时间格式的字符串转换成为 UNIX 时间戳。 -**返回结果数据类型**：BIGINT。 +**返回结果数据类型**：BIGINT, TIMESTAMP。 **应用字段**：VARCHAR, NCHAR。 @@ -476,6 +481,7 @@ TO_UNIXTIMESTAMP(expr) - 输入的日期时间字符串须符合 ISO8601/RFC3339 标准，无法转换的字符串格式将返回 NULL。 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 +- return_timestamp 指定函数返回值是否为时间戳类型，设置为1时返回 TIMESTAMP 类型，设置为0时返回 BIGINT 类型。如不指定缺省返回 BIGINT 类型。 ### 时间和日期函数 diff --git a/docs/zh/12-taos-sql/23-perf.md b/docs/zh/12-taos-sql/23-perf.md index d4ee0e178c02e65eb3f1ceaa73e170893f65cc88..fc0b8072a7ea3722990b76bea2002bae1f12d7d3 100644 --- a/docs/zh/12-taos-sql/23-perf.md +++ b/docs/zh/12-taos-sql/23-perf.md @@ -69,7 +69,7 @@ TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`，其 | 1 | consumer_id | BIGINT | 消费者的唯一 ID | | 2 | consumer_group | BINARY(192) | 消费者组 | | 3 | client_id | BINARY(192) | 用户自定义字符串，通过创建 consumer 时指定 client_id 来展示 | -| 4 | status | BINARY(20) | 消费者当前状态 | +| 4 | status | BINARY(20) | 消费者当前状态。消费者状态包括：ready(正常可用)、 lost(连接已丢失)、 rebalancing(消费者所属 vgroup 正在分配中)、unknown(未知状态)| | 5 | topics | BINARY(204) | 被订阅的 topic。若订阅多个 topic，则展示为多行 | | 6 | up_time | TIMESTAMP | 第一次连接 taosd 的时间 | | 7 | subscribe_time | TIMESTAMP | 上一次发起订阅的时间 | diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index 73fa15313b183d3731592f6c3e84e8f01e581301..a797966f57a5d9c8c942de1cce3dd3209c067de6 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -27,7 +27,7 @@ description: "TDengine 3.0 版本的语法变更说明" | - | :------- | :-------- | :------- | | 1 | ALTER ACCOUNT | 废除 | 2.x中为企业版功能，3.0不再支持。语法暂时保留了，执行报“This statement is no longer supported”错误。 | 2 | ALTER ALL DNODES | 新增 | 修改所有DNODE的参数。 -| 3 | ALTER DATABASE | 调整 | 废除
  • QUORUM：写入需要的副本确认数。3.0版本使用STRICT来指定强一致还是弱一致。3.0.0版本STRICT暂不支持修改。
  • BLOCKS：VNODE使用的内存块数。3.0版本使用BUFFER来表示VNODE写入内存池的大小。
  • UPDATE：更新操作的支持模式。3.0版本所有数据库都支持部分列更新。
  • CACHELAST：缓存最新一行数据的模式。3.0版本用CACHEMODEL代替。
  • COMP：3.0版本暂不支持修改。

    • 新增
  • CACHEMODEL：表示是否在内存中缓存子表的最近数据。
  • CACHESIZE：表示缓存子表最近数据的内存大小。
  • WAL_FSYNC_PERIOD：代替原FSYNC参数。
  • WAL_LEVEL：代替原WAL参数。
  • WAL_RETENTION_PERIOD：3.0.4.0版本新增，wal文件的额外保留策略，用于数据订阅。
  • WAL_RETENTION_SIZE：3.0.4.0版本新增，wal文件的额外保留策略，用于数据订阅。
    调整
  • REPLICA：3.0.0版本暂不支持修改。
  • KEEP：3.0版本新增支持带单位的设置方式。
+| 3 | ALTER DATABASE | 调整 | 废除
  • QUORUM：写入需要的副本确认数。3.0 版本默认行为是强一致性，且不支持修改为弱一致性。
  • BLOCKS：VNODE使用的内存块数。3.0版本使用BUFFER来表示VNODE写入内存池的大小。
  • UPDATE：更新操作的支持模式。3.0版本所有数据库都支持部分列更新。
  • CACHELAST：缓存最新一行数据的模式。3.0版本用CACHEMODEL代替。
  • COMP：3.0版本暂不支持修改。

    • 新增
  • CACHEMODEL：表示是否在内存中缓存子表的最近数据。
  • CACHESIZE：表示缓存子表最近数据的内存大小。
  • WAL_FSYNC_PERIOD：代替原FSYNC参数。
  • WAL_LEVEL：代替原WAL参数。
  • WAL_RETENTION_PERIOD：3.0.4.0版本新增，wal文件的额外保留策略，用于数据订阅。
  • WAL_RETENTION_SIZE：3.0.4.0版本新增，wal文件的额外保留策略，用于数据订阅。
    调整
  • REPLICA：3.0.0版本暂不支持修改。
  • KEEP：3.0版本新增支持带单位的设置方式。
| 4 | ALTER STABLE | 调整 | 废除
  • CHANGE TAG：修改标签列的名称。3.0版本使用RENAME TAG代替。
    新增
  • RENAME TAG：代替原CHANGE TAG子句。
  • COMMENT：修改超级表的注释。
| 5 | ALTER TABLE | 调整 | 废除
  • CHANGE TAG：修改标签列的名称。3.0版本使用RENAME TAG代替。
    新增
  • RENAME TAG：代替原CHANGE TAG子句。
  • COMMENT：修改表的注释。
  • TTL：修改表的生命周期。
| 6 | ALTER USER | 调整 | 废除
  • PRIVILEGE：修改用户权限。3.0版本使用GRANT和REVOKE来授予和回收权限。
    新增
  • ENABLE：启用或停用此用户。
  • SYSINFO：修改用户是否可查看系统信息。
diff --git a/docs/zh/14-reference/04-taosadapter.md b/docs/zh/14-reference/04-taosadapter.md index b8c5f9d647fcf920e454fabbf4da9c527b038a76..a10b5b55bc067518056b66346484f3e37bdcdc3c 100644 --- a/docs/zh/14-reference/04-taosadapter.md +++ b/docs/zh/14-reference/04-taosadapter.md @@ -54,94 +54,91 @@ taosAdapter 支持通过命令行参数、环境变量和配置文件来进行 ```shell Usage of taosAdapter: - --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") - --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) - --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") - --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) - --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" - --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") - --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) - -c, --config string config path default /etc/taos/taosadapter.toml - --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) - --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" - --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" - --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" - --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" - --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" - --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) - --help Print this help message and exit - --httpCodeServerError Use a non-200 http status code when taosd returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" - --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) - --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" - --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") - --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) - --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") - --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) - --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) - --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") - --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) - --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") - --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) - --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" - --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" - --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" - --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" - --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") - --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) - --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) - --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") - --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) - --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" - --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" - --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" - --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") - --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" - --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) - --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" - --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" - --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" - --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) - --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" - --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") - --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) - --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" - --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) - --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") - --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) - --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) - --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) - --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" - --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" - --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) - --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") - --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) - --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" - --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" - --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") - --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" - --pool.maxConnect int max connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" - --pool.maxIdle int max idle connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_IDLE" - -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) - --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) - --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) - --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) - --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") - --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) - --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) - --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) - --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) - --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) - --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) - --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) - --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") - --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) - --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") - --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" - --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" - --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") - --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) - --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" - --version Print the version and exit + --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") + --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) + --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") + --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" + --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") + --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + -c, --config string config path default /etc/taos/taosadapter.toml + --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) + --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" + --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" + --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" + --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" + --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) + --help Print this help message and exit + --httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" + --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) + --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" + --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) + --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") + --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) + --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) + --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") + --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) + --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") + --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) + --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" + --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" + --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" + --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" + --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") + --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) + --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) + --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) + --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" + --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" + --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" + --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") + --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" + --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) + --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" + --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" + --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" + --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) + --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" + --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") + --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" + --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) + --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) + --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) + --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) + --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" + --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" + --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) + --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") + --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) + --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" + --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" + --pool.maxConnect int max connections to server. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" + --pool.maxIdle int max idle connections to server. Env "TAOS_ADAPTER_POOL_MAX_IDLE" + -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) + --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) + --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) + --smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB" + --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) + --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) + --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) + --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) + --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) + --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) + --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) + --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) + --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") + --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) + --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") + --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" + --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") + --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" + --tmq.releaseIntervalMultiplierForAutocommit int When set to autocommit, the interval for message release is a multiple of the autocommit interval, with a default value of 2 and a minimum value of 1 and a maximum value of 10. Env "TAOS_ADAPTER_TMQ_RELEASE_INTERVAL_MULTIPLIER_FOR_AUTOCOMMIT" (default 2) + --version Print the version and exit ``` 备注： @@ -331,6 +328,10 @@ taosAdapter 通过参数 `restfulRowLimit` 来控制结果的返回条数，-1 taosAdapter 通过参数 `httpCodeServerError` 来设置当 C 接口返回错误时是否返回非 200 的 http 状态码。当设置为 true 时将根据 C 返回的错误码返回不同 http 状态码。具体见 [HTTP 响应码](../../connector/rest-api/#http-响应码)。 +## 配置 schemaless 写入是否自动创建 DB + +taosAdapter 从 3.0.4.0 版本开始，提供参数 `smlAutoCreateDB` 来控制在 schemaless 协议写入时是否自动创建 DB。默认值为 false 不自动创建 DB，需要用户手动创建 DB 后进行 schemaless 写入。 + ## 故障解决 您可以通过命令 `systemctl status taosadapter` 来检查 taosAdapter 运行状态。 diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index e5efd77f8050ae2305a02a5bd70367d259cb971c..fe23684fde2501ea60cccba589527fdf590d3cef 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -626,6 +626,15 @@ charset 的有效值是 UTF-8。 | 缺省值 | 1 | | 补充说明 | 不同的启动方式，生成 core 文件的目录如下：1、systemctl start taosd 启动：生成的 core 在根目录下
2、手动启动，就在 taosd 执行目录下。 | +### enableScience + +| 属性 | 说明 | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| 适用范围 | 仅客户端 TAOS-CLI 适用 | +| 含义 | 是否开启科学计数法显示浮点数 | +| 取值范围 | 0：否，1：是 | +| 缺省值 | 0 | + ### udf | 属性 | 说明 | diff --git a/docs/zh/17-operation/10-monitor.md b/docs/zh/17-operation/10-monitor.md index e936f35dcac544ad94035b5e5c9716c4aa50562e..01a225728613a1035832f5804dae5d3b6a75b875 100644 --- a/docs/zh/17-operation/10-monitor.md +++ b/docs/zh/17-operation/10-monitor.md @@ -38,3 +38,303 @@ chmod +x TDinsight.sh 运行程序并重启 Grafana 服务，打开面板：`http://localhost:3000/d/tdinsight`。 更多使用场景和限制请参考[TDinsight](/reference/tdinsight/) 文档。 + +## log 库 + +TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，可以在 taoskeeper 配置文件中修改，具体参考 [taoskeeper 文档](/reference/taosKeeper)）。taoskeeper 启动后会自动创建 log 库，并将监控数据写入到该数据库中。 + +### cluster\_info 表 + +`cluster_info` 表记录集群信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|first\_ep|VARCHAR||集群 first ep| +|first\_ep\_dnode\_id|INT||集群 first ep 的 dnode id| +|version|VARCHAR||tdengine version。例如：3.0.4.0| +|master\_uptime|FLOAT||当前 master 节点的uptime。单位：天| +|monitor_interval|INT||monitor interval。单位：秒| +|dbs\_total|INT||database 总数| +|tbs\_total|BIGINT||当前集群 table 总数| +|stbs\_total|INT||当前集群 stable 总数| +|dnodes\_total|INT||当前集群 dnode 总数| +|dnodes\_alive|INT||当前集群 dnode 存活总数| +|mnodes\_total|INT||当前集群 mnode 总数| +|mnodes\_alive|INT||当前集群 mnode 存活总数| +|vgroups\_total|INT||当前集群 vgroup 总数| +|vgroups\_alive|INT||当前集群 vgroup 存活总数| +|vnodes\_total|INT||当前集群 vnode 总数| +|vnodes\_alive|INT||当前集群 vnode 存活总数| +|connections\_total|INT||当前集群连接总数| +|topics\_total|INT||当前集群 topic 总数| +|streams\_total|INT||当前集群 stream 总数| +|protocol|INT||协议版本，目前为 1| +|cluster\_id|NCHAR|TAG|cluster id| + +### d\_info 表 + +`d_info` 表记录 dnode 状态信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|status|VARCHAR||dnode 状态| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### m\_info 表 + +`m_info` 表记录 mnode 角色信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|role|VARCHAR||mnode 角色， leader 或 follower| +|mnode\_id|INT|TAG|master node id| +|mnode\_ep|NCHAR|TAG|master node endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### dnodes\_info 表 + +`dnodes_info` 记录 dnode 信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|uptime|FLOAT||dnode uptime| +|cpu\_engine|FLOAT||taosd cpu 使用率，从 `/proc//stat` 读取| +|cpu\_system|FLOAT||服务器 cpu 使用率，从 `/proc/stat` 读取| +|cpu\_cores|FLOAT||服务器 cpu 核数| +|mem\_engine|INT||taosd 内存使用率，从 `/proc//status` 读取| +|mem\_system|INT||服务器内存使用率| +|mem\_total|INT||服务器内存总量，单位 KB| +|disk\_engine|INT||| +|disk\_used|BIGINT||data dir 挂载的磁盘使用量，单位 bytes| +|disk\_total|BIGINT||data dir 挂载的磁盘总容量，单位 bytes| +|net\_in|FLOAT||网络吞吐率，从 `/proc/net/dev` 中读取的 received bytes。单位 bytes per second| +|net\_out|FLOAT||网络吞吐率，从 `/proc/net/dev` 中读取的 transmit bytes。单位 bytes per second| +|io\_read|FLOAT||io 吞吐率，从 `/proc//io` 中读取的 rchar 与上次数值计算之后，计算得到速度。单位 bytes per second| +|io\_write|FLOAT||io 吞吐率，从 `/proc//io` 中读取的 wchar 与上次数值计算之后，计算得到速度。单位 bytes per second| +|io\_read\_disk|FLOAT||磁盘 io 吞吐率，从 `/proc//io` 中读取的 read_bytes。单位 bytes per second| +|io\_write\_disk|FLOAT||磁盘 io 吞吐率，从 `/proc//io` 中读取的 write_bytes。单位 bytes per second| +|req\_select|INT||两个间隔内发生的查询请求数目| +|req\_select\_rate|FLOAT||两个间隔内的查询请求速度 = `req_select / monitorInterval`| +|req\_insert|INT||两个间隔内发生的写入请求，包含的单条数据数目| +|req\_insert\_success|INT||两个间隔内发生的处理成功的写入请求，包含的单条数据数目| +|req\_insert\_rate|FLOAT||两个间隔内的单条数据写入速度 = `req_insert / monitorInterval`| +|req\_insert\_batch|INT||两个间隔内发生的写入请求数目| +|req\_insert\_batch\_success|INT||两个间隔内发生的成功的批量写入请求数目| +|req\_insert\_batch\_rate|FLOAT||两个间隔内的写入请求数目的速度 = `req_insert_batch / monitorInterval`| +|errors|INT||两个间隔内的出错的写入请求的数目| +|vnodes\_num|INT||dnode 上 vnodes 数量| +|masters|INT||dnode 上 master node 数量| +|has\_mnode|INT||dnode 是否包含 mnode| +|has\_qnode|INT||dnode 是否包含 qnode| +|has\_snode|INT||dnode 是否包含 snode| +|has\_bnode|INT||dnode 是否包含 bnode| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### data\_dir 表 + +`data_dir` 表记录 data 目录信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||data 目录，一般为 `/var/lib/taos`| +|level|INT||0、1、2 多级存储级别| +|avail|BIGINT||data 目录可用空间| +|used|BIGINT||data 目录已使用空间| +|total|BIGINT||data 目录空间| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### log\_dir 表 + +`log_dir` 表记录 log 目录信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||log 目录名，一般为 `/var/log/taos/`| +|avail|BIGINT||log 目录可用空间| +|used|BIGINT||log 目录已使用空间| +|total|BIGINT||log 目录空间| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### temp\_dir 表 + +`temp_dir` 表记录 temp 目录信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||temp 目录名，一般为 `/tmp/`| +|avail|BIGINT||temp 目录可用空间| +|used|BIGINT||temp 目录已使用空间| +|total|BIGINT||temp 目录空间| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### vgroups\_info 表 + +`vgroups_info` 表记录虚拟节点组信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|vgroup\_id|INT||vgroup id| +|database\_name|VARCHAR||vgroup 所属的 database 名字| +|tables\_num|BIGINT||vgroup 中 table 数量| +|status|VARCHAR||vgroup 状态| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### vnodes\_role 表 + +`vnodes_role` 表记录虚拟节点角色信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|vnode\_role|VARCHAR||vnode 角色，leader 或 follower| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### logs 表 + +`logs` 表记录登录信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|level|VARCHAR||log level| +|content|NCHAR||log content，长度不超过1024字节| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### log\_summary 表 + +`log_summary` 记录日志统计信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|error|INT||error 总数| +|info|INT||info 总数| +|debug|INT||debug 总数| +|trace|INT||trace 总数| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### grants\_info 表 + +`grants_info` 记录授权信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|expire\_time|BIGINT||认证过期时间，企业版有效，社区版为 bigint 最大值| +|timeseries\_used|BIGINT||已用测点数| +|timeseries\_total|BIGINT||总测点数，开源版本为 bigint 最大值| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### keeper\_monitor 表 + +`keeper_monitor` 记录 taoskeeper 监控数据。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|cpu|FLOAT||cpu 使用率| +|mem|FLOAT||内存使用率| +|identify|NCHAR|TAG|| + +### taosadapter\_restful\_http\_request\_total 表 + +`taosadapter_restful_http_request_total` 记录 taosadapter rest 请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|client\_ip|NCHAR|TAG|client ip| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| +|status\_code|NCHAR|TAG|status code| + +### taosadapter\_restful\_http\_request\_fail 表 + +`taosadapter_restful_http_request_fail` 记录 taosadapter rest 请求失败信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|client\_ip|NCHAR|TAG|client ip| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| +|status\_code|NCHAR|TAG|status code| + +### taosadapter\_restful\_http\_request\_in\_flight 表 + +`taosadapter_restful_http_request_in_flight` 记录 taosadapter rest 实时请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|endpoint|NCHAR|TAG|taosadpater endpoint| + +### taosadapter\_restful\_http\_request\_summary\_milliseconds 表 + +`taosadapter_restful_http_request_summary_milliseconds` 记录 taosadapter rest 请求汇总信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|count|DOUBLE||| +|sum|DOUBLE||| +|0.5|DOUBLE||| +|0.9|DOUBLE||| +|0.99|DOUBLE||| +|0.1|DOUBLE||| +|0.2|DOUBLE||| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| + +### taosadapter\_system\_mem\_percent 表 + +`taosadapter_system_mem_percent` 表记录 taosadapter 内存使用情况，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|endpoint|NCHAR|TAG|taosadpater endpoint| + +### taosadapter\_system\_cpu\_percent 表 + +`taosadapter_system_cpu_percent` 表记录 taosadapter cpu 使用情况，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|endpoint|NCHAR|TAG|taosadpater endpoint| diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index b4441ad078059f51d8632e0db9c8007db9eb804e..0974289c1fa5a1e220589af3a5d43e237b47acb6 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.0.4.0 + + + ## 3.0.3.2 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index cce6834f12af02568632949bde044b81ea718808..78926555f1f4e1d579e8b2593575c84f67d18f6a 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -10,6 +10,10 @@ taosTools 各版本安装包下载链接如下： import Release from "/components/ReleaseV3"; +## 2.4.12 + + + ## 2.4.11 diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 29182ef2779a504dcf011e94241ac079d8d13839..6fde7b48a29779b2a0c26fb58d360bbfc7ff221f 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -112,6 +112,7 @@ extern int32_t tsQueryNodeChunkSize; extern bool tsQueryUseNodeAllocator; extern bool tsKeepColumnName; extern bool tsEnableQueryHb; +extern bool tsEnableScience; extern int32_t tsRedirectPeriod; extern int32_t tsRedirectFactor; extern int32_t tsRedirectMaxPeriod; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index fcb3160f6452781821f7897d164445dc9779c1aa..34372dc2ff23bacf0e21ee459025112c28b813bb 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -26,6 +26,7 @@ extern "C" { typedef void* qTaskInfo_t; typedef void* DataSinkHandle; + struct SRpcMsg; struct SSubplan; @@ -91,6 +92,9 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); +// todo refactor +void qGetCheckpointVersion(qTaskInfo_t tinfo, int64_t* dataVer, int64_t* ckId); + /** * Set multiple input data blocks for the stream scan. * @param tinfo @@ -119,7 +123,7 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, * @param isAdd * @return */ -int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd); +int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd); /** * Create the exec task object according to task json @@ -163,6 +167,7 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo); * @return */ int32_t qAsyncKillTask(qTaskInfo_t tinfo, int32_t rspCode); + int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode); bool qTaskIsExecuting(qTaskInfo_t qinfo); @@ -182,21 +187,11 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len); int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len); STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key); -/** - * return the scan info, in the form of tuple of two items, including table uid and current timestamp - * @param tinfo - * @param uid - * @param ts - * @return - */ -int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts); -int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); +SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo); int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType); -// int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t ver); -// int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); void qStreamSetOpen(qTaskInfo_t tinfo); diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 60a4a8c6056763c6670cc209eb406c79e5ad47a6..42bc89f0b71aeba488e0e27f899c754caf8f1df9 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -189,7 +189,7 @@ bool fmIsScalarFunc(int32_t funcId); bool fmIsVectorFunc(int32_t funcId); bool fmIsIndefiniteRowsFunc(int32_t funcId); bool fmIsStringFunc(int32_t funcId); -bool fmIsDatetimeFunc(int32_t funcId); +bool fmIsDateTimeFunc(int32_t funcId); bool fmIsSelectFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId); diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h index 5703df87faedc88efed4b81bfbfdd0fddc2d561f..c825574fa6e3aea8936a3a9e5cd16224956a94a6 100644 --- a/include/libs/function/taosudf.h +++ b/include/libs/function/taosudf.h @@ -165,6 +165,8 @@ static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn *pColumn, int32_t ne if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + uint32_t extend = BitmapLen(allocCapacity) - BitmapLen(data->rowsAlloc); + memset(tmp + BitmapLen(data->rowsAlloc), 0, extend); data->fixLenCol.nullBitmap = tmp; data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity); int32_t oldLen = BitmapLen(existedRows); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index b6ada5a0c7c22bbbe1bd938faa1080464c98ae5a..cfc6ef2025d87e5dfc8e025886c62b81cf9f7cbc 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -194,6 +194,7 @@ typedef struct SRequestConnInfo { typedef void (*__freeFunc)(void* param); +// todo add creator/destroyer function typedef struct SMsgSendInfo { __async_send_cb_fn_t fp; // async callback function STargetInfo target; // for update epset diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index fd5cec293167604795961f49ca7b048102d068e8..42a7261f3889f41befa4545471bbffd40d355f4e 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -42,6 +42,7 @@ typedef struct STdbState { typedef struct { STdbState* pTdbState; int32_t number; + int64_t checkPointId; } SStreamState; SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 5b1d1fa1bcd5619e04cf23351c872714aaca31e1..103f8071914b4f9f1ac2914b7335c7217ca70546 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "executor.h" #include "os.h" +#include "executor.h" #include "query.h" #include "streamState.h" #include "tdatablock.h" @@ -50,6 +50,7 @@ enum { TASK_STATUS__RECOVER_PREPARE, TASK_STATUS__RECOVER1, TASK_STATUS__RECOVER2, + TASK_STATUS__RESTORE, // only available for source task to replay WAL from the checkpoint }; enum { @@ -103,21 +104,8 @@ typedef struct { int8_t type; } SStreamQueueItem; -#if 0 -typedef struct { - int8_t type; - int64_t ver; - int32_t* dataRef; - SSubmitReq* data; -} SStreamDataSubmit; - -typedef struct { - int8_t type; - int64_t ver; - SArray* dataRefs; // SArray - SArray* reqs; // SArray -} SStreamMergedSubmit; -#endif +typedef void FTbSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver); typedef struct { int8_t type; @@ -219,21 +207,20 @@ static FORCE_INLINE void streamQueueProcessFail(SStreamQueue* queue) { } static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) { - // return queue->qItem; } void* streamQueueNextItem(SStreamQueue* queue); -SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit); +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type); void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit); SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit); typedef struct { - char* qmsg; - // followings are not applicable to encoder and decoder - void* executor; + char* qmsg; + void* pExecutor; // not applicable to encoder and decoder + struct SWalReader* pWalReader; // not applicable to encoder and decoder } STaskExec; typedef struct { @@ -248,16 +235,13 @@ typedef struct { SUseDbRsp dbInfo; } STaskDispatcherShuffle; -typedef void FTbSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); - typedef struct { int64_t stbUid; char stbFullName[TSDB_TABLE_FNAME_LEN]; SSchemaWrapper* pSchemaWrapper; - // not applicable to encoder and decoder - void* vnode; - FTbSink* tbSinkFunc; - STSchema* pTSchema; + void* vnode; // not available to encoder and decoder + FTbSink* tbSinkFunc; + STSchema* pTSchema; } STaskSinkTb; typedef void FSmaSink(void* vnode, int64_t smaId, const SArray* data); @@ -280,24 +264,34 @@ typedef struct { SEpSet epSet; } SStreamChildEpInfo; -struct SStreamTask { - int64_t streamId; - int32_t taskId; - int32_t totalLevel; - int8_t taskLevel; - int8_t outputType; - int16_t dispatchMsgType; +typedef struct SStreamId { + int64_t streamId; + int32_t taskId; + const char* idStr; +} SStreamId; + +typedef struct SCheckpointInfo { + int64_t id; + int64_t version; // offset in WAL +} SCheckpointInfo; +typedef struct SStreamStatus { int8_t taskStatus; int8_t schedStatus; +} SStreamStatus; - // node info - int32_t selfChildId; - int32_t nodeId; - SEpSet epSet; - - int64_t recoverSnapVer; - int64_t startVer; +struct SStreamTask { + SStreamId id; + int32_t totalLevel; + int8_t taskLevel; + int8_t outputType; + int16_t dispatchMsgType; + SStreamStatus status; + int32_t selfChildId; + int32_t nodeId; + SEpSet epSet; + SCheckpointInfo chkInfo; + STaskExec exec; // fill history int8_t fillHistory; @@ -307,9 +301,6 @@ struct SStreamTask { int32_t nextCheckId; SArray* checkpointInfo; // SArray - // exec - STaskExec exec; - // output union { STaskDispatcherFixedEp fixedEpDispatcher; @@ -319,44 +310,54 @@ struct SStreamTask { STaskSinkFetch fetchSink; }; - int8_t inputStatus; - int8_t outputStatus; - - // STaosQueue* inputQueue1; - // STaosQall* inputQall; + int8_t inputStatus; + int8_t outputStatus; SStreamQueue* inputQueue; SStreamQueue* outputQueue; // trigger - int8_t triggerStatus; - int64_t triggerParam; - void* timer; - - // msg handle - SMsgCb* pMsgCb; - - // state backend - SStreamState* pState; - - // do not serialize - int32_t recoverTryingDownstream; - int32_t recoverWaitingUpstream; - int64_t checkReqId; - SArray* checkReqIds; // shuffle - int32_t refCnt; - - int64_t checkpointingId; - int32_t checkpointAlignCnt; + int8_t triggerStatus; + int64_t triggerParam; + void* timer; + SMsgCb* pMsgCb; // msg handle + SStreamState* pState; // state backend + + // the followings attributes don't be serialized + int32_t recoverTryingDownstream; + int32_t recoverWaitingUpstream; + int64_t checkReqId; + SArray* checkReqIds; // shuffle + int32_t refCnt; + int64_t checkpointingId; + int32_t checkpointAlignCnt; + struct SStreamMeta* pMeta; }; +// meta +typedef struct SStreamMeta { + char* path; + TDB* db; + TTB* pTaskDb; + TTB* pCheckpointDb; + SHashObj* pTasks; + void* ahandle; + TXN* txn; + FTaskExpand* expandFunc; + int32_t vgId; + SRWLatch lock; + int8_t walScan; + bool quit; +} SStreamMeta; + int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); -SStreamTask* tNewSStreamTask(int64_t streamId); -int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); -int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask); -void tFreeSStreamTask(SStreamTask* pTask); -int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem); +SStreamTask* tNewStreamTask(int64_t streamId); +int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); +int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); +void tFreeStreamTask(SStreamTask* pTask); +int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem); +bool tInputQueueIsFull(const SStreamTask* pTask); static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); @@ -564,40 +565,22 @@ int32_t streamAggRecoverPrepare(SStreamTask* pTask); // int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask); int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId); -// expand and deploy -typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver); - -// meta -typedef struct SStreamMeta { - char* path; - TDB* db; - TTB* pTaskDb; - TTB* pCheckpointDb; - SHashObj* pTasks; - SHashObj* pRecoverStatus; - void* ahandle; - TXN* txn; - FTaskExpand* expandFunc; - int32_t vgId; - SRWLatch lock; -} SStreamMeta; - SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId); void streamMetaClose(SStreamMeta* streamMeta); int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); -int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); -int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char* msg, int32_t msgLen); -// SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId); +int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); +int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen); +int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta); SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); -int32_t streamMetaBegin(SStreamMeta* pMeta); -int32_t streamMetaCommit(SStreamMeta* pMeta); -int32_t streamMetaRollBack(SStreamMeta* pMeta); -int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver); +int32_t streamMetaBegin(SStreamMeta* pMeta); +int32_t streamMetaCommit(SStreamMeta* pMeta); +int32_t streamMetaRollBack(SStreamMeta* pMeta); +int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index ccbc53fa5d06d2b55cf69f4efbbaa5f0186a0cc3..b51289de5e46e05d58d5af8a8b130db292439103 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -138,7 +138,8 @@ typedef struct { int8_t enableRef; } SWalFilterCond; -typedef struct { +// todo hide this struct +typedef struct SWalReader { SWal *pWal; int64_t readerId; TdFilePtr pLogFile; @@ -196,6 +197,7 @@ void walReadReset(SWalReader *pReader); int32_t walReadVer(SWalReader *pRead, int64_t ver); int32_t walReadSeekVer(SWalReader *pRead, int64_t ver); int32_t walNextValidMsg(SWalReader *pRead); +int64_t walReaderGetCurrentVer(const SWalReader* pReader); // only for tq usage void walSetReaderCapacity(SWalReader *pRead, int32_t capacity); diff --git a/packaging/debRpmAutoInstall.sh b/packaging/debRpmAutoInstall.sh index 2fe18fd7a939ddf614e7f8a565ad9448d46f4122..8fadffe4c68d05f9bb0e6f3ed160ead7d8bdb00d 100755 --- a/packaging/debRpmAutoInstall.sh +++ b/packaging/debRpmAutoInstall.sh @@ -1,7 +1,7 @@ #!/usr/bin/expect set packageName [lindex $argv 0] set packageSuffix [lindex $argv 1] -set timeout 3 +set timeout 30 if { ${packageSuffix} == "deb" } { spawn dpkg -i ${packageName} } elseif { ${packageSuffix} == "rpm"} { diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 35bea0e65ccc5070fe9d4e82adadc7132ae7cc81..7d90beac1c9e12792a4d9a1de76c06b91157049f 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -12,7 +12,7 @@ ENV TINI_VERSION v0.19.0 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${cpuType} /tini ENV DEBIAN_FRONTEND=noninteractive WORKDIR /root/ -RUN tar -zxf ${pkgFile} && cd /root/${dirName}/ && /bin/bash install.sh -e no && cd /root && rm /root/${pkgFile} && rm -rf /root/${dirName} && apt-get update && apt-get install -y locales tzdata netcat && locale-gen en_US.UTF-8 && apt-get clean && rm -rf /var/lib/apt/lists/ && chmod +x /tini +RUN tar -zxf ${pkgFile} && cd /root/${dirName}/ && /bin/bash install.sh -e no && cd /root && rm /root/${pkgFile} && rm -rf /root/${dirName} && apt-get update && apt-get install -y locales tzdata netcat curl gdb vim tmux less net-tools valgrind && locale-gen en_US.UTF-8 && apt-get clean && rm -rf /var/lib/apt/lists/ && chmod +x /tini ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" \ LC_CTYPE=en_US.UTF-8 \ diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 97226a86b5a4cdb9eac50abee4da54dfdfa46996..081383f89b358325f55cc1a7641015efcb0a4eed 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -246,7 +246,7 @@ if [ ! -f debRpmAutoInstall.sh ];then echo '#!/usr/bin/expect ' > debRpmAutoInstall.sh echo 'set packageName [lindex $argv 0]' >> debRpmAutoInstall.sh echo 'set packageSuffix [lindex $argv 1]' >> debRpmAutoInstall.sh - echo 'set timeout 3 ' >> debRpmAutoInstall.sh + echo 'set timeout 30 ' >> debRpmAutoInstall.sh echo 'if { ${packageSuffix} == "deb" } {' >> debRpmAutoInstall.sh echo ' spawn dpkg -i ${packageName} ' >> debRpmAutoInstall.sh echo '} elseif { ${packageSuffix} == "rpm"} {' >> debRpmAutoInstall.sh diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index a1a95d9f30472963ffe07d9eda4aa05aa60d9234..1b47b10520147664a3c2f3a558fed1208f84a2ca 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -575,11 +575,12 @@ function install_config() { function install_share_etc() { [ ! -d ${script_dir}/share/etc ] && return for c in `ls ${script_dir}/share/etc/`; do - if [ -e /etc/$c ]; then - out=/etc/$c.new.`date +%F` + if [ -e /etc/${clientName2}/$c ]; then + out=/etc/${clientName2}/$c.new.`date +%F` ${csudo}cp -f ${script_dir}/share/etc/$c $out ||: else - ${csudo}cp -f ${script_dir}/share/etc/$c /etc/$c ||: + ${csudo}mkdir -p /etc/${clientName2} >/dev/null 2>/dev/null ||: + ${csudo}cp -f ${script_dir}/share/etc/$c /etc/${clientName2}/$c ||: fi done diff --git a/packaging/tools/mac_before_install.txt b/packaging/tools/mac_before_install.txt index a428c612b2607ea482f0a6ddc5b26636470b72b6..4ce2374b7fed8e9cdaf50c62231df8ae6a2032e4 100644 --- a/packaging/tools/mac_before_install.txt +++ b/packaging/tools/mac_before_install.txt @@ -1,9 +1,9 @@ TDengine is an open-source, cloud-native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. With its built-in caching, stream processing, and data subscription capabilities, TDengine offers a simplified solution for time-series data processing. -To configure TDengine : edit /etc/taos/taos.cfg -To start service : launchctl start com.tdengine.taosd -To start Taos Adapter : launchctl start com.tdengine.taosadapter -To access TDengine : use taos in shell +• To configure TDengine, edit /etc/taos/taos.cfg +• To start service, run launchctl start com.tdengine.taosd +• To start Taos Adapter, run launchctl start com.tdengine.taosadapter +• To access TDengine from your local machine, run taos If you're experiencing problems installing TDengine, check the file /var/log/taos/tdengine_install.log to help troubleshoot the installation. diff --git a/packaging/tools/mac_before_install_client.txt b/packaging/tools/mac_before_install_client.txt index 0457d73c49a0bdba49864477f32d07aaa512c375..cce8191667f7570591612944f611729e9e399c31 100644 --- a/packaging/tools/mac_before_install_client.txt +++ b/packaging/tools/mac_before_install_client.txt @@ -1,9 +1,9 @@ TDengine is an open-source, cloud-native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. With its built-in caching, stream processing, and data subscription capabilities, TDengine offers a simplified solution for time-series data processing. -Once it's installed, please take the steps below: -1: open a terminal/shell in Mac -2: if connecting to Cloud Service, follow the instructions on your cloud service account and configure the environment variable -3: if connecting to another TDengine Service, you can also view help information via "taos --help" -4: execute command taos +After the installation process is complete, perform the following steps to start using TDengine: +1: Open Terminal on your Mac. +2: To connect to a TDengine server using the default settings and credentials, run the taos command. +3: To connect to a TDengine server using custom settings or credentials, run taos --help for more information. +4: To connect to TDengine Cloud, follow the instructions on the Tools - TDengine CLI page in your TDengine Cloud account. -If you're experiencing problems installing TDengine, check the file /var/log/taos/tdengine_install.log to help troubleshoot the installation. +If any issues occur during installation, check the /var/log/taos/tdengine_install.log file to troubleshoot. \ No newline at end of file diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 0dce526db6996f4a55090c3726483db83b752d2b..e4df233d678f90d55ef50ba05e088397faa8e6b0 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -150,7 +150,7 @@ fi mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm -mkdir -p ${install_dir}/share && cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||: +# mkdir -p ${install_dir}/share && cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||: if [ $adapterName != "taosadapter" ]; then mv ${install_dir}/cfg/${clientName2}adapter.toml ${install_dir}/cfg/$adapterName.toml @@ -322,6 +322,7 @@ if [[ $dbName == "taos" ]]; then mkdir -p ${install_dir}/share/ cp -Rfap ${web_dir}/admin ${install_dir}/share/ cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png + cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||: else echo "directory not found for enterprise release: ${web_dir}/admin" fi diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 86db35b4123c87c3d3f6d546caa15e70d09884e4..41f87379a9c7766348c342b4f97273cb4704ab1a 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -36,14 +36,6 @@ extern "C" { #include "tconfig.h" -#define CHECK_CODE_GOTO(expr, label) \ - do { \ - code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - goto label; \ - } \ - } while (0) - #define ERROR_MSG_BUF_DEFAULT_SIZE 512 #define HEARTBEAT_INTERVAL 1500 // ms @@ -286,28 +278,7 @@ static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { return (SReqResultInfo*)&msg->resInfo; } -static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - pRspObj->resIter++; - - if (pRspObj->resIter < pRspObj->rsp.blockNum) { - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter); - if (pRspObj->rsp.withSchema) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter); - setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); - taosMemoryFreeClear(pRspObj->resInfo.row); - taosMemoryFreeClear(pRspObj->resInfo.pCol); - taosMemoryFreeClear(pRspObj->resInfo.length); - taosMemoryFreeClear(pRspObj->resInfo.convertBuf); - taosMemoryFreeClear(pRspObj->resInfo.convertJson); - } - - setQueryResultFromRsp(&pRspObj->resInfo, pRetrieve, convertUcs4, false); - return &pRspObj->resInfo; - } - - return NULL; -} +SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4); static FORCE_INLINE SReqResultInfo* tscGetCurResInfo(TAOS_RES* res) { if (TD_RES_QUERY(res)) return &(((SRequestObj*)res)->body.resInfo); @@ -320,7 +291,6 @@ extern int32_t clientConnRefPool; extern int32_t timestampDeltaLimit; extern int64_t lastClusterId; - __async_send_cb_fn_t getMsgRspHandle(int32_t msgType); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); @@ -373,7 +343,6 @@ void taos_close_internal(void* taos); // global, called by mgmt int hbMgrInit(); void hbMgrCleanUp(); -int hbHandleRsp(SClientHbBatchRsp* hbRsp); // cluster level SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key); @@ -386,9 +355,6 @@ void stopAllRequests(SHashObj* pRequests); int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); -// --- mq -void hbMgrInitMqHbRspHandle(); - typedef struct SSqlCallbackWrapper { SParseContext* pParseCtx; SCatalogReq* pCatalogReq; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index dac44bd9c4511261124d43d19db214cac3300b49..ce174744ef740e13dffb4ab4a44cdf5a601b0f29 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1039,8 +1039,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat .sysInfo = pRequest->pTscObj->sysInfo, .allocatorId = pRequest->allocatorRefId}; - SAppInstInfo* pAppInfo = getAppInfo(pRequest); - SQueryPlan* pDag = NULL; + SQueryPlan* pDag = NULL; int64_t st = taosGetTimestampUs(); int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); @@ -1052,7 +1051,6 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat } pRequest->metric.execStart = taosGetTimestampUs(); - pRequest->metric.planCostUs = pRequest->metric.execStart - st; if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index d2a9665eee432e91547095016ad97c1f52a83319..b61335475125955b801b5b697992e79c98010431 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -1259,8 +1259,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse int code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); taos_free_result(tres); - return createSchemalessResp(env, 0, code, taos_errstr(tres)); + return jobject; } taos_free_result(tres); @@ -1286,8 +1287,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse int code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); taos_free_result(tres); - return createSchemalessResp(env, 0, code, taos_errstr(tres)); + return jobject; } taos_free_result(tres); @@ -1315,8 +1317,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse int code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); taos_free_result(tres); - return createSchemalessResp(env, 0, code, taos_errstr(tres)); + return jobject; } taos_free_result(tres); @@ -1343,8 +1346,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse int code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); taos_free_result(tres); - return createSchemalessResp(env, 0, code, taos_errstr(tres)); + return jobject; } taos_free_result(tres); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 8c706223189f97d4491331082b8e0d715079ce47..ceca06e309bb10bcbbb5df9cd8b870dcc2eee6e3 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -210,6 +210,11 @@ typedef struct { tmq_t* pTmq; } SMqCommitCbParam; +typedef struct SSyncCommitInfo { + tsem_t sem; + int32_t code; +} SSyncCommitInfo; + static int32_t doAskEp(tmq_t* tmq); static int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg); static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet); @@ -521,11 +526,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN return TSDB_CODE_OUT_OF_MEMORY; } - pMsgSendInfo->msgInfo = (SDataBuf){ - .pData = buf, - .len = sizeof(SMsgHead) + len, - .handle = NULL, - }; + pMsgSendInfo->msgInfo = (SDataBuf) { .pData = buf, .len = sizeof(SMsgHead) + len, .handle = NULL }; pMsgSendInfo->requestId = generateRequestId(); pMsgSendInfo->requestObjRefId = 0; @@ -786,11 +787,7 @@ void tmqSendHbReq(void* param, void* tmrId) { goto OVER; } - sendInfo->msgInfo = (SDataBuf){ - .pData = pReq, - .len = tlen, - .handle = NULL, - }; + sendInfo->msgInfo = (SDataBuf){ .pData = pReq, .len = tlen, .handle = NULL }; sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; @@ -2126,13 +2123,8 @@ void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* } } -typedef struct SSyncCommitInfo { - tsem_t sem; - int32_t code; -} SSyncCommitInfo; - -static void commitCallBackFn(tmq_t* pTmq, int32_t code, void* param) { - SSyncCommitInfo* pInfo = (SSyncCommitInfo*)param; +static void commitCallBackFn(tmq_t *pTmq, int32_t code, void* param) { + SSyncCommitInfo* pInfo = (SSyncCommitInfo*) param; pInfo->code = code; tsem_post(&pInfo->sem); } @@ -2309,3 +2301,26 @@ void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, cons waitingRspNum); } } + +SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { + SMqRspObj* pRspObj = (SMqRspObj*)res; + pRspObj->resIter++; + + if (pRspObj->resIter < pRspObj->rsp.blockNum) { + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter); + if (pRspObj->rsp.withSchema) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter); + setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); + taosMemoryFreeClear(pRspObj->resInfo.row); + taosMemoryFreeClear(pRspObj->resInfo.pCol); + taosMemoryFreeClear(pRspObj->resInfo.length); + taosMemoryFreeClear(pRspObj->resInfo.convertBuf); + taosMemoryFreeClear(pRspObj->resInfo.convertJson); + } + + setQueryResultFromRsp(&pRspObj->resInfo, pRetrieve, convertUcs4, false); + return &pRspObj->resInfo; + } + + return NULL; +} \ No newline at end of file diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b2cf380f3b378710c728186d0e6d5097cfda91a3..f4d2ed01b079b11ff00c7d021a918f5dbe543240 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -221,6 +221,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c } uint8_t* p = (uint8_t*)pSource->nullbitmap; + pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] &= (0B11111111 << shiftBits); // clear remind bits pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] |= (p[0] >> remindBits); // copy remind bits if (BitmapLen(numOfRow1) == BitmapLen(total)) { @@ -232,6 +233,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)]; int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1); + memset(start, 0, overCount); while (i < len) { // size limit of pSource->nullbitmap if (i >= 1) { start[i - 1] |= (p[i] >> remindBits); // copy remind bits @@ -309,9 +311,11 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int pColumnInfoData->pData = tmp; if (BitmapLen(numOfRow1) < BitmapLen(finalNumOfRows)) { char* btmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(finalNumOfRows)); + if (btmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } uint32_t extend = BitmapLen(finalNumOfRows) - BitmapLen(numOfRow1); memset(btmp + BitmapLen(numOfRow1), 0, extend); - pColumnInfoData->nullbitmap = btmp; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index af2da6ae2af49a6a6e581ee7b1d097d1896ff7ca..da4a91223815af34c5dfb679046a9e81bbe46edc 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -105,6 +105,7 @@ int32_t tsQueryPolicy = 1; int32_t tsQueryRspPolicy = 0; int64_t tsQueryMaxConcurrentTables = 200; // unit is TSDB_TABLE_NUM_UNIT bool tsEnableQueryHb = false; +bool tsEnableScience = false; // on taos-cli show float and doulbe with scientific notation if true int32_t tsQuerySmaOptimize = 0; int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data. bool tsQueryPlannerTrace = false; @@ -117,6 +118,7 @@ int32_t tsRedirectMaxPeriod = 1000; int32_t tsMaxRetryWaitTime = 10000; bool tsUseAdapter = false; + /* * denote if the server needs to compress response message at the application layer to client, including query rsp, * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. @@ -328,6 +330,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, 1) != 0) return -1; if (cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, false) != 0) return -1; + if (cfgAddBool(pCfg, "enableScience", tsEnableScience, false) != 0) return -1; if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1; if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1; if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, true) != 0) return -1; @@ -730,6 +733,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; + tsEnableScience = cfgGetItem(pCfg, "enableScience")->bval; tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; diff --git a/source/dnode/mgmt/mgmt_snode/src/smInt.c b/source/dnode/mgmt/mgmt_snode/src/smInt.c index 28097311ac3530788d83c965ed4918d4621effc4..e2223497671362e1c8af19e3481c797e8a2b3972 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smInt.c +++ b/source/dnode/mgmt/mgmt_snode/src/smInt.c @@ -55,6 +55,7 @@ int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { smClose(pMgmt); return -1; } + tmsgReportStartup("snode-impl", "initialized"); if (smStartWorker(pMgmt) != 0) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 4c5b1246e7f21f2a9ca8862e8cc2580407d13264..0244a4fd6e34c8f362047a9829ecc56285ff31fa 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" +#include "vnd.h" SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { SVnodeObj *pVnode = NULL; @@ -78,6 +79,11 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) { char path[TSDB_FILENAME_LEN] = {0}; + bool atExit = true; + + if (vnodeIsLeader(pVnode->pImpl)) { + vnodeProposeCommitOnNeed(pVnode->pImpl, atExit); + } taosThreadRwlockWrlock(&pMgmt->lock); taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 756ca008b06eaa19b2871295d816452210cb1427..da08bd01aca0ef60674ec6bb14c2c1cb1ebc781f 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -163,8 +163,8 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { - dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, - terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); + dGWarn("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, + terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); terrno = (terrno != 0) ? terrno : -1; return terrno; } diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 493ba48601382bdd25ce95b1639708c9f25c202d..b9aa8eb674b9b6f760f0c782f2a9e7aabe55eeb0 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -5,6 +5,7 @@ ENDIF () IF (TD_ENTERPRISE) LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/privilege/src/privilege.c) LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndDb.c) + LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndVgroup.c) ENDIF () add_library(mnode STATIC ${MNODE_SRC}) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 94584dfe58f3bd40dc4df5b67cde6c415b071693..4d05637a2b3ab872d5a0ba99c022e21f79ca9b05 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -67,7 +67,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { return 0; } -static SClusterObj *mndAcquireCluster(SMnode *pMnode) { +static SClusterObj *mndAcquireCluster(SMnode *pMnode, void **ppIter) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; @@ -76,23 +76,27 @@ static SClusterObj *mndAcquireCluster(SMnode *pMnode) { pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pCluster); if (pIter == NULL) break; + *ppIter = pIter; + return pCluster; } return NULL; } -static void mndReleaseCluster(SMnode *pMnode, SClusterObj *pCluster) { +static void mndReleaseCluster(SMnode *pMnode, SClusterObj *pCluster, void *pIter) { SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pCluster); } int64_t mndGetClusterId(SMnode *pMnode) { int64_t clusterId = 0; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { clusterId = pCluster->id; - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } return clusterId; @@ -100,10 +104,11 @@ int64_t mndGetClusterId(SMnode *pMnode) { int64_t mndGetClusterCreateTime(SMnode *pMnode) { int64_t createTime = 0; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { createTime = pCluster->createdTime; - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } return createTime; @@ -121,10 +126,11 @@ static int32_t mndGetClusterUpTimeImp(SClusterObj *pCluster) { float mndGetClusterUpTime(SMnode *pMnode) { int64_t upTime = 0; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { upTime = mndGetClusterUpTimeImp(pCluster); - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } return upTime / 86400.0f; @@ -321,11 +327,12 @@ static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter) { static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SClusterObj clusterObj = {0}; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { memcpy(&clusterObj, pCluster, sizeof(SClusterObj)); clusterObj.upTime += tsUptimeInterval; - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } if (clusterObj.id <= 0) { diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index fb81a764f133f53f786b1d227e6c8d80f25d9c49..c69f08eb6b244b358fa73312d5a86e7ff4818097 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -70,7 +70,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI32(pEncoder, innerSz) < 0) return -1; for (int32_t j = 0; j < innerSz; j++) { SStreamTask *pTask = taosArrayGetP(pArray, j); - if (tEncodeSStreamTask(pEncoder, pTask) < 0) return -1; + if (tEncodeStreamTask(pEncoder, pTask) < 0) return -1; } } @@ -130,7 +130,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { taosArrayDestroy(pArray); return -1; } - if (tDecodeSStreamTask(pDecoder, pTask) < 0) { + if (tDecodeStreamTask(pDecoder, pTask) < 0) { taosMemoryFree(pTask); taosArrayDestroy(pArray); return -1; @@ -158,7 +158,10 @@ void tFreeStreamObj(SStreamObj *pStream) { taosMemoryFree(pStream->sql); taosMemoryFree(pStream->ast); taosMemoryFree(pStream->physicalPlan); - if (pStream->outputSchema.nCols) taosMemoryFree(pStream->outputSchema.pSchema); + + if (pStream->outputSchema.nCols) { + taosMemoryFree(pStream->outputSchema.pSchema); + } int32_t sz = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < sz; i++) { @@ -166,11 +169,14 @@ void tFreeStreamObj(SStreamObj *pStream) { int32_t taskSz = taosArrayGetSize(pLevel); for (int32_t j = 0; j < taskSz; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); - tFreeSStreamTask(pTask); + tFreeStreamTask(pTask); } + taosArrayDestroy(pLevel); } + taosArrayDestroy(pStream->tasks); + // tagSchema.pSchema if (pStream->tagSchema.nCols > 0) { taosMemoryFree(pStream->tagSchema.pSchema); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index d1671aa12a5b4c5dfbc2a3a1ede49825b7e384a2..734f624be0e19c942c10244f28263570d6ea4504 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -138,7 +138,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream for (int32_t j = 0; j < sinkLvSize; j++) { SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j); if (pLastLevelTask->nodeId == pVgInfo->vgId) { - pVgInfo->taskId = pLastLevelTask->taskId; + pVgInfo->taskId = pLastLevelTask->id.taskId; break; } } @@ -149,7 +149,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream SArray* pArray = taosArrayGetP(pStream->tasks, 0); // one sink only SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0); - pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId; + pTask->fixedEpDispatcher.taskId = lastLevelTask->id.taskId; pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet; } @@ -224,7 +224,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { continue; } - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid); if (pTask == NULL) { sdbRelease(pSdb, pVgroup); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -260,7 +260,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { SArray* tasks = taosArrayGetP(pStream->tasks, 0); - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -350,12 +350,13 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } - pInnerTask = tNewSStreamTask(pStream->uid); + pInnerTask = tNewStreamTask(pStream->uid); if (pInnerTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; qDestroyQueryPlan(pPlan); return -1; } + pInnerTask->fillHistory = pStream->fillHistory; mndAddTaskToTaskSet(taskInnerLevel, pInnerTask); @@ -421,7 +422,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { continue; } - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; sdbRelease(pSdb, pVgroup); @@ -440,7 +441,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; pTask->outputType = TASK_OUTPUT__FIXED_DISPATCH; - pTask->fixedEpDispatcher.taskId = pInnerTask->taskId; + pTask->fixedEpDispatcher.taskId = pInnerTask->id.taskId; pTask->fixedEpDispatcher.nodeId = pInnerTask->nodeId; pTask->fixedEpDispatcher.epSet = pInnerTask->epSet; @@ -460,7 +461,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { pEpInfo->childId = pTask->selfChildId; pEpInfo->epSet = pTask->epSet; pEpInfo->nodeId = pTask->nodeId; - pEpInfo->taskId = pTask->taskId; + pEpInfo->taskId = pTask->id.taskId; taosArrayPush(pInnerTask->childEpInfo, &pEpInfo); sdbRelease(pSdb, pVgroup); } @@ -491,7 +492,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { continue; } - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid); if (pTask == NULL) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index ff759f5e785728998d6b8c90824a244cd80ff3f6..76bb144fcb0b33ff5a832b01c9dd0b9ad9b95175 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -35,12 +35,12 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); -static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); +static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq); -// static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq); -/*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/ +static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq); +static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq); static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); @@ -418,7 +418,7 @@ FAIL: int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { SEncoder encoder; tEncoderInit(&encoder, NULL, 0); - tEncodeSStreamTask(&encoder, pTask); + tEncodeStreamTask(&encoder, pTask); int32_t size = encoder.pos; int32_t tlen = sizeof(SMsgHead) + size; tEncoderClear(&encoder); @@ -430,7 +430,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { ((SMsgHead *)buf)->vgId = htonl(pTask->nodeId); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncoderInit(&encoder, abuf, size); - tEncodeSStreamTask(&encoder, pTask); + tEncodeStreamTask(&encoder, pTask); tEncoderClear(&encoder); STransAction action = {0}; @@ -601,7 +601,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { return -1; } pReq->head.vgId = htonl(pTask->nodeId); - pReq->taskId = pTask->taskId; + pReq->taskId = pTask->id.taskId; STransAction action = {0}; memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); action.pCont = pReq; @@ -1209,7 +1209,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock // task id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pTask->taskId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pTask->id.taskId, false); // node type char nodeType[20 + VARSTR_HEADER_SIZE] = {0}; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 6a745a0a4f94ff4668e55927b99596bfd705d5ca..49921c9a1a0572c8498ca1ff01fb5491b23cd595 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -671,7 +671,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { // possibly no vg is changed // when each topic is re-balanced, issue an trans to save the results in sdb. if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) { - mError("mq re-balance persist output error, possibly vnode splitted or dropped"); + mError("mq re-balance persist output error, possibly vnode splitted or dropped,msg:%s", terrstr()); } taosArrayDestroy(rebOutput.newConsumers); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 3a1c4ce58f23cc951bf4ab41014eb89bde443d8b..d08227927aae2d0d6fd2443df36650e8445a08f8 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1168,15 +1168,13 @@ static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int3 sprintf(sql, "error"); } - // char *obj = taosMemoryMalloc(sqlLen + VARSTR_HEADER_SIZE + 1); char obj[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(obj, sql, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, *numOfRows, (const char *)obj, false); - // taosMemoryFree(obj); } else { - char condition[20] = {0}; + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, *numOfRows, (const char *)condition, false); @@ -1257,12 +1255,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); - char tableName[20] = {0}; + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char condition[20] = {0}; + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); @@ -1292,12 +1290,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); - char tableName[20] = {0}; + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char condition[20] = {0}; + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); @@ -1329,12 +1327,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false); - char tableName[20] = {0}; + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char condition[20] = {0}; + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 31924e04717e0bfffb485b7726a533454d718194..ed1fddb63f9310b5c0aae6d99d249c47a20a2a7d 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1891,56 +1891,17 @@ int32_t mndAddVgroupBalanceToTrans(SMnode *pMnode, SVgObj *pVgroup, STrans *pTra return 0; } -int32_t mndProcessVgroupBalanceLeaderMsg(SRpcMsg *pReq) { - int32_t code = -1; - - SBalanceVgroupLeaderReq req = {0}; - if (tDeserializeSBalanceVgroupLeaderReq(pReq->pCont, pReq->contLen, &req) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return code; - } - - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - - int32_t total = sdbGetSize(pSdb, SDB_VGROUP); - if(total <= 0) { - terrno = TSDB_CODE_TSC_INVALID_OPERATION; - return code; - } - - STrans *pTrans = NULL; - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "bal-vg-leader"); - if (pTrans == NULL) goto _OVER; - mndTransSetSerial(pTrans); - mInfo("trans:%d, used to balance vgroup leader", pTrans->id); - - void *pIter = NULL; - int32_t count = 0; - while (1) { - SVgObj *pVgroup = NULL; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - - if(mndAddVgroupBalanceToTrans(pMnode, pVgroup, pTrans) == 0){ - count++; - } +extern int32_t mndProcessVgroupBalanceLeaderMsgImp(SRpcMsg *pReq); - sdbRelease(pSdb, pVgroup); - } - - if(count == 0) { - terrno = TSDB_CODE_TSC_INVALID_OPERATION; - goto _OVER; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - code = 0; +int32_t mndProcessVgroupBalanceLeaderMsg(SRpcMsg *pReq) { + return mndProcessVgroupBalanceLeaderMsgImp(pReq); +} -_OVER: - mndTransDrop(pTrans); - return code; +#ifndef TD_ENTERPRISE +int32_t mndProcessVgroupBalanceLeaderMsgImp(SRpcMsg *pReq) { + return 0; } +#endif static int32_t mndCheckDnodeMemory(SMnode *pMnode, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pOldVgroup, SVgObj *pNewVgroup, SArray *pArray) { diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 3d1b356f8c995847c1001cd7b288f7e7cac8fd89..cefc4fa63eb7ba79a46a8527ba0f7c2652b0e6ab 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -32,6 +32,7 @@ void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) { tDecoderClear(&decoder); goto FAIL; } + tDecoderClear(&decoder); int32_t taskId = req.taskId; @@ -65,7 +66,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); pTask->refCnt = 1; - pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; + pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputQueue = streamQueueOpen(); pTask->outputQueue = streamQueueOpen(); @@ -76,21 +77,19 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; pTask->pMsgCb = &pSnode->msgCb; - pTask->startVer = ver; + pTask->chkInfo.version = ver; + pTask->pMeta = pSnode->pMeta; pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1); if (pTask->pState == NULL) { return -1; } - SReadHandle mgHandle = { - .vnode = NULL, - .numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo), - .pStateBackend = pTask->pState, - }; + int32_t numOfChildEp = taosArrayGetSize(pTask->childEpInfo); + SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState }; - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, 0); - ASSERT(pTask->exec.executor); + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, 0); + ASSERT(pTask->exec.pExecutor); streamSetupTrigger(pTask); return 0; @@ -140,9 +139,10 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { if (pTask == NULL) { return -1; } + SDecoder decoder; tDecoderInit(&decoder, (uint8_t *)msg, msgLen); - code = tDecodeSStreamTask(&decoder, pTask); + code = tDecodeStreamTask(&decoder, pTask); if (code < 0) { tDecoderClear(&decoder); taosMemoryFree(pTask); @@ -153,7 +153,7 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); // 2.save task - code = streamMetaAddTask(pSnode->pMeta, -1, pTask); + code = streamMetaAddDeployedTask(pSnode->pMeta, -1, pTask); if (code < 0) { return -1; } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 52df20877a68ccde0c43e54de4c8e5b45a31a791..8c0e5c35cd46e83a3aa3ba124fbf4adafbe607f5 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -57,6 +57,7 @@ target_sources( # tq "src/tq/tq.c" + "src/tq/tqUtil.c" "src/tq/tqScan.c" "src/tq/tqMeta.c" "src/tq/tqRead.c" @@ -64,6 +65,7 @@ target_sources( "src/tq/tqPush.c" "src/tq/tqSink.c" "src/tq/tqCommit.c" + "src/tq/tqRestore.c" "src/tq/tqSnapshot.c" "src/tq/tqOffsetSnapshot.c" ) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 83a126c9c5e6df1dcfc6f14fedf27341e69cd6a4..7a020b9e4725f2143f832642d4df9156e22d8c13 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -92,7 +92,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); -void vnodeProposeCommitOnNeed(SVnode *pVnode); +void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit); // meta typedef struct SMeta SMeta; // todo: remove @@ -257,15 +257,16 @@ void tqCloseReader(STqReader *); void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); -int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); void tqNextBlock(STqReader *pReader, SFetchRet *ret); +int32_t extractSubmitMsgFromWal(SWalReader *pReader, SPackedData *pPackedData); -int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); +int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); // int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); -bool tqNextDataBlock2(STqReader *pReader); +bool tqNextDataBlock(STqReader *pReader); bool tqNextDataBlockFilterOut2(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock2(SSDataBlock *pBlock, STqReader *pReader, SSubmitTbData **pSubmitTbDataRet); int32_t tqRetrieveTaosxBlock2(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 9037644602034819fd1da260c1cef02b7ae29a97..c007f8479035c2ceaa2528c4093e52a7fa306456 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -80,7 +80,7 @@ typedef struct { typedef struct { int8_t subType; - STqReader* pExecReader; + STqReader* pTqReader; qTaskInfo_t task; union { STqExecCol execCol; @@ -128,6 +128,10 @@ typedef struct { tmr_h timer; } STqMgmt; +typedef struct { + int32_t size; +} STqOffsetHead; + static STqMgmt tqMgmt = {0}; int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle); @@ -154,10 +158,6 @@ int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_ int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key); int32_t tqMetaRestoreCheckInfo(STQ* pTq); -typedef struct { - int32_t size; -} STqOffsetHead; - STqOffsetStore* tqOffsetOpen(STQ* pTq); void tqOffsetClose(STqOffsetStore*); STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey); @@ -176,6 +176,18 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); // tqStream int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); +int32_t tqStreamTasksScanWal(STQ* pTq); + +// tq util +void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId); +int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver); +int32_t launchTaskForWalBlock(SStreamTask* pTask, SFetchRet* pRet, STqOffset* pOffset); +int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); + +void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver); +void saveOffsetForAllTasks(STQ* pTq, int64_t ver); +void initOffsetForAllRestoreTasks(STQ* pTq); +int32_t transferToWalReadTask(SStreamMeta* pStreamMeta, SArray* pTaskList); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 134909090f1c14f2972b98bf00d9d7c0486b5588..ae65e2ba3f199e53e682e6f429fbfc22c794e7fa 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -96,7 +96,7 @@ int32_t vnodeGetBatchMeta(SVnode* pVnode, SRpcMsg* pMsg); // vnodeCommit.c int32_t vnodeBegin(SVnode* pVnode); -int32_t vnodeShouldCommit(SVnode* pVnode); +int32_t vnodeShouldCommit(SVnode* pVnode, bool atExit); void vnodeUpdCommitSched(SVnode* pVnode); void vnodeRollback(SVnode* pVnode); int32_t vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg); @@ -115,7 +115,6 @@ void vnodeSyncClose(SVnode* pVnode); void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg, int32_t code); bool vnodeIsLeader(SVnode* pVnode); bool vnodeIsRoleLeader(SVnode* pVnode); -int vnodeShouldCommit(SVnode* pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index a3f2c4559f94e36efd68bca259bcfee85650774c..b3c8d77ac50cb9a98faacb92f5491b2634d6800f 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -193,9 +193,10 @@ void tqCleanUp(); STQ* tqOpen(const char* path, SVnode* pVnode); void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); -int tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, +int tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type); -int tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer); +int tqUnregisterPushHandle(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer); +int tqStartStreamTasks(STQ* pTq); // restore all stream tasks after vnode launching completed. int tqCommit(STQ*); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 9501bf4b8ef6055f2e4f28242f8f259896623725..795f281ab2107acdce93ec2373ab99e1391b61ba 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -531,10 +531,11 @@ static void freePayload(const void* key, size_t keyLen, void* value) { return; } - SHashObj* pHashObj = (SHashObj*)p[0]; + SHashObj* pHashObj = (SHashObj*)p[0]; + STagFilterResEntry** pEntry = taosHashGet(pHashObj, &p[1], sizeof(uint64_t)); - { + if (pEntry != NULL && (*pEntry) != NULL) { int64_t st = taosGetTimestampUs(); SListIter iter = {0}; @@ -547,9 +548,9 @@ static void freePayload(const void* key, size_t keyLen, void* value) { void* tmp = tdListPopNode(&((*pEntry)->list), pNode); taosMemoryFree(tmp); - int64_t et = taosGetTimestampUs(); - metaInfo("clear items in cache, remain cached item:%d, elapsed time:%.2fms", listNEles(&((*pEntry)->list)), - (et - st) / 1000.0); + double el = (taosGetTimestampUs() - st) / 1000.0; + metaInfo("clear items in meta-cache, remain cached item:%d, elapsed time:%.2fms", listNEles(&((*pEntry)->list)), + el); break; } } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index c75c675ec39946e4f23d005a8050d35b52eca762..ce987ca88ec930db76fcb560ff79f758e2bed1a2 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -168,7 +168,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pRSmaInfo->taskInfo[i]) { - if ((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0) { + if ((terrno = qUpdateTableListForStreamScanner(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0) { tdReleaseRSmaInfo(pSma, pRSmaInfo); smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i, terrstr()); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index abc8a2636997c5c2b0e6919431674ed5c2f032a7..1230a352d93c7fcea6342a7430634e56d17d31c3 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -18,6 +18,7 @@ // 0: not init // 1: already inited // 2: wait to be inited or cleaup +#define WAL_READ_TASKS_ID (-1) int32_t tqInit() { int8_t old; @@ -61,12 +62,12 @@ static void destroyTqHandle(void* data) { if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { taosMemoryFreeClear(pData->execHandle.execCol.qmsg); } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__DB) { - tqCloseReader(pData->execHandle.pExecReader); + tqCloseReader(pData->execHandle.pTqReader); walCloseReader(pData->pWalReader); taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid); } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { walCloseReader(pData->pWalReader); - tqCloseReader(pData->execHandle.pExecReader); + tqCloseReader(pData->execHandle.pTqReader); } } @@ -82,12 +83,18 @@ static void tqPushEntryFree(void* data) { taosMemoryFree(p); } +static bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) { + return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG && + pLeft->val.version <= pRight->val.version; +} + STQ* tqOpen(const char* path, SVnode* pVnode) { STQ* pTq = taosMemoryCalloc(1, sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pTq->path = taosStrdup(path); pTq->pVnode = pVnode; pTq->walLogLastVer = pVnode->pWal->vers.lastVer; @@ -138,44 +145,6 @@ void tqClose(STQ* pTq) { taosMemoryFree(pTq); } -int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp) { - int32_t len = 0; - int32_t code = 0; - tEncodeSize(tEncodeSMqMetaRsp, pRsp, len, code); - if (code < 0) { - return -1; - } - int32_t tlen = sizeof(SMqRspHead) + len; - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - return -1; - } - - ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_META_RSP; - ((SMqRspHead*)buf)->epoch = pReq->epoch; - ((SMqRspHead*)buf)->consumerId = pReq->consumerId; - - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - - SEncoder encoder = {0}; - tEncoderInit(&encoder, abuf, len); - tEncodeSMqMetaRsp(&encoder, pRsp); - tEncoderClear(&encoder); - - SRpcMsg resp = { - .info = pMsg->info, - .pCont = buf, - .contLen = tlen, - .code = 0, - }; - tmsgSendRsp(&resp); - - tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type %d, offset type:%d", - TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->rspOffset.type); - - return 0; -} - static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type) { int32_t len = 0; @@ -253,11 +222,6 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con return 0; } -static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) { - return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG && - pLeft->val.version <= pRight->val.version; -} - int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { STqOffset offset = {0}; int32_t vgId = TD_VID(pTq->pVnode); @@ -330,318 +294,6 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { return 0; } -static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) { - pRsp->reqOffset = pReq->reqOffset; - - pRsp->blockData = taosArrayInit(0, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); - - if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL) { - return -1; - } - - pRsp->withTbName = 0; - pRsp->withSchema = false; - return 0; -} - -static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { - pRsp->reqOffset = pReq->reqOffset; - - pRsp->withTbName = 1; - pRsp->withSchema = 1; - pRsp->blockData = taosArrayInit(0, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); - pRsp->blockTbName = taosArrayInit(0, sizeof(void*)); - pRsp->blockSchema = taosArrayInit(0, sizeof(void*)); - - if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) { - return -1; - } - - return 0; -} - -static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, - SRpcMsg* pMsg, bool* pBlockReturned) { - uint64_t consumerId = pRequest->consumerId; - STqOffsetVal reqOffset = pRequest->reqOffset; - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); - int32_t vgId = TD_VID(pTq->pVnode); - - *pBlockReturned = false; - - // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. - if (pOffset != NULL) { - *pOffsetVal = pOffset->val; - - char formatBuf[80]; - tFormatOffset(formatBuf, 80, pOffsetVal); - tqDebug("tmq poll: consumer:0x%" PRIx64 - ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%" PRIx64, - consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId); - return 0; - } else { - // no poll occurs in this vnode for this topic, let's seek to the right offset value. - if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { - if (pRequest->useSnapshot) { - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey:%s, vgId:%d, (earliest) set offset to be snapshot", - consumerId, pHandle->subKey, vgId); - - if (pHandle->fetchMeta) { - tqOffsetResetToMeta(pOffsetVal, 0); - } else { - tqOffsetResetToData(pOffsetVal, 0, 0); - } - } else { - pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); - if (pHandle->pRef == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - // offset set to previous version when init - tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); - } - } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); - - tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, - pHandle->subKey, vgId, dataRsp.rspOffset.version); - int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); - tDeleteSMqDataRsp(&dataRsp); - - *pBlockReturned = true; - return code; - } else { - STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pRequest); - tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - - *pBlockReturned = true; - return code; - } - } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { - tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 - " in vg %d, subkey %s, reset none failed", - pHandle->subKey, consumerId, vgId, pRequest->subKey); - terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; - return -1; - } - } - - return 0; -} - -#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) - -static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, - SRpcMsg* pMsg, STqOffsetVal* pOffset) { - uint64_t consumerId = pRequest->consumerId; - int32_t vgId = TD_VID(pTq->pVnode); - - SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); - - // lock - taosWLockLatch(&pTq->lock); - - qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); - int code = tqScanData(pTq, pHandle, &dataRsp, pOffset); - if (code != 0) { - goto end; - } - - // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && - dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { - code = tqRegisterPushEntry(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); - taosWUnLockLatch(&pTq->lock); - return code; - } - - code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP); - - // NOTE: this pHandle->consumerId may have been changed already. - -end : { - char buf[80] = {0}; - tFormatOffset(buf, 80, &dataRsp.rspOffset); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 - " code:%d", - consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); - taosWUnLockLatch(&pTq->lock); - tDeleteSMqDataRsp(&dataRsp); -} - return code; -} - -static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, - SRpcMsg* pMsg, STqOffsetVal* offset) { - int code = 0; - int32_t vgId = TD_VID(pTq->pVnode); - SWalCkHead* pCkHead = NULL; - SMqMetaRsp metaRsp = {0}; - STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pRequest); - - if (offset->type != TMQ_OFFSET__LOG) { - if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { - return -1; - } - - if (metaRsp.metaRspLen > 0) { - code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp); - tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 - ",ts:%" PRId64, - pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, - metaRsp.rspOffset.ts); - taosMemoryFree(metaRsp.metaRsp); - tDeleteSTaosxRsp(&taosxRsp); - return code; - } - - tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 - ",ts:%" PRId64, - pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, - taosxRsp.rspOffset.uid, taosxRsp.rspOffset.ts); - if (taosxRsp.blockNum > 0) { - code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - return code; - } else { - *offset = taosxRsp.rspOffset; - } - } - - if (offset->type == TMQ_OFFSET__LOG) { - int64_t fetchVer = offset->version + 1; - pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); - if (pCkHead == NULL) { - tDeleteSTaosxRsp(&taosxRsp); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - walSetReaderCapacity(pHandle->pWalReader, 2048); - int totalRows = 0; - while (1) { - int32_t savedEpoch = atomic_load_32(&pHandle->epoch); - if (savedEpoch > pRequest->epoch) { - tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey:%s vgId:%d offset %" PRId64 - ", found new consumer epoch %d, discard req epoch %d", - pRequest->consumerId, pRequest->epoch, pHandle->subKey, vgId, fetchVer, savedEpoch, pRequest->epoch); - break; - } - - if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead, pRequest->reqId) < 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); - return code; - } - - SWalCont* pHead = &pCkHead->head; - tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", - pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); - - // process meta - if (pHead->msgType != TDMT_VND_SUBMIT) { - if (totalRows > 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer - 1); - code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); - return code; - } - - tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); - tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer); - metaRsp.resMsgType = pHead->msgType; - metaRsp.metaRspLen = pHead->bodyLen; - metaRsp.metaRsp = pHead->body; - if (tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp) < 0) { - code = -1; - taosMemoryFreeClear(pCkHead); - tDeleteSTaosxRsp(&taosxRsp); - return code; - } - code = 0; - taosMemoryFreeClear(pCkHead); - tDeleteSTaosxRsp(&taosxRsp); - return code; - } - - // process data - SPackedData submit = { - .msgStr = POINTER_SHIFT(pHead->body, sizeof(SSubmitReq2Msg)), - .msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg), - .ver = pHead->version, - }; - - if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows) < 0) { - tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId, - pRequest->subKey); - taosMemoryFreeClear(pCkHead); - tDeleteSTaosxRsp(&taosxRsp); - return -1; - } - - if (totalRows >= 4096 || taosxRsp.createTableNum > 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); - return code; - } else { - fetchVer++; - } - } - } - - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); - return 0; -} - -static int32_t doPollDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) { - int32_t code = -1; - STqOffsetVal offset = {0}; - STqOffsetVal reqOffset = pRequest->reqOffset; - - // 1. reset the offset if needed - if (IS_OFFSET_RESET_TYPE(reqOffset.type)) { - // handle the reset offset cases, according to the consumer's choice. - bool blockReturned = false; - code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned); - if (code != 0) { - return code; - } - - // empty block returned, quit - if (blockReturned) { - return 0; - } - } else { // use the consumer specified offset - // the offset value can not be monotonious increase?? - offset = reqOffset; - } - - // this is a normal subscribe requirement - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &offset); - } - - // todo handle the case where re-balance occurs. - // for taosx - return extractDataAndRspForDbStbSubscribe(pTq, pHandle, pRequest, pMsg, &offset); -} - int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { SMqPollReq req = {0}; if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { @@ -689,7 +341,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); - return doPollDataForMq(pTq, pHandle, &req, pMsg); + return tqExtractDataForMq(pTq, pHandle, &req, pMsg); } int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { @@ -815,10 +467,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg &pHandle->execHandle.numOfCols, req.newConsumerId); void* scanner = NULL; qExtractStreamScanner(pHandle->execHandle.task, &scanner); - pHandle->execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); + pHandle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); - pHandle->execHandle.pExecReader = tqOpenReader(pVnode); + pHandle->execHandle.pTqReader = tqOpenReader(pVnode); pHandle->execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); @@ -837,8 +489,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); } - pHandle->execHandle.pExecReader = tqOpenReader(pVnode); - tqReaderSetTbUidList(pHandle->execHandle.pExecReader, tbUidList); + pHandle->execHandle.pTqReader = tqOpenReader(pVnode); + tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, @@ -874,7 +526,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg atomic_store_32(&pHandle->epoch, -1); // remove if it has been register in the push manager, and return one empty block to consumer - tqUnregisterPushEntry(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); + tqUnregisterPushHandle(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_add_fetch_32(&pHandle->epoch, 1); @@ -896,16 +548,14 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { -#if 0 - if (pTask->taskLevel == TASK_LEVEL__AGG) { - A(taosArrayGetSize(pTask->childEpInfo) != 0); - } -#endif + // todo extract method + char buf[128] = {0}; + sprintf(buf, "0x%"PRIx64"-%d", pTask->id.streamId, pTask->id.taskId); int32_t vgId = TD_VID(pTq->pVnode); + pTask->id.idStr = taosStrdup(buf); pTask->refCnt = 1; - pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; - + pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputQueue = streamQueueOpen(); pTask->outputQueue = streamQueueOpen(); @@ -916,11 +566,14 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; pTask->pMsgCb = &pTq->pVnode->msgCb; - pTask->startVer = ver; + pTask->pMeta = pTq->pStreamMeta; + pTask->chkInfo.version = ver; // expand executor if (pTask->fillHistory) { - pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; + pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; + } else { + pTask->status.taskStatus = TASK_STATUS__RESTORE; } if (pTask->taskLevel == TASK_LEVEL__SOURCE) { @@ -930,14 +583,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { } SReadHandle handle = { - .meta = pTq->pVnode->pMeta, - .vnode = pTq->pVnode, - .initTqReader = 1, - .pStateBackend = pTask->pState, - }; - - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); - if (pTask->exec.executor == NULL) { + .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState}; + + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); + if (pTask->exec.pExecutor == NULL) { return -1; } @@ -946,14 +595,12 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { if (pTask->pState == NULL) { return -1; } - SReadHandle mgHandle = { - .vnode = NULL, - .numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo), - .pStateBackend = pTask->pState, - }; - - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId); - if (pTask->exec.executor == NULL) { + + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo); + SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState}; + + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId); + if (pTask->exec.pExecutor == NULL) { return -1; } } @@ -974,16 +621,20 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { ver1 = info.skmVer; } - pTask->tbSink.pTSchema = - tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, ver1); - if (pTask->tbSink.pTSchema == NULL) { + SSchemaWrapper* pschemaWrapper = pTask->tbSink.pSchemaWrapper; + pTask->tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1); + if(pTask->tbSink.pTSchema == NULL) { return -1; } } + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); + } + streamSetupTrigger(pTask); - tqInfo("expand stream task on vg %d, task id %d, child id %d, level %d", vgId, pTask->taskId, pTask->selfChildId, - pTask->taskLevel); + tqInfo("vgId:%d expand stream task, s-task:%s, ver:%" PRId64 " child id:%d, level:%d", vgId, pTask->id.idStr, + pTask->chkInfo.version, pTask->selfChildId, pTask->taskLevel); return 0; } @@ -1006,18 +657,24 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { .upstreamNodeId = req.upstreamNodeId, .upstreamTaskId = req.upstreamTaskId, }; + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - if (pTask && atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL) { - rsp.status = 1; + if (pTask) { + rsp.status = (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL) ? 1 : 0; + streamMetaReleaseTask(pTq->pStreamMeta, pTask); + + tqDebug("tq recv task check req(reqId:0x%" PRIx64 + ") %d at node %d task status:%d, check req from task %d at node %d, rsp status %d", + rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, pTask->status.taskStatus, rsp.upstreamTaskId, + rsp.upstreamNodeId, rsp.status); } else { rsp.status = 0; + tqDebug("tq recv task check(taskId:%d not built yet) req(reqId:0x%" PRIx64 + ") %d at node %d, check req from task %d at node %d, rsp status %d", + taskId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, + rsp.status); } - if (pTask) streamMetaReleaseTask(pTq->pStreamMeta, pTask); - - tqDebug("tq recv task check req(reqId:0x%" PRIx64 ") %d at node %d check req from task %d at node %d, status %d", - rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); - SEncoder encoder; int32_t code; int32_t len; @@ -1035,13 +692,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { tEncodeSStreamTaskCheckRsp(&encoder, &rsp); tEncoderClear(&encoder); - SRpcMsg rspMsg = { - .code = 0, - .pCont = buf, - .contLen = sizeof(SMsgHead) + len, - .info = pMsg->info, - }; - + SRpcMsg rspMsg = { .code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info }; tmsgSendRsp(&rspMsg); return 0; } @@ -1057,8 +708,8 @@ int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t sversion, char* msg, int32 tDecoderClear(&decoder); return -1; } - tDecoderClear(&decoder); + tDecoderClear(&decoder); tqDebug("tq recv task check rsp(reqId:0x%" PRIx64 ") %d at node %d check req from task %d at node %d, status %d", rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); @@ -1090,17 +741,20 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - code = tDecodeSStreamTask(&decoder, pTask); + code = tDecodeStreamTask(&decoder, pTask); if (code < 0) { tDecoderClear(&decoder); taosMemoryFree(pTask); return -1; } + tDecoderClear(&decoder); // 2.save task - code = streamMetaAddTask(pTq->pStreamMeta, sversion, pTask); + code = streamMetaAddDeployedTask(pTq->pStreamMeta, sversion, pTask); if (code < 0) { + tqError("vgId:%d failed to add s-task:%s, total:%d", TD_VID(pTq->pVnode), pTask->id.idStr, + streamMetaGetNumOfTasks(pTq->pStreamMeta)); return -1; } @@ -1109,6 +763,8 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms streamTaskCheckDownstream(pTask, sversion); } + tqDebug("vgId:%d s-task:%s is deployed and add meta from mnd, status:%d, total:%d", TD_VID(pTq->pVnode), + pTask->id.idStr, pTask->status.taskStatus, streamMetaGetNumOfTasks(pTq->pStreamMeta)); return 0; } @@ -1124,7 +780,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { } // check param - int64_t fillVer1 = pTask->startVer; + int64_t fillVer1 = pTask->chkInfo.version; if (fillVer1 <= 0) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); return -1; @@ -1133,7 +789,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { // do recovery step 1 streamSourceRecoverScanStep1(pTask); - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } @@ -1148,7 +804,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { return 0; } @@ -1190,7 +846,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } @@ -1309,7 +965,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; - qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); + qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->id.taskId, ver); if (!failed) { SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); @@ -1318,8 +974,8 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { pRefBlock->dataRef = pRef; atomic_add_fetch_32(pRefBlock->dataRef, 1); - if (tAppendDataForStream(pTask, (SStreamQueueItem*)pRefBlock) < 0) { - qError("stream task input del failed, task id %d", pTask->taskId); + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) { + qError("stream task input del failed, task id %d", pTask->id.taskId); atomic_sub_fetch_32(pRef, 1); taosFreeQitem(pRefBlock); @@ -1327,7 +983,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } if (streamSchedExec(pTask) < 0) { - qError("stream task launch failed, task id %d", pTask->taskId); + qError("stream task launch failed, task id %d", pTask->id.taskId); continue; } @@ -1353,13 +1009,13 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { taosArrayPush(pStreamBlock->blocks, &block); if (!failed) { - if (tAppendDataForStream(pTask, (SStreamQueueItem*)pStreamBlock) < 0) { - qError("stream task input del failed, task id %d", pTask->taskId); + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pStreamBlock) < 0) { + qError("stream task input del failed, task id %d", pTask->id.taskId); continue; } if (streamSchedExec(pTask) < 0) { - qError("stream task launch failed, task id %d", pTask->taskId); + qError("stream task launch failed, task id %d", pTask->id.taskId); continue; } } else { @@ -1372,17 +1028,32 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { return 0; } +static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTask* pTask, SStreamDataSubmit2* pSubmit, + const char* key, int64_t ver) { + doSaveTaskOffset(pOffsetStore, key, ver); + int32_t code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pSubmit, ver); + + // remove the offset, if all functions are completed successfully. + if (code == TSDB_CODE_SUCCESS) { + tqOffsetDelete(pOffsetStore, key); + } + + return code; +} + int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { +#if 0 void* pIter = NULL; - bool succ = true; - - SStreamDataSubmit2* pSubmit = streamDataSubmitNew(submit); + SStreamDataSubmit2* pSubmit = streamDataSubmitNew(submit, STREAM_INPUT__DATA_SUBMIT); if (pSubmit == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to create data submit for stream since out of memory"); - succ = false; + saveOffsetForAllTasks(pTq, submit.ver); + return -1; } + SArray* pInputQueueFullTasks = taosArrayInit(4, POINTER_BYTES); + while (1) { pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); if (pIter == NULL) { @@ -1394,47 +1065,75 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { continue; } - if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { - tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->taskId, pTask->taskStatus); + if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId, + pTask->status.taskStatus); continue; } - tqDebug("data submit enqueue stream task:%d, ver: %" PRId64, pTask->taskId, submit.ver); - if (succ) { - int32_t code = tAppendDataForStream(pTask, (SStreamQueueItem*)pSubmit); - if (code < 0) { - // let's handle the back pressure + // check if offset value exists + char key[128] = {0}; + createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); - tqError("stream task:%d failed to put into queue for, too many", pTask->taskId); - continue; - } + if (tInputQueueIsFull(pTask)) { + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); - if (streamSchedExec(pTask) < 0) { - tqError("stream task:%d launch failed, code:%s", pTask->taskId, tstrerror(terrno)); - continue; + int64_t ver = submit.ver; + if (pOffset == NULL) { + doSaveTaskOffset(pTq->pOffsetStore, key, submit.ver); + } else { + ver = pOffset->val.version; } - } else { - streamTaskInputFail(pTask); + + tqDebug("s-task:%s input queue is full, discard submit block, ver:%" PRId64, pTask->id.idStr, ver); + taosArrayPush(pInputQueueFullTasks, &pTask); + continue; } - } - if (pSubmit != NULL) { - streamDataSubmitDestroy(pSubmit); - taosFreeQitem(pSubmit); + // check if offset value exists + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); + ASSERT(pOffset == NULL); + + addSubmitBlockNLaunchTask(pTq->pOffsetStore, pTask, pSubmit, key, submit.ver); } - return succ ? 0 : -1; + streamDataSubmitDestroy(pSubmit); + taosFreeQitem(pSubmit); +#endif + + tqStartStreamTasks(pTq); + return 0; } int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRunReq* pReq = pMsg->pCont; - int32_t taskId = pReq->taskId; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - if (pTask) { - streamProcessRunReq(pTask); + + int32_t taskId = pReq->taskId; + int32_t vgId = TD_VID(pTq->pVnode); + + if (taskId == WAL_READ_TASKS_ID) { // all tasks are extracted submit data from the wal + tqStreamTasksScanWal(pTq); + return 0; + } + + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); + if (pTask != NULL) { + if (pTask->status.taskStatus == TASK_STATUS__NORMAL) { + tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr); + streamProcessRunReq(pTask); + } else if (pTask->status.taskStatus == TASK_STATUS__RESTORE) { + tqDebug("vgId:%d s-task:%s start to process block from wal, last chk point:%" PRId64, vgId, + pTask->id.idStr, pTask->chkInfo.version); + streamProcessRunReq(pTask); + } else { + tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr); + } + streamMetaReleaseTask(pTq->pStreamMeta, pTask); + tqStartStreamTasks(pTq); return 0; } else { + tqError("vgId:%d failed to found s-task, taskId:%d", vgId, taskId); return -1; } } @@ -1447,14 +1146,10 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecodeStreamDispatchReq(&decoder, &req); - int32_t taskId = req.taskId; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.taskId); if (pTask) { - SRpcMsg rsp = { - .info = pMsg->info, - .code = 0, - }; + SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; streamProcessDispatchReq(pTask, &req, &rsp, exec); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; @@ -1467,7 +1162,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t taskId = ntohl(pRsp->upstreamTaskId); SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - tqDebug("recv dispatch rsp, code: %x", pMsg->code); + tqDebug("recv dispatch rsp, code:%x", pMsg->code); if (pTask) { streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamMetaReleaseTask(pTq->pStreamMeta, pTask); @@ -1495,10 +1190,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { int32_t taskId = req.dstTaskId; SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); if (pTask) { - SRpcMsg rsp = { - .info = pMsg->info, - .code = 0, - }; + SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; streamProcessRetrieveReq(pTask, &req, &rsp); streamMetaReleaseTask(pTq->pStreamMeta, pTask); tDeleteStreamRetrieveReq(&req); @@ -1534,10 +1226,7 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); if (pTask) { - SRpcMsg rsp = { - .info = pMsg->info, - .code = 0, - }; + SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; streamProcessDispatchReq(pTask, &req, &rsp, false); streamMetaReleaseTask(pTq->pStreamMeta, pTask); rpcFreeCont(pMsg->pCont); @@ -1554,10 +1243,7 @@ FAIL: SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp)); if (pRspHead == NULL) { - SRpcMsg rsp = { - .code = TSDB_CODE_OUT_OF_MEMORY, - .info = pMsg->info, - }; + SRpcMsg rsp = { .code = TSDB_CODE_OUT_OF_MEMORY, .info = pMsg->info }; tqDebug("send dispatch error rsp, code: %x", code); tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); @@ -1575,11 +1261,7 @@ FAIL: pRsp->inputStatus = TASK_OUTPUT_STATUS__NORMAL; SRpcMsg rsp = { - .code = code, - .info = pMsg->info, - .contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp), - .pCont = pRspHead, - }; + .code = code, .info = pMsg->info, .contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp), .pCont = pRspHead}; tqDebug("send dispatch error rsp, code: %x", code); tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); @@ -1588,3 +1270,40 @@ FAIL: } int32_t tqCheckLogInWal(STQ* pTq, int64_t sversion) { return sversion <= pTq->walLogLastVer; } + +int32_t tqStartStreamTasks(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + + SStreamMeta* pMeta = pTq->pStreamMeta; + taosWLockLatch(&pMeta->lock); + pMeta->walScan += 1; + + if (pMeta->walScan > 1) { + tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScan); + taosWUnLockLatch(&pTq->pStreamMeta->lock); + return 0; + } + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed restore stream tasks, code:%s", vgId, terrstr(terrno)); + taosWUnLockLatch(&pTq->pStreamMeta->lock); + return -1; + } + + int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pTasks); + + tqInfo("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks); + initOffsetForAllRestoreTasks(pTq); + + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = WAL_READ_TASKS_ID; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); + taosWUnLockLatch(&pTq->pStreamMeta->lock); + + return 0; +} diff --git a/source/dnode/vnode/src/tq/tqCommit.c b/source/dnode/vnode/src/tq/tqCommit.c index 7fc66c49192eb011ce674bc7a2d396825ba1b435..0f5daa31ad412c8ed0773cca8e09cdeeb0fa80f5 100644 --- a/source/dnode/vnode/src/tq/tqCommit.c +++ b/source/dnode/vnode/src/tq/tqCommit.c @@ -16,10 +16,13 @@ #include "tq.h" int tqCommit(STQ* pTq) { +#if 0 + // stream meta commit does not be aligned to the vnode commit if (streamMetaCommit(pTq->pStreamMeta) < 0) { tqError("vgId:%d, failed to commit stream meta since %s", TD_VID(pTq->pVnode), terrstr()); return -1; } +#endif return tqOffsetCommitFile(pTq->pOffsetStore); } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 7b0cdab2f8ac7e49ef7f18fc5e971a0399969d65..cd8cefb30741d2d42891380ede286aab6df91f0a 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -320,15 +320,15 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { code = -1; goto end; } - handle.execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); - if (handle.execHandle.pExecReader == NULL) { + handle.execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); + if (handle.execHandle.pTqReader == NULL) { tqError("cannot extract exec reader for %s", handle.subKey); code = -1; goto end; } } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__DB) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode); + handle.execHandle.pTqReader = tqOpenReader(pTq->pVnode); buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta, (SSnapContext**)(&reader.sContext)); @@ -343,8 +343,8 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); } - handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode); - tqReaderSetTbUidList(handle.execHandle.pExecReader, tbUidList); + handle.execHandle.pTqReader = tqOpenReader(pTq->pVnode); + tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 66d1ac2c7e530e4b8c732deaaa83afc6191b5e44..e8051a140616d2c8296a42c0ca4dca11c3816cf9 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -128,31 +128,35 @@ int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) { } int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { - if (!pStore->needCommit) return 0; + if (!pStore->needCommit) { + return 0; + } + // TODO file name should be with a newer version char* fname = tqOffsetBuildFName(pStore->pTq->path, 0); TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); - - int32_t err = terrno; - const char* errStr = tstrerror(err); - int32_t sysErr = errno; - const char* sysErrStr = strerror(errno); - tqError("vgId:%d, cannot open file %s when commit offset since %s", pStore->pTq->pVnode->config.vgId, fname, - sysErrStr); + const char* err = strerror(errno); + tqError("vgId:%d, failed to open offset file %s, since %s", TD_VID(pStore->pTq->pVnode), fname, err); taosMemoryFree(fname); return -1; } + taosMemoryFree(fname); + void* pIter = NULL; while (1) { pIter = taosHashIterate(pStore->pHash, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + STqOffset* pOffset = (STqOffset*)pIter; int32_t bodyLen; int32_t code; tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code); + if (code < 0) { taosHashCancelIterate(pStore->pHash, pIter); return -1; @@ -166,6 +170,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { SEncoder encoder; tEncoderInit(&encoder, abuf, bodyLen); tEncodeSTqOffset(&encoder, pOffset); + // write file int64_t writeLen; if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) { @@ -174,8 +179,10 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { taosMemoryFree(buf); return -1; } + taosMemoryFree(buf); } + // close and rename file taosCloseFile(&pFile); pStore->needCommit = 0; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index a406a793dc7d80c6399deb3da811adf31b6627a6..7a1a6b74548e285b82663ce2c5b473b825518531 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -323,15 +323,22 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v taosWUnLockLatch(&pTq->lock); } - // push data for stream processing - if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode)) { + tqDebug("handle submit, restore:%d, size:%d", pTq->pVnode->restored, (int)taosHashGetSize(pTq->pStreamMeta->pTasks)); + + // push data for stream processing: + // 1. the vnode has already been restored. + // 2. the vnode should be the leader. + // 3. the stream is not suspended yet. + if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode) && pTq->pVnode->restored) { if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) { return 0; } if (msgType == TDMT_VND_SUBMIT) { +#if 0 void* data = taosMemoryMalloc(len); if (data == NULL) { + // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", vgId); return -1; @@ -340,7 +347,10 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v memcpy(data, pReq, len); SPackedData submit = {.msgStr = data, .msgLen = len, .ver = ver}; - tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq); + tqDebug("vgId:%d tq copy submit msg:%p len:%d ver:%" PRId64 " from %p for stream", vgId, data, len, ver, pReq); + tqProcessSubmitReq(pTq, submit); +#endif + SPackedData submit = {0}; tqProcessSubmitReq(pTq, submit); } @@ -352,7 +362,7 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v return 0; } -int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, +int32_t tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type) { uint64_t consumerId = pRequest->consumerId; int32_t vgId = TD_VID(pTq->pVnode); @@ -389,7 +399,7 @@ int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, return 0; } -int32_t tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) { +int32_t tqUnregisterPushHandle(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) { int32_t vgId = TD_VID(pTq->pVnode); STqPushEntry** pEntry = taosHashGet(pTq->pPushMgr, pKey, keyLen); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 72b478e6bfcf58d4aab59fb776ad91bb9a193180..25ab7209d2ca33c928ba5b648348bcc329189d1f 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -113,7 +113,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) { } SMetaReader mr = {0}; - metaReaderInit(&mr, pHandle->execHandle.pExecReader->pVnodeMeta, 0); + metaReaderInit(&mr, pHandle->execHandle.pTqReader->pVnodeMeta, 0); if (metaGetTableEntryByName(&mr, req.tbName) < 0) { metaReaderClear(&mr); @@ -262,8 +262,6 @@ STqReader* tqOpenReader(SVnode* pVnode) { } pReader->pVnodeMeta = pVnode->pMeta; - /*pReader->pMsg = NULL;*/ -// pReader->ver = -1; pReader->pColIdList = NULL; pReader->cachedSchemaVer = 0; pReader->cachedSchemaSuid = 0; @@ -298,7 +296,29 @@ int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) { if (walReadSeekVer(pReader->pWalReader, ver) < 0) { return -1; } - tqDebug("tmq poll: wal reader seek to ver success ver:%"PRId64" %s", ver, id); + tqDebug("wal reader seek to ver:%"PRId64" %s", ver, id); + return 0; +} + +int32_t extractSubmitMsgFromWal(SWalReader* pReader, SPackedData* pPackedData) { + if (walNextValidMsg(pReader) < 0) { + return -1; + } + + void* pBody = POINTER_SHIFT(pReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + int32_t len = pReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); + int64_t ver = pReader->pHead->head.version; + + void* data = taosMemoryMalloc(len); + if (data == NULL) { + // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", 0); + return -1; + } + + memcpy(data, pBody, len); + *pPackedData = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data}; return 0; } @@ -309,26 +329,28 @@ void tqNextBlock(STqReader* pReader, SFetchRet* ret) { ret->fetchType = FETCH_TYPE__NONE; return; } - void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + + void* pBody = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); int64_t ver = pReader->pWalReader->pHead->head.version; - tqReaderSetSubmitReq2(pReader, body, bodyLen, ver); + tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver); } - while (tqNextDataBlock2(pReader)) { + while (tqNextDataBlock(pReader)) { memset(&ret->data, 0, sizeof(SSDataBlock)); int32_t code = tqRetrieveDataBlock2(&ret->data, pReader, NULL); if (code != 0 || ret->data.info.rows == 0) { continue; } + ret->fetchType = FETCH_TYPE__DATA; return; } } } -int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { +int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { pReader->msg2.msgStr = msgStr; pReader->msg2.msgLen = msgLen; pReader->msg2.ver = ver; @@ -345,7 +367,7 @@ int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, return 0; } -bool tqNextDataBlock2(STqReader* pReader) { +bool tqNextDataBlock(STqReader* pReader) { if (pReader->msg2.msgStr == NULL) { return false; } @@ -354,13 +376,20 @@ bool tqNextDataBlock2(STqReader* pReader) { while (pReader->nextBlk < blockSz) { tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, pReader->msg2.ver, pReader->nextBlk); + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - if (pReader->tbIdHash == NULL) return true; + if (pReader->tbIdHash == NULL) { + return true; + } void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); if (ret != NULL) { + tqDebug("tq reader block found, ver:%"PRId64", uid:%"PRId64, pReader->msg2.ver, pSubmitTbData->uid); return true; + } else { + tqDebug("tq reader discard block, uid:%"PRId64", continue", pSubmitTbData->uid); } + pReader->nextBlk++; } @@ -427,7 +456,10 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); pReader->nextBlk++; - if (pSubmitTbDataRet) *pSubmitTbDataRet = pSubmitTbData; + if (pSubmitTbDataRet) { + *pSubmitTbDataRet = pSubmitTbData; + } + int32_t sversion = pSubmitTbData->sver; int64_t suid = pSubmitTbData->suid; int64_t uid = pSubmitTbData->uid; @@ -900,7 +932,7 @@ int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) { return 0; } -int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) { +int tqReaderAddTbUidList(STqReader* pReader, const SArray* pTableUidList) { if (pReader->tbIdHash == NULL) { pReader->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (pReader->tbIdHash == NULL) { @@ -909,8 +941,9 @@ int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) { } } - for (int i = 0; i < taosArrayGetSize(tbUidList); i++) { - int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); + int32_t numOfTables = taosArrayGetSize(pTableUidList); + for (int i = 0; i < numOfTables; i++) { + int64_t* pKey = (int64_t*)taosArrayGet(pTableUidList, i); taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0); } @@ -926,30 +959,34 @@ int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { return 0; } +// todo update the table list in wal reader int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { - void* pIter = NULL; + void* pIter = NULL; + int32_t vgId = TD_VID(pTq->pVnode); + + // update the table list for each consumer handle while (1) { pIter = taosHashIterate(pTq->pHandle, pIter); if (pIter == NULL) { break; } - STqHandle* pExec = (STqHandle*)pIter; - if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - int32_t code = qUpdateQualifiedTableId(pExec->execHandle.task, tbUidList, isAdd); + STqHandle* pTqHandle = (STqHandle*)pIter; + if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + int32_t code = qUpdateTableListForStreamScanner(pTqHandle->execHandle.task, tbUidList, isAdd); if (code != 0) { - tqError("update qualified table error for %s", pExec->subKey); + tqError("update qualified table error for %s", pTqHandle->subKey); continue; } - } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) { + } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { if (!isAdd) { int32_t sz = taosArrayGetSize(tbUidList); for (int32_t i = 0; i < sz; i++) { int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - taosHashPut(pExec->execHandle.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0); + taosHashPut(pTqHandle->execHandle.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0); } } - } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { + } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (isAdd) { SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); SMetaReader mr = {0}; @@ -964,35 +1001,43 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } tDecoderClear(&mr.coder); - - if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pExec->execHandle.execTb.suid) { + if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pTqHandle->execHandle.execTb.suid) { tqDebug("table uid %" PRId64 " does not add to tq handle", *id); continue; } + tqDebug("table uid %" PRId64 " add to tq handle", *id); taosArrayPush(qa, id); } + metaReaderClear(&mr); if (taosArrayGetSize(qa) > 0) { - tqReaderAddTbUidList(pExec->execHandle.pExecReader, qa); + tqReaderAddTbUidList(pTqHandle->execHandle.pTqReader, qa); } + taosArrayDestroy(qa); } else { - tqReaderRemoveTbUidList(pExec->execHandle.pExecReader, tbUidList); + tqReaderRemoveTbUidList(pTqHandle->execHandle.pTqReader, tbUidList); } } } + + // update the table list handle for each stream scanner/wal reader while (1) { pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd); + int32_t code = qUpdateTableListForStreamScanner(pTask->exec.pExecutor, tbUidList, isAdd); if (code != 0) { - tqError("update qualified table error for stream task %d", pTask->taskId); + tqError("vgId:%d, s-task:%s update qualified table error for stream task", vgId, pTask->id.idStr); continue; } } } + return 0; } diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c new file mode 100644 index 0000000000000000000000000000000000000000..6ed74ddcc3b10ef5705480e6773ea12c4fd1047c --- /dev/null +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tq.h" + +static int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle); +static int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList); + +// this function should be executed by stream threads. +// there is a case that the WAL increases more fast than the restore procedure, and this restore procedure +// will not stop eventually. +int tqStreamTasksScanWal(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + SStreamMeta* pMeta = pTq->pStreamMeta; + int64_t st = taosGetTimestampMs(); + + while (1) { + tqInfo("vgId:%d continue check if data in wal are available", vgId); + + // check all restore tasks + bool allFull = true; + streamTaskReplayWal(pTq->pStreamMeta, pTq->pOffsetStore, &allFull); + + int32_t times = 0; + + if (allFull) { + taosWLockLatch(&pMeta->lock); + pMeta->walScan -= 1; + times = pMeta->walScan; + + if (pMeta->walScan <= 0) { + taosWUnLockLatch(&pMeta->lock); + break; + } + + taosWUnLockLatch(&pMeta->lock); + tqInfo("vgId:%d scan wal for stream tasks for %d times", vgId, times); + } + } + + double el = (taosGetTimestampMs() - st) / 1000.0; + tqInfo("vgId:%d scan wal for stream tasks completed, elapsed time:%.2f sec", vgId, el); + + // restore wal scan flag +// atomic_store_8(&pTq->pStreamMeta->walScan, 0); + return 0; +} + +//int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) { +// int32_t numOfTask = taosArrayGetSize(pTaskList); +// if (numOfTask <= 0) { +// return TSDB_CODE_SUCCESS; +// } +// +// // todo: add lock +// for (int32_t i = 0; i < numOfTask; ++i) { +// SStreamTask* pTask = taosArrayGetP(pTaskList, i); +// tqDebug("vgId:%d transfer s-task:%s state restore -> ready, checkpoint:%" PRId64 " checkpoint id:%" PRId64, +// pStreamMeta->vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->chkInfo.id); +// taosHashRemove(pStreamMeta->pWalReadTasks, &pTask->id.taskId, sizeof(pTask->id.taskId)); +// +// // NOTE: do not change the following order +// atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); +// taosHashPut(pStreamMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES); +// } +// +// return TSDB_CODE_SUCCESS; +//} + +int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) { + void* pIter = NULL; + int32_t vgId = pStreamMeta->vgId; + + *pScanIdle = true; + + bool allWalChecked = true; + tqDebug("vgId:%d start to check wal to extract new submit block", vgId); + + while (1) { + pIter = taosHashIterate(pStreamMeta->pTasks, pIter); + if (pIter == NULL) { + break; + } + + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + continue; + } + + if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || + pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, + pTask->status.taskStatus); + continue; + } + + // check if offset value exists + char key[128] = {0}; + createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); + + if (tInputQueueIsFull(pTask)) { + tqDebug("vgId:%d s-task:%s input queue is full, do nothing", vgId, pTask->id.idStr); + continue; + } + + *pScanIdle = false; + + // check if offset value exists + STqOffset* pOffset = tqOffsetRead(pOffsetStore, key); + ASSERT(pOffset != NULL); + + // seek the stored version and extract data from WAL + int32_t code = walReadSeekVer(pTask->exec.pWalReader, pOffset->val.version); + if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit + continue; + } + + // append the data for the stream + tqDebug("vgId:%d wal reader seek to ver:%" PRId64 " %s", vgId, pOffset->val.version, pTask->id.idStr); + + SPackedData packData = {0}; + code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData); + if (code != TSDB_CODE_SUCCESS) { // failed, continue + continue; + } + + SStreamDataSubmit2* p = streamDataSubmitNew(packData, STREAM_INPUT__DATA_SUBMIT); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("%s failed to create data submit for stream since out of memory", pTask->id.idStr); + continue; + } + + allWalChecked = false; + + tqDebug("s-task:%s submit data extracted from WAL", pTask->id.idStr); + code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver); + if (code == TSDB_CODE_SUCCESS) { + pOffset->val.version = walReaderGetCurrentVer(pTask->exec.pWalReader); + tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr, + pOffset->val.version); + } else { + // do nothing + } + + streamDataSubmitDestroy(p); + taosFreeQitem(p); + } + + if (allWalChecked) { + *pScanIdle = true; + } + return 0; +} + diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index f01e169a53e206ad2852c41c48e076b5524119c4..27db66f048df70ebeaa0e566a591f3ce1ffbcebd 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -38,7 +38,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t } static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp) { - SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader->pSchemaWrapper); + SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pTqReader->pSchemaWrapper); if (pSW == NULL) { return -1; } @@ -137,7 +137,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta if (pDataBlock != NULL && pDataBlock->info.rows > 0) { if (pRsp->withTbName) { if (pOffset->type == TMQ_OFFSET__LOG) { - int64_t uid = pExec->pExecReader->lastBlkUid; + int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, 1) < 0) { continue; } @@ -203,9 +203,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR SArray* pSchemas = taosArrayInit(0, sizeof(void*)); if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { - STqReader* pReader = pExec->pExecReader; - tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); - while (tqNextDataBlock2(pReader)) { + STqReader* pReader = pExec->pTqReader; + tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver); + while (tqNextDataBlock(pReader)) { taosArrayClear(pBlocks); taosArrayClear(pSchemas); SSubmitTbData* pSubmitTbDataRet = NULL; @@ -213,7 +213,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->lastBlkUid; + int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); @@ -262,8 +262,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR } } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { - STqReader* pReader = pExec->pExecReader; - tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); + STqReader* pReader = pExec->pTqReader; + tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver); while (tqNextDataBlockFilterOut2(pReader, pExec->execDb.pFilterOutTbUid)) { taosArrayClear(pBlocks); taosArrayClear(pSchemas); @@ -272,7 +272,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->lastBlkUid; + int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 4645df5b67c121065b0844a7bca345205479e1b2..62b81305b7233edd48e7dab796975b1ab69a7ad3 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -87,7 +87,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d return; } - tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + tqDebug("vgId:%d, s-task:%s write into table, block num: %d", TD_VID(pVnode), pTask->id.idStr, blockSz); for (int32_t i = 0; i < blockSz; i++) { bool createTb = true; SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); @@ -382,7 +382,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* int32_t blockSz = taosArrayGetSize(pBlocks); - tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + tqDebug("vgId:%d, s-task:%s write results blocks:%d into table", TD_VID(pVnode), pTask->id.idStr, blockSz); void* pBuf = NULL; SArray* tagArray = NULL; @@ -475,11 +475,9 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* } for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId); - STagVal tagVal = { - .cid = pTSchema->numOfCols + step, - .type = pTagData->info.type, - }; - void* pData = colDataGetData(pTagData, rowId); + + STagVal tagVal = {.cid = pTSchema->numOfCols + step, .type = pTagData->info.type}; + void* pData = colDataGetData(pTagData, rowId); if (colDataIsNull_s(pTagData, rowId)) { continue; } else if (IS_VAR_DATA_TYPE(pTagData->info.type)) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..4c37e1052f75305c24f3e233274762cdfdff0fff --- /dev/null +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -0,0 +1,462 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tq.h" + +#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) + +static int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp); + +// stream_task:stream_id:task_id +void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) { + int32_t n = 12; + char* p = dst; + + memcpy(p, "stream_task:", n); + p += n; + + int32_t inc = tintToHex(streamId, p); + p += inc; + + *(p++) = ':'; + tintToHex(taskId, p); +} + +int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) { + int32_t code = tAppendDataToInputQueue(pTask, pQueueItem); + if (code < 0) { + tqError("s-task:%s failed to put into queue, too many, next start ver:%" PRId64, pTask->id.idStr, ver); + return -1; + } + + if (streamSchedExec(pTask) < 0) { + tqError("stream task:%d failed to be launched, code:%s", pTask->id.taskId, tstrerror(terrno)); + return -1; + } + + return TSDB_CODE_SUCCESS; +} + +void initOffsetForAllRestoreTasks(STQ* pTq) { + void* pIter = NULL; + + while(1) { + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); + if (pIter == NULL) { + break; + } + + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + continue; + } + + if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("s-task:%s skip push data, since not ready, status %d", pTask->id.idStr, pTask->status.taskStatus); + continue; + } + + char key[128] = {0}; + createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); + + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); + if (pOffset == NULL) { + doSaveTaskOffset(pTq->pOffsetStore, key, pTask->chkInfo.version); + } + } +} + +void saveOffsetForAllTasks(STQ* pTq, int64_t ver) { + void* pIter = NULL; + + while(1) { + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); + if (pIter == NULL) { + break; + } + + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + continue; + } + + if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, + pTask->status.taskStatus); + continue; + } + + char key[128] = {0}; + createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); + + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); + if (pOffset == NULL) { + doSaveTaskOffset(pTq->pOffsetStore, key, ver); + } + } +} + +void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver) { + STqOffset offset = {0}; + tqOffsetResetToLog(&offset.val, ver); + + tstrncpy(offset.subKey, pKey, tListLen(offset.subKey)); + + // keep the offset info in the offset store + tqOffsetWrite(pOffsetStore, &offset); +} + +static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) { + pRsp->reqOffset = pReq->reqOffset; + + pRsp->blockData = taosArrayInit(0, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); + + if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL) { + return -1; + } + + pRsp->withTbName = 0; + pRsp->withSchema = false; + return 0; +} + +static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { + pRsp->reqOffset = pReq->reqOffset; + + pRsp->withTbName = 1; + pRsp->withSchema = 1; + pRsp->blockData = taosArrayInit(0, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); + pRsp->blockTbName = taosArrayInit(0, sizeof(void*)); + pRsp->blockSchema = taosArrayInit(0, sizeof(void*)); + + if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) { + return -1; + } + + return 0; +} + +static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, + SRpcMsg* pMsg, bool* pBlockReturned) { + uint64_t consumerId = pRequest->consumerId; + STqOffsetVal reqOffset = pRequest->reqOffset; + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); + int32_t vgId = TD_VID(pTq->pVnode); + + *pBlockReturned = false; + + // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. + if (pOffset != NULL) { + *pOffsetVal = pOffset->val; + + char formatBuf[80]; + tFormatOffset(formatBuf, 80, pOffsetVal); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%"PRIx64, + consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId); + return 0; + } else { + // no poll occurs in this vnode for this topic, let's seek to the right offset value. + if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { + if (pRequest->useSnapshot) { + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey:%s, vgId:%d, (earliest) set offset to be snapshot", + consumerId, pHandle->subKey, vgId); + + if (pHandle->fetchMeta) { + tqOffsetResetToMeta(pOffsetVal, 0); + } else { + tqOffsetResetToData(pOffsetVal, 0, 0); + } + } else { + pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); + if (pHandle->pRef == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + // offset set to previous version when init + tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); + } + } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); + + tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, + pHandle->subKey, vgId, dataRsp.rspOffset.version); + int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); + tDeleteSMqDataRsp(&dataRsp); + + *pBlockReturned = true; + return code; + } else { + STaosxRsp taosxRsp = {0}; + tqInitTaosxRsp(&taosxRsp, pRequest); + tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); + int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); + tDeleteSTaosxRsp(&taosxRsp); + + *pBlockReturned = true; + return code; + } + } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { + tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", + pHandle->subKey, consumerId, vgId, pRequest->subKey); + terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; + return -1; + } + } + + return 0; +} + +static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, + SRpcMsg* pMsg, STqOffsetVal* pOffset) { + uint64_t consumerId = pRequest->consumerId; + int32_t vgId = TD_VID(pTq->pVnode); + + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); + + // lock + taosWLockLatch(&pTq->lock); + + qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); + int code = tqScanData(pTq, pHandle, &dataRsp, pOffset); + if(code != 0) { + goto end; + } + + // till now, all data has been transferred to consumer, new data needs to push client once arrived. + if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && + dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { + code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); + taosWUnLockLatch(&pTq->lock); + return code; + } + + + code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP); + + // NOTE: this pHandle->consumerId may have been changed already. + + end: + { + char buf[80] = {0}; + tFormatOffset(buf, 80, &dataRsp.rspOffset); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 " code:%d", + consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); + taosWUnLockLatch(&pTq->lock); + tDeleteSMqDataRsp(&dataRsp); + } + return code; +} + + +static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal *offset) { + int code = 0; + int32_t vgId = TD_VID(pTq->pVnode); + SWalCkHead* pCkHead = NULL; + SMqMetaRsp metaRsp = {0}; + STaosxRsp taosxRsp = {0}; + tqInitTaosxRsp(&taosxRsp, pRequest); + + if (offset->type != TMQ_OFFSET__LOG) { + if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { + return -1; + } + + if (metaRsp.metaRspLen > 0) { + code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp); + tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 ",ts:%" PRId64, + pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, metaRsp.rspOffset.ts); + taosMemoryFree(metaRsp.metaRsp); + tDeleteSTaosxRsp(&taosxRsp); + return code; + } + + tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 + ",ts:%" PRId64,pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid,taosxRsp.rspOffset.ts); + if (taosxRsp.blockNum > 0) { + code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); + tDeleteSTaosxRsp(&taosxRsp); + return code; + }else { + *offset = taosxRsp.rspOffset; + } + } + + + if (offset->type == TMQ_OFFSET__LOG) { + int64_t fetchVer = offset->version + 1; + pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); + if (pCkHead == NULL) { + tDeleteSTaosxRsp(&taosxRsp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + walSetReaderCapacity(pHandle->pWalReader, 2048); + int totalRows = 0; + while (1) { + int32_t savedEpoch = atomic_load_32(&pHandle->epoch); + if (savedEpoch > pRequest->epoch) { + tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey:%s vgId:%d offset %" PRId64 + ", found new consumer epoch %d, discard req epoch %d", pRequest->consumerId, pRequest->epoch, pHandle->subKey, vgId, fetchVer, savedEpoch, pRequest->epoch); + break; + } + + if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead, pRequest->reqId) < 0) { + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); + code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); + tDeleteSTaosxRsp(&taosxRsp); + taosMemoryFreeClear(pCkHead); + return code; + } + + SWalCont* pHead = &pCkHead->head; + tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", pRequest->consumerId, + pRequest->epoch, vgId, fetchVer, pHead->msgType); + + // process meta + if (pHead->msgType != TDMT_VND_SUBMIT) { + if(totalRows > 0) { + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer - 1); + code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); + tDeleteSTaosxRsp(&taosxRsp); + taosMemoryFreeClear(pCkHead); + return code; + } + + tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); + tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer); + metaRsp.resMsgType = pHead->msgType; + metaRsp.metaRspLen = pHead->bodyLen; + metaRsp.metaRsp = pHead->body; + if (tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp) < 0) { + code = -1; + taosMemoryFreeClear(pCkHead); + tDeleteSTaosxRsp(&taosxRsp); + return code; + } + code = 0; + taosMemoryFreeClear(pCkHead); + tDeleteSTaosxRsp(&taosxRsp); + return code; + } + + // process data + SPackedData submit = { + .msgStr = POINTER_SHIFT(pHead->body, sizeof(SSubmitReq2Msg)), + .msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg), + .ver = pHead->version, + }; + + if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows) < 0) { + tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId, + pRequest->subKey); + taosMemoryFreeClear(pCkHead); + tDeleteSTaosxRsp(&taosxRsp); + return -1; + } + + if (totalRows >= 4096 || taosxRsp.createTableNum > 0) { + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); + code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); + tDeleteSTaosxRsp(&taosxRsp); + taosMemoryFreeClear(pCkHead); + return code; + } else { + fetchVer++; + } + } + } + + tDeleteSTaosxRsp(&taosxRsp); + taosMemoryFreeClear(pCkHead); + return 0; +} + +int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) { + int32_t code = -1; + STqOffsetVal offset = {0}; + STqOffsetVal reqOffset = pRequest->reqOffset; + + // 1. reset the offset if needed + if (IS_OFFSET_RESET_TYPE(reqOffset.type)) { + // handle the reset offset cases, according to the consumer's choice. + bool blockReturned = false; + code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned); + if (code != 0) { + return code; + } + + // empty block returned, quit + if (blockReturned) { + return 0; + } + } else { // use the consumer specified offset + // the offset value can not be monotonious increase?? + offset = reqOffset; + } + + // this is a normal subscribe requirement + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &offset); + } + + // todo handle the case where re-balance occurs. + // for taosx + return extractDataAndRspForDbStbSubscribe(pTq, pHandle, pRequest, pMsg, &offset); +} + +int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp) { + int32_t len = 0; + int32_t code = 0; + tEncodeSize(tEncodeSMqMetaRsp, pRsp, len, code); + if (code < 0) { + return -1; + } + int32_t tlen = sizeof(SMqRspHead) + len; + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + return -1; + } + + ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_META_RSP; + ((SMqRspHead*)buf)->epoch = pReq->epoch; + ((SMqRspHead*)buf)->consumerId = pReq->consumerId; + + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + + SEncoder encoder = {0}; + tEncoderInit(&encoder, abuf, len); + tEncodeSMqMetaRsp(&encoder, pRsp); + tEncoderClear(&encoder); + + SRpcMsg resp = { + .info = pMsg->info, + .pCont = buf, + .contLen = tlen, + .code = 0, + }; + tmsgSendRsp(&resp); + + tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type %d, offset type:%d", + TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->rspOffset.type); + + return 0; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index f61b1bfcdbe731a2948763dfc2f6e191d1192aa4..f86efb7c5bfd23813f8095df77ba1a94836eab6c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -35,7 +35,11 @@ _err: static void tsdbCloseBICache(STsdb *pTsdb) { SLRUCache *pCache = pTsdb->biCache; if (pCache) { + int32_t elems = taosLRUCacheGetElems(pCache); + tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems); taosLRUCacheEraseUnrefEntries(pCache); + elems = taosLRUCacheGetElems(pCache); + tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems); taosLRUCacheCleanup(pCache); @@ -1112,7 +1116,12 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie * &state->blockIdx); */ state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ); - if (!state->pBlockIdx) { /* + if (!state->pBlockIdx) { + tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + + state->aBlockIdxHandle = NULL; + state->aBlockIdx = NULL; + /* tsdbDataFReaderClose(state->pDataFReader); *state->pDataFReader = NULL; resetLastBlockLoadInfo(state->pLoadInfo);*/ @@ -1761,11 +1770,14 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo hasRow = true; - code = updateTSchema(TSDBROW_SVERSION(pRow), pr, uid); - if (TSDB_CODE_SUCCESS != code) { - goto _err; + int32_t sversion = TSDBROW_SVERSION(pRow); + if (sversion != -1) { + code = updateTSchema(sversion, pr, uid); + if (TSDB_CODE_SUCCESS != code) { + goto _err; + } + pTSchema = pr->pCurrSchema; } - pTSchema = pr->pCurrSchema; int16_t nCol = pTSchema->numOfCols; TSKEY rowTs = TSDBROW_TS(pRow); @@ -1915,11 +1927,14 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach hasRow = true; - code = updateTSchema(TSDBROW_SVERSION(pRow), pr, uid); - if (TSDB_CODE_SUCCESS != code) { - goto _err; + int32_t sversion = TSDBROW_SVERSION(pRow); + if (sversion != -1) { + code = updateTSchema(sversion, pr, uid); + if (TSDB_CODE_SUCCESS != code) { + goto _err; + } + pTSchema = pr->pCurrSchema; } - pTSchema = pr->pCurrSchema; int16_t nCol = pTSchema->numOfCols; TSKEY rowTs = TSDBROW_TS(pRow); @@ -2223,6 +2238,7 @@ int32_t tsdbCacheGetBlockIdx(SLRUCache *pCache, SDataFReader *pFileReader, LRUHa taosThreadMutexUnlock(&pTsdb->biMutex); } + tsdbTrace("bi cache:%p, ref", pCache); *handle = h; return code; @@ -2232,6 +2248,7 @@ int32_t tsdbBICacheRelease(SLRUCache *pCache, LRUHandle *h) { int32_t code = 0; taosLRUCacheRelease(pCache, h, false); + tsdbTrace("bi cache:%p, release", pCache); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 30b9cacd4b62cb7f95216d99cb29c6bfb2f1a65d..964ff2e297b2daeffba50fbb585ba75575b97b7f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -367,15 +367,6 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 p->ts = pCol->ts; p->colVal = pCol->colVal; singleTableLastTs = pCol->ts; - - // only set value for last row query - if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST_ROW)) { - if (taosArrayGetSize(pTableUidList) == 0) { - taosArrayPush(pTableUidList, &pKeyInfo->uid); - } else { - taosArraySet(pTableUidList, 0, &pKeyInfo->uid); - } - } } } else { SLastCol* p = taosArrayGet(pLastCols, slotId); @@ -422,6 +413,12 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } + if (taosArrayGetSize(pTableUidList) == 0) { + taosArrayPush(pTableUidList, &pKeyInfo->uid); + } else { + taosArraySet(pTableUidList, 0, &pKeyInfo->uid); + } + tsdbCacheRelease(lruCache, h); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 996b2e4e45118a5ecc82ac633a572094b47da7f1..89686c3d33f672bb030bc6bec4190e820638dd57 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -831,6 +831,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, // this block belongs to a table that is not queried. STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockIdx->uid, pReader->idStr); if (pScanInfo == NULL) { + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); return terrno; } @@ -2088,7 +2089,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2112,7 +2113,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } } } else { // not merge block data - int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2320,7 +2321,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, pRow, pSchema); } else { - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2352,7 +2353,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, piRow, piSchema); } else { init = true; - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2575,7 +2576,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3242,8 +3243,8 @@ static int32_t readRowsCountFromFiles(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; while (1) { - bool hasNext = false; - int32_t code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext); + bool hasNext = false; + code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext); if (code) { return code; } @@ -3515,8 +3516,8 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_ int64_t startVer = (pCond->startVersion == -1) ? 0 : pCond->startVersion; int64_t endVer = 0; - if (pCond->endVersion == - -1) { // user not specified end version, set current maximum version of vnode as the endVersion + if (pCond->endVersion == -1) { + // user not specified end version, set current maximum version of vnode as the endVersion endVer = pVnode->state.applied; } else { endVer = (pCond->endVersion > pVnode->state.applied) ? pVnode->state.applied : pCond->endVersion; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index dd11134bd0e5a10b498406451be10b9aeda6fae7..8e778da877af5affc40e81e06c3ba7048f7ae439 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1259,6 +1259,11 @@ void tBlockDataReset(SBlockData *pBlockData) { pBlockData->suid = 0; pBlockData->uid = 0; pBlockData->nRow = 0; + for (int32_t i = 0; i < pBlockData->nColData; i++) { + tColDataDestroy(&pBlockData->aColData[i]); + } + pBlockData->nColData = 0; + taosMemoryFreeClear(pBlockData->aColData); } void tBlockDataClear(SBlockData *pBlockData) { diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 09f1ca78776784673a92f2a23d65314bf14b360c..3eb813f3948ae1f5434564400d10ae4f0e1701c0 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -149,7 +149,7 @@ void vnodeUpdCommitSched(SVnode *pVnode) { pVnode->commitSched.maxWaitMs = tsVndCommitMaxIntervalMs + (randNum % tsVndCommitMaxIntervalMs); } -int vnodeShouldCommit(SVnode *pVnode) { +int vnodeShouldCommit(SVnode *pVnode, bool atExit) { SVCommitSched *pSched = &pVnode->commitSched; int64_t nowMs = taosGetMonoTimestampMs(); bool diskAvail = osDataSpaceAvailable(); @@ -158,7 +158,8 @@ int vnodeShouldCommit(SVnode *pVnode) { taosThreadMutexLock(&pVnode->mutex); if (pVnode->inUse && diskAvail) { needCommit = - ((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)); + ((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)) || + ((pVnode->inUse->size > 0) && atExit); } taosThreadMutexUnlock(&pVnode->mutex); return needCommit; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 65fac5a4752c2b6c6584534031e27e1ed302192d..b62bf27def54c75558bd184a330655a6745d871d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -400,7 +400,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } break; case TDMT_STREAM_TASK_DEPLOY: { - if (tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) { + if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) { goto _err; } } break; @@ -447,13 +447,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp walApplyVer(pVnode->pWal, version); - /*vInfo("vgId:%d, push msg begin", pVnode->config.vgId);*/ if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { /*vInfo("vgId:%d, push msg end", pVnode->config.vgId);*/ vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } - /*vInfo("vgId:%d, push msg end", pVnode->config.vgId);*/ // commit if need if (needCommit) { @@ -541,13 +539,10 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return vnodeGetBatchMeta(pVnode, pMsg); case TDMT_VND_TMQ_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg); - case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); -#if 1 case TDMT_STREAM_TASK_DISPATCH: return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true); -#endif case TDMT_STREAM_TASK_CHECK: return tqProcessStreamTaskCheckReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index d681f5b65e2228e6d2b16c88688ba672ad26564d..d4a394b584edb4886984f285f261e1115cc4c444 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -129,8 +129,8 @@ static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak return code; } -void vnodeProposeCommitOnNeed(SVnode *pVnode) { - if (!vnodeShouldCommit(pVnode)) { +void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit) { + if (!vnodeShouldCommit(pVnode, atExit)) { return; } @@ -145,18 +145,20 @@ void vnodeProposeCommitOnNeed(SVnode *pVnode) { rpcMsg.pCont = pHead; rpcMsg.info.noResp = 1; + vInfo("vgId:%d, propose vnode commit", pVnode->config.vgId); bool isWeak = false; - if (vnodeProposeMsg(pVnode, &rpcMsg, isWeak) < 0) { - vTrace("vgId:%d, failed to propose vnode commit since %s", pVnode->config.vgId, terrstr()); - goto _out; - } - vInfo("vgId:%d, proposed vnode commit", pVnode->config.vgId); + if (!atExit) { + if (vnodeProposeMsg(pVnode, &rpcMsg, isWeak) < 0) { + vTrace("vgId:%d, failed to propose vnode commit since %s", pVnode->config.vgId, terrstr()); + } + rpcFreeCont(rpcMsg.pCont); + rpcMsg.pCont = NULL; + } else { + tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &rpcMsg); + } -_out: vnodeUpdCommitSched(pVnode); - rpcFreeCont(rpcMsg.pCont); - rpcMsg.pCont = NULL; } #if BATCH_ENABLE @@ -236,7 +238,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) continue; } - vnodeProposeCommitOnNeed(pVnode); + bool atExit = false; + vnodeProposeCommitOnNeed(pVnode, atExit); code = vnodePreProcessWriteMsg(pVnode, pMsg); if (code != 0) { @@ -288,7 +291,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) continue; } - vnodeProposeCommitOnNeed(pVnode); + bool atExit = false; + vnodeProposeCommitOnNeed(pVnode, atExit); code = vnodePreProcessWriteMsg(pVnode, pMsg); if (code != 0) { @@ -549,6 +553,9 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) pVnode->restored = true; vInfo("vgId:%d, sync restore finished", pVnode->config.vgId); + + // start to restore all stream tasks + tqStartStreamTasks(pVnode->pTq); } static void vnodeBecomeFollower(const SSyncFSM *pFsm) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 7d90560c3322c844c792c3fac6da80ef8b0d6959..2cb6626b0327ca549a0ccd809b98481802c1412d 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -127,14 +127,9 @@ enum { }; typedef struct { - // TODO remove prepareStatus - // STqOffsetVal prepareStatus; // for tmq - STqOffsetVal currentOffset; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - // int8_t returned; - int64_t snapshotVer; - // const SSubmitReq* pReq; - + STqOffsetVal currentOffset; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int64_t snapshotVer; SPackedData submit; SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; @@ -144,6 +139,8 @@ typedef struct { int64_t fillHistoryVer1; int64_t fillHistoryVer2; SStreamState* pState; + int64_t dataVersion; + int64_t checkPointId; } SStreamTaskInfo; typedef struct { @@ -191,7 +188,6 @@ enum { OP_OPENED = 0x1, OP_RES_TO_RETURN = 0x5, OP_EXEC_DONE = 0x9, - // OP_EXEC_RECV = 0x11, }; typedef struct SOperatorFpSet { @@ -560,6 +556,7 @@ typedef struct SStreamIntervalOperatorInfo { uint64_t numOfDatapack; SArray* pUpdated; SSHashObj* pUpdatedMap; + int64_t dataVersion; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { @@ -609,6 +606,7 @@ typedef struct SStreamSessionAggOperatorInfo { bool ignoreExpiredDataSaved; SArray* pUpdated; SSHashObj* pStUpdated; + int64_t dataVersion; } SStreamSessionAggOperatorInfo; typedef struct SStreamStateAggOperatorInfo { @@ -627,6 +625,7 @@ typedef struct SStreamStateAggOperatorInfo { bool ignoreExpiredDataSaved; SArray* pUpdated; SSHashObj* pSeUpdated; + int64_t dataVersion; } SStreamStateAggOperatorInfo; typedef struct SStreamPartitionOperatorInfo { @@ -827,7 +826,7 @@ void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); void doDestroyTask(SExecTaskInfo* pTaskInfo); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); -char* buildTaskId(uint64_t taskId, uint64_t queryId); +void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 204a18ed4ef5119c64adce3e4db4c9b5500ac2a4..2df24463b7609ac37fcb426190f1c24a06b6d67f 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -264,8 +264,6 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pInfo->pRes->info.id.groupId = pKeyInfo->groupId; if (taosArrayGetSize(pInfo->pUidList) > 0) { - ASSERT((pInfo->retrieveType & CACHESCAN_RETRIEVE_LAST_ROW) == CACHESCAN_RETRIEVE_LAST_ROW); - pInfo->pRes->info.id.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, 0); code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, pInfo->pRes->info.rows, GET_TASKID(pTaskInfo), NULL); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 0229631d404b31926866d3bd56df9f6e433f3406..6e3a7d872538e227493915c3beccd4ffafd00f14 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -127,7 +127,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu pOperator->status = OP_NOT_OPENED; SStreamScanInfo* pInfo = pOperator->info; - qDebug("task stream set total blocks:%d %s", (int32_t)numOfBlocks, id); + qDebug("s-task set source blocks:%d %s", (int32_t)numOfBlocks, id); ASSERT(pInfo->validBlockIndex == 0 && taosArrayGetSize(pInfo->pBlockLists) == 0); if (type == STREAM_INPUT__MERGED_SUBMIT) { @@ -173,9 +173,7 @@ void doSetTaskId(SOperatorInfo* pOperator) { void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { SExecTaskInfo* pTaskInfo = tinfo; pTaskInfo->id.queryId = queryId; - - taosMemoryFreeClear(pTaskInfo->id.str); - pTaskInfo->id.str = buildTaskId(taskId, queryId); + buildTaskId(taskId, queryId, pTaskInfo->id.str); // set the idstr for tsdbReader doSetTaskId(pTaskInfo->pRoot); @@ -198,6 +196,13 @@ int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { return code; } +void qGetCheckpointVersion(qTaskInfo_t tinfo, int64_t* dataVer, int64_t* ckId) { + SExecTaskInfo* pTaskInfo = tinfo; + *dataVer = pTaskInfo->streamInfo.dataVersion; + *ckId = pTaskInfo->streamInfo.checkPointId; +} + + int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { return TSDB_CODE_APP_ERROR; @@ -363,27 +368,23 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S return qa; } -int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) { +int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + const char* id = GET_TASKID(pTaskInfo); + int32_t code = 0; if (isAdd) { - qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), pTaskInfo->id.str); + qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), id); } // traverse to the stream scanner node to add this table id - SOperatorInfo* pInfo = pTaskInfo->pRoot; - while (pInfo->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - pInfo = pInfo->pDownstream[0]; - } - - int32_t code = 0; + SOperatorInfo* pInfo = extractOperatorInTree(pTaskInfo->pRoot, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, id); SStreamScanInfo* pScanInfo = pInfo->info; + if (isAdd) { // add new table id SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo)); int32_t numOfQualifiedTables = taosArrayGetSize(qa); - - qDebug(" %d qualified child tables added into stream scanner", numOfQualifiedTables); - + qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id); code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(qa); @@ -424,19 +425,6 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo } } -#if 0 - bool exists = false; - for (int32_t k = 0; k < taosArrayGetSize(pListInfo->pTableList); ++k) { - STableKeyInfo* pKeyInfo = taosArrayGet(pListInfo->pTableList, k); - if (pKeyInfo->uid == keyInfo.uid) { - qWarn("ignore duplicated query table uid:%" PRIu64 " added, %s", pKeyInfo->uid, pTaskInfo->id.str); - exists = true; - } - } - - if (!exists) { -#endif - tableListAddTableInfo(pTableListInfo, keyInfo.uid, keyInfo.groupId); } @@ -447,7 +435,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo taosArrayDestroy(qa); } else { // remove the table id in current list - qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList)); + qDebug("%d remove child tables from the stream scanner, %s", (int32_t)taosArrayGetSize(tableIdList), id); taosWLockLatch(&pTaskInfo->lock); code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList); taosWUnLockLatch(&pTaskInfo->lock); @@ -1273,3 +1261,22 @@ void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { rpcFreeCont(pMsg->pCont); destroySendMsgInfo(pSendInfo); } + +SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = tinfo; + SArray* plist = getTableListInfo(pTaskInfo); + + // only extract table in the first elements + STableListInfo* pTableListInfo = taosArrayGetP(plist, 0); + + SArray* pUidList = taosArrayInit(10, sizeof(uint64_t)); + + int32_t numOfTables = tableListGetSize(pTableListInfo); + for(int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pKeyInfo = tableListGetInfo(pTableListInfo, i); + taosArrayPush(pUidList, &pKeyInfo->uid); + } + + taosArrayDestroy(plist); + return pUidList; +} diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a0697a71026688883bc3865e9e1df6860cb94989..7594079cfb03fbe7b8a3a06b21aa6b8b625deb23 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1151,8 +1151,8 @@ void cleanupExprSupp(SExprSupp* pSupp) { void cleanupBasicInfo(SOptrBasicInfo* pInfo) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -char* buildTaskId(uint64_t taskId, uint64_t queryId) { - char* p = taosMemoryMalloc(64); +void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst) { + char* p = dst; int32_t offset = 6; memcpy(p, "TID:0x", offset); @@ -1163,7 +1163,6 @@ char* buildTaskId(uint64_t taskId, uint64_t queryId) { offset += tintToHex(queryId, &p[offset]); p[offset] = 0; - return p; } SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, @@ -1185,7 +1184,9 @@ SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t v taosInitRWLatch(&pTaskInfo->lock); pTaskInfo->id.vgId = vgId; pTaskInfo->id.queryId = queryId; - pTaskInfo->id.str = buildTaskId(taskId, queryId); + + pTaskInfo->id.str = taosMemoryMalloc(64); + buildTaskId(taskId, queryId, pTaskInfo->id.str); return pTaskInfo; } @@ -2008,7 +2009,11 @@ void qStreamCloseTsdbReader(void* task) { } static void extractTableList(SArray* pList, const SOperatorInfo* pOperator) { - if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { + if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pScanInfo = pOperator->info; + STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info; + taosArrayPush(pList, &pTableScanInfo->base.pTableListInfo); + } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pScanInfo = pOperator->info; taosArrayPush(pList, &pScanInfo->base.pTableListInfo); } else { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8bf2c0d67310a1008a73775d2c568001bb64f1c4..ae396a4c6811fa4c7ab2e11ec8a720cd0d35cf07 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -33,7 +33,7 @@ int32_t scanDebug = 0; -#define MULTI_READER_MAX_TABLE_NUM 5000 +#define MULTI_READER_MAX_TABLE_NUM 5000 #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) @@ -330,7 +330,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, pBlockInfo->id.uid); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); pCost->skipBlocks += 1; tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; @@ -341,7 +341,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { @@ -708,9 +708,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { // todo refactor /*pTableScanInfo->lastStatus.uid = pBlock->info.id.uid;*/ /*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/ -// pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; -// pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; -// pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; + // pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; + // pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; + // pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; return pBlock; } @@ -900,7 +900,7 @@ static void destroyTableScanOperatorInfo(void* param) { SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { - int32_t code = 0; + int32_t code = 0; STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -912,7 +912,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; int32_t numOfCols = 0; - code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->base.matchInfo); + code = + extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->base.matchInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1636,7 +1637,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.submit.msgStr != NULL) { if (pInfo->tqReader->msg2.msgStr == NULL) { SPackedData submit = pTaskInfo->streamInfo.submit; - if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { + if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { qError("submit msg messed up when initing stream submit block %p", submit.msgStr); return NULL; } @@ -1645,7 +1646,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pRes); SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - while (tqNextDataBlock2(pInfo->tqReader)) { + while (tqNextDataBlock(pInfo->tqReader)) { SSDataBlock block = {0}; int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader, NULL); @@ -1668,8 +1669,9 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - qDebug("queue scan tsdb return %"PRId64" rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, pResult->info.rows, - pResult->info.window.skey, pResult->info.window.ekey, pInfo->tqReader->pWalReader->curVersion); + qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, + pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, + pInfo->tqReader->pWalReader->curVersion); tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pResult->info.id.uid, pResult->info.window.ekey); return pResult; } @@ -1687,17 +1689,21 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { while (1) { SFetchRet ret = {0}; tqNextBlock(pInfo->tqReader, &ret); - tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1); //curVersion move to next, so currentOffset = curVersion - 1 + tqOffsetResetToLog( + &pTaskInfo->streamInfo.currentOffset, + pInfo->tqReader->pWalReader->curVersion - 1); // curVersion move to next, so currentOffset = curVersion - 1 if (ret.fetchType == FETCH_TYPE__DATA) { - qDebug("doQueueScan get data from log %"PRId64" rows, version:%" PRId64, ret.data.info.rows, pTaskInfo->streamInfo.currentOffset.version); + qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, ret.data.info.rows, + pTaskInfo->streamInfo.currentOffset.version); blockDataCleanup(pInfo->pRes); setBlockIntoRes(pInfo, &ret.data, true); if (pInfo->pRes->info.rows > 0) { - qDebug("doQueueScan get data from log %"PRId64" rows, return, version:%" PRId64, pInfo->pRes->info.rows, pTaskInfo->streamInfo.currentOffset.version); + qDebug("doQueueScan get data from log %" PRId64 " rows, return, version:%" PRId64, pInfo->pRes->info.rows, + pTaskInfo->streamInfo.currentOffset.version); return pInfo->pRes; } - }else if(ret.fetchType == FETCH_TYPE__NONE){ + } else if (ret.fetchType == FETCH_TYPE__NONE) { qDebug("doQueueScan get none from log, return, version:%" PRId64, pTaskInfo->streamInfo.currentOffset.version); return NULL; } @@ -1806,7 +1812,6 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { /*resetTableScanInfo(pTSInfo, pWin);*/ tsdbReaderClose(pTSInfo->base.dataReader); - qDebug("4"); pTSInfo->base.dataReader = NULL; pInfo->pTableScanOp->status = OP_OPENED; @@ -1889,7 +1894,6 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; tsdbReaderClose(pTSInfo->base.dataReader); - qDebug("5"); pTSInfo->base.dataReader = NULL; @@ -1916,6 +1920,7 @@ FETCH_NEXT_BLOCK: if (pBlock->info.parTbName[0]) { streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); } + // TODO move into scan pBlock->info.calWin.skey = INT64_MIN; pBlock->info.calWin.ekey = INT64_MAX; @@ -2058,7 +2063,7 @@ FETCH_NEXT_BLOCK: int32_t current = pInfo->validBlockIndex++; SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); - if (tqReaderSetSubmitReq2(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { + if (tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { qError("submit msg messed up when initing stream submit block %p, current %d, total %d", pSubmit, current, totBlockNum); continue; @@ -2067,7 +2072,7 @@ FETCH_NEXT_BLOCK: blockDataCleanup(pInfo->pRes); - while (tqNextDataBlock2(pInfo->tqReader)) { + while (tqNextDataBlock(pInfo->tqReader)) { SSDataBlock block = {0}; int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader, NULL); @@ -2176,7 +2181,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { } SMetaTableInfo mtInfo = getUidfromSnapShot(pInfo->sContext); - STqOffsetVal offset = {0}; + STqOffsetVal offset = {0}; if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); tqOffsetResetToLog(&offset, pInfo->sContext->snapVersion); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b42b9c467e2df3b55a5496d359f71a9f84a80ce9..007a6f63d16d59da4b648f43803a058963a6c487 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2333,9 +2333,15 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN return startPos; } +static void setStreamDataVersion(SExecTaskInfo* pTaskInfo, int64_t version, int64_t ckId) { + pTaskInfo->streamInfo.dataVersion = version; + pTaskInfo->streamInfo.checkPointId = ckId; +} + static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t groupId, SSHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; + pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -2432,10 +2438,8 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pSDataBlock->info.rows, numOfOutput); - SWinKey key = { - .ts = nextWin.skey, - .groupId = groupId, - }; + + SWinKey key = { .ts = nextWin.skey, .groupId = groupId }; saveOutputBuf(pInfo->pState, &key, pResult, pInfo->aggSup.resultRowSize); releaseOutputBuf(pInfo->pState, &key, pResult); if (pInfo->delKey.ts > key.ts) { @@ -2503,6 +2507,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamIntervalOperator(pInfo); + setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId); qDebug("===stream===clear semi operator"); } else { deleteIntervalDiscBuf(pInfo->pState, pInfo->pPullDataMap, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark, @@ -2776,6 +2781,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->numOfDatapack = 0; pInfo->pUpdated = NULL; pInfo->pUpdatedMap = NULL; + pInfo->dataVersion = 0; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -3126,6 +3132,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData int32_t rows = pSDataBlock->info.rows; int32_t winRows = 0; + pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); + SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = NULL; @@ -3589,6 +3597,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->ignoreExpiredDataSaved = false; pInfo->pUpdated = NULL; pInfo->pStUpdated = NULL; + pInfo->dataVersion = 0; setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -3899,6 +3908,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t winRows = 0; + + pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); + if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; @@ -4115,6 +4127,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->ignoreExpiredDataSaved = false; pInfo->pUpdated = NULL; pInfo->pSeUpdated = NULL; + pInfo->dataVersion = 0; setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -4750,6 +4763,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { &pInfo->delKey); setOperatorCompleted(pOperator); streamStateCommit(pTaskInfo->streamInfo.pState); + setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId); return NULL; } @@ -4771,6 +4785,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { pInfo->numOfDatapack = 0; break; } + pInfo->numOfDatapack++; printDataBlock(pBlock, "single interval recv"); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 69951f680e745a47c6c7e31947f27f67cb37f97c..fe98a1dd537af5fc48b66c431dcfe524fdefe972 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1933,14 +1933,35 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l } static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - if (1 != LIST_LENGTH(pFunc->pParameterList)) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + int16_t resType = TSDB_DATA_TYPE_BIGINT; + + if (1 != numOfParams && 2 != numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - if (!IS_STR_DATA_TYPE(((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type)) { + uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_STR_DATA_TYPE(para1Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + if (2 == numOfParams) { + uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + if (!IS_INTEGER_TYPE(para2Type)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); + if (pValue->datum.i == 1) { + resType = TSDB_DATA_TYPE_TIMESTAMP; + } else if (pValue->datum.i == 0) { + resType = TSDB_DATA_TYPE_BIGINT; + } else { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "TO_UNIXTIMESTAMP function second parameter should be 0/1"); + } + } + // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); @@ -1948,7 +1969,7 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int return code; } - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType}; return TSDB_CODE_SUCCESS; } @@ -3279,7 +3300,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_irowts", .type = FUNCTION_TYPE_IROWTS, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC|FUNC_MGT_KEEP_ORDER_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, .translateFunc = translateTimePseudoColumn, .getEnvFunc = getTimePseudoFuncEnv, .initFunc = NULL, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 01f81c9a99e9d00528691abec5a345881d9a1991..a8ecd9b0a2f0da0900673448233c8cfc51af01c4 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3353,7 +3353,7 @@ int32_t spreadFunction(SqlFunctionCtx* pCtx) { goto _spread_over; } double tmin = 0.0, tmax = 0.0; - if (IS_SIGNED_NUMERIC_TYPE(type)) { + if (IS_SIGNED_NUMERIC_TYPE(type) || IS_TIMESTAMP_TYPE(type)) { tmin = (double)GET_INT64_VAL(&pAgg->min); tmax = (double)GET_INT64_VAL(&pAgg->max); } else if (IS_FLOAT_TYPE(type)) { diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 94ab616dda984f8ebb87c4a8d709851292cd2881..18f6e8050b56e1c73a28cf64bb24c1da3844651f 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -174,6 +174,8 @@ bool fmIsSelectFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); } +bool fmIsDateTimeFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_DATETIME_FUNC); } + bool fmIsPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); } bool fmIsScanPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCAN_PC_FUNC); } @@ -184,6 +186,7 @@ bool fmIsWindowClauseFunc(int32_t funcId) { return fmIsAggFunc(funcId) || fmIsWi bool fmIsIndefiniteRowsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_INDEFINITE_ROWS_FUNC); } + bool fmIsSpecialDataRequiredFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SPECIAL_DATA_REQUIRED); } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 2269ad7f6a8d5e5d1c67b70be661168907820797..8c8b99a6f804b2b7200e0641ca3d5bdb2660292c 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -343,7 +343,7 @@ typedef struct SUdfcFuncStub { char udfName[TSDB_FUNC_NAME_LEN + 1]; UdfcFuncHandle handle; int32_t refCount; - int64_t lastRefTime; + int64_t createTime; } SUdfcFuncStub; typedef struct SUdfcProxy { @@ -363,6 +363,7 @@ typedef struct SUdfcProxy { uv_mutex_t udfStubsMutex; SArray *udfStubs; // SUdfcFuncStub + SArray *expiredUdfStubs; //SUdfcFuncStub uv_mutex_t udfcUvMutex; int8_t initialized; @@ -959,7 +960,7 @@ int32_t udfcOpen(); int32_t udfcClose(); int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle); -void releaseUdfFuncHandle(char *udfName); +void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle); int32_t cleanUpUdfs(); bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); @@ -967,6 +968,8 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pRes int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); +void cleanupNotExpiredUdfs(); +void cleanupExpiredUdfs(); int compareUdfcFuncSub(const void *elem1, const void *elem2) { SUdfcFuncStub *stub1 = (SUdfcFuncStub *)elem1; SUdfcFuncStub *stub2 = (SUdfcFuncStub *)elem2; @@ -982,16 +985,24 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { if (stubIndex != -1) { SUdfcFuncStub *foundStub = taosArrayGet(gUdfcProxy.udfStubs, stubIndex); UdfcFuncHandle handle = foundStub->handle; - if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { - *pHandle = foundStub->handle; - ++foundStub->refCount; - foundStub->lastRefTime = taosGetTimestampUs(); - uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); - return 0; + int64_t currUs = taosGetTimestampUs(); + bool expired = (currUs - foundStub->createTime) >= 10 * 1000 * 1000; + if (!expired) { + if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { + *pHandle = foundStub->handle; + ++foundStub->refCount; + uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); + return 0; + } else { + fnInfo("udf invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", udfName, + foundStub->refCount, foundStub->createTime); + taosArrayRemove(gUdfcProxy.udfStubs, stubIndex); + } } else { - fnInfo("invalid handle for %s, refCount: %d, last ref time: %" PRId64 ". remove it from cache", udfName, - foundStub->refCount, foundStub->lastRefTime); + fnInfo("udf handle expired for %s, will setup udf. move it to expired list", udfName); taosArrayRemove(gUdfcProxy.udfStubs, stubIndex); + taosArrayPush(gUdfcProxy.expiredUdfStubs, foundStub); + taosArraySort(gUdfcProxy.expiredUdfStubs, compareUdfcFuncSub); } } *pHandle = NULL; @@ -1001,7 +1012,7 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { strncpy(stub.udfName, udfName, TSDB_FUNC_NAME_LEN); stub.handle = *pHandle; ++stub.refCount; - stub.lastRefTime = taosGetTimestampUs(); + stub.createTime = taosGetTimestampUs(); taosArrayPush(gUdfcProxy.udfStubs, &stub); taosArraySort(gUdfcProxy.udfStubs, compareUdfcFuncSub); } else { @@ -1012,32 +1023,51 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { return code; } -void releaseUdfFuncHandle(char *udfName) { +void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle) { uv_mutex_lock(&gUdfcProxy.udfStubsMutex); SUdfcFuncStub key = {0}; strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN); SUdfcFuncStub *foundStub = taosArraySearch(gUdfcProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); - if (!foundStub) { + SUdfcFuncStub *expiredStub = taosArraySearch(gUdfcProxy.expiredUdfStubs, &key, compareUdfcFuncSub, TD_EQ); + if (!foundStub && !expiredStub) { uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return; } - if (foundStub->refCount > 0) { + if (foundStub != NULL && foundStub->handle == handle && foundStub->refCount > 0) { --foundStub->refCount; } + if (expiredStub != NULL && expiredStub->handle == handle && expiredStub->refCount > 0) { + --expiredStub->refCount; + } uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); } -int32_t cleanUpUdfs() { - int8_t initialized = atomic_load_8(&gUdfcProxy.initialized); - if (!initialized) { - return TSDB_CODE_SUCCESS; +void cleanupExpiredUdfs() { + int32_t i = 0; + SArray *expiredUdfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); + while (i < taosArrayGetSize(gUdfcProxy.expiredUdfStubs)) { + SUdfcFuncStub *stub = taosArrayGet(gUdfcProxy.expiredUdfStubs, i); + if (stub->refCount == 0) { + fnInfo("tear down udf. expired. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); + doTeardownUdf(stub->handle); + } else { + fnInfo("udf still in use. expired. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName, + stub->refCount, stub->createTime, stub->handle); + UdfcFuncHandle handle = stub->handle; + if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { + taosArrayPush(expiredUdfStubs, stub); + } else { + fnInfo("udf invalid handle for %s, expired. refCount: %d, create time: %" PRId64 ". remove it from cache", + stub->udfName, stub->refCount, stub->createTime); + } + } + ++i; } + taosArrayDestroy(gUdfcProxy.expiredUdfStubs); + gUdfcProxy.expiredUdfStubs = expiredUdfStubs; +} - uv_mutex_lock(&gUdfcProxy.udfStubsMutex); - if (gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) { - uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); - return TSDB_CODE_SUCCESS; - } +void cleanupNotExpiredUdfs() { SArray *udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); int32_t i = 0; while (i < taosArrayGetSize(gUdfcProxy.udfStubs)) { @@ -1046,20 +1076,38 @@ int32_t cleanUpUdfs() { fnInfo("tear down udf. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); doTeardownUdf(stub->handle); } else { - fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %" PRId64 ", handle: %p", stub->udfName, - stub->refCount, stub->lastRefTime, stub->handle); + fnInfo("udf still in use. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName, + stub->refCount, stub->createTime, stub->handle); UdfcFuncHandle handle = stub->handle; if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { taosArrayPush(udfStubs, stub); } else { - fnInfo("udf invalid handle for %s, refCount: %d, last ref time: %" PRId64 ". remove it from cache", - stub->udfName, stub->refCount, stub->lastRefTime); + fnInfo("udf invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", + stub->udfName, stub->refCount, stub->createTime); } } ++i; } taosArrayDestroy(gUdfcProxy.udfStubs); gUdfcProxy.udfStubs = udfStubs; +} + +int32_t cleanUpUdfs() { + int8_t initialized = atomic_load_8(&gUdfcProxy.initialized); + if (!initialized) { + return TSDB_CODE_SUCCESS; + } + + uv_mutex_lock(&gUdfcProxy.udfStubsMutex); + if ((gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) && + (gUdfcProxy.expiredUdfStubs == NULL || taosArrayGetSize(gUdfcProxy.expiredUdfStubs) == 0)) { + uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); + return TSDB_CODE_SUCCESS; + } + + cleanupNotExpiredUdfs(); + cleanupExpiredUdfs(); + uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return 0; } @@ -1075,7 +1123,7 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, code = doCallUdfScalarFunc(handle, input, numOfCols, output); if (code != TSDB_CODE_SUCCESS) { fnError("udfc scalar function execution failure"); - releaseUdfFuncHandle(udfName); + releaseUdfFuncHandle(udfName, handle); return code; } @@ -1089,7 +1137,7 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; } } - releaseUdfFuncHandle(udfName); + releaseUdfFuncHandle(udfName, handle); return code; } @@ -1122,7 +1170,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult SUdfInterBuf buf = {0}; if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); return false; } if (buf.bufLen <= session->bufSize) { @@ -1131,10 +1179,10 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult udfRes->interResNum = buf.numOfResult; } else { fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); return false; } - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); freeUdfInterBuf(&buf); return true; } @@ -1191,7 +1239,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { taosArrayDestroy(pTempBlock->pDataBlock); taosMemoryFree(pTempBlock); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); freeUdfInterBuf(&newState); return udfCode; } @@ -1236,7 +1284,7 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock) { freeUdfInterBuf(&resultBuf); int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); return udfCallCode == 0 ? numOfResults : udfCallCode; } @@ -1663,6 +1711,7 @@ int32_t udfcOpen() { uv_barrier_wait(&proxy->initBarrier); uv_mutex_init(&proxy->udfStubsMutex); proxy->udfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub)); + proxy->expiredUdfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub)); uv_mutex_init(&proxy->udfcUvMutex); fnInfo("udfc initialized") return 0; } @@ -1679,6 +1728,7 @@ int32_t udfcClose() { uv_thread_join(&udfc->loopThread); uv_mutex_destroy(&udfc->taskQueueMutex); uv_barrier_destroy(&udfc->initBarrier); + taosArrayDestroy(udfc->expiredUdfStubs); taosArrayDestroy(udfc->udfStubs); uv_mutex_destroy(&udfc->udfStubsMutex); uv_mutex_destroy(&udfc->udfcUvMutex); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index aa72309c62f7008c1191b2640444af5f51f0cd7c..5034af2f8264dab3312950369c15d4fb890b38b7 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -591,7 +591,7 @@ SUdf *udfdNewUdf(const char *udfName) { SUdf *udfdGetOrCreateUdf(const char *udfName) { uv_mutex_lock(&global.udfsMutex); SUdf **pUdfHash = taosHashGet(global.udfsHash, udfName, strlen(udfName)); - int64_t currTime = taosGetTimestampSec(); + int64_t currTime = taosGetTimestampMs(); bool expired = false; if (pUdfHash) { expired = currTime - (*pUdfHash)->lastFetchTime > 10 * 1000; // 10s @@ -688,6 +688,8 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { output.colMeta.type = udf->outputType; output.colMeta.precision = 0; output.colMeta.scale = 0; + udfColEnsureCapacity(&output, call->block.info.rows); + SUdfDataBlock input = {0}; convertDataBlockToUdfDataBlock(&call->block, &input); code = udf->scriptPlugin->udfScalarProcFunc(&input, &output, udf->scriptUdfCtx); diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c index 7798a0bf3d957c03240a1f22914a54d0fdcf5b30..5b950879962b56dc38a1c26f8665c13cd6c29c9c 100644 --- a/source/libs/function/test/udf1.c +++ b/source/libs/function/test/udf1.c @@ -24,7 +24,7 @@ DLL_EXPORT int32_t udf1(SUdfDataBlock *block, SUdfColumn *resultCol) { } } if (j == block->numOfCols) { - int32_t luckyNum = 88; + int32_t luckyNum = 1; udfColDataSet(resultCol, i, (char *)&luckyNum, false); } } diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index 4e23999ec2c32e2c82a6b0c569b3ff5c60770ef7..156744ef1dceb9e47860de780904c54997023f5c 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -136,6 +136,7 @@ static bool logicConditionNodeEqual(const SLogicConditionNode* a, const SLogicCo static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) { COMPARE_SCALAR_FIELD(funcId); + COMPARE_STRING_FIELD(functionName); COMPARE_NODE_LIST_FIELD(pParameterList); return true; } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index f99976e0df2480817f268f2438ec53db0a62f570..d7a6baaffeacd0f9c009467b3f41e4a49d335263 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -1047,8 +1047,8 @@ sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. fill_opt(A) ::= . { A = NULL; } fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); } -fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA literal_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } -fill_opt(A) ::= FILL NK_LP VALUE_F NK_COMMA literal_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); } +fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } +fill_opt(A) ::= FILL NK_LP VALUE_F NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); } %type fill_mode { EFillMode } %destructor fill_mode { } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index fd656a5ab70f6f441ad67edcaf9bad1799e5a183..25e92a55ec2adce02505fc4b3c78615764ece5b9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -645,6 +645,10 @@ static bool isSelectStmt(SNode* pCurrStmt) { return NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt); } +static bool isDeleteStmt(SNode* pCurrStmt) { + return NULL != pCurrStmt && QUERY_NODE_DELETE_STMT == nodeType(pCurrStmt); +} + static bool isSetOperator(SNode* pCurrStmt) { return NULL != pCurrStmt && QUERY_NODE_SET_OPERATOR == nodeType(pCurrStmt); } @@ -669,6 +673,9 @@ static uint8_t getPrecisionFromCurrStmt(SNode* pCurrStmt, uint8_t defaultVal) { if (NULL != pCurrStmt && QUERY_NODE_CREATE_STREAM_STMT == nodeType(pCurrStmt)) { return getPrecisionFromCurrStmt(((SCreateStreamStmt*)pCurrStmt)->pQuery, defaultVal); } + if (isDeleteStmt(pCurrStmt)) { + return ((SDeleteStmt*)pCurrStmt)->precision; + } return defaultVal; } @@ -688,6 +695,10 @@ static bool isWindowPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsWindowPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); } +static bool isInterpPseudoColumnFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsInterpPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); +} + static bool isTimelineFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsTimelineFunc(((SFunctionNode*)pNode)->funcId)); } @@ -1235,6 +1246,10 @@ static int32_t calcTypeBytes(SDataType dt) { } static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { + if (pVal->translate) { + return TSDB_CODE_SUCCESS; + } + SDataType dt = pVal->node.resType; dt.bytes = calcTypeBytes(dt); return translateValueImpl(pCxt, pVal, dt, false); @@ -1295,7 +1310,8 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { } static EDealRes haveVectorFunction(SNode* pNode, void* pContext) { - if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode)) { + if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) || + isInterpPseudoColumnFunc(pNode)) { *((bool*)pContext) = true; return DEAL_RES_END; } @@ -1522,6 +1538,21 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc return TSDB_CODE_SUCCESS; } +static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsInterpPseudoColumnFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + if (!isSelectStmt(pCxt->pCurrStmt)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, + "%s must be used in select statements", pFunc->functionName); + } + if (pCxt->currClause == SQL_CLAUSE_WHERE) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, + "%s is not allowed in where clause", pFunc->functionName); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsTimelineFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1539,6 +1570,21 @@ static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFu return TSDB_CODE_SUCCESS; } +static int32_t translateDateTimeFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsDateTimeFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + + if (!isSelectStmt(pCxt->pCurrStmt)) { + return TSDB_CODE_SUCCESS; + } + + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; + pFunc->node.resType.precision = pSelect->precision; + + return TSDB_CODE_SUCCESS; +} + static bool hasFillClause(SNode* pCurrStmt) { if (!isSelectStmt(pCurrStmt)) { return false; @@ -1678,6 +1724,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { SSelectStmt* pSelect = (SSelectStmt*)pCurrStmt; pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId); + if (fmIsIndefiniteRowsFunc(pFunc->funcId)) { pSelect->hasIndefiniteRowsFunc = true; pSelect->returnRows = fmGetFuncReturnRows(pFunc); @@ -1692,7 +1739,8 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { pSelect->hasUniqueFunc = pSelect->hasUniqueFunc ? true : (FUNCTION_TYPE_UNIQUE == pFunc->funcType); pSelect->hasTailFunc = pSelect->hasTailFunc ? true : (FUNCTION_TYPE_TAIL == pFunc->funcType); pSelect->hasInterpFunc = pSelect->hasInterpFunc ? true : (FUNCTION_TYPE_INTERP == pFunc->funcType); - pSelect->hasInterpPseudoColFunc = pSelect->hasInterpPseudoColFunc ? true : fmIsInterpPseudoColumnFunc(pFunc->funcId); + pSelect->hasInterpPseudoColFunc = + pSelect->hasInterpPseudoColFunc ? true : fmIsInterpPseudoColumnFunc(pFunc->funcId); pSelect->hasLastRowFunc = pSelect->hasLastRowFunc ? true : (FUNCTION_TYPE_LAST_ROW == pFunc->funcType); pSelect->hasLastFunc = pSelect->hasLastFunc ? true : (FUNCTION_TYPE_LAST == pFunc->funcType); pSelect->hasTimeLineFunc = pSelect->hasTimeLineFunc ? true : fmIsTimelineFunc(pFunc->funcId); @@ -1819,9 +1867,15 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SFunctionNode* p if (TSDB_CODE_SUCCESS == code) { code = translateInterpFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = translateInterpPseudoColumnFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == code) { code = translateTimelineFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = translateDateTimeFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == code) { code = translateBlockDistFunc(pCxt, pFunc); } @@ -2877,6 +2931,11 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList if (TSDB_CODE_SUCCESS == code) { code = scalarCalculateConstants(pCaseFunc, &pCell->pNode); } + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE != nodeType(pCell->pNode)) { + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value is just a constant"); + } else if (TSDB_CODE_SUCCESS != code) { + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); + } return code; } @@ -2893,9 +2952,9 @@ static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeL if (fillNo >= LIST_LENGTH(pFillValues->pNodeList)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch"); } - if (TSDB_CODE_SUCCESS != - convertFillValue(pCxt, ((SExprNode*)pProject)->resType, pFillValues->pNodeList, fillNo)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); + int32_t code = convertFillValue(pCxt, ((SExprNode*)pProject)->resType, pFillValues->pNodeList, fillNo); + if (TSDB_CODE_SUCCESS != code) { + return code; } ++fillNo; } @@ -2976,12 +3035,13 @@ static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect } static int32_t translateHaving(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (NULL == pSelect->pGroupByList && NULL != pSelect->pHaving) { + if (NULL == pSelect->pGroupByList && NULL == pSelect->pPartitionByList && NULL == pSelect->pWindow && + NULL != pSelect->pHaving) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); } pCxt->currClause = SQL_CLAUSE_HAVING; int32_t code = translateExpr(pCxt, &pSelect->pHaving); - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && (NULL != pSelect->pGroupByList || NULL != pSelect->pWindow)) { code = checkExprForGroupBy(pCxt, &pSelect->pHaving); } return code; @@ -3372,7 +3432,8 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE); } if (pSelect->hasInterpPseudoColFunc) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "Has Interp pseudo column(s) but missing interp function"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, + "Has Interp pseudo column(s) but missing interp function"); } return TSDB_CODE_SUCCESS; } @@ -3741,6 +3802,7 @@ static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) { pCxt->pCurrStmt = (SNode*)pDelete; int32_t code = translateFrom(pCxt, pDelete->pFromTable); if (TSDB_CODE_SUCCESS == code) { + pDelete->precision = ((STableNode*)pDelete->pFromTable)->precision; code = translateDeleteWhere(pCxt, pDelete); } pCxt->currClause = SQL_CLAUSE_SELECT; @@ -5143,26 +5205,32 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); } - if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped"); - } - int32_t tagsLen = 0; for (int32_t i = 0; i < pTableMeta->tableInfo.numOfTags; ++i) { tagsLen += pTagsSchema[i].bytes; } if (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType || - TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType) { + TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType || TSDB_ALTER_TABLE_DROP_COLUMN == pStmt->alterType || + TSDB_ALTER_TABLE_DROP_TAG == pStmt->alterType) { if (TSDB_SUPER_TABLE != pTableMeta->tableType) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Table is not super table"); } const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); - } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || - pSchema->bytes >= calcTypeBytes(pStmt->dataType)) { + return generateSyntaxErrMsg( + &pCxt->msgBuf, + (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType || TSDB_ALTER_TABLE_DROP_COLUMN == pStmt->alterType) + ? TSDB_CODE_PAR_INVALID_COLUMN + : TSDB_CODE_PAR_INVALID_TAG_NAME, + pStmt->colName); + } + + if ((TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType || + TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType) && + (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || + pSchema->bytes >= calcTypeBytes(pStmt->dataType))) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL); } @@ -5207,6 +5275,10 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable } } + if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped"); + } + return TSDB_CODE_SUCCESS; } @@ -5679,6 +5751,14 @@ static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pSt static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) { // The statement is executed directly on the client without constructing a message. + if ('\0' != pStmt->value[0]) { + return TSDB_CODE_SUCCESS; + } + char* p = strchr(pStmt->config, ' '); + if (NULL != p) { + *p = 0; + strcpy(pStmt->value, p + 1); + } return TSDB_CODE_SUCCESS; } @@ -8257,10 +8337,6 @@ static void destoryAlterTbReq(SVAlterTbReq* pReq) { static int32_t rewriteAlterTableImpl(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, SQuery* pQuery) { - if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped"); - } - if (TSDB_SUPER_TABLE == pTableMeta->tableType) { return TSDB_CODE_SUCCESS; } else if (TSDB_CHILD_TABLE != pTableMeta->tableType && TSDB_NORMAL_TABLE != pTableMeta->tableType) { diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 9ad5bcf644a93718f6e754f3f806c404ac48bf85..a38e2368ddd23702b0e2b5a993b2e11da1fbc950 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -217,331 +217,343 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2905) +#define YY_ACTTAB_COUNT (3030) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 2116, 1881, 2010, 499, 432, 1877, 500, 1758, 431, 2102, - /* 10 */ 670, 2051, 46, 44, 1646, 1722, 2102, 2008, 640, 2098, - /* 20 */ 393, 504, 1495, 1520, 39, 38, 2098, 501, 45, 43, - /* 30 */ 42, 41, 40, 1576, 1791, 1493, 2134, 2010, 1520, 132, - /* 40 */ 131, 130, 129, 128, 127, 126, 125, 124, 2084, 384, - /* 50 */ 669, 591, 2007, 640, 2275, 2094, 2100, 374, 244, 1571, - /* 60 */ 28, 1944, 2094, 2100, 375, 19, 663, 652, 371, 2281, - /* 70 */ 184, 638, 1501, 663, 2276, 617, 1942, 628, 140, 1869, - /* 80 */ 507, 2115, 107, 500, 1758, 2151, 36, 296, 169, 2117, - /* 90 */ 673, 2119, 2120, 668, 168, 663, 1734, 758, 141, 9, + /* 0 */ 2116, 209, 2010, 1881, 432, 502, 1883, 1765, 431, 2102, + /* 10 */ 670, 2010, 48, 46, 1646, 1722, 2098, 2008, 640, 2098, + /* 20 */ 393, 1522, 1495, 384, 41, 40, 2007, 640, 47, 45, + /* 30 */ 44, 43, 42, 1576, 1791, 1493, 2134, 180, 1520, 132, + /* 40 */ 131, 130, 129, 128, 127, 126, 125, 124, 2084, 1931, + /* 50 */ 669, 591, 2094, 2100, 2275, 2094, 2100, 388, 398, 1571, + /* 60 */ 30, 1937, 1939, 663, 499, 19, 663, 500, 1758, 2281, + /* 70 */ 184, 168, 1501, 1734, 2276, 617, 1520, 628, 140, 1869, + /* 80 */ 507, 2115, 107, 500, 1758, 2151, 38, 296, 169, 2117, + /* 90 */ 673, 2119, 2120, 668, 655, 663, 2176, 758, 141, 9, /* 100 */ 15, 735, 734, 733, 732, 403, 1884, 731, 730, 144, /* 110 */ 725, 724, 723, 722, 721, 720, 719, 157, 715, 714, /* 120 */ 713, 402, 401, 710, 709, 708, 707, 706, 592, 2241, - /* 130 */ 398, 2219, 1320, 1937, 1939, 123, 1578, 1579, 122, 121, + /* 130 */ 62, 2219, 1320, 1938, 1939, 123, 1578, 1579, 122, 121, /* 140 */ 120, 119, 118, 117, 116, 115, 114, 1311, 695, 694, /* 150 */ 693, 1315, 692, 1317, 1318, 691, 688, 2216, 1326, 685, - /* 160 */ 1328, 1329, 682, 679, 177, 652, 1551, 1561, 2280, 1409, - /* 170 */ 1410, 2275, 1577, 1580, 1944, 653, 1892, 630, 182, 2212, - /* 180 */ 2213, 356, 138, 2217, 358, 1993, 1496, 2279, 1494, 1942, - /* 190 */ 1720, 2276, 2278, 133, 287, 288, 516, 39, 38, 286, - /* 200 */ 537, 45, 43, 42, 41, 40, 278, 62, 703, 155, + /* 160 */ 1328, 1329, 682, 679, 504, 652, 1551, 1561, 2280, 277, + /* 170 */ 501, 2275, 1577, 1580, 1944, 653, 1892, 630, 182, 2212, + /* 180 */ 2213, 356, 138, 2217, 1409, 1410, 1496, 2279, 1494, 1942, + /* 190 */ 1720, 2276, 2278, 133, 287, 288, 1989, 41, 40, 286, + /* 200 */ 537, 47, 45, 44, 43, 42, 278, 52, 703, 155, /* 210 */ 154, 700, 699, 698, 152, 1499, 1500, 1794, 1550, 1553, /* 220 */ 1554, 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, 1569, - /* 230 */ 1570, 1572, 1573, 1574, 1575, 2, 46, 44, 425, 1169, - /* 240 */ 1522, 341, 62, 1518, 393, 49, 1495, 62, 611, 93, + /* 230 */ 1570, 1572, 1573, 1574, 1575, 2, 48, 46, 425, 1169, + /* 240 */ 192, 341, 62, 1518, 393, 51, 1495, 62, 87, 93, /* 250 */ 469, 2116, 616, 483, 350, 2275, 482, 1576, 177, 1493, - /* 260 */ 406, 670, 427, 423, 405, 45, 43, 42, 41, 40, - /* 270 */ 615, 184, 452, 50, 484, 2276, 617, 454, 1171, 1994, - /* 280 */ 1174, 1175, 180, 1571, 555, 554, 553, 2134, 1723, 19, - /* 290 */ 106, 545, 137, 549, 1931, 1520, 1501, 548, 1605, 2084, + /* 260 */ 406, 670, 427, 423, 405, 47, 45, 44, 43, 42, + /* 270 */ 615, 184, 452, 1888, 484, 2276, 617, 454, 1171, 1994, + /* 280 */ 1174, 1175, 187, 1571, 555, 554, 553, 2134, 1723, 19, + /* 290 */ 106, 545, 137, 549, 551, 550, 1501, 548, 1605, 2084, /* 300 */ 103, 669, 547, 552, 366, 365, 1521, 591, 546, 123, /* 310 */ 2275, 1639, 122, 121, 120, 119, 118, 117, 116, 115, /* 320 */ 114, 758, 359, 101, 15, 2281, 184, 430, 2280, 429, /* 330 */ 2276, 617, 2115, 1191, 442, 1190, 2151, 653, 1892, 110, /* 340 */ 2117, 673, 2119, 2120, 668, 1522, 663, 1885, 226, 143, /* 350 */ 438, 150, 2175, 2204, 1606, 133, 428, 387, 2200, 187, - /* 360 */ 1578, 1579, 542, 480, 1519, 1192, 474, 473, 472, 471, + /* 360 */ 1578, 1579, 542, 480, 652, 1192, 474, 473, 472, 471, /* 370 */ 468, 467, 466, 465, 464, 460, 459, 458, 457, 340, - /* 380 */ 449, 448, 447, 209, 444, 443, 357, 502, 277, 1765, - /* 390 */ 1551, 1561, 2280, 338, 187, 2275, 1577, 1580, 1650, 187, - /* 400 */ 555, 554, 553, 705, 1520, 1938, 1939, 545, 137, 549, - /* 410 */ 1496, 2279, 1494, 548, 606, 2276, 2277, 1868, 547, 552, - /* 420 */ 366, 365, 1354, 1355, 546, 187, 1708, 1264, 35, 391, + /* 380 */ 449, 448, 447, 177, 444, 443, 357, 653, 1892, 638, + /* 390 */ 1551, 1561, 2280, 338, 187, 2275, 1577, 1580, 165, 187, + /* 400 */ 555, 554, 553, 358, 1993, 189, 1895, 545, 137, 549, + /* 410 */ 1496, 2279, 1494, 548, 606, 2276, 2277, 2070, 547, 552, + /* 420 */ 366, 365, 1354, 1355, 546, 2279, 1708, 1264, 37, 391, /* 430 */ 1600, 1601, 1602, 1603, 1604, 1608, 1609, 1610, 1611, 1499, - /* 440 */ 1500, 1552, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, + /* 440 */ 1500, 1974, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, /* 450 */ 1560, 665, 661, 1569, 1570, 1572, 1573, 1574, 1575, 2, - /* 460 */ 12, 46, 44, 227, 1266, 1495, 167, 1521, 211, 393, - /* 470 */ 2116, 1495, 502, 1833, 1765, 1674, 578, 652, 1493, 172, - /* 480 */ 631, 705, 1576, 476, 1493, 533, 529, 525, 521, 224, - /* 490 */ 2219, 39, 38, 277, 1745, 45, 43, 42, 41, 40, - /* 500 */ 612, 607, 600, 1523, 1523, 66, 2134, 1883, 1571, 551, - /* 510 */ 550, 1501, 1520, 610, 19, 1501, 2215, 2098, 2084, 639, + /* 460 */ 12, 48, 46, 227, 1266, 1495, 413, 1521, 610, 393, + /* 470 */ 2116, 1495, 652, 196, 167, 1674, 516, 1520, 1493, 172, + /* 480 */ 631, 1833, 1576, 476, 1493, 533, 529, 525, 521, 224, + /* 490 */ 253, 41, 40, 277, 2134, 47, 45, 44, 43, 42, + /* 500 */ 612, 607, 600, 1523, 611, 66, 2134, 2103, 1571, 12, + /* 510 */ 1191, 10, 1190, 84, 19, 1501, 83, 2098, 2084, 639, /* 520 */ 669, 1501, 603, 602, 1672, 1673, 1675, 1676, 1677, 88, - /* 530 */ 39, 38, 222, 12, 45, 43, 42, 41, 40, 2134, - /* 540 */ 758, 1944, 2084, 200, 199, 543, 758, 1744, 381, 15, - /* 550 */ 1552, 2115, 1607, 2094, 2100, 2151, 1942, 49, 110, 2117, - /* 560 */ 673, 2119, 2120, 668, 663, 663, 475, 1262, 166, 514, - /* 570 */ 181, 2003, 2204, 316, 39, 38, 387, 2200, 45, 43, - /* 580 */ 42, 41, 40, 1426, 1427, 1578, 1579, 314, 73, 186, - /* 590 */ 1743, 72, 62, 609, 560, 2084, 1191, 2230, 1190, 221, - /* 600 */ 215, 62, 87, 639, 220, 12, 512, 10, 2219, 570, - /* 610 */ 207, 495, 493, 490, 696, 1551, 1561, 396, 360, 1425, - /* 620 */ 1428, 1577, 1580, 240, 213, 162, 33, 1887, 1192, 1496, - /* 630 */ 717, 1494, 372, 1894, 2214, 1496, 1612, 1494, 2084, 563, - /* 640 */ 1942, 2067, 653, 1892, 557, 653, 1892, 1944, 1742, 239, - /* 650 */ 62, 255, 193, 637, 386, 2003, 2102, 541, 1499, 1500, - /* 660 */ 189, 540, 1942, 55, 1499, 1500, 2098, 1550, 1553, 1554, + /* 530 */ 41, 40, 222, 91, 47, 45, 44, 43, 42, 1868, + /* 540 */ 758, 211, 1192, 200, 199, 502, 758, 1765, 609, 15, + /* 550 */ 1650, 2115, 51, 2094, 2100, 2151, 1520, 62, 110, 2117, + /* 560 */ 673, 2119, 2120, 668, 663, 663, 475, 2051, 166, 514, + /* 570 */ 181, 2003, 2204, 316, 41, 40, 387, 2200, 47, 45, + /* 580 */ 44, 43, 42, 1426, 1427, 1578, 1579, 314, 73, 186, + /* 590 */ 657, 72, 2176, 718, 560, 1854, 456, 2230, 1745, 221, + /* 600 */ 215, 62, 705, 705, 220, 455, 512, 14, 13, 570, + /* 610 */ 207, 495, 493, 490, 244, 1551, 1561, 396, 385, 1425, + /* 620 */ 1428, 1577, 1580, 240, 213, 162, 165, 653, 1892, 1496, + /* 630 */ 1177, 1494, 372, 1894, 1894, 1496, 1519, 1494, 616, 563, + /* 640 */ 1942, 2275, 653, 1892, 557, 57, 2084, 1519, 1944, 239, + /* 650 */ 62, 255, 44, 43, 42, 371, 615, 184, 1499, 1500, + /* 660 */ 436, 2276, 617, 1942, 1499, 1500, 1523, 1550, 1553, 1554, /* 670 */ 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, 1569, 1570, - /* 680 */ 1572, 1573, 1574, 1575, 2, 46, 44, 1581, 109, 70, - /* 690 */ 1944, 2116, 69, 393, 1879, 1495, 2084, 397, 653, 1892, - /* 700 */ 1715, 631, 2094, 2100, 388, 1942, 1576, 1741, 1493, 187, - /* 710 */ 718, 32, 1854, 663, 628, 140, 436, 39, 38, 653, - /* 720 */ 1892, 45, 43, 42, 41, 40, 1834, 2134, 81, 80, - /* 730 */ 435, 87, 1571, 191, 164, 729, 727, 437, 639, 2084, - /* 740 */ 1875, 669, 39, 38, 187, 1501, 45, 43, 42, 41, - /* 750 */ 40, 653, 1892, 187, 339, 2084, 1888, 421, 1974, 1989, - /* 760 */ 419, 415, 411, 408, 428, 1177, 1740, 653, 1892, 446, - /* 770 */ 758, 1519, 2115, 47, 653, 1892, 2151, 2116, 1739, 110, - /* 780 */ 2117, 673, 2119, 2120, 668, 461, 663, 670, 648, 1767, - /* 790 */ 2003, 181, 462, 2204, 34, 1643, 1738, 387, 2200, 1714, - /* 800 */ 39, 38, 187, 192, 45, 43, 42, 41, 40, 1578, - /* 810 */ 1579, 653, 1892, 2134, 2084, 275, 2212, 627, 2231, 134, - /* 820 */ 626, 569, 2275, 628, 140, 2084, 2084, 669, 1896, 515, - /* 830 */ 1523, 653, 1892, 196, 567, 1685, 565, 615, 184, 1551, - /* 840 */ 1561, 1989, 2276, 617, 2084, 1577, 1580, 39, 38, 1889, - /* 850 */ 420, 45, 43, 42, 41, 40, 1867, 142, 2115, 1496, - /* 860 */ 2175, 1494, 2151, 653, 1892, 110, 2117, 673, 2119, 2120, - /* 870 */ 668, 1944, 663, 84, 242, 249, 83, 2295, 241, 2204, - /* 880 */ 363, 245, 165, 387, 2200, 194, 1943, 634, 1499, 1500, - /* 890 */ 1895, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, - /* 900 */ 665, 661, 1569, 1570, 1572, 1573, 1574, 1575, 2, 46, - /* 910 */ 44, 2116, 655, 456, 2176, 653, 1892, 393, 1737, 1495, - /* 920 */ 1736, 670, 455, 2238, 183, 2212, 2213, 253, 138, 2217, - /* 930 */ 1576, 2116, 1493, 587, 591, 1174, 1175, 2275, 90, 345, - /* 940 */ 1662, 667, 370, 657, 571, 2176, 364, 2134, 362, 361, - /* 950 */ 1733, 539, 2281, 184, 1586, 1735, 1571, 2276, 617, 2084, - /* 960 */ 1520, 669, 653, 1892, 653, 1892, 2084, 2134, 2084, 1501, - /* 970 */ 91, 1275, 541, 42, 41, 40, 540, 2279, 254, 2084, - /* 980 */ 632, 669, 636, 2103, 1274, 703, 155, 154, 700, 699, - /* 990 */ 698, 152, 2115, 2098, 758, 2077, 2151, 47, 2084, 110, - /* 1000 */ 2117, 673, 2119, 2120, 668, 2116, 663, 2078, 14, 13, - /* 1010 */ 1732, 2295, 2115, 2204, 1731, 670, 2151, 387, 2200, 332, - /* 1020 */ 2117, 673, 2119, 2120, 668, 666, 663, 654, 2169, 2094, - /* 1030 */ 2100, 39, 38, 1578, 1579, 45, 43, 42, 41, 40, - /* 1040 */ 663, 2134, 591, 385, 1642, 2275, 485, 1989, 620, 653, - /* 1050 */ 1892, 165, 660, 2084, 591, 669, 664, 2275, 2084, 1894, - /* 1060 */ 2281, 184, 2084, 1551, 1561, 2276, 617, 291, 697, 1577, - /* 1070 */ 1580, 1935, 2281, 184, 2224, 1639, 396, 2276, 617, 1717, - /* 1080 */ 1718, 653, 1892, 1496, 165, 1494, 2115, 653, 1892, 1279, - /* 1090 */ 2151, 198, 1894, 170, 2117, 673, 2119, 2120, 668, 650, - /* 1100 */ 663, 576, 1278, 628, 140, 651, 1552, 52, 1619, 3, - /* 1110 */ 243, 701, 1499, 1500, 1935, 1550, 1553, 1554, 1555, 1556, + /* 680 */ 1572, 1573, 1574, 1575, 2, 48, 46, 1581, 109, 70, + /* 690 */ 1944, 2116, 69, 393, 619, 1495, 1744, 381, 653, 1892, + /* 700 */ 1715, 631, 1552, 1221, 187, 1942, 1576, 1619, 1493, 187, + /* 710 */ 2219, 34, 729, 727, 628, 140, 437, 41, 40, 653, + /* 720 */ 1892, 47, 45, 44, 43, 42, 578, 2134, 81, 80, + /* 730 */ 435, 541, 1571, 191, 164, 540, 2215, 446, 639, 2084, + /* 740 */ 1222, 669, 41, 40, 2084, 1501, 47, 45, 44, 43, + /* 750 */ 42, 653, 1892, 187, 339, 142, 697, 421, 2175, 1935, + /* 760 */ 419, 415, 411, 408, 428, 12, 243, 653, 1892, 461, + /* 770 */ 758, 1501, 2115, 49, 653, 1892, 2151, 2116, 639, 110, + /* 780 */ 2117, 673, 2119, 2120, 668, 462, 663, 670, 637, 1767, + /* 790 */ 2003, 181, 515, 2204, 36, 396, 87, 387, 2200, 1714, + /* 800 */ 41, 40, 187, 165, 47, 45, 44, 43, 42, 1578, + /* 810 */ 1579, 1894, 360, 2134, 193, 275, 2212, 627, 2231, 134, + /* 820 */ 626, 1887, 2275, 628, 140, 2084, 696, 669, 648, 1944, + /* 830 */ 2003, 653, 1892, 1468, 1469, 1685, 386, 615, 184, 1551, + /* 840 */ 1561, 1989, 2276, 617, 1942, 1577, 1580, 41, 40, 1889, + /* 850 */ 717, 47, 45, 44, 43, 42, 1867, 590, 2115, 1496, + /* 860 */ 1275, 1494, 2151, 653, 1892, 110, 2117, 673, 2119, 2120, + /* 870 */ 668, 1944, 663, 1274, 242, 1944, 1989, 2295, 241, 2204, + /* 880 */ 363, 245, 397, 387, 2200, 194, 1943, 634, 1499, 1500, + /* 890 */ 1942, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, + /* 900 */ 665, 661, 1569, 1570, 1572, 1573, 1574, 1575, 2, 48, + /* 910 */ 46, 2116, 1743, 1174, 1175, 653, 1892, 393, 1742, 1495, + /* 920 */ 198, 670, 1877, 2238, 183, 2212, 2213, 2067, 138, 2217, + /* 930 */ 1576, 2116, 1493, 587, 591, 485, 1523, 2275, 90, 345, + /* 940 */ 1662, 667, 370, 1643, 571, 1741, 364, 2134, 362, 361, + /* 950 */ 1740, 539, 2281, 184, 1879, 569, 1571, 2276, 617, 2084, + /* 960 */ 2084, 669, 653, 1892, 653, 1892, 2084, 2134, 567, 1501, + /* 970 */ 565, 701, 541, 1875, 1935, 702, 540, 55, 1935, 2084, + /* 980 */ 632, 669, 636, 2219, 1896, 703, 155, 154, 700, 699, + /* 990 */ 698, 152, 2115, 2084, 758, 2077, 2151, 49, 2084, 110, + /* 1000 */ 2117, 673, 2119, 2120, 668, 2116, 663, 2078, 54, 2214, + /* 1010 */ 3, 2295, 2115, 2204, 1520, 670, 2151, 387, 2200, 332, + /* 1020 */ 2117, 673, 2119, 2120, 668, 666, 663, 654, 2169, 653, + /* 1030 */ 1892, 41, 40, 1578, 1579, 47, 45, 44, 43, 42, + /* 1040 */ 399, 2134, 591, 1739, 249, 2275, 1586, 291, 165, 653, + /* 1050 */ 1892, 153, 1520, 2084, 591, 669, 1894, 2275, 653, 1892, + /* 1060 */ 2281, 184, 1279, 1551, 1561, 2276, 617, 650, 1607, 1577, + /* 1070 */ 1580, 232, 2281, 184, 230, 1278, 651, 2276, 617, 2224, + /* 1080 */ 1639, 653, 1892, 1496, 254, 1494, 2115, 1738, 653, 1892, + /* 1090 */ 2151, 2084, 439, 170, 2117, 673, 2119, 2120, 668, 297, + /* 1100 */ 663, 576, 1737, 628, 140, 440, 400, 1736, 56, 146, + /* 1110 */ 1733, 135, 1499, 1500, 1732, 1550, 1553, 1554, 1555, 1556, /* 1120 */ 1557, 1558, 1559, 1560, 665, 661, 1569, 1570, 1572, 1573, - /* 1130 */ 1574, 1575, 2, 46, 44, 653, 1892, 653, 1892, 1730, - /* 1140 */ 1729, 393, 399, 1495, 618, 2296, 1728, 2116, 591, 2070, - /* 1150 */ 165, 2275, 1727, 297, 1576, 400, 1493, 670, 1894, 2251, - /* 1160 */ 153, 623, 439, 1726, 487, 1725, 2281, 184, 1870, 252, - /* 1170 */ 310, 2276, 617, 1921, 2116, 440, 702, 1468, 1469, 1935, - /* 1180 */ 1571, 74, 232, 2134, 670, 230, 598, 2084, 2084, 146, - /* 1190 */ 573, 135, 572, 1501, 2084, 2084, 616, 669, 413, 2275, - /* 1200 */ 2084, 590, 1597, 544, 185, 2212, 2213, 1781, 138, 2217, - /* 1210 */ 2134, 2084, 148, 2084, 615, 184, 2244, 54, 758, 2276, - /* 1220 */ 617, 15, 2084, 234, 669, 1260, 233, 272, 2115, 556, - /* 1230 */ 82, 1504, 2151, 153, 2116, 110, 2117, 673, 2119, 2120, - /* 1240 */ 668, 1503, 663, 604, 670, 236, 619, 2295, 235, 2204, - /* 1250 */ 1774, 1772, 225, 387, 2200, 2115, 153, 1578, 1579, 2151, - /* 1260 */ 64, 711, 110, 2117, 673, 2119, 2120, 668, 238, 663, - /* 1270 */ 2134, 237, 558, 561, 2295, 64, 2204, 259, 621, 266, - /* 1280 */ 387, 2200, 2084, 1240, 669, 2105, 2135, 1551, 1561, 1998, - /* 1290 */ 1463, 1221, 404, 1577, 1580, 1759, 703, 155, 154, 700, - /* 1300 */ 699, 698, 152, 14, 13, 153, 1764, 1496, 48, 1494, - /* 1310 */ 1932, 284, 2234, 1466, 71, 2115, 151, 1671, 153, 2151, - /* 1320 */ 629, 53, 169, 2117, 673, 2119, 2120, 668, 1222, 663, - /* 1330 */ 48, 1768, 1670, 64, 261, 48, 1499, 1500, 2107, 1550, + /* 1130 */ 1574, 1575, 2, 48, 46, 2084, 420, 487, 1870, 1731, + /* 1140 */ 1730, 393, 35, 1495, 618, 2296, 1729, 2116, 591, 623, + /* 1150 */ 2084, 2275, 1612, 1728, 1576, 2084, 1493, 670, 2084, 2251, + /* 1160 */ 1552, 620, 2084, 1727, 1726, 1725, 2281, 184, 74, 543, + /* 1170 */ 310, 2276, 617, 1921, 2116, 234, 236, 1504, 233, 235, + /* 1180 */ 1571, 544, 148, 2134, 670, 1781, 598, 2084, 2084, 153, + /* 1190 */ 252, 1262, 1642, 1501, 2084, 2084, 238, 669, 1552, 237, + /* 1200 */ 573, 2084, 572, 1260, 185, 2212, 2213, 556, 138, 2217, + /* 1210 */ 2134, 2084, 2084, 2084, 153, 50, 50, 82, 758, 1774, + /* 1220 */ 259, 15, 2084, 1772, 669, 1717, 1718, 664, 2115, 1834, + /* 1230 */ 14, 13, 2151, 153, 2116, 110, 2117, 673, 2119, 2120, + /* 1240 */ 668, 558, 663, 1503, 670, 561, 1463, 2295, 50, 2204, + /* 1250 */ 2105, 284, 660, 387, 2200, 2115, 71, 1578, 1579, 2151, + /* 1260 */ 151, 153, 110, 2117, 673, 2119, 2120, 668, 2244, 663, + /* 1270 */ 2134, 1466, 1671, 1670, 2295, 64, 2204, 261, 50, 1735, + /* 1280 */ 387, 2200, 2084, 225, 669, 272, 604, 1551, 1561, 2135, + /* 1290 */ 635, 266, 404, 1577, 1580, 1998, 703, 155, 154, 700, + /* 1300 */ 699, 698, 152, 2107, 1759, 1423, 1932, 1496, 289, 1494, + /* 1310 */ 2234, 50, 711, 645, 677, 2115, 151, 293, 1305, 2151, + /* 1320 */ 1768, 153, 169, 2117, 673, 2119, 2120, 668, 1764, 663, + /* 1330 */ 1507, 629, 1613, 136, 1240, 1562, 1499, 1500, 151, 1550, /* 1340 */ 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, - /* 1350 */ 1569, 1570, 1572, 1573, 1574, 1575, 2, 271, 274, 390, - /* 1360 */ 389, 677, 635, 2242, 151, 1423, 153, 2116, 289, 1509, - /* 1370 */ 136, 645, 151, 293, 712, 1305, 1, 670, 5, 2269, - /* 1380 */ 1576, 753, 1502, 407, 1507, 412, 354, 309, 1446, 441, - /* 1390 */ 1613, 304, 1562, 624, 1506, 197, 1238, 2116, 1523, 1999, - /* 1400 */ 445, 478, 450, 2134, 1518, 463, 1571, 670, 1991, 2223, - /* 1410 */ 470, 477, 479, 488, 489, 2084, 202, 669, 1332, 1501, - /* 1420 */ 486, 1336, 201, 1343, 491, 492, 204, 1341, 494, 156, - /* 1430 */ 496, 1524, 497, 2134, 4, 498, 505, 506, 1526, 508, - /* 1440 */ 212, 1521, 509, 214, 659, 2084, 1525, 669, 2115, 510, - /* 1450 */ 1527, 511, 2151, 217, 513, 110, 2117, 673, 2119, 2120, - /* 1460 */ 668, 219, 663, 85, 86, 2116, 1194, 2295, 517, 2204, - /* 1470 */ 223, 536, 534, 387, 2200, 670, 535, 538, 2115, 344, - /* 1480 */ 2060, 1882, 2151, 2057, 229, 110, 2117, 673, 2119, 2120, - /* 1490 */ 668, 1878, 663, 112, 577, 575, 231, 2295, 89, 2204, - /* 1500 */ 158, 2134, 159, 387, 2200, 149, 1880, 1876, 160, 2056, - /* 1510 */ 246, 161, 582, 2084, 580, 669, 305, 581, 585, 250, - /* 1520 */ 1453, 588, 8, 605, 2250, 248, 643, 595, 2249, 586, - /* 1530 */ 257, 601, 614, 1510, 2235, 1505, 2245, 376, 608, 2226, - /* 1540 */ 265, 173, 596, 594, 260, 268, 2115, 267, 593, 622, - /* 1550 */ 2151, 2298, 625, 110, 2117, 673, 2119, 2120, 668, 269, - /* 1560 */ 663, 377, 1513, 1515, 2274, 2179, 139, 2204, 270, 1639, - /* 1570 */ 1522, 387, 2200, 2116, 2220, 661, 1569, 1570, 1572, 1573, - /* 1580 */ 1574, 1575, 279, 670, 633, 380, 1528, 96, 2004, 306, - /* 1590 */ 641, 642, 2018, 2116, 2017, 2016, 307, 646, 273, 383, - /* 1600 */ 98, 647, 308, 670, 61, 100, 1893, 2185, 102, 2134, - /* 1610 */ 1936, 311, 754, 1855, 2076, 755, 675, 757, 51, 346, - /* 1620 */ 2075, 2084, 347, 669, 2074, 315, 300, 335, 320, 2134, - /* 1630 */ 313, 334, 78, 2071, 409, 1486, 410, 1487, 324, 190, - /* 1640 */ 414, 2084, 416, 669, 2069, 417, 418, 2068, 355, 2066, - /* 1650 */ 422, 2065, 424, 2064, 2115, 79, 426, 1449, 2151, 1448, - /* 1660 */ 2030, 110, 2117, 673, 2119, 2120, 668, 2029, 663, 2028, - /* 1670 */ 433, 434, 2027, 2177, 2115, 2204, 1400, 2116, 2151, 387, - /* 1680 */ 2200, 110, 2117, 673, 2119, 2120, 668, 670, 663, 2026, - /* 1690 */ 1982, 1981, 1979, 656, 145, 2204, 1978, 1977, 1980, 387, - /* 1700 */ 2200, 2116, 1976, 1975, 1973, 1972, 1971, 195, 451, 1970, - /* 1710 */ 453, 670, 1984, 2134, 1969, 1968, 1967, 1966, 1965, 1964, - /* 1720 */ 1963, 1962, 1961, 1960, 1959, 2084, 1958, 669, 1957, 1956, - /* 1730 */ 1955, 1954, 1953, 1952, 1983, 147, 1951, 2134, 1950, 1949, - /* 1740 */ 1948, 1947, 481, 1946, 1945, 1797, 203, 1402, 342, 2084, - /* 1750 */ 1796, 669, 1795, 205, 206, 343, 1793, 1276, 2115, 1280, - /* 1760 */ 1754, 1176, 2151, 218, 2024, 111, 2117, 673, 2119, 2120, - /* 1770 */ 668, 178, 663, 1753, 2047, 2037, 2025, 1272, 2116, 2204, - /* 1780 */ 2002, 1871, 2115, 2203, 2200, 76, 2151, 77, 670, 111, - /* 1790 */ 2117, 673, 2119, 2120, 668, 208, 663, 216, 2104, 210, - /* 1800 */ 1792, 2116, 179, 2204, 503, 1790, 518, 658, 2200, 520, - /* 1810 */ 519, 670, 1788, 522, 2134, 1214, 523, 1786, 524, 526, - /* 1820 */ 1784, 527, 528, 530, 2116, 532, 2084, 1771, 669, 1770, - /* 1830 */ 1750, 531, 1873, 1348, 670, 1347, 1872, 2134, 1263, 1261, - /* 1840 */ 726, 1259, 1258, 1257, 1256, 1255, 728, 2116, 1250, 2084, - /* 1850 */ 1252, 669, 1782, 1251, 1249, 367, 1775, 670, 1773, 671, - /* 1860 */ 2134, 228, 368, 2151, 369, 559, 111, 2117, 673, 2119, - /* 1870 */ 2120, 668, 2084, 663, 669, 63, 562, 1749, 564, 1748, - /* 1880 */ 2204, 566, 2115, 2134, 349, 2200, 2151, 1747, 568, 111, - /* 1890 */ 2117, 673, 2119, 2120, 668, 2084, 663, 669, 113, 1473, - /* 1900 */ 1475, 27, 1472, 2204, 2046, 2115, 1459, 1455, 2201, 2151, - /* 1910 */ 2116, 67, 326, 2117, 673, 2119, 2120, 668, 1457, 663, - /* 1920 */ 670, 2036, 1477, 583, 2023, 2021, 2280, 20, 2115, 1687, - /* 1930 */ 56, 17, 2151, 6, 29, 170, 2117, 673, 2119, 2120, - /* 1940 */ 668, 7, 663, 589, 256, 584, 2134, 597, 258, 599, - /* 1950 */ 59, 382, 60, 373, 163, 613, 1669, 171, 2084, 251, - /* 1960 */ 669, 262, 30, 263, 1661, 264, 21, 65, 92, 2105, - /* 1970 */ 31, 1707, 1708, 22, 1702, 2116, 1701, 378, 1706, 1705, - /* 1980 */ 379, 1636, 1635, 2022, 276, 667, 2020, 2297, 2019, 2001, - /* 1990 */ 58, 2115, 94, 95, 174, 2151, 2116, 282, 333, 2117, - /* 2000 */ 673, 2119, 2120, 668, 283, 663, 670, 23, 644, 2116, - /* 2010 */ 1667, 2134, 285, 290, 68, 2000, 97, 292, 295, 670, - /* 2020 */ 103, 13, 24, 2084, 1511, 669, 1588, 99, 1587, 11, - /* 2030 */ 1543, 1598, 2134, 2154, 175, 1566, 1564, 392, 662, 188, - /* 2040 */ 57, 1563, 672, 674, 2084, 2134, 669, 18, 37, 16, - /* 2050 */ 394, 25, 676, 1535, 1333, 26, 2115, 2084, 395, 669, - /* 2060 */ 2151, 579, 678, 332, 2117, 673, 2119, 2120, 668, 1330, - /* 2070 */ 663, 680, 2170, 681, 683, 1327, 684, 2115, 686, 761, - /* 2080 */ 1321, 2151, 687, 689, 333, 2117, 673, 2119, 2120, 668, - /* 2090 */ 2115, 663, 1319, 303, 2151, 690, 2116, 333, 2117, 673, - /* 2100 */ 2119, 2120, 668, 1325, 663, 104, 670, 1324, 298, 176, - /* 2110 */ 105, 1342, 1323, 75, 1338, 751, 747, 743, 739, 301, - /* 2120 */ 2116, 1322, 1212, 704, 1246, 1245, 1244, 1270, 1243, 1242, - /* 2130 */ 670, 1241, 2134, 1239, 1237, 1236, 1235, 1233, 716, 299, - /* 2140 */ 1232, 1231, 1230, 1229, 2084, 1228, 669, 1227, 1265, 1267, - /* 2150 */ 1224, 1223, 1218, 1220, 1219, 1789, 2134, 1217, 736, 108, - /* 2160 */ 737, 1787, 294, 738, 740, 742, 1785, 744, 2084, 1783, - /* 2170 */ 669, 748, 1769, 746, 741, 752, 745, 574, 750, 1166, - /* 2180 */ 749, 2151, 1746, 2116, 328, 2117, 673, 2119, 2120, 668, - /* 2190 */ 302, 663, 756, 670, 649, 1497, 312, 759, 760, 1721, - /* 2200 */ 1721, 2115, 1721, 1721, 1721, 2151, 1721, 2116, 317, 2117, - /* 2210 */ 673, 2119, 2120, 668, 1721, 663, 1721, 670, 1721, 2134, - /* 2220 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 281, - /* 2230 */ 1721, 2084, 1721, 669, 280, 1721, 1721, 1721, 1721, 1721, - /* 2240 */ 1721, 1721, 1721, 2134, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2250 */ 1721, 1721, 1721, 2116, 247, 2084, 1721, 669, 1721, 1721, - /* 2260 */ 1721, 1721, 1721, 670, 2115, 1721, 1721, 1721, 2151, 2116, - /* 2270 */ 1721, 318, 2117, 673, 2119, 2120, 668, 1721, 663, 670, - /* 2280 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 2134, - /* 2290 */ 1721, 1721, 2151, 1721, 1721, 319, 2117, 673, 2119, 2120, - /* 2300 */ 668, 2084, 663, 669, 1721, 2134, 1721, 1721, 1721, 1721, - /* 2310 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, - /* 2320 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, - /* 2330 */ 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, 670, - /* 2340 */ 1721, 325, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, - /* 2350 */ 2115, 1721, 1721, 2116, 2151, 1721, 1721, 329, 2117, 673, - /* 2360 */ 2119, 2120, 668, 670, 663, 2134, 1721, 1721, 1721, 1721, - /* 2370 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, - /* 2380 */ 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2134, - /* 2390 */ 670, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, - /* 2400 */ 1721, 2084, 1721, 669, 1721, 1721, 1721, 1721, 1721, 670, - /* 2410 */ 2115, 1721, 1721, 1721, 2151, 1721, 2134, 321, 2117, 673, - /* 2420 */ 2119, 2120, 668, 1721, 663, 1721, 1721, 1721, 2084, 1721, - /* 2430 */ 669, 1721, 1721, 1721, 2115, 2134, 1721, 1721, 2151, 1721, - /* 2440 */ 1721, 330, 2117, 673, 2119, 2120, 668, 2084, 663, 669, - /* 2450 */ 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2460 */ 670, 2115, 1721, 1721, 1721, 2151, 2116, 1721, 322, 2117, - /* 2470 */ 673, 2119, 2120, 668, 1721, 663, 670, 1721, 1721, 1721, - /* 2480 */ 2115, 1721, 1721, 1721, 2151, 1721, 2134, 331, 2117, 673, - /* 2490 */ 2119, 2120, 668, 1721, 663, 1721, 2116, 1721, 2084, 1721, - /* 2500 */ 669, 1721, 2134, 1721, 1721, 1721, 670, 1721, 1721, 1721, - /* 2510 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 2116, 1721, 1721, - /* 2520 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, - /* 2530 */ 1721, 2115, 2134, 1721, 1721, 2151, 1721, 1721, 323, 2117, - /* 2540 */ 673, 2119, 2120, 668, 2084, 663, 669, 2115, 1721, 1721, - /* 2550 */ 1721, 2151, 1721, 2134, 336, 2117, 673, 2119, 2120, 668, - /* 2560 */ 1721, 663, 1721, 2116, 1721, 2084, 1721, 669, 1721, 1721, - /* 2570 */ 1721, 1721, 1721, 670, 1721, 1721, 1721, 2115, 1721, 1721, - /* 2580 */ 1721, 2151, 1721, 1721, 337, 2117, 673, 2119, 2120, 668, - /* 2590 */ 1721, 663, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 2134, - /* 2600 */ 1721, 1721, 2151, 1721, 1721, 2128, 2117, 673, 2119, 2120, - /* 2610 */ 668, 2084, 663, 669, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2620 */ 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2630 */ 1721, 1721, 1721, 670, 1721, 1721, 2116, 1721, 1721, 1721, - /* 2640 */ 1721, 1721, 1721, 1721, 2115, 1721, 670, 1721, 2151, 1721, - /* 2650 */ 1721, 2127, 2117, 673, 2119, 2120, 668, 1721, 663, 2134, - /* 2660 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2670 */ 1721, 2084, 2134, 669, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2680 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 2116, 1721, 1721, - /* 2690 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, - /* 2700 */ 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, 1721, - /* 2710 */ 1721, 2126, 2117, 673, 2119, 2120, 668, 2115, 663, 1721, - /* 2720 */ 1721, 2151, 1721, 2134, 351, 2117, 673, 2119, 2120, 668, - /* 2730 */ 1721, 663, 1721, 2116, 1721, 2084, 1721, 669, 1721, 1721, - /* 2740 */ 1721, 1721, 1721, 670, 1721, 1721, 1721, 1721, 2116, 1721, - /* 2750 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, - /* 2760 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 2134, - /* 2770 */ 1721, 1721, 2151, 1721, 1721, 352, 2117, 673, 2119, 2120, - /* 2780 */ 668, 2084, 663, 669, 2134, 1721, 1721, 1721, 1721, 1721, - /* 2790 */ 1721, 1721, 1721, 1721, 2116, 1721, 2084, 1721, 669, 1721, - /* 2800 */ 1721, 1721, 1721, 1721, 670, 1721, 1721, 1721, 1721, 2116, - /* 2810 */ 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, 670, - /* 2820 */ 1721, 348, 2117, 673, 2119, 2120, 668, 1721, 663, 2115, - /* 2830 */ 2134, 1721, 1721, 2151, 1721, 1721, 353, 2117, 673, 2119, - /* 2840 */ 2120, 668, 2084, 663, 669, 2134, 1721, 1721, 1721, 1721, - /* 2850 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, - /* 2860 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2870 */ 1721, 1721, 1721, 1721, 1721, 671, 1721, 1721, 1721, 2151, - /* 2880 */ 1721, 1721, 328, 2117, 673, 2119, 2120, 668, 1721, 663, - /* 2890 */ 2115, 1721, 1721, 1721, 2151, 1721, 1721, 327, 2117, 673, - /* 2900 */ 2119, 2120, 668, 1721, 663, + /* 1350 */ 1569, 1570, 1572, 1573, 1574, 1575, 2, 274, 271, 390, + /* 1360 */ 389, 712, 1, 2242, 5, 407, 412, 2116, 309, 1509, + /* 1370 */ 753, 1332, 354, 1336, 1446, 304, 197, 670, 1343, 2269, + /* 1380 */ 1576, 624, 1502, 1238, 441, 1523, 1999, 445, 450, 478, + /* 1390 */ 1341, 621, 1518, 463, 1991, 156, 1506, 2116, 470, 477, + /* 1400 */ 479, 488, 1597, 2134, 489, 486, 1571, 670, 201, 2223, + /* 1410 */ 202, 491, 492, 1524, 497, 2084, 204, 669, 494, 1501, + /* 1420 */ 496, 4, 498, 505, 506, 1526, 508, 212, 1521, 509, + /* 1430 */ 214, 1525, 1527, 2134, 510, 513, 511, 217, 1194, 517, + /* 1440 */ 112, 534, 535, 219, 659, 2084, 85, 669, 2115, 86, + /* 1450 */ 223, 536, 2151, 538, 344, 110, 2117, 673, 2119, 2120, + /* 1460 */ 668, 575, 663, 2060, 89, 2116, 577, 2295, 305, 2204, + /* 1470 */ 1882, 229, 1878, 387, 2200, 670, 231, 158, 2115, 246, + /* 1480 */ 159, 1880, 2151, 1876, 160, 110, 2117, 673, 2119, 2120, + /* 1490 */ 668, 161, 663, 581, 2057, 580, 250, 2295, 1453, 2204, + /* 1500 */ 588, 2134, 605, 387, 2200, 2056, 585, 595, 643, 601, + /* 1510 */ 2250, 2235, 582, 2084, 2249, 669, 248, 149, 2245, 8, + /* 1520 */ 376, 257, 608, 614, 586, 260, 596, 594, 2226, 593, + /* 1530 */ 267, 265, 377, 1510, 625, 1505, 2298, 173, 622, 1639, + /* 1540 */ 268, 139, 1522, 633, 380, 279, 2115, 96, 1528, 2004, + /* 1550 */ 2151, 270, 646, 110, 2117, 673, 2119, 2120, 668, 306, + /* 1560 */ 663, 641, 1513, 1515, 2220, 2179, 647, 2204, 98, 642, + /* 1570 */ 307, 387, 2200, 2116, 2018, 661, 1569, 1570, 1572, 1573, + /* 1580 */ 1574, 1575, 1893, 670, 100, 61, 269, 2017, 2185, 102, + /* 1590 */ 2016, 383, 308, 2116, 675, 1936, 1855, 300, 311, 2274, + /* 1600 */ 754, 757, 273, 670, 755, 53, 335, 320, 346, 2134, + /* 1610 */ 347, 313, 315, 2076, 334, 324, 2075, 2074, 78, 2071, + /* 1620 */ 409, 2084, 410, 669, 1486, 1487, 190, 414, 2069, 2134, + /* 1630 */ 416, 417, 418, 2068, 355, 2066, 422, 2065, 2064, 426, + /* 1640 */ 424, 2084, 79, 669, 1449, 1448, 2030, 2029, 2028, 433, + /* 1650 */ 434, 2027, 2026, 1400, 2115, 1982, 1981, 1979, 2151, 1978, + /* 1660 */ 1977, 110, 2117, 673, 2119, 2120, 668, 145, 663, 1980, + /* 1670 */ 1976, 1975, 1973, 2177, 2115, 2204, 195, 2116, 2151, 387, + /* 1680 */ 2200, 110, 2117, 673, 2119, 2120, 668, 670, 663, 1972, + /* 1690 */ 1971, 1970, 453, 656, 1984, 2204, 451, 1969, 1968, 387, + /* 1700 */ 2200, 2116, 1967, 1966, 1965, 1964, 1963, 1962, 1961, 1960, + /* 1710 */ 1959, 670, 1958, 2134, 1957, 1956, 1955, 147, 1954, 1953, + /* 1720 */ 1952, 1983, 1951, 1950, 1949, 2084, 1948, 669, 1947, 1946, + /* 1730 */ 1945, 481, 342, 1402, 343, 1276, 1797, 2134, 203, 1280, + /* 1740 */ 1796, 205, 1795, 1793, 206, 1754, 2104, 179, 1176, 2084, + /* 1750 */ 218, 669, 1753, 2047, 2037, 2025, 2024, 2002, 2115, 1871, + /* 1760 */ 1272, 1214, 2151, 1792, 1790, 111, 2117, 673, 2119, 2120, + /* 1770 */ 668, 76, 663, 208, 503, 210, 77, 178, 2116, 2204, + /* 1780 */ 216, 518, 2115, 2203, 2200, 1788, 2151, 519, 670, 111, + /* 1790 */ 2117, 673, 2119, 2120, 668, 522, 663, 520, 523, 1786, + /* 1800 */ 524, 2116, 526, 2204, 528, 1784, 530, 658, 2200, 1771, + /* 1810 */ 1770, 670, 532, 1750, 2134, 527, 1873, 531, 1348, 1347, + /* 1820 */ 1872, 1263, 1261, 1259, 2116, 1258, 2084, 1257, 669, 1256, + /* 1830 */ 1255, 1252, 1782, 726, 670, 728, 1251, 2134, 1249, 1250, + /* 1840 */ 367, 1775, 1773, 368, 559, 369, 562, 2116, 63, 2084, + /* 1850 */ 1749, 669, 1748, 1747, 228, 568, 113, 670, 2046, 671, + /* 1860 */ 2134, 564, 566, 2151, 1473, 374, 111, 2117, 673, 2119, + /* 1870 */ 2120, 668, 2084, 663, 669, 1475, 1477, 29, 1472, 67, + /* 1880 */ 2204, 1459, 2115, 2134, 349, 2200, 2151, 1457, 375, 111, + /* 1890 */ 2117, 673, 2119, 2120, 668, 2084, 663, 669, 1455, 2036, + /* 1900 */ 583, 2023, 2021, 2204, 20, 2115, 2280, 31, 2201, 2151, + /* 1910 */ 2116, 599, 333, 2117, 673, 2119, 2120, 668, 251, 663, + /* 1920 */ 670, 17, 6, 1687, 256, 7, 589, 264, 2115, 597, + /* 1930 */ 58, 258, 2151, 163, 21, 333, 2117, 673, 2119, 2120, + /* 1940 */ 668, 22, 663, 2105, 584, 1669, 2134, 171, 373, 262, + /* 1950 */ 263, 32, 33, 65, 24, 1661, 92, 23, 2084, 1702, + /* 1960 */ 669, 1701, 1707, 1708, 378, 1706, 1705, 379, 276, 1636, + /* 1970 */ 1635, 60, 174, 2022, 2020, 2116, 2019, 2001, 95, 94, + /* 1980 */ 282, 25, 2000, 283, 285, 670, 1667, 290, 68, 99, + /* 1990 */ 644, 2115, 97, 295, 292, 2151, 2116, 13, 326, 2117, + /* 2000 */ 673, 2119, 2120, 668, 103, 663, 670, 26, 1511, 2116, + /* 2010 */ 1588, 2134, 1587, 1598, 175, 11, 2154, 188, 1566, 667, + /* 2020 */ 662, 1543, 39, 2084, 59, 669, 674, 676, 18, 1564, + /* 2030 */ 1338, 672, 2134, 395, 298, 1563, 16, 382, 27, 680, + /* 2040 */ 28, 613, 1535, 1333, 2084, 2134, 669, 678, 683, 1330, + /* 2050 */ 1327, 681, 684, 686, 689, 1321, 2115, 2084, 687, 669, + /* 2060 */ 2151, 1319, 690, 170, 2117, 673, 2119, 2120, 668, 104, + /* 2070 */ 663, 105, 1342, 1246, 704, 75, 1212, 2115, 1325, 1324, + /* 2080 */ 1245, 2151, 1244, 2116, 333, 2117, 673, 2119, 2120, 668, + /* 2090 */ 2115, 663, 1243, 670, 2151, 1242, 1241, 332, 2117, 673, + /* 2100 */ 2119, 2120, 668, 2116, 663, 1323, 2170, 1322, 1239, 1237, + /* 2110 */ 1236, 1235, 1270, 670, 299, 2297, 716, 1233, 1232, 2134, + /* 2120 */ 1231, 1230, 1229, 1228, 392, 1227, 1267, 1265, 1224, 1223, + /* 2130 */ 1220, 2084, 1219, 669, 1218, 1217, 1789, 736, 737, 2134, + /* 2140 */ 738, 1787, 740, 742, 394, 741, 1785, 745, 744, 746, + /* 2150 */ 1783, 2084, 748, 669, 749, 750, 1769, 752, 1166, 1746, + /* 2160 */ 302, 756, 1721, 759, 2115, 1497, 312, 760, 2151, 1721, + /* 2170 */ 1721, 333, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, + /* 2180 */ 1721, 1721, 1721, 1721, 2115, 1721, 579, 1721, 2151, 1721, + /* 2190 */ 1721, 333, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, + /* 2200 */ 2116, 1721, 1721, 1721, 761, 1721, 1721, 1721, 1721, 1721, + /* 2210 */ 670, 1721, 1721, 1721, 1721, 1721, 2116, 1721, 303, 1721, + /* 2220 */ 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, 1721, + /* 2230 */ 1721, 1721, 1721, 1721, 176, 1721, 2134, 1721, 1721, 1721, + /* 2240 */ 751, 747, 743, 739, 301, 1721, 2116, 1721, 2084, 1721, + /* 2250 */ 669, 1721, 2134, 1721, 1721, 1721, 670, 1721, 1721, 1721, + /* 2260 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 1721, 1721, 1721, + /* 2270 */ 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, + /* 2280 */ 1721, 574, 2134, 1721, 108, 2151, 670, 294, 328, 2117, + /* 2290 */ 673, 2119, 2120, 668, 2084, 663, 669, 2115, 1721, 1721, + /* 2300 */ 1721, 2151, 1721, 1721, 317, 2117, 673, 2119, 2120, 668, + /* 2310 */ 1721, 663, 2134, 1721, 1721, 1721, 1721, 1721, 1721, 649, + /* 2320 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 2115, 1721, 1721, + /* 2330 */ 1721, 2151, 1721, 1721, 318, 2117, 673, 2119, 2120, 668, + /* 2340 */ 2116, 663, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2350 */ 670, 1721, 1721, 2116, 281, 1721, 1721, 2115, 1721, 280, + /* 2360 */ 1721, 2151, 1721, 670, 319, 2117, 673, 2119, 2120, 668, + /* 2370 */ 1721, 663, 1721, 1721, 1721, 1721, 2134, 1721, 1721, 247, + /* 2380 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 2134, + /* 2390 */ 669, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2400 */ 1721, 2084, 1721, 669, 2116, 1721, 1721, 1721, 1721, 1721, + /* 2410 */ 1721, 1721, 1721, 1721, 670, 1721, 1721, 2116, 1721, 1721, + /* 2420 */ 1721, 2115, 1721, 1721, 1721, 2151, 1721, 670, 325, 2117, + /* 2430 */ 673, 2119, 2120, 668, 2115, 663, 1721, 1721, 2151, 1721, + /* 2440 */ 2134, 329, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, + /* 2450 */ 1721, 1721, 2084, 2134, 669, 1721, 1721, 1721, 1721, 1721, + /* 2460 */ 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, 1721, 1721, + /* 2470 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2480 */ 1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 670, 2151, + /* 2490 */ 1721, 1721, 321, 2117, 673, 2119, 2120, 668, 2115, 663, + /* 2500 */ 1721, 1721, 2151, 1721, 1721, 330, 2117, 673, 2119, 2120, + /* 2510 */ 668, 1721, 663, 1721, 2134, 1721, 1721, 1721, 1721, 1721, + /* 2520 */ 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, 2116, + /* 2530 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, + /* 2540 */ 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, 1721, + /* 2550 */ 1721, 1721, 1721, 1721, 670, 1721, 1721, 1721, 1721, 2115, + /* 2560 */ 1721, 1721, 1721, 2151, 1721, 2134, 322, 2117, 673, 2119, + /* 2570 */ 2120, 668, 1721, 663, 1721, 1721, 1721, 2084, 1721, 669, + /* 2580 */ 2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2590 */ 2116, 1721, 2084, 1721, 669, 1721, 1721, 1721, 1721, 1721, + /* 2600 */ 670, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2610 */ 2115, 1721, 1721, 1721, 2151, 1721, 1721, 331, 2117, 673, + /* 2620 */ 2119, 2120, 668, 1721, 663, 2115, 2134, 1721, 1721, 2151, + /* 2630 */ 1721, 1721, 323, 2117, 673, 2119, 2120, 668, 2084, 663, + /* 2640 */ 669, 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2650 */ 1721, 670, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2660 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, + /* 2670 */ 1721, 2115, 1721, 1721, 1721, 2151, 1721, 2134, 336, 2117, + /* 2680 */ 673, 2119, 2120, 668, 1721, 663, 1721, 2116, 1721, 2084, + /* 2690 */ 1721, 669, 1721, 1721, 2134, 1721, 1721, 670, 1721, 1721, + /* 2700 */ 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, 1721, + /* 2710 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2720 */ 1721, 1721, 2115, 2134, 1721, 1721, 2151, 1721, 1721, 337, + /* 2730 */ 2117, 673, 2119, 2120, 668, 2084, 663, 669, 1721, 2115, + /* 2740 */ 1721, 1721, 1721, 2151, 1721, 1721, 2128, 2117, 673, 2119, + /* 2750 */ 2120, 668, 1721, 663, 1721, 2116, 1721, 1721, 1721, 1721, + /* 2760 */ 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, 2115, 1721, + /* 2770 */ 1721, 1721, 2151, 1721, 2116, 2127, 2117, 673, 2119, 2120, + /* 2780 */ 668, 1721, 663, 1721, 670, 1721, 1721, 2116, 1721, 1721, + /* 2790 */ 1721, 2134, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, + /* 2800 */ 1721, 1721, 1721, 2084, 1721, 669, 1721, 1721, 1721, 1721, + /* 2810 */ 2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2820 */ 1721, 1721, 2084, 2134, 669, 1721, 1721, 1721, 1721, 1721, + /* 2830 */ 1721, 1721, 1721, 1721, 1721, 2084, 2115, 669, 1721, 1721, + /* 2840 */ 2151, 1721, 1721, 2126, 2117, 673, 2119, 2120, 668, 1721, + /* 2850 */ 663, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, + /* 2860 */ 2116, 1721, 351, 2117, 673, 2119, 2120, 668, 2115, 663, + /* 2870 */ 670, 1721, 2151, 2116, 1721, 352, 2117, 673, 2119, 2120, + /* 2880 */ 668, 1721, 663, 670, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2890 */ 1721, 1721, 1721, 1721, 1721, 1721, 2134, 1721, 670, 1721, + /* 2900 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 2134, + /* 2910 */ 669, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2920 */ 1721, 2084, 1721, 669, 2134, 1721, 1721, 1721, 1721, 1721, + /* 2930 */ 1721, 1721, 1721, 1721, 2116, 1721, 2084, 1721, 669, 1721, + /* 2940 */ 1721, 2115, 1721, 1721, 670, 2151, 1721, 1721, 348, 2117, + /* 2950 */ 673, 2119, 2120, 668, 2115, 663, 1721, 1721, 2151, 1721, + /* 2960 */ 1721, 353, 2117, 673, 2119, 2120, 668, 1721, 663, 671, + /* 2970 */ 2134, 1721, 1721, 2151, 1721, 1721, 328, 2117, 673, 2119, + /* 2980 */ 2120, 668, 2084, 663, 669, 1721, 1721, 1721, 1721, 1721, + /* 2990 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 3000 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 3010 */ 1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, + /* 3020 */ 1721, 1721, 327, 2117, 673, 2119, 2120, 668, 1721, 663, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 333, 370, 383, 337, 400, 370, 340, 341, 404, 371, - /* 10 */ 343, 365, 12, 13, 14, 0, 371, 398, 399, 381, - /* 20 */ 20, 14, 22, 20, 8, 9, 381, 20, 12, 13, - /* 30 */ 14, 15, 16, 33, 0, 35, 369, 383, 20, 24, - /* 40 */ 25, 26, 27, 28, 29, 30, 31, 32, 381, 395, - /* 50 */ 383, 447, 398, 399, 450, 417, 418, 419, 412, 59, - /* 60 */ 44, 369, 417, 418, 419, 65, 428, 20, 376, 465, - /* 70 */ 466, 20, 72, 428, 470, 471, 384, 342, 343, 0, + /* 0 */ 333, 338, 383, 370, 400, 342, 371, 344, 404, 371, + /* 10 */ 343, 383, 12, 13, 14, 0, 381, 398, 399, 381, + /* 20 */ 20, 20, 22, 395, 8, 9, 398, 399, 12, 13, + /* 30 */ 14, 15, 16, 33, 0, 35, 369, 368, 20, 24, + /* 40 */ 25, 26, 27, 28, 29, 30, 31, 32, 381, 380, + /* 50 */ 383, 447, 417, 418, 450, 417, 418, 419, 379, 59, + /* 60 */ 44, 382, 383, 428, 337, 65, 428, 340, 341, 465, + /* 70 */ 466, 332, 72, 334, 470, 471, 20, 342, 343, 0, /* 80 */ 337, 414, 348, 340, 341, 418, 436, 437, 421, 422, - /* 90 */ 423, 424, 425, 426, 332, 428, 334, 97, 364, 39, + /* 90 */ 423, 424, 425, 426, 432, 428, 434, 97, 364, 39, /* 100 */ 100, 67, 68, 69, 70, 71, 372, 73, 74, 75, /* 110 */ 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, /* 120 */ 86, 87, 88, 89, 90, 91, 92, 93, 461, 462, - /* 130 */ 379, 420, 97, 382, 383, 21, 136, 137, 24, 25, + /* 130 */ 100, 420, 97, 382, 383, 21, 136, 137, 24, 25, /* 140 */ 26, 27, 28, 29, 30, 31, 32, 112, 113, 114, /* 150 */ 115, 116, 117, 118, 119, 120, 121, 446, 123, 124, - /* 160 */ 125, 126, 127, 128, 369, 20, 166, 167, 447, 166, - /* 170 */ 167, 450, 172, 173, 369, 342, 343, 442, 443, 444, - /* 180 */ 445, 376, 447, 448, 389, 390, 186, 466, 188, 384, - /* 190 */ 330, 470, 471, 360, 130, 131, 64, 8, 9, 135, + /* 160 */ 125, 126, 127, 128, 14, 20, 166, 167, 447, 168, + /* 170 */ 20, 450, 172, 173, 369, 342, 343, 442, 443, 444, + /* 180 */ 445, 376, 447, 448, 166, 167, 186, 466, 188, 384, + /* 190 */ 330, 470, 471, 360, 130, 131, 343, 8, 9, 135, /* 200 */ 367, 12, 13, 14, 15, 16, 59, 100, 129, 130, /* 210 */ 131, 132, 133, 134, 135, 215, 216, 0, 218, 219, /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, /* 230 */ 230, 231, 232, 233, 234, 235, 12, 13, 181, 4, - /* 240 */ 20, 18, 100, 20, 20, 100, 22, 100, 20, 102, + /* 240 */ 387, 18, 100, 20, 20, 100, 22, 100, 350, 102, /* 250 */ 27, 333, 447, 30, 65, 450, 33, 33, 369, 35, /* 260 */ 400, 343, 205, 206, 404, 12, 13, 14, 15, 16, - /* 270 */ 465, 466, 49, 100, 51, 470, 471, 54, 43, 390, - /* 280 */ 45, 46, 368, 59, 67, 68, 69, 369, 0, 65, - /* 290 */ 100, 74, 75, 76, 380, 20, 72, 80, 109, 381, + /* 270 */ 465, 466, 49, 375, 51, 470, 471, 54, 43, 390, + /* 280 */ 45, 46, 252, 59, 67, 68, 69, 369, 0, 65, + /* 290 */ 100, 74, 75, 76, 355, 356, 72, 80, 109, 381, /* 300 */ 110, 383, 85, 86, 87, 88, 20, 447, 91, 21, /* 310 */ 450, 251, 24, 25, 26, 27, 28, 29, 30, 31, /* 320 */ 32, 97, 99, 348, 100, 465, 466, 185, 3, 187, @@ -550,378 +562,390 @@ static const YYCODETYPE yy_lookahead[] = { /* 350 */ 342, 433, 434, 435, 165, 360, 214, 439, 440, 252, /* 360 */ 136, 137, 367, 140, 20, 52, 143, 144, 145, 146, /* 370 */ 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, - /* 380 */ 157, 158, 159, 338, 161, 162, 163, 342, 168, 344, - /* 390 */ 166, 167, 447, 385, 252, 450, 172, 173, 14, 252, - /* 400 */ 67, 68, 69, 64, 20, 382, 383, 74, 75, 76, + /* 380 */ 157, 158, 159, 369, 161, 162, 163, 342, 343, 20, + /* 390 */ 166, 167, 447, 385, 252, 450, 172, 173, 369, 252, + /* 400 */ 67, 68, 69, 389, 390, 360, 377, 74, 75, 76, /* 410 */ 186, 466, 188, 80, 171, 470, 471, 0, 85, 86, - /* 420 */ 87, 88, 136, 137, 91, 252, 101, 35, 239, 240, + /* 420 */ 87, 88, 136, 137, 91, 3, 101, 35, 239, 240, /* 430 */ 241, 242, 243, 244, 245, 246, 247, 248, 249, 215, - /* 440 */ 216, 166, 218, 219, 220, 221, 222, 223, 224, 225, + /* 440 */ 216, 0, 218, 219, 220, 221, 222, 223, 224, 225, /* 450 */ 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, - /* 460 */ 236, 12, 13, 33, 72, 22, 351, 20, 338, 20, - /* 470 */ 333, 22, 342, 358, 344, 215, 111, 20, 35, 49, - /* 480 */ 343, 64, 33, 81, 35, 55, 56, 57, 58, 59, - /* 490 */ 420, 8, 9, 168, 333, 12, 13, 14, 15, 16, - /* 500 */ 257, 258, 259, 20, 20, 4, 369, 371, 59, 355, - /* 510 */ 356, 72, 20, 343, 65, 72, 446, 381, 381, 342, + /* 460 */ 236, 12, 13, 33, 72, 22, 49, 20, 343, 20, + /* 470 */ 333, 22, 20, 59, 351, 215, 64, 20, 35, 49, + /* 480 */ 343, 358, 33, 81, 35, 55, 56, 57, 58, 59, + /* 490 */ 59, 8, 9, 168, 369, 12, 13, 14, 15, 16, + /* 500 */ 257, 258, 259, 20, 20, 4, 369, 371, 59, 236, + /* 510 */ 20, 238, 22, 99, 65, 72, 102, 381, 381, 342, /* 520 */ 383, 72, 262, 263, 264, 265, 266, 267, 268, 99, - /* 530 */ 8, 9, 102, 236, 12, 13, 14, 15, 16, 369, - /* 540 */ 97, 369, 381, 141, 142, 13, 97, 333, 376, 100, - /* 550 */ 166, 414, 165, 417, 418, 418, 384, 100, 421, 422, - /* 560 */ 423, 424, 425, 426, 428, 428, 164, 35, 18, 392, + /* 530 */ 8, 9, 102, 102, 12, 13, 14, 15, 16, 0, + /* 540 */ 97, 338, 52, 141, 142, 342, 97, 344, 423, 100, + /* 550 */ 14, 414, 100, 417, 418, 418, 20, 100, 421, 422, + /* 560 */ 423, 424, 425, 426, 428, 428, 164, 365, 18, 392, /* 570 */ 433, 394, 435, 23, 8, 9, 439, 440, 12, 13, /* 580 */ 14, 15, 16, 136, 137, 136, 137, 37, 38, 452, - /* 590 */ 333, 41, 100, 423, 4, 381, 20, 460, 22, 169, - /* 600 */ 170, 100, 350, 342, 174, 236, 176, 238, 420, 19, - /* 610 */ 60, 61, 62, 63, 111, 166, 167, 361, 366, 172, - /* 620 */ 173, 172, 173, 33, 194, 369, 239, 375, 52, 186, - /* 630 */ 72, 188, 376, 377, 446, 186, 249, 188, 381, 49, - /* 640 */ 384, 0, 342, 343, 54, 342, 343, 369, 333, 59, - /* 650 */ 100, 168, 168, 392, 376, 394, 371, 129, 215, 216, - /* 660 */ 360, 133, 384, 360, 215, 216, 381, 218, 219, 220, + /* 590 */ 432, 41, 434, 357, 4, 359, 155, 460, 333, 169, + /* 600 */ 170, 100, 64, 64, 174, 164, 176, 1, 2, 19, + /* 610 */ 60, 61, 62, 63, 412, 166, 167, 361, 361, 172, + /* 620 */ 173, 172, 173, 33, 194, 369, 369, 342, 343, 186, + /* 630 */ 14, 188, 376, 377, 377, 186, 20, 188, 447, 49, + /* 640 */ 384, 450, 342, 343, 54, 360, 381, 20, 369, 59, + /* 650 */ 100, 168, 14, 15, 16, 376, 465, 466, 215, 216, + /* 660 */ 360, 470, 471, 384, 215, 216, 20, 218, 219, 220, /* 670 */ 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, /* 680 */ 231, 232, 233, 234, 235, 12, 13, 14, 138, 99, - /* 690 */ 369, 333, 102, 20, 370, 22, 381, 376, 342, 343, - /* 700 */ 178, 343, 417, 418, 419, 384, 33, 333, 35, 252, - /* 710 */ 357, 2, 359, 428, 342, 343, 360, 8, 9, 342, - /* 720 */ 343, 12, 13, 14, 15, 16, 358, 369, 178, 179, - /* 730 */ 180, 350, 59, 183, 168, 355, 356, 360, 342, 381, - /* 740 */ 370, 383, 8, 9, 252, 72, 12, 13, 14, 15, - /* 750 */ 16, 342, 343, 252, 204, 381, 375, 207, 0, 343, - /* 760 */ 210, 211, 212, 213, 214, 14, 333, 342, 343, 360, - /* 770 */ 97, 20, 414, 100, 342, 343, 418, 333, 333, 421, + /* 690 */ 369, 333, 102, 20, 272, 22, 333, 376, 342, 343, + /* 700 */ 178, 343, 166, 35, 252, 384, 33, 101, 35, 252, + /* 710 */ 420, 2, 355, 356, 342, 343, 360, 8, 9, 342, + /* 720 */ 343, 12, 13, 14, 15, 16, 111, 369, 178, 179, + /* 730 */ 180, 129, 59, 183, 168, 133, 446, 360, 342, 381, + /* 740 */ 72, 383, 8, 9, 381, 72, 12, 13, 14, 15, + /* 750 */ 16, 342, 343, 252, 204, 431, 378, 207, 434, 381, + /* 760 */ 210, 211, 212, 213, 214, 236, 130, 342, 343, 360, + /* 770 */ 97, 72, 414, 100, 342, 343, 418, 333, 342, 421, /* 780 */ 422, 423, 424, 425, 426, 360, 428, 343, 392, 345, - /* 790 */ 394, 433, 360, 435, 2, 4, 333, 439, 440, 277, - /* 800 */ 8, 9, 252, 387, 12, 13, 14, 15, 16, 136, - /* 810 */ 137, 342, 343, 369, 381, 443, 444, 445, 460, 447, - /* 820 */ 448, 21, 450, 342, 343, 381, 381, 383, 370, 360, - /* 830 */ 20, 342, 343, 59, 34, 101, 36, 465, 466, 166, - /* 840 */ 167, 343, 470, 471, 381, 172, 173, 8, 9, 360, - /* 850 */ 209, 12, 13, 14, 15, 16, 0, 431, 414, 186, - /* 860 */ 434, 188, 418, 342, 343, 421, 422, 423, 424, 425, - /* 870 */ 426, 369, 428, 99, 131, 370, 102, 433, 135, 435, - /* 880 */ 37, 360, 369, 439, 440, 387, 384, 400, 215, 216, - /* 890 */ 377, 218, 219, 220, 221, 222, 223, 224, 225, 226, + /* 790 */ 394, 433, 360, 435, 2, 361, 350, 439, 440, 277, + /* 800 */ 8, 9, 252, 369, 12, 13, 14, 15, 16, 136, + /* 810 */ 137, 377, 366, 369, 168, 443, 444, 445, 460, 447, + /* 820 */ 448, 375, 450, 342, 343, 381, 111, 383, 392, 369, + /* 830 */ 394, 342, 343, 197, 198, 101, 376, 465, 466, 166, + /* 840 */ 167, 343, 470, 471, 384, 172, 173, 8, 9, 360, + /* 850 */ 72, 12, 13, 14, 15, 16, 0, 48, 414, 186, + /* 860 */ 22, 188, 418, 342, 343, 421, 422, 423, 424, 425, + /* 870 */ 426, 369, 428, 35, 131, 369, 343, 433, 135, 435, + /* 880 */ 37, 360, 376, 439, 440, 387, 384, 400, 215, 216, + /* 890 */ 384, 218, 219, 220, 221, 222, 223, 224, 225, 226, /* 900 */ 227, 228, 229, 230, 231, 232, 233, 234, 235, 12, - /* 910 */ 13, 333, 432, 155, 434, 342, 343, 20, 333, 22, - /* 920 */ 333, 343, 164, 345, 443, 444, 445, 59, 447, 448, - /* 930 */ 33, 333, 35, 360, 447, 45, 46, 450, 195, 196, - /* 940 */ 101, 343, 199, 432, 201, 434, 103, 369, 105, 106, - /* 950 */ 333, 108, 465, 466, 14, 334, 59, 470, 471, 381, - /* 960 */ 20, 383, 342, 343, 342, 343, 381, 369, 381, 72, - /* 970 */ 102, 22, 129, 14, 15, 16, 133, 3, 168, 381, - /* 980 */ 360, 383, 360, 371, 35, 129, 130, 131, 132, 133, + /* 910 */ 13, 333, 333, 45, 46, 342, 343, 20, 333, 22, + /* 920 */ 387, 343, 370, 345, 443, 444, 445, 0, 447, 448, + /* 930 */ 33, 333, 35, 360, 447, 97, 20, 450, 195, 196, + /* 940 */ 101, 343, 199, 4, 201, 333, 103, 369, 105, 106, + /* 950 */ 333, 108, 465, 466, 370, 21, 59, 470, 471, 381, + /* 960 */ 381, 383, 342, 343, 342, 343, 381, 369, 34, 72, + /* 970 */ 36, 378, 129, 370, 381, 378, 133, 168, 381, 381, + /* 980 */ 360, 383, 360, 420, 370, 129, 130, 131, 132, 133, /* 990 */ 134, 135, 414, 381, 97, 400, 418, 100, 381, 421, - /* 1000 */ 422, 423, 424, 425, 426, 333, 428, 400, 1, 2, - /* 1010 */ 333, 433, 414, 435, 333, 343, 418, 439, 440, 421, - /* 1020 */ 422, 423, 424, 425, 426, 427, 428, 429, 430, 417, - /* 1030 */ 418, 8, 9, 136, 137, 12, 13, 14, 15, 16, - /* 1040 */ 428, 369, 447, 361, 253, 450, 97, 343, 44, 342, - /* 1050 */ 343, 369, 65, 381, 447, 383, 370, 450, 381, 377, - /* 1060 */ 465, 466, 381, 166, 167, 470, 471, 360, 378, 172, - /* 1070 */ 173, 381, 465, 466, 250, 251, 361, 470, 471, 136, - /* 1080 */ 137, 342, 343, 186, 369, 188, 414, 342, 343, 22, - /* 1090 */ 418, 387, 377, 421, 422, 423, 424, 425, 426, 360, - /* 1100 */ 428, 400, 35, 342, 343, 360, 166, 42, 101, 44, - /* 1110 */ 130, 378, 215, 216, 381, 218, 219, 220, 221, 222, + /* 1000 */ 422, 423, 424, 425, 426, 333, 428, 400, 42, 446, + /* 1010 */ 44, 433, 414, 435, 20, 343, 418, 439, 440, 421, + /* 1020 */ 422, 423, 424, 425, 426, 427, 428, 429, 430, 342, + /* 1030 */ 343, 8, 9, 136, 137, 12, 13, 14, 15, 16, + /* 1040 */ 361, 369, 447, 333, 370, 450, 14, 360, 369, 342, + /* 1050 */ 343, 44, 20, 381, 447, 383, 377, 450, 342, 343, + /* 1060 */ 465, 466, 22, 166, 167, 470, 471, 360, 165, 172, + /* 1070 */ 173, 104, 465, 466, 107, 35, 360, 470, 471, 250, + /* 1080 */ 251, 342, 343, 186, 168, 188, 414, 333, 342, 343, + /* 1090 */ 418, 381, 22, 421, 422, 423, 424, 425, 426, 360, + /* 1100 */ 428, 400, 333, 342, 343, 35, 360, 333, 101, 42, + /* 1110 */ 333, 44, 215, 216, 333, 218, 219, 220, 221, 222, /* 1120 */ 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, - /* 1130 */ 233, 234, 235, 12, 13, 342, 343, 342, 343, 333, - /* 1140 */ 333, 20, 361, 22, 472, 473, 333, 333, 447, 0, - /* 1150 */ 369, 450, 333, 360, 33, 360, 35, 343, 377, 345, - /* 1160 */ 44, 44, 22, 333, 97, 333, 465, 466, 0, 405, - /* 1170 */ 362, 470, 471, 365, 333, 35, 378, 197, 198, 381, - /* 1180 */ 59, 111, 104, 369, 343, 107, 345, 381, 381, 42, - /* 1190 */ 200, 44, 202, 72, 381, 381, 447, 383, 49, 450, - /* 1200 */ 381, 48, 215, 13, 443, 444, 445, 0, 447, 448, - /* 1210 */ 369, 381, 44, 381, 465, 466, 391, 101, 97, 470, - /* 1220 */ 471, 100, 381, 104, 383, 35, 107, 474, 414, 22, - /* 1230 */ 160, 35, 418, 44, 333, 421, 422, 423, 424, 425, - /* 1240 */ 426, 35, 428, 463, 343, 104, 272, 433, 107, 435, - /* 1250 */ 0, 0, 346, 439, 440, 414, 44, 136, 137, 418, - /* 1260 */ 44, 13, 421, 422, 423, 424, 425, 426, 104, 428, - /* 1270 */ 369, 107, 22, 22, 433, 44, 435, 44, 274, 457, - /* 1280 */ 439, 440, 381, 35, 383, 47, 369, 166, 167, 391, - /* 1290 */ 101, 35, 346, 172, 173, 341, 129, 130, 131, 132, - /* 1300 */ 133, 134, 135, 1, 2, 44, 343, 186, 44, 188, - /* 1310 */ 380, 44, 391, 101, 44, 414, 44, 101, 44, 418, - /* 1320 */ 449, 168, 421, 422, 423, 424, 425, 426, 72, 428, - /* 1330 */ 44, 0, 101, 44, 101, 44, 215, 216, 100, 218, + /* 1130 */ 233, 234, 235, 12, 13, 381, 209, 97, 0, 333, + /* 1140 */ 333, 20, 239, 22, 472, 473, 333, 333, 447, 44, + /* 1150 */ 381, 450, 249, 333, 33, 381, 35, 343, 381, 345, + /* 1160 */ 166, 44, 381, 333, 333, 333, 465, 466, 111, 13, + /* 1170 */ 362, 470, 471, 365, 333, 104, 104, 35, 107, 107, + /* 1180 */ 59, 13, 44, 369, 343, 0, 345, 381, 381, 44, + /* 1190 */ 405, 35, 253, 72, 381, 381, 104, 383, 166, 107, + /* 1200 */ 200, 381, 202, 35, 443, 444, 445, 22, 447, 448, + /* 1210 */ 369, 381, 381, 381, 44, 44, 44, 160, 97, 0, + /* 1220 */ 44, 100, 381, 0, 383, 136, 137, 370, 414, 358, + /* 1230 */ 1, 2, 418, 44, 333, 421, 422, 423, 424, 425, + /* 1240 */ 426, 22, 428, 35, 343, 22, 101, 433, 44, 435, + /* 1250 */ 47, 44, 65, 439, 440, 414, 44, 136, 137, 418, + /* 1260 */ 44, 44, 421, 422, 423, 424, 425, 426, 391, 428, + /* 1270 */ 369, 101, 101, 101, 433, 44, 435, 101, 44, 334, + /* 1280 */ 439, 440, 381, 346, 383, 474, 463, 166, 167, 369, + /* 1290 */ 101, 457, 346, 172, 173, 391, 129, 130, 131, 132, + /* 1300 */ 133, 134, 135, 100, 341, 101, 380, 186, 101, 188, + /* 1310 */ 391, 44, 13, 101, 44, 414, 44, 101, 101, 418, + /* 1320 */ 0, 44, 421, 422, 423, 424, 425, 426, 343, 428, + /* 1330 */ 188, 449, 101, 44, 35, 101, 215, 216, 44, 218, /* 1340 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 441, 467, 12, - /* 1360 */ 13, 44, 101, 462, 44, 101, 44, 333, 101, 22, - /* 1370 */ 44, 101, 44, 101, 13, 101, 451, 343, 254, 345, - /* 1380 */ 33, 50, 35, 416, 188, 49, 415, 101, 184, 388, - /* 1390 */ 101, 402, 101, 276, 188, 42, 35, 333, 20, 391, - /* 1400 */ 388, 165, 386, 369, 20, 342, 59, 343, 342, 345, - /* 1410 */ 388, 386, 386, 98, 354, 381, 342, 383, 101, 72, - /* 1420 */ 96, 101, 353, 101, 95, 352, 342, 101, 342, 101, - /* 1430 */ 342, 20, 335, 369, 48, 339, 335, 339, 20, 409, - /* 1440 */ 350, 20, 383, 350, 97, 381, 20, 383, 414, 344, - /* 1450 */ 20, 401, 418, 350, 344, 421, 422, 423, 424, 425, - /* 1460 */ 426, 350, 428, 350, 350, 333, 53, 433, 342, 435, - /* 1470 */ 350, 335, 347, 439, 440, 343, 347, 369, 414, 335, - /* 1480 */ 381, 369, 418, 381, 369, 421, 422, 423, 424, 425, - /* 1490 */ 426, 369, 428, 342, 413, 203, 369, 433, 100, 435, - /* 1500 */ 369, 369, 369, 439, 440, 411, 369, 369, 369, 381, - /* 1510 */ 348, 369, 408, 381, 191, 383, 409, 192, 383, 348, - /* 1520 */ 190, 342, 269, 261, 456, 407, 260, 381, 456, 406, - /* 1530 */ 396, 381, 177, 186, 391, 188, 391, 381, 381, 459, - /* 1540 */ 458, 456, 271, 270, 396, 454, 414, 455, 255, 273, - /* 1550 */ 418, 475, 275, 421, 422, 423, 424, 425, 426, 453, - /* 1560 */ 428, 278, 215, 216, 469, 433, 343, 435, 416, 251, - /* 1570 */ 20, 439, 440, 333, 420, 228, 229, 230, 231, 232, - /* 1580 */ 233, 234, 348, 343, 342, 344, 20, 348, 394, 396, - /* 1590 */ 381, 381, 381, 333, 381, 381, 396, 170, 468, 381, - /* 1600 */ 348, 393, 365, 343, 100, 348, 343, 438, 100, 369, - /* 1610 */ 381, 342, 36, 359, 0, 336, 373, 335, 403, 397, - /* 1620 */ 0, 381, 397, 383, 0, 331, 348, 410, 363, 369, - /* 1630 */ 349, 363, 42, 0, 35, 35, 208, 35, 363, 35, - /* 1640 */ 208, 381, 35, 383, 0, 35, 208, 0, 208, 0, - /* 1650 */ 35, 0, 22, 0, 414, 195, 35, 188, 418, 186, - /* 1660 */ 0, 421, 422, 423, 424, 425, 426, 0, 428, 0, - /* 1670 */ 182, 181, 0, 433, 414, 435, 47, 333, 418, 439, + /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 467, 441, 12, + /* 1360 */ 13, 13, 451, 462, 254, 416, 49, 333, 101, 22, + /* 1370 */ 50, 101, 415, 101, 184, 402, 42, 343, 101, 345, + /* 1380 */ 33, 276, 35, 35, 388, 20, 391, 388, 386, 165, + /* 1390 */ 101, 274, 20, 342, 342, 101, 188, 333, 388, 386, + /* 1400 */ 386, 98, 215, 369, 354, 96, 59, 343, 353, 345, + /* 1410 */ 342, 95, 352, 20, 335, 381, 342, 383, 342, 72, + /* 1420 */ 342, 48, 339, 335, 339, 20, 409, 350, 20, 383, + /* 1430 */ 350, 20, 20, 369, 344, 344, 401, 350, 53, 342, + /* 1440 */ 342, 347, 347, 350, 97, 381, 350, 383, 414, 350, + /* 1450 */ 350, 335, 418, 369, 335, 421, 422, 423, 424, 425, + /* 1460 */ 426, 203, 428, 381, 100, 333, 413, 433, 409, 435, + /* 1470 */ 369, 369, 369, 439, 440, 343, 369, 369, 414, 348, + /* 1480 */ 369, 369, 418, 369, 369, 421, 422, 423, 424, 425, + /* 1490 */ 426, 369, 428, 192, 381, 191, 348, 433, 190, 435, + /* 1500 */ 342, 369, 261, 439, 440, 381, 383, 381, 260, 381, + /* 1510 */ 456, 391, 408, 381, 456, 383, 407, 411, 391, 269, + /* 1520 */ 381, 396, 381, 177, 406, 396, 271, 270, 459, 255, + /* 1530 */ 455, 458, 278, 186, 275, 188, 475, 456, 273, 251, + /* 1540 */ 454, 343, 20, 342, 344, 348, 414, 348, 20, 394, + /* 1550 */ 418, 416, 170, 421, 422, 423, 424, 425, 426, 396, + /* 1560 */ 428, 381, 215, 216, 420, 433, 393, 435, 348, 381, + /* 1570 */ 396, 439, 440, 333, 381, 228, 229, 230, 231, 232, + /* 1580 */ 233, 234, 343, 343, 348, 100, 453, 381, 438, 100, + /* 1590 */ 381, 381, 365, 333, 373, 381, 359, 348, 342, 469, + /* 1600 */ 36, 335, 468, 343, 336, 403, 410, 363, 397, 369, + /* 1610 */ 397, 349, 331, 0, 363, 363, 0, 0, 42, 0, + /* 1620 */ 35, 381, 208, 383, 35, 35, 35, 208, 0, 369, + /* 1630 */ 35, 35, 208, 0, 208, 0, 35, 0, 0, 35, + /* 1640 */ 22, 381, 195, 383, 188, 186, 0, 0, 0, 182, + /* 1650 */ 181, 0, 0, 47, 414, 0, 0, 0, 418, 0, + /* 1660 */ 0, 421, 422, 423, 424, 425, 426, 42, 428, 0, + /* 1670 */ 0, 0, 0, 433, 414, 435, 155, 333, 418, 439, /* 1680 */ 440, 421, 422, 423, 424, 425, 426, 343, 428, 0, - /* 1690 */ 0, 0, 0, 433, 42, 435, 0, 0, 0, 439, - /* 1700 */ 440, 333, 0, 0, 0, 0, 0, 155, 35, 0, - /* 1710 */ 155, 343, 0, 369, 0, 0, 0, 0, 0, 0, + /* 1690 */ 0, 0, 155, 433, 0, 435, 35, 0, 0, 439, + /* 1700 */ 440, 333, 0, 0, 0, 0, 0, 0, 0, 0, + /* 1710 */ 0, 343, 0, 369, 0, 0, 0, 42, 0, 0, /* 1720 */ 0, 0, 0, 0, 0, 381, 0, 383, 0, 0, - /* 1730 */ 0, 0, 0, 0, 0, 42, 0, 369, 0, 0, - /* 1740 */ 0, 0, 139, 0, 0, 0, 59, 22, 48, 381, - /* 1750 */ 0, 383, 0, 59, 59, 48, 0, 22, 414, 22, - /* 1760 */ 0, 14, 418, 177, 0, 421, 422, 423, 424, 425, - /* 1770 */ 426, 44, 428, 0, 0, 0, 0, 35, 333, 435, - /* 1780 */ 0, 0, 414, 439, 440, 39, 418, 39, 343, 421, - /* 1790 */ 422, 423, 424, 425, 426, 42, 428, 39, 47, 40, - /* 1800 */ 0, 333, 47, 435, 47, 0, 35, 439, 440, 39, - /* 1810 */ 49, 343, 0, 35, 369, 66, 49, 0, 39, 35, - /* 1820 */ 0, 49, 39, 35, 333, 39, 381, 0, 383, 0, - /* 1830 */ 0, 49, 0, 35, 343, 22, 0, 369, 35, 35, - /* 1840 */ 44, 35, 35, 35, 35, 35, 44, 333, 22, 381, - /* 1850 */ 35, 383, 0, 35, 35, 22, 0, 343, 0, 414, - /* 1860 */ 369, 107, 22, 418, 22, 51, 421, 422, 423, 424, - /* 1870 */ 425, 426, 381, 428, 383, 109, 35, 0, 35, 0, - /* 1880 */ 435, 35, 414, 369, 439, 440, 418, 0, 22, 421, - /* 1890 */ 422, 423, 424, 425, 426, 381, 428, 383, 20, 35, - /* 1900 */ 35, 100, 35, 435, 0, 414, 193, 35, 440, 418, - /* 1910 */ 333, 100, 421, 422, 423, 424, 425, 426, 22, 428, - /* 1920 */ 343, 0, 101, 22, 0, 0, 3, 44, 414, 101, - /* 1930 */ 168, 256, 418, 48, 100, 421, 422, 423, 424, 425, - /* 1940 */ 426, 48, 428, 175, 100, 168, 369, 98, 101, 96, - /* 1950 */ 44, 374, 44, 168, 189, 464, 101, 100, 381, 170, - /* 1960 */ 383, 100, 100, 44, 101, 47, 256, 3, 100, 47, - /* 1970 */ 44, 101, 101, 44, 35, 333, 35, 35, 35, 35, - /* 1980 */ 35, 101, 101, 0, 47, 343, 0, 473, 0, 0, - /* 1990 */ 44, 414, 100, 39, 47, 418, 333, 47, 421, 422, - /* 2000 */ 423, 424, 425, 426, 101, 428, 343, 100, 171, 333, - /* 2010 */ 101, 369, 100, 100, 100, 0, 39, 169, 47, 343, - /* 2020 */ 110, 2, 44, 381, 22, 383, 98, 100, 98, 237, - /* 2030 */ 22, 215, 369, 100, 47, 101, 101, 374, 100, 47, - /* 2040 */ 250, 101, 217, 111, 381, 369, 383, 256, 100, 100, - /* 2050 */ 374, 100, 35, 101, 101, 100, 414, 381, 35, 383, - /* 2060 */ 418, 1, 100, 421, 422, 423, 424, 425, 426, 101, - /* 2070 */ 428, 35, 430, 100, 35, 101, 100, 414, 35, 19, - /* 2080 */ 101, 418, 100, 35, 421, 422, 423, 424, 425, 426, - /* 2090 */ 414, 428, 101, 33, 418, 100, 333, 421, 422, 423, - /* 2100 */ 424, 425, 426, 122, 428, 100, 343, 122, 44, 49, - /* 2110 */ 100, 35, 122, 100, 22, 55, 56, 57, 58, 59, - /* 2120 */ 333, 122, 66, 65, 35, 35, 35, 72, 35, 35, - /* 2130 */ 343, 35, 369, 35, 35, 35, 35, 35, 94, 44, - /* 2140 */ 35, 35, 22, 35, 381, 35, 383, 35, 35, 72, - /* 2150 */ 35, 35, 22, 35, 35, 0, 369, 35, 35, 99, - /* 2160 */ 49, 0, 102, 39, 35, 39, 0, 35, 381, 0, - /* 2170 */ 383, 35, 0, 39, 49, 35, 49, 414, 39, 35, - /* 2180 */ 49, 418, 0, 333, 421, 422, 423, 424, 425, 426, - /* 2190 */ 22, 428, 21, 343, 134, 22, 22, 21, 20, 476, - /* 2200 */ 476, 414, 476, 476, 476, 418, 476, 333, 421, 422, - /* 2210 */ 423, 424, 425, 426, 476, 428, 476, 343, 476, 369, - /* 2220 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 169, - /* 2230 */ 476, 381, 476, 383, 174, 476, 476, 476, 476, 476, - /* 2240 */ 476, 476, 476, 369, 476, 476, 476, 476, 476, 476, - /* 2250 */ 476, 476, 476, 333, 194, 381, 476, 383, 476, 476, - /* 2260 */ 476, 476, 476, 343, 414, 476, 476, 476, 418, 333, - /* 2270 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 343, - /* 2280 */ 476, 476, 476, 476, 476, 476, 476, 476, 414, 369, - /* 2290 */ 476, 476, 418, 476, 476, 421, 422, 423, 424, 425, - /* 2300 */ 426, 381, 428, 383, 476, 369, 476, 476, 476, 476, - /* 2310 */ 476, 476, 476, 476, 476, 476, 476, 381, 476, 383, - /* 2320 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 333, - /* 2330 */ 476, 476, 476, 476, 414, 476, 476, 476, 418, 343, - /* 2340 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, - /* 2350 */ 414, 476, 476, 333, 418, 476, 476, 421, 422, 423, - /* 2360 */ 424, 425, 426, 343, 428, 369, 476, 476, 476, 476, - /* 2370 */ 476, 476, 476, 476, 476, 476, 476, 381, 476, 383, - /* 2380 */ 333, 476, 476, 476, 476, 476, 476, 476, 476, 369, - /* 2390 */ 343, 476, 476, 476, 476, 476, 476, 476, 476, 333, - /* 2400 */ 476, 381, 476, 383, 476, 476, 476, 476, 476, 343, - /* 2410 */ 414, 476, 476, 476, 418, 476, 369, 421, 422, 423, - /* 2420 */ 424, 425, 426, 476, 428, 476, 476, 476, 381, 476, - /* 2430 */ 383, 476, 476, 476, 414, 369, 476, 476, 418, 476, - /* 2440 */ 476, 421, 422, 423, 424, 425, 426, 381, 428, 383, - /* 2450 */ 333, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2460 */ 343, 414, 476, 476, 476, 418, 333, 476, 421, 422, - /* 2470 */ 423, 424, 425, 426, 476, 428, 343, 476, 476, 476, - /* 2480 */ 414, 476, 476, 476, 418, 476, 369, 421, 422, 423, - /* 2490 */ 424, 425, 426, 476, 428, 476, 333, 476, 381, 476, - /* 2500 */ 383, 476, 369, 476, 476, 476, 343, 476, 476, 476, - /* 2510 */ 476, 476, 476, 476, 381, 476, 383, 333, 476, 476, - /* 2520 */ 476, 476, 476, 476, 476, 476, 476, 343, 476, 476, - /* 2530 */ 476, 414, 369, 476, 476, 418, 476, 476, 421, 422, - /* 2540 */ 423, 424, 425, 426, 381, 428, 383, 414, 476, 476, - /* 2550 */ 476, 418, 476, 369, 421, 422, 423, 424, 425, 426, - /* 2560 */ 476, 428, 476, 333, 476, 381, 476, 383, 476, 476, - /* 2570 */ 476, 476, 476, 343, 476, 476, 476, 414, 476, 476, - /* 2580 */ 476, 418, 476, 476, 421, 422, 423, 424, 425, 426, - /* 2590 */ 476, 428, 476, 476, 476, 476, 476, 476, 414, 369, - /* 2600 */ 476, 476, 418, 476, 476, 421, 422, 423, 424, 425, - /* 2610 */ 426, 381, 428, 383, 476, 476, 476, 476, 476, 476, - /* 2620 */ 476, 476, 476, 333, 476, 476, 476, 476, 476, 476, - /* 2630 */ 476, 476, 476, 343, 476, 476, 333, 476, 476, 476, - /* 2640 */ 476, 476, 476, 476, 414, 476, 343, 476, 418, 476, - /* 2650 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 369, - /* 2660 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2670 */ 476, 381, 369, 383, 476, 476, 476, 476, 476, 476, - /* 2680 */ 476, 476, 476, 476, 381, 476, 383, 333, 476, 476, - /* 2690 */ 476, 476, 476, 476, 476, 476, 476, 343, 476, 476, - /* 2700 */ 476, 476, 476, 476, 414, 476, 476, 476, 418, 476, - /* 2710 */ 476, 421, 422, 423, 424, 425, 426, 414, 428, 476, - /* 2720 */ 476, 418, 476, 369, 421, 422, 423, 424, 425, 426, - /* 2730 */ 476, 428, 476, 333, 476, 381, 476, 383, 476, 476, - /* 2740 */ 476, 476, 476, 343, 476, 476, 476, 476, 333, 476, - /* 2750 */ 476, 476, 476, 476, 476, 476, 476, 476, 343, 476, - /* 2760 */ 476, 476, 476, 476, 476, 476, 476, 476, 414, 369, - /* 2770 */ 476, 476, 418, 476, 476, 421, 422, 423, 424, 425, - /* 2780 */ 426, 381, 428, 383, 369, 476, 476, 476, 476, 476, - /* 2790 */ 476, 476, 476, 476, 333, 476, 381, 476, 383, 476, - /* 2800 */ 476, 476, 476, 476, 343, 476, 476, 476, 476, 333, - /* 2810 */ 476, 476, 476, 476, 414, 476, 476, 476, 418, 343, - /* 2820 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 414, - /* 2830 */ 369, 476, 476, 418, 476, 476, 421, 422, 423, 424, - /* 2840 */ 425, 426, 381, 428, 383, 369, 476, 476, 476, 476, - /* 2850 */ 476, 476, 476, 476, 476, 476, 476, 381, 476, 383, - /* 2860 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2870 */ 476, 476, 476, 476, 476, 414, 476, 476, 476, 418, - /* 2880 */ 476, 476, 421, 422, 423, 424, 425, 426, 476, 428, - /* 2890 */ 414, 476, 476, 476, 418, 476, 476, 421, 422, 423, - /* 2900 */ 424, 425, 426, 476, 428, + /* 1730 */ 0, 139, 48, 22, 48, 22, 0, 369, 59, 22, + /* 1740 */ 0, 59, 0, 0, 59, 0, 47, 47, 14, 381, + /* 1750 */ 177, 383, 0, 0, 0, 0, 0, 0, 414, 0, + /* 1760 */ 35, 66, 418, 0, 0, 421, 422, 423, 424, 425, + /* 1770 */ 426, 39, 428, 42, 47, 40, 39, 44, 333, 435, + /* 1780 */ 39, 35, 414, 439, 440, 0, 418, 49, 343, 421, + /* 1790 */ 422, 423, 424, 425, 426, 35, 428, 39, 49, 0, + /* 1800 */ 39, 333, 35, 435, 39, 0, 35, 439, 440, 0, + /* 1810 */ 0, 343, 39, 0, 369, 49, 0, 49, 35, 22, + /* 1820 */ 0, 35, 35, 35, 333, 35, 381, 35, 383, 35, + /* 1830 */ 35, 35, 0, 44, 343, 44, 35, 369, 35, 22, + /* 1840 */ 22, 0, 0, 22, 51, 22, 35, 333, 109, 381, + /* 1850 */ 0, 383, 0, 0, 107, 22, 20, 343, 0, 414, + /* 1860 */ 369, 35, 35, 418, 35, 374, 421, 422, 423, 424, + /* 1870 */ 425, 426, 381, 428, 383, 35, 101, 100, 35, 100, + /* 1880 */ 435, 193, 414, 369, 439, 440, 418, 22, 374, 421, + /* 1890 */ 422, 423, 424, 425, 426, 381, 428, 383, 35, 0, + /* 1900 */ 22, 0, 0, 435, 44, 414, 3, 100, 440, 418, + /* 1910 */ 333, 96, 421, 422, 423, 424, 425, 426, 170, 428, + /* 1920 */ 343, 256, 48, 101, 100, 48, 175, 47, 414, 98, + /* 1930 */ 168, 101, 418, 189, 44, 421, 422, 423, 424, 425, + /* 1940 */ 426, 44, 428, 47, 168, 101, 369, 100, 168, 100, + /* 1950 */ 44, 100, 44, 3, 44, 101, 100, 256, 381, 35, + /* 1960 */ 383, 35, 101, 101, 35, 35, 35, 35, 47, 101, + /* 1970 */ 101, 44, 47, 0, 0, 333, 0, 0, 39, 100, + /* 1980 */ 47, 100, 0, 101, 100, 343, 101, 100, 100, 100, + /* 1990 */ 171, 414, 39, 47, 169, 418, 333, 2, 421, 422, + /* 2000 */ 423, 424, 425, 426, 110, 428, 343, 44, 22, 333, + /* 2010 */ 98, 369, 98, 215, 47, 237, 100, 47, 101, 343, + /* 2020 */ 100, 22, 100, 381, 250, 383, 111, 35, 256, 101, + /* 2030 */ 22, 217, 369, 35, 44, 101, 100, 374, 100, 35, + /* 2040 */ 100, 464, 101, 101, 381, 369, 383, 100, 35, 101, + /* 2050 */ 101, 100, 100, 35, 35, 101, 414, 381, 100, 383, + /* 2060 */ 418, 101, 100, 421, 422, 423, 424, 425, 426, 100, + /* 2070 */ 428, 100, 35, 35, 65, 100, 66, 414, 122, 122, + /* 2080 */ 35, 418, 35, 333, 421, 422, 423, 424, 425, 426, + /* 2090 */ 414, 428, 35, 343, 418, 35, 35, 421, 422, 423, + /* 2100 */ 424, 425, 426, 333, 428, 122, 430, 122, 35, 35, + /* 2110 */ 35, 35, 72, 343, 44, 473, 94, 35, 35, 369, + /* 2120 */ 35, 22, 35, 35, 374, 35, 72, 35, 35, 35, + /* 2130 */ 35, 381, 35, 383, 22, 35, 0, 35, 49, 369, + /* 2140 */ 39, 0, 35, 39, 374, 49, 0, 49, 35, 39, + /* 2150 */ 0, 381, 35, 383, 49, 39, 0, 35, 35, 0, + /* 2160 */ 22, 21, 476, 21, 414, 22, 22, 20, 418, 476, + /* 2170 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2180 */ 476, 476, 476, 476, 414, 476, 1, 476, 418, 476, + /* 2190 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2200 */ 333, 476, 476, 476, 19, 476, 476, 476, 476, 476, + /* 2210 */ 343, 476, 476, 476, 476, 476, 333, 476, 33, 476, + /* 2220 */ 476, 476, 476, 476, 476, 476, 343, 476, 476, 476, + /* 2230 */ 476, 476, 476, 476, 49, 476, 369, 476, 476, 476, + /* 2240 */ 55, 56, 57, 58, 59, 476, 333, 476, 381, 476, + /* 2250 */ 383, 476, 369, 476, 476, 476, 343, 476, 476, 476, + /* 2260 */ 476, 476, 476, 476, 381, 476, 383, 476, 476, 476, + /* 2270 */ 476, 476, 476, 476, 476, 476, 333, 476, 476, 476, + /* 2280 */ 476, 414, 369, 476, 99, 418, 343, 102, 421, 422, + /* 2290 */ 423, 424, 425, 426, 381, 428, 383, 414, 476, 476, + /* 2300 */ 476, 418, 476, 476, 421, 422, 423, 424, 425, 426, + /* 2310 */ 476, 428, 369, 476, 476, 476, 476, 476, 476, 134, + /* 2320 */ 476, 476, 476, 476, 381, 476, 383, 414, 476, 476, + /* 2330 */ 476, 418, 476, 476, 421, 422, 423, 424, 425, 426, + /* 2340 */ 333, 428, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2350 */ 343, 476, 476, 333, 169, 476, 476, 414, 476, 174, + /* 2360 */ 476, 418, 476, 343, 421, 422, 423, 424, 425, 426, + /* 2370 */ 476, 428, 476, 476, 476, 476, 369, 476, 476, 194, + /* 2380 */ 476, 476, 476, 476, 476, 476, 476, 476, 381, 369, + /* 2390 */ 383, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2400 */ 476, 381, 476, 383, 333, 476, 476, 476, 476, 476, + /* 2410 */ 476, 476, 476, 476, 343, 476, 476, 333, 476, 476, + /* 2420 */ 476, 414, 476, 476, 476, 418, 476, 343, 421, 422, + /* 2430 */ 423, 424, 425, 426, 414, 428, 476, 476, 418, 476, + /* 2440 */ 369, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2450 */ 476, 476, 381, 369, 383, 476, 476, 476, 476, 476, + /* 2460 */ 476, 476, 476, 476, 476, 381, 476, 383, 476, 476, + /* 2470 */ 476, 476, 476, 476, 476, 476, 476, 476, 333, 476, + /* 2480 */ 476, 476, 476, 476, 476, 414, 476, 476, 343, 418, + /* 2490 */ 476, 476, 421, 422, 423, 424, 425, 426, 414, 428, + /* 2500 */ 476, 476, 418, 476, 476, 421, 422, 423, 424, 425, + /* 2510 */ 426, 476, 428, 476, 369, 476, 476, 476, 476, 476, + /* 2520 */ 476, 476, 476, 476, 476, 476, 381, 476, 383, 333, + /* 2530 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 343, + /* 2540 */ 476, 476, 476, 476, 333, 476, 476, 476, 476, 476, + /* 2550 */ 476, 476, 476, 476, 343, 476, 476, 476, 476, 414, + /* 2560 */ 476, 476, 476, 418, 476, 369, 421, 422, 423, 424, + /* 2570 */ 425, 426, 476, 428, 476, 476, 476, 381, 476, 383, + /* 2580 */ 369, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2590 */ 333, 476, 381, 476, 383, 476, 476, 476, 476, 476, + /* 2600 */ 343, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2610 */ 414, 476, 476, 476, 418, 476, 476, 421, 422, 423, + /* 2620 */ 424, 425, 426, 476, 428, 414, 369, 476, 476, 418, + /* 2630 */ 476, 476, 421, 422, 423, 424, 425, 426, 381, 428, + /* 2640 */ 383, 333, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2650 */ 476, 343, 476, 476, 476, 476, 476, 476, 333, 476, + /* 2660 */ 476, 476, 476, 476, 476, 476, 476, 476, 343, 476, + /* 2670 */ 476, 414, 476, 476, 476, 418, 476, 369, 421, 422, + /* 2680 */ 423, 424, 425, 426, 476, 428, 476, 333, 476, 381, + /* 2690 */ 476, 383, 476, 476, 369, 476, 476, 343, 476, 476, + /* 2700 */ 476, 476, 476, 476, 476, 476, 381, 476, 383, 476, + /* 2710 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2720 */ 476, 476, 414, 369, 476, 476, 418, 476, 476, 421, + /* 2730 */ 422, 423, 424, 425, 426, 381, 428, 383, 476, 414, + /* 2740 */ 476, 476, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2750 */ 425, 426, 476, 428, 476, 333, 476, 476, 476, 476, + /* 2760 */ 476, 476, 476, 476, 476, 343, 476, 476, 414, 476, + /* 2770 */ 476, 476, 418, 476, 333, 421, 422, 423, 424, 425, + /* 2780 */ 426, 476, 428, 476, 343, 476, 476, 333, 476, 476, + /* 2790 */ 476, 369, 476, 476, 476, 476, 476, 343, 476, 476, + /* 2800 */ 476, 476, 476, 381, 476, 383, 476, 476, 476, 476, + /* 2810 */ 369, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2820 */ 476, 476, 381, 369, 383, 476, 476, 476, 476, 476, + /* 2830 */ 476, 476, 476, 476, 476, 381, 414, 383, 476, 476, + /* 2840 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, + /* 2850 */ 428, 476, 476, 476, 476, 414, 476, 476, 476, 418, + /* 2860 */ 333, 476, 421, 422, 423, 424, 425, 426, 414, 428, + /* 2870 */ 343, 476, 418, 333, 476, 421, 422, 423, 424, 425, + /* 2880 */ 426, 476, 428, 343, 476, 476, 476, 476, 333, 476, + /* 2890 */ 476, 476, 476, 476, 476, 476, 369, 476, 343, 476, + /* 2900 */ 476, 476, 476, 476, 476, 476, 476, 476, 381, 369, + /* 2910 */ 383, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2920 */ 476, 381, 476, 383, 369, 476, 476, 476, 476, 476, + /* 2930 */ 476, 476, 476, 476, 333, 476, 381, 476, 383, 476, + /* 2940 */ 476, 414, 476, 476, 343, 418, 476, 476, 421, 422, + /* 2950 */ 423, 424, 425, 426, 414, 428, 476, 476, 418, 476, + /* 2960 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 414, + /* 2970 */ 369, 476, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2980 */ 425, 426, 381, 428, 383, 476, 476, 476, 476, 476, + /* 2990 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3000 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3010 */ 476, 476, 476, 476, 476, 414, 476, 476, 476, 418, + /* 3020 */ 476, 476, 421, 422, 423, 424, 425, 426, 476, 428, }; #define YY_SHIFT_COUNT (761) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2182) +#define YY_SHIFT_MAX (2185) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 550, 0, 224, 0, 449, 449, 449, 449, 449, 449, /* 10 */ 449, 449, 449, 449, 449, 449, 673, 897, 897, 1121, /* 20 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, /* 30 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 457, - /* 50 */ 492, 142, 145, 147, 107, 173, 107, 145, 145, 1347, - /* 60 */ 1347, 1347, 107, 1347, 1347, 501, 107, 18, 447, 47, - /* 70 */ 47, 447, 235, 235, 3, 286, 7, 7, 47, 47, - /* 80 */ 47, 47, 47, 47, 47, 51, 47, 47, 132, 18, - /* 90 */ 47, 47, 228, 47, 18, 47, 51, 47, 51, 18, - /* 100 */ 47, 47, 18, 47, 18, 18, 18, 47, 339, 223, + /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, + /* 50 */ 897, 452, 457, 142, 145, 147, 30, 107, 30, 145, + /* 60 */ 145, 1347, 30, 1347, 1347, 501, 30, 56, 447, 344, + /* 70 */ 344, 447, 235, 235, 18, 286, 150, 150, 344, 344, + /* 80 */ 344, 344, 344, 344, 344, 369, 344, 344, 412, 56, + /* 90 */ 344, 344, 484, 344, 56, 344, 369, 344, 369, 56, + /* 100 */ 344, 344, 56, 344, 56, 56, 56, 344, 538, 223, /* 110 */ 189, 189, 333, 114, 443, 443, 443, 443, 443, 443, /* 120 */ 443, 443, 443, 443, 443, 443, 443, 443, 443, 443, - /* 130 */ 443, 443, 443, 843, 325, 3, 286, 392, 220, 220, - /* 140 */ 220, 417, 369, 369, 392, 344, 344, 344, 132, 365, - /* 150 */ 297, 18, 439, 18, 439, 439, 503, 558, 35, 35, - /* 160 */ 35, 35, 35, 35, 35, 35, 2060, 217, 288, 483, - /* 170 */ 522, 260, 313, 243, 384, 940, 576, 484, 890, 751, - /* 180 */ 528, 810, 824, 60, 974, 824, 1065, 791, 275, 1124, - /* 190 */ 1336, 1204, 1353, 1378, 1353, 1236, 1384, 1384, 1353, 1236, - /* 200 */ 1236, 1315, 1324, 1384, 1329, 1384, 1384, 1384, 1411, 1386, - /* 210 */ 1411, 1386, 1418, 132, 1421, 132, 1426, 1430, 132, 1426, - /* 220 */ 132, 132, 132, 1384, 132, 1413, 1413, 1411, 18, 18, - /* 230 */ 18, 18, 18, 18, 18, 18, 18, 18, 18, 1384, - /* 240 */ 1411, 439, 439, 439, 1292, 1398, 1418, 339, 1325, 1323, - /* 250 */ 1421, 339, 1330, 1384, 1378, 1378, 439, 1262, 1266, 439, - /* 260 */ 1262, 1266, 439, 439, 18, 1253, 1355, 1262, 1271, 1273, - /* 270 */ 1293, 1124, 1283, 1277, 1276, 1318, 344, 1550, 1384, 1426, - /* 280 */ 339, 339, 1566, 1266, 439, 439, 439, 439, 439, 1266, - /* 290 */ 439, 1427, 339, 503, 339, 344, 1504, 1508, 439, 558, - /* 300 */ 1384, 339, 1576, 1411, 2905, 2905, 2905, 2905, 2905, 2905, - /* 310 */ 2905, 2905, 2905, 34, 430, 15, 590, 734, 16, 839, + /* 130 */ 443, 443, 443, 843, 325, 18, 286, 392, 1, 1, + /* 140 */ 1, 539, 273, 273, 392, 627, 627, 627, 412, 615, + /* 150 */ 529, 56, 699, 56, 699, 699, 715, 778, 35, 35, + /* 160 */ 35, 35, 35, 35, 35, 35, 2185, 217, 288, 483, + /* 170 */ 522, 260, 313, 243, 536, 1032, 490, 646, 868, 616, + /* 180 */ 602, 916, 829, 60, 422, 829, 966, 939, 994, 1110, + /* 190 */ 1317, 1190, 1334, 1365, 1334, 1224, 1372, 1372, 1334, 1224, + /* 200 */ 1224, 1303, 1309, 1372, 1316, 1372, 1372, 1372, 1393, 1373, + /* 210 */ 1393, 1373, 1405, 412, 1408, 412, 1411, 1412, 412, 1411, + /* 220 */ 412, 412, 412, 1372, 412, 1385, 1385, 1393, 56, 56, + /* 230 */ 56, 56, 56, 56, 56, 56, 56, 56, 56, 1372, + /* 240 */ 1393, 699, 699, 699, 1258, 1364, 1405, 538, 1301, 1304, + /* 250 */ 1408, 538, 1308, 1372, 1365, 1365, 699, 1241, 1248, 699, + /* 260 */ 1241, 1248, 699, 699, 56, 1250, 1346, 1241, 1255, 1257, + /* 270 */ 1274, 1110, 1254, 1259, 1265, 1288, 627, 1522, 1372, 1411, + /* 280 */ 538, 538, 1528, 1248, 699, 699, 699, 699, 699, 1248, + /* 290 */ 699, 1382, 538, 715, 538, 627, 1485, 1489, 699, 778, + /* 300 */ 1372, 538, 1564, 1393, 3030, 3030, 3030, 3030, 3030, 3030, + /* 310 */ 3030, 3030, 3030, 34, 430, 15, 590, 734, 16, 839, /* 320 */ 79, 709, 792, 566, 856, 1023, 1023, 1023, 1023, 1023, /* 330 */ 1023, 1023, 1023, 1023, 1167, 743, 253, 253, 402, 57, - /* 340 */ 758, 774, 949, 1067, 800, 980, 64, 64, 959, 1007, - /* 350 */ 387, 959, 959, 959, 1149, 641, 1116, 1140, 1147, 1070, - /* 360 */ 1168, 1078, 1119, 1141, 1164, 532, 1190, 1207, 1250, 1251, - /* 370 */ 990, 1189, 1212, 868, 1216, 1231, 1233, 943, 1004, 1117, - /* 380 */ 1153, 1261, 1264, 1267, 1270, 1272, 1274, 1302, 1289, 1196, - /* 390 */ 1206, 987, 1291, 1238, 1286, 1317, 1320, 1322, 1326, 1328, - /* 400 */ 190, 1248, 1361, 1256, 1331, 1614, 1620, 1624, 1590, 1633, - /* 410 */ 1599, 1428, 1600, 1602, 1604, 1432, 1644, 1607, 1610, 1438, - /* 420 */ 1647, 1440, 1649, 1615, 1651, 1630, 1653, 1621, 1460, 1469, - /* 430 */ 1473, 1660, 1667, 1669, 1488, 1490, 1672, 1689, 1629, 1690, - /* 440 */ 1691, 1692, 1652, 1696, 1697, 1698, 1702, 1703, 1704, 1705, - /* 450 */ 1706, 1552, 1673, 1709, 1555, 1712, 1714, 1715, 1716, 1717, - /* 460 */ 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1726, 1728, 1729, - /* 470 */ 1730, 1693, 1731, 1732, 1733, 1734, 1736, 1738, 1725, 1739, - /* 480 */ 1740, 1741, 1603, 1743, 1744, 1735, 1700, 1737, 1707, 1745, - /* 490 */ 1687, 1742, 1750, 1694, 1752, 1695, 1756, 1760, 1753, 1746, - /* 500 */ 1727, 1751, 1755, 1747, 1757, 1773, 1759, 1748, 1774, 1775, - /* 510 */ 1776, 1758, 1586, 1764, 1780, 1781, 1749, 1800, 1805, 1771, - /* 520 */ 1761, 1770, 1812, 1778, 1767, 1779, 1817, 1784, 1772, 1783, - /* 530 */ 1820, 1788, 1782, 1786, 1827, 1829, 1830, 1832, 1766, 1754, - /* 540 */ 1798, 1813, 1836, 1803, 1804, 1806, 1807, 1808, 1809, 1810, - /* 550 */ 1796, 1802, 1815, 1818, 1826, 1819, 1852, 1833, 1856, 1840, - /* 560 */ 1814, 1858, 1842, 1841, 1877, 1843, 1879, 1846, 1887, 1866, - /* 570 */ 1878, 1864, 1865, 1867, 1821, 1801, 1904, 1762, 1811, 1713, - /* 580 */ 1872, 1896, 1921, 1765, 1901, 1777, 1789, 1924, 1925, 1785, - /* 590 */ 1768, 1923, 1883, 1675, 1834, 1828, 1844, 1885, 1849, 1893, - /* 600 */ 1853, 1847, 1906, 1908, 1855, 1857, 1861, 1862, 1863, 1919, - /* 610 */ 1918, 1922, 1868, 1926, 1710, 1870, 1871, 1964, 1929, 1791, - /* 620 */ 1939, 1941, 1942, 1943, 1944, 1945, 1880, 1881, 1937, 1790, - /* 630 */ 1946, 1947, 1983, 1986, 1988, 1989, 1892, 1954, 1751, 1950, - /* 640 */ 1907, 1903, 1909, 1912, 1913, 1837, 1914, 2015, 1977, 1848, - /* 650 */ 1927, 1910, 1751, 1971, 1978, 1928, 1792, 1930, 2019, 2002, - /* 660 */ 1816, 1933, 1934, 1938, 1935, 1948, 1940, 1987, 1949, 1951, - /* 670 */ 1992, 1952, 2008, 1825, 1955, 1932, 1953, 2017, 2023, 1962, - /* 680 */ 1968, 2036, 1973, 1974, 2039, 1976, 1979, 2043, 1982, 1991, - /* 690 */ 2048, 1995, 1981, 1985, 1990, 1999, 2005, 2064, 2010, 2076, - /* 700 */ 2013, 2064, 2064, 2092, 2056, 2058, 2089, 2090, 2091, 2093, - /* 710 */ 2094, 2096, 2098, 2099, 2100, 2101, 2055, 2044, 2095, 2102, - /* 720 */ 2105, 2106, 2120, 2108, 2110, 2112, 2077, 1796, 2113, 1802, - /* 730 */ 2115, 2116, 2118, 2119, 2130, 2122, 2155, 2123, 2111, 2124, - /* 740 */ 2161, 2129, 2125, 2126, 2166, 2132, 2127, 2134, 2169, 2136, - /* 750 */ 2131, 2139, 2172, 2140, 2144, 2182, 2168, 2171, 2173, 2174, - /* 760 */ 2176, 2178, + /* 340 */ 441, 414, 838, 1040, 934, 636, 64, 64, 638, 606, + /* 350 */ 903, 638, 638, 638, 417, 927, 1007, 1070, 1067, 1057, + /* 360 */ 1138, 967, 1071, 1072, 1092, 1156, 1168, 1185, 1219, 1223, + /* 370 */ 1000, 1145, 1170, 431, 1171, 1172, 1176, 1089, 1117, 1105, + /* 380 */ 809, 1189, 1204, 1207, 1212, 1216, 1217, 1229, 1231, 1142, + /* 390 */ 1208, 1187, 1234, 1203, 1267, 1270, 1272, 1277, 1289, 1294, + /* 400 */ 190, 1299, 1348, 668, 1320, 1613, 1616, 1617, 1576, 1619, + /* 410 */ 1585, 1414, 1589, 1590, 1591, 1419, 1628, 1595, 1596, 1424, + /* 420 */ 1633, 1426, 1635, 1601, 1637, 1618, 1638, 1604, 1447, 1456, + /* 430 */ 1459, 1646, 1647, 1648, 1467, 1469, 1651, 1652, 1606, 1655, + /* 440 */ 1656, 1657, 1625, 1659, 1660, 1669, 1670, 1671, 1672, 1689, + /* 450 */ 1690, 1521, 1661, 1691, 1537, 1694, 1697, 1698, 1702, 1703, + /* 460 */ 1704, 1705, 1706, 1707, 1708, 1709, 1710, 1712, 1714, 1715, + /* 470 */ 1716, 1675, 1718, 1719, 1720, 1721, 1722, 1723, 1711, 1724, + /* 480 */ 1726, 1728, 1592, 1729, 1730, 1713, 1684, 1717, 1686, 1736, + /* 490 */ 1679, 1725, 1740, 1682, 1742, 1685, 1743, 1745, 1731, 1732, + /* 500 */ 1733, 1699, 1700, 1734, 1727, 1752, 1735, 1737, 1753, 1754, + /* 510 */ 1755, 1741, 1573, 1756, 1757, 1759, 1695, 1763, 1764, 1746, + /* 520 */ 1738, 1758, 1785, 1760, 1749, 1761, 1799, 1767, 1766, 1765, + /* 530 */ 1805, 1771, 1768, 1773, 1809, 1810, 1813, 1816, 1739, 1747, + /* 540 */ 1783, 1797, 1820, 1786, 1787, 1788, 1790, 1792, 1794, 1795, + /* 550 */ 1789, 1791, 1796, 1801, 1817, 1803, 1832, 1818, 1841, 1821, + /* 560 */ 1793, 1842, 1823, 1811, 1850, 1826, 1852, 1827, 1853, 1833, + /* 570 */ 1836, 1829, 1840, 1843, 1775, 1777, 1858, 1762, 1779, 1688, + /* 580 */ 1863, 1865, 1899, 1744, 1878, 1776, 1748, 1901, 1902, 1780, + /* 590 */ 1751, 1903, 1860, 1665, 1807, 1822, 1824, 1874, 1831, 1877, + /* 600 */ 1815, 1830, 1890, 1897, 1844, 1847, 1849, 1851, 1854, 1906, + /* 610 */ 1880, 1896, 1856, 1908, 1701, 1861, 1862, 1950, 1910, 1772, + /* 620 */ 1924, 1926, 1929, 1930, 1931, 1932, 1868, 1869, 1921, 1774, + /* 630 */ 1927, 1925, 1973, 1974, 1976, 1977, 1879, 1939, 1699, 1933, + /* 640 */ 1881, 1882, 1885, 1884, 1887, 1819, 1888, 1982, 1953, 1825, + /* 650 */ 1889, 1894, 1699, 1946, 1963, 1912, 1778, 1914, 1995, 1986, + /* 660 */ 1798, 1916, 1917, 1920, 1928, 1922, 1934, 1967, 1936, 1938, + /* 670 */ 1970, 1941, 1999, 1814, 1940, 1915, 1942, 1992, 1998, 1947, + /* 680 */ 1948, 2004, 1951, 1949, 2013, 1952, 1954, 2018, 1958, 1960, + /* 690 */ 2019, 1962, 1956, 1957, 1983, 1985, 1969, 1990, 1971, 2037, + /* 700 */ 1975, 1990, 1990, 2008, 2010, 2009, 2038, 2045, 2047, 2057, + /* 710 */ 2060, 2061, 2073, 2074, 2075, 2076, 2040, 2022, 2070, 2082, + /* 720 */ 2083, 2085, 2099, 2087, 2088, 2090, 2054, 1789, 2092, 1791, + /* 730 */ 2093, 2094, 2095, 2097, 2112, 2100, 2136, 2102, 2089, 2101, + /* 740 */ 2141, 2107, 2096, 2104, 2146, 2113, 2098, 2110, 2150, 2117, + /* 750 */ 2105, 2116, 2156, 2122, 2123, 2159, 2138, 2140, 2143, 2144, + /* 760 */ 2142, 2147, }; #define YY_REDUCE_COUNT (312) #define YY_REDUCE_MIN (-396) -#define YY_REDUCE_MAX (2476) +#define YY_REDUCE_MAX (2601) static const short yy_reduce_ofst[] = { /* 0 */ -140, 137, -82, 358, 444, 578, 814, 841, 1034, 1064, /* 10 */ 1132, 1240, 1260, 1344, 1368, 1445, 598, -333, 672, 1468, - /* 20 */ 901, 1491, 1514, 1577, 1642, 1663, 1676, 1763, 1787, 1850, - /* 30 */ 1874, 1920, 1936, 1996, 2020, 2047, 2066, 2117, 2133, 2163, - /* 40 */ 2184, 2230, 2290, 2303, 2354, 2400, 2415, 2461, 2476, 372, - /* 50 */ -195, -396, -265, 487, 595, 607, 701, 481, 761, -362, - /* 60 */ -355, 285, 749, 136, 612, -279, -55, 256, -346, -167, - /* 70 */ -5, -381, -334, -257, -205, -249, 45, 130, 300, 303, - /* 80 */ 356, 377, 409, 425, 432, 177, 469, 489, 252, -308, - /* 90 */ 521, 573, 170, 620, 172, 622, 261, 707, 396, 682, - /* 100 */ 739, 745, 278, 793, 715, 321, 781, 795, -266, 8, - /* 110 */ -350, -350, 115, -238, 161, 214, 257, 315, 374, 433, - /* 120 */ 445, 463, 585, 587, 617, 677, 681, 806, 807, 813, - /* 130 */ 819, 830, 832, -86, -289, -111, 23, 154, -289, 70, - /* 140 */ 188, -25, 480, 511, 380, 416, 498, 704, 381, -354, - /* 150 */ 426, 513, 690, 502, 733, 798, 808, 353, -369, -365, - /* 160 */ 324, 370, 458, 505, 686, 458, 764, 368, 621, 825, - /* 170 */ 753, 780, 906, 822, 917, 917, 946, 898, 954, 963, - /* 180 */ 930, 921, 871, 871, 891, 871, 916, 925, 917, 967, - /* 190 */ 971, 989, 1001, 1008, 1012, 1016, 1063, 1066, 1022, 1025, - /* 200 */ 1026, 1060, 1069, 1074, 1073, 1084, 1086, 1088, 1097, 1096, - /* 210 */ 1101, 1098, 1030, 1090, 1059, 1093, 1105, 1050, 1103, 1110, - /* 220 */ 1111, 1113, 1114, 1126, 1120, 1125, 1129, 1136, 1108, 1112, - /* 230 */ 1115, 1122, 1127, 1131, 1133, 1137, 1138, 1139, 1142, 1151, - /* 240 */ 1144, 1099, 1102, 1128, 1081, 1094, 1107, 1162, 1104, 1118, - /* 250 */ 1135, 1171, 1123, 1179, 1143, 1145, 1146, 1068, 1134, 1150, - /* 260 */ 1072, 1148, 1156, 1157, 917, 1080, 1082, 1085, 1092, 1091, - /* 270 */ 1106, 1152, 1076, 1095, 1130, 871, 1223, 1154, 1242, 1241, - /* 280 */ 1234, 1239, 1194, 1193, 1209, 1210, 1211, 1213, 1214, 1200, - /* 290 */ 1218, 1208, 1252, 1237, 1257, 1263, 1169, 1243, 1229, 1254, - /* 300 */ 1269, 1278, 1279, 1282, 1215, 1217, 1222, 1225, 1265, 1268, - /* 310 */ 1275, 1281, 1294, + /* 20 */ 901, 1491, 1514, 1577, 1642, 1663, 1676, 1750, 1770, 1867, + /* 30 */ 1883, 1913, 1943, 2007, 2020, 2071, 2084, 2145, 2196, 2211, + /* 40 */ 2257, 2308, 2325, 2354, 2422, 2441, 2454, 2527, 2540, 2555, + /* 50 */ 2601, 372, -195, -396, -265, 487, 595, 607, 701, 481, + /* 60 */ 761, -362, 191, -365, 136, -279, -55, 256, -372, -167, + /* 70 */ -5, -381, -273, -257, 14, -321, -337, 203, 45, 285, + /* 80 */ 300, 356, 377, 409, 425, 177, 432, 489, 446, 279, + /* 90 */ 521, 573, 125, 620, 321, 622, 396, 687, 436, 257, + /* 100 */ 707, 716, 460, 739, 434, 506, 679, 746, -266, 8, + /* 110 */ -350, -350, 123, -261, 265, 363, 579, 585, 612, 617, + /* 120 */ 710, 754, 769, 774, 777, 781, 806, 807, 813, 820, + /* 130 */ 830, 831, 832, -331, -289, -111, -249, -61, -289, 290, + /* 140 */ 563, -25, -338, 158, 357, -147, 498, 533, -102, 202, + /* 150 */ 324, 29, 378, 502, 593, 597, 808, 236, -367, 552, + /* 160 */ 584, 603, 614, 674, 857, 614, 785, 871, 945, 877, + /* 170 */ 811, 823, 937, 834, 920, 920, 946, 904, 963, 985, + /* 180 */ 926, 919, 882, 882, 890, 882, 917, 911, 920, 949, + /* 190 */ 957, 973, 996, 995, 999, 1002, 1051, 1052, 1010, 1013, + /* 200 */ 1014, 1050, 1055, 1068, 1060, 1074, 1076, 1078, 1079, 1083, + /* 210 */ 1088, 1085, 1017, 1077, 1046, 1080, 1090, 1035, 1087, 1091, + /* 220 */ 1093, 1096, 1099, 1097, 1100, 1094, 1095, 1116, 1084, 1101, + /* 230 */ 1102, 1103, 1107, 1108, 1111, 1112, 1114, 1115, 1122, 1098, + /* 240 */ 1119, 1082, 1113, 1124, 1053, 1106, 1059, 1131, 1104, 1109, + /* 250 */ 1123, 1148, 1118, 1158, 1120, 1127, 1126, 1054, 1125, 1128, + /* 260 */ 1058, 1129, 1139, 1141, 920, 1069, 1073, 1081, 1075, 1086, + /* 270 */ 1133, 1135, 1061, 1130, 1134, 882, 1198, 1144, 1201, 1200, + /* 280 */ 1197, 1199, 1155, 1163, 1180, 1188, 1193, 1206, 1209, 1174, + /* 290 */ 1210, 1173, 1220, 1227, 1236, 1239, 1150, 1221, 1214, 1237, + /* 300 */ 1256, 1249, 1268, 1266, 1202, 1196, 1211, 1213, 1244, 1251, + /* 310 */ 1252, 1262, 1281, }; static const YYACTIONTYPE yy_default[] = { /* 0 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, @@ -2454,8 +2478,8 @@ static const char *const yyRuleName[] = { /* 531 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", /* 532 */ "fill_opt ::=", /* 533 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 534 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 535 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP", + /* 534 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP", + /* 535 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP", /* 536 */ "fill_mode ::= NONE", /* 537 */ "fill_mode ::= PREV", /* 538 */ "fill_mode ::= NULL", @@ -3663,8 +3687,8 @@ static const struct { { 396, -4 }, /* (531) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ { 456, 0 }, /* (532) fill_opt ::= */ { 456, -4 }, /* (533) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 456, -6 }, /* (534) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 456, -6 }, /* (535) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ + { 456, -6 }, /* (534) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ + { 456, -6 }, /* (535) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ { 463, -1 }, /* (536) fill_mode ::= NONE */ { 463, -1 }, /* (537) fill_mode ::= PREV */ { 463, -1 }, /* (538) fill_mode ::= NULL */ @@ -5351,10 +5375,10 @@ static YYACTIONTYPE yy_reduce( case 533: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ { yymsp[-3].minor.yy448 = createFillNode(pCxt, yymsp[-1].minor.yy46, NULL); } break; - case 534: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + case 534: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ { yymsp[-5].minor.yy448 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy432)); } break; - case 535: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ + case 535: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ { yymsp[-5].minor.yy448 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy432)); } break; case 536: /* fill_mode ::= NONE */ diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index c9ee83a64757b6fe951d93fd9c9d917ba357f58b..6544898be9ff389cf441d8824ef06b235fd7cc35 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -740,6 +740,13 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm code = createColumnByRewriteExprs(pWindow->pFuncs, &pWindow->node.pTargets); } + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { + pWindow->node.pConditions = nodesCloneNode(pSelect->pHaving); + if (NULL == pWindow->node.pConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + pSelect->hasAggFuncs = false; if (TSDB_CODE_SUCCESS == code) { @@ -1132,6 +1139,14 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS } } + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving && !pSelect->hasAggFuncs && NULL == pSelect->pGroupByList && + NULL == pSelect->pWindow) { + pPartition->node.pConditions = nodesCloneNode(pSelect->pHaving); + if (NULL == pPartition->node.pConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pPartition; } else { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 52bb03466c2800c93f663f706300eb2b7695d8ae..07ea110d7e68382a4872e4560a21eca90ac10332 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1095,7 +1095,7 @@ static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool group *pNotOptimize = false; return TSDB_CODE_SUCCESS; } - + switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: { SScanLogicNode* pScan = (SScanLogicNode*)pNode; @@ -2139,7 +2139,7 @@ typedef struct SLastRowScanOptLastParaCkCxt { bool hasCol; } SLastRowScanOptLastParaCkCxt; -static EDealRes lastRowScanOptLastParaCheckImpl(SNode* pNode, void* pContext) { +static EDealRes lastRowScanOptLastParaIsTagImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SLastRowScanOptLastParaCkCxt* pCxt = pContext; if (COLUMN_TYPE_TAG == ((SColumnNode*)pNode)->colType || COLUMN_TYPE_TBNAME == ((SColumnNode*)pNode)->colType) { @@ -2152,10 +2152,10 @@ static EDealRes lastRowScanOptLastParaCheckImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool lastRowScanOptLastParaCheck(SNode* pExpr) { +static bool lastRowScanOptLastParaIsTag(SNode* pExpr) { SLastRowScanOptLastParaCkCxt cxt = {.hasTag = false, .hasCol = false}; - nodesWalkExpr(pExpr, lastRowScanOptLastParaCheckImpl, &cxt); - return !cxt.hasTag && cxt.hasCol; + nodesWalkExpr(pExpr, lastRowScanOptLastParaIsTagImpl, &cxt); + return cxt.hasTag && !cxt.hasCol; } static bool hasSuitableCache(int8_t cacheLastMode, bool hasLastRow, bool hasLast) { @@ -2195,15 +2195,19 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) { SFunctionNode* pAggFunc = (SFunctionNode*)pFunc; if (FUNCTION_TYPE_LAST == pAggFunc->funcType) { - if (hasSelectFunc || !lastRowScanOptLastParaCheck(nodesListGetNode(pAggFunc->pParameterList, 0))) { + if (hasSelectFunc || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) { return false; } hasLastFunc = true; - } else if (FUNCTION_TYPE_SELECT_VALUE == pAggFunc->funcType || FUNCTION_TYPE_GROUP_KEY == pAggFunc->funcType) { + } else if (FUNCTION_TYPE_SELECT_VALUE == pAggFunc->funcType) { if (hasLastFunc) { return false; } hasSelectFunc = true; + } else if (FUNCTION_TYPE_GROUP_KEY == pAggFunc->funcType) { + if (!lastRowScanOptLastParaIsTag(nodesListGetNode(pAggFunc->pParameterList, 0))) { + return false; + } } else if (FUNCTION_TYPE_LAST_ROW != pAggFunc->funcType) { return false; } @@ -2237,7 +2241,7 @@ static EDealRes lastRowScanOptSetColDataType(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols) { +static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols, bool erase) { SNode* pTarget = NULL; WHERE_EACH(pTarget, pTargets) { bool found = false; @@ -2249,7 +2253,7 @@ static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCo break; } } - if (!found) { + if (!found && erase) { ERASE_NODE(pTargets); continue; } @@ -2290,9 +2294,8 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic pScan->igLastNull = pAgg->hasLast ? true : false; if (NULL != cxt.pLastCols) { cxt.doAgg = false; - lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols); - NODES_DESTORY_LIST(pScan->pScanPseudoCols); - lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols); + lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols, true); + lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols, false); nodesClearList(cxt.pLastCols); } pAgg->hasLastRow = false; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index fe01977b2e2803e6797df2bc730177eb28aa6617..0521076d23d9662219574388e86960b5dddbea6f 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1038,23 +1038,40 @@ bool sclContainsAggFuncNode(SNode *pNode) { return aggFunc; } +static uint8_t sclGetOpValueNodeTsPrecision(SNode *pLeft, SNode *pRight) { + uint8_t lPrec = ((SExprNode *)pLeft)->resType.precision; + uint8_t rPrec = ((SExprNode *)pRight)->resType.precision; + + uint8_t lType = ((SExprNode *)pLeft)->resType.type; + uint8_t rType = ((SExprNode *)pRight)->resType.type; + + if (TSDB_DATA_TYPE_TIMESTAMP == lType && TSDB_DATA_TYPE_TIMESTAMP == rType) { + return TMAX(lPrec, rPrec); + } else if (TSDB_DATA_TYPE_TIMESTAMP == lType && TSDB_DATA_TYPE_TIMESTAMP != rType) { + return lPrec; + } else if (TSDB_DATA_TYPE_TIMESTAMP == rType && TSDB_DATA_TYPE_TIMESTAMP != lType) { + return rPrec; + } + + return 0; +} int32_t sclConvertOpValueNodeTs(SOperatorNode *node, SScalarCtx *ctx) { int32_t code = 0; - + if (node->pLeft && SCL_IS_VAR_VALUE_NODE(node->pLeft)) { if (node->pRight && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pRight)->resType.type)) { - SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pRight)->resType.precision, (SValueNode*)node->pLeft)); + SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pLeft)); } } else if (node->pRight && SCL_IS_NOTNULL_CONST_NODE(node->pRight)) { if (node->pLeft && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pLeft)->resType.type)) { if (SCL_IS_VAR_VALUE_NODE(node->pRight)) { - SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pLeft)->resType.precision, (SValueNode*)node->pRight)); + SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pRight)); } else if (QUERY_NODE_NODE_LIST == node->pRight->type) { SNode* pNode; FOREACH(pNode, ((SNodeListNode*)node->pRight)->pNodeList) { if (SCL_IS_VAR_VALUE_NODE(pNode)) { - SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pLeft)->resType.precision, (SValueNode*)pNode)); + SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode*)pNode)); } } } @@ -1077,7 +1094,7 @@ int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) { if (NULL == node->pCase) { return TSDB_CODE_SUCCESS; } - + if (SCL_IS_VAR_VALUE_NODE(node->pCase)) { SNode* pNode; FOREACH(pNode, node->pWhenThenList) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 24b25cec80ce58390b7a83d71a607bbaba18fe50..835264eabe165f9eaf2fe3869ebb03e13cf2dd05 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1124,7 +1124,8 @@ _end: int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(pInput); int64_t timePrec; - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + int32_t idx = (inputNum == 2) ? 1 : 2; + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[idx]), pInput[idx].columnData->pData); for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index ceddf4f2153735c4d8423dde806272419dc35fc5..790547bb6154e6a53847f609d918fe4fac784c29 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories( target_link_libraries( stream PUBLIC tdb - PRIVATE os util transport qcom executor + PRIVATE os util transport qcom executor wal ) if(${BUILD_TEST}) diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 66496f11f8b2da8ad46d91637efdeb1444f26319..876b80697a9679125350f8b9b2a1e26e4fdb0572 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -44,7 +44,7 @@ int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, SEpSet* pEpSet); -SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem); #ifdef __cplusplus } diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 361cd2cacc1d2ba963b65387d2ce90e6b1f77ab1..7171b529129858db7c455f33422b2f83e8e9677c 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -16,6 +16,8 @@ #include "streamInc.h" #include "ttimer.h" +#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 2000 + int32_t streamInit() { int8_t old; while (1) { @@ -50,7 +52,7 @@ void streamCleanUp() { void streamSchedByTimer(void* param, void* tmrId) { SStreamTask* pTask = (void*)param; - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(NULL, pTask); return; } @@ -64,15 +66,16 @@ void streamSchedByTimer(void* param, void* tmrId) { taosFreeQitem(trigger); return; } - trigger->pBlock->info.type = STREAM_GET_ALL; + trigger->pBlock->info.type = STREAM_GET_ALL; atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE); - if (tAppendDataForStream(pTask, (SStreamQueueItem*)trigger) < 0) { + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)trigger) < 0) { taosFreeQitem(trigger); taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); return; } + streamSchedExec(pTask); } @@ -91,31 +94,33 @@ int32_t streamSetupTrigger(SStreamTask* pTask) { int32_t streamSchedExec(SStreamTask* pTask) { int8_t schedStatus = - atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__WAITING); + atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__WAITING); if (schedStatus == TASK_SCHED_STATUS__INACTIVE) { SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); if (pRunReq == NULL) { - atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE); + terrno = TSDB_CODE_OUT_OF_MEMORY; + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); return -1; } pRunReq->head.vgId = pTask->nodeId; - pRunReq->streamId = pTask->streamId; - pRunReq->taskId = pTask->taskId; + pRunReq->streamId = pTask->id.streamId; + pRunReq->taskId = pTask->id.taskId; SRpcMsg msg = { .msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq) }; tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &msg); + qDebug("trigger to run s-task:%s", pTask->id.idStr); } return 0; } -int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SRpcMsg* pRsp) { +int32_t streamTaskEnqueueBlocks(SStreamTask* pTask, const SStreamDispatchReq* pReq, SRpcMsg* pRsp) { SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); int8_t status; - // enqueue + // enqueue data block if (pData != NULL) { pData->type = STREAM_INPUT__DATA_BLOCK; pData->srcVgId = pReq->dataSrcVgId; @@ -123,10 +128,10 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR /*pData->blocks = pReq->data;*/ /*pBlock->sourceVer = pReq->sourceVer;*/ streamDispatchReqToData(pReq, pData); - if (tAppendDataForStream(pTask, (SStreamQueueItem*)pData) == 0) { + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pData) == 0) { status = TASK_INPUT_STATUS__NORMAL; - } else { - status = TASK_INPUT_STATUS__FAILED; + } else { // input queue is full, upstream is blocked now + status = TASK_INPUT_STATUS__BLOCKED; } } else { streamTaskInputFail(pTask); @@ -142,10 +147,12 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR pCont->upstreamNodeId = htonl(pReq->upstreamNodeId); pCont->upstreamTaskId = htonl(pReq->upstreamTaskId); pCont->downstreamNodeId = htonl(pTask->nodeId); - pCont->downstreamTaskId = htonl(pTask->taskId); + pCont->downstreamTaskId = htonl(pTask->id.taskId); pRsp->pCont = buf; + pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); tmsgSendRsp(pRsp); + return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1; } @@ -155,7 +162,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, // enqueue if (pData != NULL) { - qDebug("task %d(child %d) recv retrieve req from task %d, reqId %" PRId64, pTask->taskId, pTask->selfChildId, + qDebug("task %d(child %d) recv retrieve req from task %d, reqId %" PRId64, pTask->id.taskId, pTask->selfChildId, pReq->srcTaskId, pReq->reqId); pData->type = STREAM_INPUT__DATA_RETRIEVE; @@ -164,7 +171,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, /*pData->blocks = pReq->data;*/ /*pBlock->sourceVer = pReq->sourceVer;*/ streamRetrieveReqToData(pReq, pData); - if (tAppendDataForStream(pTask, (SStreamQueueItem*)pData) == 0) { + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pData) == 0) { status = TASK_INPUT_STATUS__NORMAL; } else { status = TASK_INPUT_STATUS__FAILED; @@ -205,10 +212,10 @@ int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) { } int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) { - qDebug("task %d receive dispatch req from node %d task %d", pTask->taskId, pReq->upstreamNodeId, + qDebug("vgId:%d s-task:%s receive dispatch req from taskId:%d", pReq->upstreamNodeId, pTask->id.idStr, pReq->upstreamTaskId); - streamTaskEnqueue(pTask, pReq, pRsp); + streamTaskEnqueueBlocks(pTask, pReq, pRsp); tDeleteStreamDispatchReq(pReq); if (exec) { @@ -228,13 +235,14 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { ASSERT(pRsp->inputStatus == TASK_OUTPUT_STATUS__NORMAL || pRsp->inputStatus == TASK_OUTPUT_STATUS__BLOCKED); - - qDebug("task %d receive dispatch rsp, code: %x", pTask->taskId, code); + qDebug("s-task:%s receive dispatch rsp, code: %x", pTask->id.idStr, code); if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); - qDebug("task %d is shuffle, left waiting rsp %d", pTask->taskId, leftRsp); - if (leftRsp > 0) return 0; + qDebug("task %d is shuffle, left waiting rsp %d", pTask->id.taskId, leftRsp); + if (leftRsp > 0) { + return 0; + } } int8_t old = atomic_exchange_8(&pTask->outputStatus, pRsp->inputStatus); @@ -261,7 +269,7 @@ int32_t streamProcessRunReq(SStreamTask* pTask) { } int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) { - qDebug("task %d receive retrieve req from node %d task %d", pTask->taskId, pReq->srcNodeId, pReq->srcTaskId); + qDebug("task %d receive retrieve req from node %d task %d", pTask->id.taskId, pReq->srcNodeId, pReq->srcTaskId); streamTaskEnqueueRetrieve(pTask, pReq, pRsp); @@ -275,26 +283,43 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, S return 0; } -int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem) { +bool tInputQueueIsFull(const SStreamTask* pTask) { + return taosQueueItemSize((pTask->inputQueue->queue)) >= STREAM_TASK_INPUT_QUEUEU_CAPACITY; +} + +int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { int8_t type = pItem->type; if (type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit2* pSubmitBlock = streamSubmitBlockClone((SStreamDataSubmit2*)pItem); if (pSubmitBlock == NULL) { - qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); + qDebug("task %d %p submit enqueue failed since out of memory", pTask->id.taskId, pTask); terrno = TSDB_CODE_OUT_OF_MEMORY; atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); return -1; } int32_t total = taosQueueItemSize(pTask->inputQueue->queue) + 1; - qDebug("stream task:%d %p submit enqueue %p %p %p msgLen:%d ver:%" PRId64 ", total in queue:%d", pTask->taskId, - pTask, pItem, pSubmitBlock, pSubmitBlock->submit.msgStr, pSubmitBlock->submit.msgLen, + qDebug("s-task:%s submit enqueue %p %p msgLen:%d ver:%" PRId64 ", total in queue:%d", pTask->id.idStr, + pItem, pSubmitBlock->submit.msgStr, pSubmitBlock->submit.msgLen, pSubmitBlock->submit.ver, total); + if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) { + qError("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY); + streamDataSubmitDestroy(pSubmitBlock); + return -1; + } + taosWriteQitem(pTask->inputQueue->queue, pSubmitBlock); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { + int32_t total = taosQueueItemSize(pTask->inputQueue->queue) + 1; + if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) { + qError("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY); + return -1; + } + + qDebug("s-task:%s data block enqueue, total in queue:%d", pTask->id.idStr, total); taosWriteQitem(pTask->inputQueue->queue, pItem); } else if (type == STREAM_INPUT__CHECKPOINT) { taosWriteQitem(pTask->inputQueue->queue, pItem); @@ -307,7 +332,6 @@ int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem) { } #if 0 - // TODO: back pressure atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL); #endif diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 3fba1cb556a259b290dcca8764499d3d589e1da8..ae616260f3b38038f103b661b4a05aabace9ac31 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -67,9 +67,8 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock return 0; } -SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit) { +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type) { SStreamDataSubmit2* pDataSubmit = (SStreamDataSubmit2*)taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); - if (pDataSubmit == NULL) { return NULL; } @@ -82,7 +81,7 @@ SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit) { pDataSubmit->submit = submit; *pDataSubmit->dataRef = 1; // initialize the reference count to be 1 - pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT; + pDataSubmit->type = type; return pDataSubmit; } @@ -139,28 +138,27 @@ SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit) { return pSubmitClone; } -SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { - ASSERT(elem); - if (dst->type == STREAM_INPUT__DATA_BLOCK && elem->type == STREAM_INPUT__DATA_BLOCK) { +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem) { + if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; - SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)elem; + SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)pElem; taosArrayAddAll(pBlock->blocks, pBlockSrc->blocks); taosArrayDestroy(pBlockSrc->blocks); - taosFreeQitem(elem); + taosFreeQitem(pElem); return dst; - } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { + } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)dst; - SStreamDataSubmit2* pBlockSrc = (SStreamDataSubmit2*)elem; + SStreamDataSubmit2* pBlockSrc = (SStreamDataSubmit2*)pElem; streamMergeSubmit(pMerged, pBlockSrc); - taosFreeQitem(elem); + taosFreeQitem(pElem); return dst; - } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { + } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamMergedSubmit2* pMerged = streamMergedSubmitNew(); ASSERT(pMerged); streamMergeSubmit(pMerged, (SStreamDataSubmit2*)dst); - streamMergeSubmit(pMerged, (SStreamDataSubmit2*)elem); + streamMergeSubmit(pMerged, (SStreamDataSubmit2*)pElem); taosFreeQitem(dst); - taosFreeQitem(elem); + taosFreeQitem(pElem); return (SStreamQueueItem*)pMerged; } else { return NULL; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 7e7c23f98addfae5689093bedb1a86196cd380c9..a9f6d29bf5f51cf5826fac3c092b3bd1feff0aa7 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -121,9 +121,9 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); SStreamRetrieveReq req = { - .streamId = pTask->streamId, + .streamId = pTask->id.streamId, .srcNodeId = pTask->nodeId, - .srcTaskId = pTask->taskId, + .srcTaskId = pTask->id.taskId, .pRetrieve = pRetrieve, .retrieveLen = dataStrLen, }; @@ -168,7 +168,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) } buf = NULL; - qDebug("task %d(child %d) send retrieve req to task %d at node %d, reqId %" PRId64, pTask->taskId, + qDebug("s-task:%s (child %d) send retrieve req to task %d at node %d, reqId %" PRId64, pTask->id.idStr, pTask->selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId); } code = 0; @@ -238,7 +238,8 @@ int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* msg.pCont = buf; msg.msgType = TDMT_STREAM_TASK_CHECK; - qDebug("dispatch from task %d to task %d node %d: check msg", pTask->taskId, pReq->downstreamTaskId, nodeId); + qDebug("dispatch from s-task:%s to downstream s-task:%"PRIx64":%d node %d: check msg", pTask->id.idStr, + pReq->streamId, pReq->downstreamTaskId, nodeId); tmsgSendReq(pEpSet, &msg); @@ -282,7 +283,7 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov tmsgSendReq(pEpSet, &msg); - qDebug("dispatch from task %d to task %d node %d: recover finish msg", pTask->taskId, pReq->taskId, vgId); + qDebug("dispatch from task %d to task %d node %d: recover finish msg", pTask->id.taskId, pReq->taskId, vgId); return 0; FAIL: @@ -319,8 +320,7 @@ int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* p msg.pCont = buf; msg.msgType = pTask->dispatchMsgType; - qDebug("dispatch from task %d to task %d node %d: data msg", pTask->taskId, pReq->taskId, vgId); - + qDebug("dispatch from s-task:%s to taskId:%d vgId:%d data msg", pTask->id.idStr, pReq->taskId, vgId); tmsgSendReq(pEpSet, &msg); code = 0; @@ -382,9 +382,9 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { SStreamDispatchReq req = { - .streamId = pTask->streamId, + .streamId = pTask->id.streamId, .dataSrcVgId = pData->srcVgId, - .upstreamTaskId = pTask->taskId, + .upstreamTaskId = pTask->id.taskId, .upstreamChildId = pTask->selfChildId, .upstreamNodeId = pTask->nodeId, .blockNum = blockNum, @@ -402,14 +402,15 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat goto FAIL_FIXED_DISPATCH; } } + int32_t vgId = pTask->fixedEpDispatcher.nodeId; SEpSet* pEpSet = &pTask->fixedEpDispatcher.epSet; int32_t downstreamTaskId = pTask->fixedEpDispatcher.taskId; req.taskId = downstreamTaskId; - qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId, - downstreamTaskId, vgId); + qDebug("s-task:%s (child taskId:%d) dispatch blocks:%d to down stream s-task:%d in vgId:%d", pTask->id.idStr, + pTask->selfChildId, blockNum, downstreamTaskId, vgId); if (streamDispatchOneDataReq(pTask, &req, vgId, pEpSet) < 0) { goto FAIL_FIXED_DISPATCH; @@ -432,9 +433,9 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat } for (int32_t i = 0; i < vgSz; i++) { - pReqs[i].streamId = pTask->streamId; + pReqs[i].streamId = pTask->id.streamId; pReqs[i].dataSrcVgId = pData->srcVgId; - pReqs[i].upstreamTaskId = pTask->taskId; + pReqs[i].upstreamTaskId = pTask->id.taskId; pReqs[i].upstreamChildId = pTask->selfChildId; pReqs[i].upstreamNodeId = pTask->nodeId; pReqs[i].blockNum = 0; @@ -494,6 +495,8 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat int32_t streamDispatch(SStreamTask* pTask) { ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH); + qDebug("s-task:%s try to dispatch intermediate result block to downstream, numofBlocks in outputQ:%d", pTask->id.idStr, + taosQueueItemSize(pTask->outputQueue->queue)); int8_t old = atomic_val_compare_exchange_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL, TASK_OUTPUT_STATUS__WAIT); @@ -503,13 +506,12 @@ int32_t streamDispatch(SStreamTask* pTask) { SStreamDataBlock* pBlock = streamQueueNextItem(pTask->outputQueue); if (pBlock == NULL) { - qDebug("stream stop dispatching since no output: task %d", pTask->taskId); + qDebug("s-task:%s stream stop dispatching since no output in output queue", pTask->id.idStr); atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); return 0; } - ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - qDebug("stream dispatching: task %d", pTask->taskId); + ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); int32_t code = 0; if (streamDispatchAllBlocks(pTask, pBlock) < 0) { @@ -518,6 +520,7 @@ int32_t streamDispatch(SStreamTask* pTask) { atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); goto FREE; } + FREE: taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(pBlock); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 6ef327049c954d886d2c04bb5cc14eeb8524112e..3d896c08ac006799b7d096165f15cc94f8a528b5 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -18,70 +18,82 @@ #define STREAM_EXEC_MAX_BATCH_NUM 100 static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) { - int32_t code; - void* exec = pTask->exec.executor; - while(pTask->taskLevel == TASK_LEVEL__SOURCE && atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { - qError("stream task wait for the end of fill history"); - taosMsleep(2); - continue; + int32_t code = TSDB_CODE_SUCCESS; + void* pExecutor = pTask->exec.pExecutor; + + while (pTask->taskLevel == TASK_LEVEL__SOURCE) { + int8_t status = atomic_load_8(&pTask->status.taskStatus); + if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__RESTORE) { + qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr, + atomic_load_8(&pTask->status.taskStatus)); + taosMsleep(2); + } else { + break; + } } // set input const SStreamQueueItem* pItem = (const SStreamQueueItem*)data; if (pItem->type == STREAM_INPUT__GET_RES) { const SStreamTrigger* pTrigger = (const SStreamTrigger*)data; - qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); + qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); const SStreamDataSubmit2* pSubmit = (const SStreamDataSubmit2*)data; - qDebug("stream task:%d %p set submit input %p %p %d %" PRId64, pTask->taskId, pTask, pSubmit, pSubmit->submit.msgStr, + qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); + qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, pSubmit, pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); - qSetMultiStreamInput(exec, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data; - SArray* blocks = pBlock->blocks; - qDebug("task %d %p set ssdata input", pTask->taskId, pTask); - qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK); + + SArray* pBlockList = pBlock->blocks; + int32_t numOfBlocks = taosArrayGetSize(pBlockList); + qDebug("s-task:%s set sdata blocks as input num:%d, ver:%"PRId64, pTask->id.idStr, numOfBlocks, pBlock->sourceVer); + qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { const SStreamMergedSubmit2* pMerged = (const SStreamMergedSubmit2*)data; - SArray* blocks = pMerged->submits; - qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size); - qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT); + + SArray* pBlockList = pMerged->submits; + int32_t numOfBlocks = taosArrayGetSize(pBlockList); + qDebug("st-task:%s %p set submit input (merged), batch num:%d", pTask->id.idStr, pTask, numOfBlocks); + qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)data; - qSetMultiStreamInput(exec, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK); + qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else { ASSERT(0); } - // exec + // pExecutor while (1) { - if (pTask->taskStatus == TASK_STATUS__DROPPING) { + if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { return 0; } SSDataBlock* output = NULL; uint64_t ts = 0; - if ((code = qExecTask(exec, &output, &ts)) < 0) { + if ((code = qExecTask(pExecutor, &output, &ts)) < 0) { if (code == TSDB_CODE_QRY_IN_EXEC) { - resetTaskInfo(exec); + resetTaskInfo(pExecutor); } - /*ASSERT(false);*/ - qError("unexpected stream execution, stream %" PRId64 " task: %d, since %s", pTask->streamId, pTask->taskId, - terrstr()); + + qError("unexpected stream execution, s-task:%s since %s", pTask->id.idStr, terrstr()); continue; } + if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - SSDataBlock block = {0}; + SSDataBlock block = {0}; + const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)data; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); + assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); block.info.type = STREAM_PULL_OVER; block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); - qDebug("task %d(child %d) processed retrieve, reqId %" PRId64, pTask->taskId, pTask->selfChildId, + qDebug("task %d(child %d) processed retrieve, reqId %" PRId64, pTask->id.taskId, pTask->selfChildId, pRetrieveBlock->reqId); } break; @@ -94,20 +106,21 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* continue; } - qDebug("task %d(child %d) executed and get block", pTask->taskId, pTask->selfChildId); + qDebug("task %d(child %d) executed and get block", pTask->id.taskId, pTask->selfChildId); SSDataBlock block = {0}; assignOneDataBlock(&block, output); block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); } + return 0; } int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; qSetStreamOpOpen(exec); bool finished = false; @@ -121,7 +134,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { int32_t batchCnt = 0; while (1) { - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { taosArrayDestroy(pRes); return 0; } @@ -147,17 +160,17 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { batchCnt++; - qDebug("task %d scan exec block num %d, block limit %d", pTask->taskId, batchCnt, batchSz); + qDebug("task %d scan exec block num %d, block limit %d", pTask->id.taskId, batchCnt, batchSz); if (batchCnt >= batchSz) break; } if (taosArrayGetSize(pRes) == 0) { if (finished) { taosArrayDestroy(pRes); - qDebug("task %d finish recover exec task ", pTask->taskId); + qDebug("task %d finish recover exec task ", pTask->id.taskId); break; } else { - qDebug("task %d continue recover exec task ", pTask->taskId); + qDebug("task %d continue recover exec task ", pTask->id.taskId); continue; } } @@ -173,7 +186,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { streamTaskOutput(pTask, qRes); if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - qDebug("task %d scan exec dispatch block num %d", pTask->taskId, batchCnt); + qDebug("task %d scan exec dispatch block num %d", pTask->id.taskId, batchCnt); streamDispatch(pTask); } if (finished) break; @@ -186,7 +199,7 @@ int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) { // fetch all queue item, merge according to batchLimit int32_t numOfItems = taosReadAllQitems(pTask->inputQueue1, pTask->inputQall); if (numOfItems == 0) { - qDebug("task: %d, stream task exec over, queue empty", pTask->taskId); + qDebug("task: %d, stream task exec over, queue empty", pTask->id.taskId); return 0; } SStreamQueueItem* pMerged = NULL; @@ -221,106 +234,141 @@ int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) { int32_t streamExecForAll(SStreamTask* pTask) { while (1) { - int32_t batchCnt = 1; - void* input = NULL; + int32_t batchSize = 1; + void* pInput = NULL; + + // merge multiple input data if possible in the input queue. while (1) { SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); if (qItem == NULL) { - qDebug("stream task exec over, queue empty, task: %d", pTask->taskId); +// qDebug("s-task:%s extract data from input queue, queue is empty, abort", pTask->id.idStr); break; } - if (input == NULL) { - input = qItem; + + if (pInput == NULL) { + pInput = qItem; streamQueueProcessSuccess(pTask->inputQueue); if (pTask->taskLevel == TASK_LEVEL__SINK) { break; } } else { - void* newRet; - if ((newRet = streamMergeQueueItem(input, qItem)) == NULL) { + void* newRet = NULL; + if ((newRet = streamMergeQueueItem(pInput, qItem)) == NULL) { streamQueueProcessFail(pTask->inputQueue); break; } else { - batchCnt++; - input = newRet; + batchSize++; + pInput = newRet; streamQueueProcessSuccess(pTask->inputQueue); - if (batchCnt > STREAM_EXEC_MAX_BATCH_NUM) { + if (batchSize > STREAM_EXEC_MAX_BATCH_NUM) { break; } } } } - if (pTask->taskStatus == TASK_STATUS__DROPPING) { - if (input) streamFreeQitem(input); + if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { + if (pInput) { + streamFreeQitem(pInput); + } return 0; } - if (input == NULL) { + if (pInput == NULL) { break; } if (pTask->taskLevel == TASK_LEVEL__SINK) { - ASSERT(((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_BLOCK); - streamTaskOutput(pTask, input); + ASSERT(((SStreamQueueItem*)pInput)->type == STREAM_INPUT__DATA_BLOCK); + qDebug("s-task:%s sink node start to sink result. numOfBlocks:%d", pTask->id.idStr, batchSize); + streamTaskOutput(pTask, pInput); continue; } SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + qDebug("s-task:%s exec begin, numOfBlocks:%d", pTask->id.idStr, batchSize); + + streamTaskExecImpl(pTask, pInput, pRes); + + int64_t ckId = 0; + int64_t dataVer = 0; + qGetCheckpointVersion(pTask->exec.pExecutor, &dataVer, &ckId); + if (dataVer > pTask->chkInfo.version) { // save it since the checkpoint is updated + qDebug("s-task:%s exec end, start to update check point, ver from %" PRId64 " to %" PRId64 + ", checkPoint id:%" PRId64 " -> %" PRId64, + pTask->id.idStr, pTask->chkInfo.version, dataVer, pTask->chkInfo.id, ckId); - qDebug("stream task:%d exec begin, msg batch: %d", pTask->taskId, batchCnt); - streamTaskExecImpl(pTask, input, pRes); + pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId}; - qDebug("stream task:%d exec end", pTask->taskId); + taosWLockLatch(&pTask->pMeta->lock); + streamMetaSaveTask(pTask->pMeta, pTask); + if (streamMetaCommit(pTask->pMeta) < 0) { + taosWUnLockLatch(&pTask->pMeta->lock); + qError("s-task:%s failed to commit stream meta, since %s", pTask->id.idStr, terrstr()); + return -1; + } else { + taosWUnLockLatch(&pTask->pMeta->lock); + qDebug("s-task:%s update checkpoint ver succeed", pTask->id.idStr); + } + } else { + qDebug("s-task:%s exec end", pTask->id.idStr); + } if (taosArrayGetSize(pRes) != 0) { SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); if (qRes == NULL) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - streamFreeQitem(input); + streamFreeQitem(pInput); return -1; } + qRes->type = STREAM_INPUT__DATA_BLOCK; qRes->blocks = pRes; - if (((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)input; + if (((SStreamQueueItem*)pInput)->type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)pInput; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; - } else if (((SStreamQueueItem*)input)->type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)input; + } else if (((SStreamQueueItem*)pInput)->type == STREAM_INPUT__MERGED_SUBMIT) { + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)pInput; qRes->childId = pTask->selfChildId; qRes->sourceVer = pMerged->ver; } if (streamTaskOutput(pTask, qRes) < 0) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - streamFreeQitem(input); + streamFreeQitem(pInput); taosFreeQitem(qRes); return -1; } } else { taosArrayDestroy(pRes); } - streamFreeQitem(input); + streamFreeQitem(pInput); } return 0; } int32_t streamTryExec(SStreamTask* pTask) { + // this function may be executed by multi-threads, so status check is required. int8_t schedStatus = - atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE); + atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE); + if (schedStatus == TASK_SCHED_STATUS__WAITING) { int32_t code = streamExecForAll(pTask); if (code < 0) { - atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__FAILED); + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__FAILED); return -1; } - atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE); + + // todo the task should be commit here + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + qDebug("s-task:%s exec completed", pTask->id.idStr); if (!taosQueueEmpty(pTask->inputQueue->queue)) { streamSchedExec(pTask); } } + return 0; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 03391c0ba2b5dd52e4442c4f172c27e7c474c37a..51cc3157805e8e03de266ec3821d51a38f4a6ec4 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -24,6 +24,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + int32_t len = strlen(path) + 20; char* streamPath = taosMemoryCalloc(1, len); sprintf(streamPath, "%s/%s", path, "stream"); @@ -50,7 +51,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } - pMeta->pTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + _hash_fn_t fp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); + pMeta->pTasks = taosHashInit(64, fp, true, HASH_ENTRY_LOCK); if (pMeta->pTasks == NULL) { goto _err; } @@ -59,9 +61,10 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } + pMeta->vgId = vgId; pMeta->ahandle = ahandle; pMeta->expandFunc = expandFunc; - + taosInitRWLatch(&pMeta->lock); return pMeta; _err: @@ -81,19 +84,28 @@ void streamMetaClose(SStreamMeta* pMeta) { tdbClose(pMeta->db); void* pIter = NULL; +// while(pMeta->walScan) { +// qDebug("wait stream daemon quit"); +// taosMsleep(100); +// } + while (1) { pIter = taosHashIterate(pMeta->pTasks, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->timer) { taosTmrStop(pTask->timer); pTask->timer = NULL; } - tFreeSStreamTask(pTask); + + tFreeStreamTask(pTask); /*streamMetaReleaseTask(pMeta, pTask);*/ } + taosHashCleanup(pMeta->pTasks); - taosHashCleanup(pMeta->pRecoverStatus); taosMemoryFree(pMeta->path); taosMemoryFree(pMeta); } @@ -106,7 +118,7 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg, } SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - if (tDecodeSStreamTask(&decoder, pTask) < 0) { + if (tDecodeStreamTask(&decoder, pTask) < 0) { tDecoderClear(&decoder); goto FAIL; } @@ -117,12 +129,12 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg, goto FAIL; } - if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { + if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { goto FAIL; } - if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, pMeta->txn) < 0) { - taosHashRemove(pMeta->pTasks, &pTask->taskId, sizeof(int32_t)); + if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), msg, msgLen, pMeta->txn) < 0) { + taosHashRemove(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t)); ASSERT(0); goto FAIL; } @@ -130,7 +142,7 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg, return 0; FAIL: - if (pTask) tFreeSStreamTask(pTask); + if (pTask) tFreeStreamTask(pTask); return -1; } #endif @@ -139,7 +151,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { void* buf = NULL; int32_t len; int32_t code; - tEncodeSize(tEncodeSStreamTask, pTask, len, code); + tEncodeSize(tEncodeStreamTask, pTask, len, code); if (code < 0) { return -1; } @@ -150,10 +162,10 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, len); - tEncodeSStreamTask(&encoder, pTask); + tEncodeStreamTask(&encoder, pTask); tEncoderClear(&encoder); - if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) { + if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) { return -1; } @@ -161,8 +173,8 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { return 0; } -#if 1 -int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { +// add to the ready tasks hash map, not the restored tasks hash map +int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { return -1; } @@ -171,39 +183,24 @@ int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { return -1; } - taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)); - + taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, POINTER_BYTES); return 0; } -#endif -#if 0 -SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId) { - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - ASSERT((*ppTask)->taskId == taskId); - return *ppTask; - } else { - return NULL; - } +int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta) { + return (int32_t) taosHashGetSize(pMeta->pTasks); } -#endif SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) { taosRLockLatch(&pMeta->lock); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - SStreamTask* pTask = *ppTask; - if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__DROPPING) { - atomic_add_fetch_32(&pTask->refCnt, 1); - taosRUnLockLatch(&pMeta->lock); - return pTask; - } else { - taosRUnLockLatch(&pMeta->lock); - return NULL; - } + if (ppTask != NULL && (atomic_load_8(&((*ppTask)->status.taskStatus)) != TASK_STATUS__DROPPING)) { + atomic_add_fetch_32(&(*ppTask)->refCnt, 1); + taosRUnLockLatch(&pMeta->lock); + return *ppTask; } + taosRUnLockLatch(&pMeta->lock); return NULL; } @@ -212,8 +209,8 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) { int32_t left = atomic_sub_fetch_32(&pTask->refCnt, 1); ASSERT(left >= 0); if (left == 0) { - ASSERT(atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING); - tFreeSStreamTask(pTask); + ASSERT(atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING); + tFreeStreamTask(pTask); } } @@ -227,7 +224,7 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { * taosTmrStop(pTask->timer);*/ /*pTask->timer = NULL;*/ /*}*/ - atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING); taosWLockLatch(&pMeta->lock); streamMetaReleaseTask(pMeta, pTask); @@ -245,9 +242,12 @@ int32_t streamMetaBegin(SStreamMeta* pMeta) { int32_t streamMetaCommit(SStreamMeta* pMeta) { if (tdbCommit(pMeta->db, pMeta->txn) < 0) { + ASSERT(0); return -1; } + if (tdbPostCommit(pMeta->db, pMeta->txn) < 0) { + ASSERT(0); return -1; } @@ -293,25 +293,27 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { return -1; } tDecoderInit(&decoder, (uint8_t*)pVal, vLen); - tDecodeSStreamTask(&decoder, pTask); + tDecodeStreamTask(&decoder, pTask); tDecoderClear(&decoder); - if (pMeta->expandFunc(pMeta->ahandle, pTask, -1) < 0) { + // todo set correct initial version. + if (pMeta->expandFunc(pMeta->ahandle, pTask, 0) < 0) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); return -1; } - if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { + if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); return -1; } - /*pTask->taskStatus = TASK_STATUS__NORMAL;*/ + + /*pTask->status.taskStatus = TASK_STATUS__NORMAL;*/ if (pTask->fillHistory) { - pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; + pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; streamTaskCheckDownstream(pTask, ver); } } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 87058bf490d43f2cbbd1b18eb8411c034b40a9cd..03afc0692d8ce98b9e82fe2ec2ee02da80307119 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -16,9 +16,10 @@ #include "streamInc.h" int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) { - qDebug("task %d at node %d launch recover", pTask->taskId, pTask->nodeId); + qDebug("s-task:%s at node %d launch recover", pTask->id.idStr, pTask->nodeId); + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_PREPARE); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__RECOVER_PREPARE); streamSetParamForRecover(pTask); streamSourceRecoverPrepareStep1(pTask, version); @@ -33,34 +34,31 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) { memcpy(serializedReq, &req, len); - SRpcMsg rpcMsg = { - .contLen = len, - .pCont = serializedReq, - .msgType = TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, - }; - + SRpcMsg rpcMsg = { .contLen = len, .pCont = serializedReq, .msgType = TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE }; if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) { /*ASSERT(0);*/ } } else if (pTask->taskLevel == TASK_LEVEL__AGG) { - atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); streamSetParamForRecover(pTask); streamAggRecoverPrepare(pTask); } else if (pTask->taskLevel == TASK_LEVEL__SINK) { - atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); } + return 0; } // checkstatus int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { SStreamTaskCheckReq req = { - .streamId = pTask->streamId, - .upstreamTaskId = pTask->taskId, + .streamId = pTask->id.streamId, + .upstreamTaskId = pTask->id.taskId, .upstreamNodeId = pTask->nodeId, .childId = pTask->selfChildId, }; + // serialize if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { req.reqId = tGenIdPI64(); @@ -68,7 +66,7 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { req.downstreamTaskId = pTask->fixedEpDispatcher.taskId; pTask->checkReqId = req.reqId; - qDebug("task %d at node %d check downstream task %d at node %d", pTask->taskId, pTask->nodeId, req.downstreamTaskId, + qDebug("task %d at node %d check downstream task %d at node %d", pTask->id.taskId, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); streamDispatchOneCheckReq(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { @@ -83,12 +81,12 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { taosArrayPush(pTask->checkReqIds, &req.reqId); req.downstreamNodeId = pVgInfo->vgId; req.downstreamTaskId = pVgInfo->taskId; - qDebug("task %d at node %d check downstream task %d at node %d (shuffle)", pTask->taskId, pTask->nodeId, + qDebug("task %d at node %d check downstream task %d at node %d (shuffle)", pTask->id.taskId, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); streamDispatchOneCheckReq(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); } } else { - qDebug("task %d at node %d direct launch recover since no downstream", pTask->taskId, pTask->nodeId); + qDebug("task %d at node %d direct launch recover since no downstream", pTask->id.taskId, pTask->nodeId); streamTaskLaunchRecover(pTask, version); } return 0; @@ -104,7 +102,7 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp .downstreamNodeId = pRsp->downstreamNodeId, .childId = pRsp->childId, }; - qDebug("task %d at node %d check downstream task %d at node %d (recheck)", pTask->taskId, pTask->nodeId, + qDebug("task %d at node %d check downstream task %d at node %d (recheck)", pTask->id.taskId, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet); @@ -122,12 +120,13 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp } int32_t streamProcessTaskCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq) { - return atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL; + return atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL; } int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version) { qDebug("task %d at node %d recv check rsp from task %d at node %d: status %d", pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status); + if (pRsp->status == 1) { if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { bool found = false; @@ -138,7 +137,11 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* break; } } - if (!found) return -1; + + if (!found) { + return -1; + } + int32_t left = atomic_sub_fetch_32(&pTask->recoverTryingDownstream, 1); ASSERT(left >= 0); if (left == 0) { @@ -147,7 +150,10 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* streamTaskLaunchRecover(pTask, version); } } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - if (pRsp->reqId != pTask->checkReqId) return -1; + if (pRsp->reqId != pTask->checkReqId) { + return -1; + } + streamTaskLaunchRecover(pTask, version); } else { ASSERT(0); @@ -160,28 +166,29 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* // common int32_t streamSetParamForRecover(SStreamTask* pTask) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; return qStreamSetParamForRecover(exec); } int32_t streamRestoreParam(SStreamTask* pTask) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; return qStreamRestoreParam(exec); } + int32_t streamSetStatusNormal(SStreamTask* pTask) { - atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); return 0; } // source int32_t streamSourceRecoverPrepareStep1(SStreamTask* pTask, int64_t ver) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; return qStreamSourceRecoverStep1(exec, ver); } int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamRecoverStep1Req* pReq) { pReq->msgHead.vgId = pTask->nodeId; - pReq->streamId = pTask->streamId; - pReq->taskId = pTask->taskId; + pReq->streamId = pTask->id.streamId; + pReq->taskId = pTask->id.taskId; return 0; } @@ -192,13 +199,13 @@ int32_t streamSourceRecoverScanStep1(SStreamTask* pTask) { int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* pReq) { pReq->msgHead.vgId = pTask->nodeId; - pReq->streamId = pTask->streamId; - pReq->taskId = pTask->taskId; + pReq->streamId = pTask->id.streamId; + pReq->taskId = pTask->id.taskId; return 0; } int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; if (qStreamSourceRecoverStep2(exec, ver) < 0) { } return streamScanExec(pTask, 100); @@ -206,7 +213,7 @@ int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) { int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { SStreamRecoverFinishReq req = { - .streamId = pTask->streamId, + .streamId = pTask->id.streamId, .childId = pTask->selfChildId, }; // serialize @@ -227,13 +234,13 @@ int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { // agg int32_t streamAggRecoverPrepare(SStreamTask* pTask) { - void* exec = pTask->exec.executor; pTask->recoverWaitingUpstream = taosArrayGetSize(pTask->childEpInfo); + qDebug("s-task:%s wait for %d upstreams", pTask->id.idStr, pTask->recoverWaitingUpstream); return 0; } int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; if (qStreamRestoreParam(exec) < 0) { return -1; } @@ -247,6 +254,7 @@ int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) { int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) { if (pTask->taskLevel == TASK_LEVEL__AGG) { int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingUpstream, 1); + qDebug("s-task:%s remain unfinished child tasks:%d", pTask->id.idStr, left); ASSERT(left >= 0); if (left == 0) { streamAggChildrenRecoverFinish(pTask); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 411726075e1b4488292868bb0bdf3e7ec942aa1e..7bea989e3a63be5110deb9e8dbb5b00ac6eed643 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -121,7 +121,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int char statePath[1024]; if (!specPath) { - sprintf(statePath, "%s/%d", path, pTask->taskId); + sprintf(statePath, "%s/%d", path, pTask->id.taskId); } else { memset(statePath, 0, 1024); tstrncpy(statePath, path, 1024); @@ -193,6 +193,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int } pState->pTdbState->pOwner = pTask; + pState->checkPointId = 0; return pState; @@ -243,6 +244,7 @@ int32_t streamStateCommit(SStreamState* pState) { TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } + pState->checkPointId++; return 0; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index e9aba0bc3941d501ba25a5ea28aaf02e11d42cbd..67c60008fdb98b5f341be5271668a17bd3563c92 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -15,15 +15,22 @@ #include "executor.h" #include "tstream.h" +#include "wal.h" -SStreamTask* tNewSStreamTask(int64_t streamId) { +SStreamTask* tNewStreamTask(int64_t streamId) { SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { return NULL; } - pTask->taskId = tGenIdPI32(); - pTask->streamId = streamId; - pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; + + pTask->id.taskId = tGenIdPI32(); + pTask->id.streamId = streamId; + + char buf[128] = {0}; + sprintf(buf, "0x%"PRIx64"-%d", pTask->id.streamId, pTask->id.taskId); + + pTask->id.idStr = taosStrdup(buf); + pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; @@ -48,24 +55,24 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) { return 0; } -int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { +int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; + if (tEncodeI64(pEncoder, pTask->id.streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->id.taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->totalLevel) < 0) return -1; if (tEncodeI8(pEncoder, pTask->taskLevel) < 0) return -1; if (tEncodeI8(pEncoder, pTask->outputType) < 0) return -1; if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->taskStatus) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->schedStatus) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->status.taskStatus) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->status.schedStatus) < 0) return -1; if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; - if (tEncodeI64(pEncoder, pTask->recoverSnapVer) < 0) return -1; - if (tEncodeI64(pEncoder, pTask->startVer) < 0) return -1; + if (tEncodeI64(pEncoder, pTask->chkInfo.id) < 0) return -1; + if (tEncodeI64(pEncoder, pTask->chkInfo.version) < 0) return -1; if (tEncodeI8(pEncoder, pTask->fillHistory) < 0) return -1; int32_t epSz = taosArrayGetSize(pTask->childEpInfo); @@ -101,24 +108,24 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { return pEncoder->pos; } -int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { +int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; + if (tDecodeI64(pDecoder, &pTask->id.streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->id.taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->totalLevel) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->taskLevel) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->outputType) < 0) return -1; if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->taskStatus) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->schedStatus) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->status.taskStatus) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->status.schedStatus) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; - if (tDecodeI64(pDecoder, &pTask->recoverSnapVer) < 0) return -1; - if (tDecodeI64(pDecoder, &pTask->startVer) < 0) return -1; + if (tDecodeI64(pDecoder, &pTask->chkInfo.id) < 0) return -1; + if (tDecodeI64(pDecoder, &pTask->chkInfo.version) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->fillHistory) < 0) return -1; int32_t epSz; @@ -162,24 +169,47 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { return 0; } -void tFreeSStreamTask(SStreamTask* pTask) { - qDebug("free stream task %d", pTask->taskId); - if (pTask->inputQueue) streamQueueClose(pTask->inputQueue); - if (pTask->outputQueue) streamQueueClose(pTask->outputQueue); - if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg); - if (pTask->exec.executor) qDestroyTask(pTask->exec.executor); +void tFreeStreamTask(SStreamTask* pTask) { + qDebug("free s-task:%s", pTask->id.idStr); + + if (pTask->inputQueue) { + streamQueueClose(pTask->inputQueue); + } + if (pTask->outputQueue) { + streamQueueClose(pTask->outputQueue); + } + if (pTask->exec.qmsg) { + taosMemoryFree(pTask->exec.qmsg); + } + + if (pTask->exec.pExecutor) { + qDestroyTask(pTask->exec.pExecutor); + pTask->exec.pExecutor = NULL; + } + + if (pTask->exec.pWalReader != NULL) { + walCloseReader(pTask->exec.pWalReader); + } + taosArrayDestroyP(pTask->childEpInfo, taosMemoryFree); if (pTask->outputType == TASK_OUTPUT__TABLE) { tDeleteSSchemaWrapper(pTask->tbSink.pSchemaWrapper); taosMemoryFree(pTask->tbSink.pTSchema); } + if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { taosArrayDestroy(pTask->shuffleDispatcher.dbInfo.pVgroupInfos); taosArrayDestroy(pTask->checkReqIds); pTask->checkReqIds = NULL; } - if (pTask->pState) streamStateClose(pTask->pState); + if (pTask->pState) { + streamStateClose(pTask->pState); + } + + if (pTask->id.idStr != NULL) { + taosMemoryFree((void*)pTask->id.idStr); + } taosMemoryFree(pTask); } diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index 68db811b1273128a679573a41c5e669df531c83a..d709e33cd46aa091255afa84a3e3aac91bbe1d03 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -77,18 +77,19 @@ static FORCE_INLINE int32_t syncLogReplGetNextRetryBackoff(SSyncLogReplMgr* pMgr SyncTerm syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); -int32_t syncLogReplReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogReplReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier); -int32_t syncLogReplReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogReplReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); +int32_t syncLogReplDoOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); + +int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, SRaftId* pDestId, + bool* pBarrier); int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg); -int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); // SSyncLogBuffer SSyncLogBuffer* syncLogBufferCreate(); @@ -100,6 +101,7 @@ int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode); int64_t syncLogBufferGetEndIndex(SSyncLogBuffer* pBuf); SyncTerm syncLogBufferGetLastMatchTerm(SSyncLogBuffer* pBuf); bool syncLogBufferIsEmpty(SSyncLogBuffer* pBuf); + int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry); int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevTerm); int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* pMatchTerm); diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index a39e043c52a0016a6e7242572667a9d02d62be5e..f9447e016882f9c219a74ef96394d00535bc411f 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -45,7 +45,7 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) void syncEntryDestroy(SSyncRaftEntry* pEntry); void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg); // step 7 -static FORCE_INLINE bool syncLogIsReplicationBarrier(SSyncRaftEntry* pEntry) { +static FORCE_INLINE bool syncLogReplBarrier(SSyncRaftEntry* pEntry) { return pEntry->originalRpcType == TDMT_SYNC_NOOP; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 8927e409e1d6127353ae42b381dcd87a9b98a5f8..966b3ed09380fc16410b83a35008d74227bed1af 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -617,7 +617,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_ sNTrace(pSyncNode, "propose msg, type:%s", TMSG_INFO(pMsg->msgType)); code = (*pSyncNode->syncEqMsg)(pSyncNode->msgcb, &rpcMsg); if (code != 0) { - sError("vgId:%d, failed to propose msg while enqueue since %s", pSyncNode->vgId, terrstr()); + sWarn("vgId:%d, failed to propose msg while enqueue since %s", pSyncNode->vgId, terrstr()); (void)syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 04e52b3f493fc304dcde395afb879a586d152aa6..6bebef77dc75ac0ec0adc0273d23bec4789cf527 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -633,7 +633,7 @@ int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; if (pMgr->retryBackoff == SYNC_MAX_RETRY_BACKOFF) { syncLogReplReset(pMgr); - sWarn("vgId:%d, reset sync log repl mgr since retry backoff exceeding limit. peer:%" PRIx64, pNode->vgId, + sWarn("vgId:%d, reset sync log repl since retry backoff exceeding limit. peer:%" PRIx64, pNode->vgId, pDestId->addr); return -1; } @@ -658,15 +658,15 @@ int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->states[pos].acked) { if (pMgr->matchIndex < index && pMgr->states[pos].timeMs + (syncGetRetryMaxWaitMs() << 3) < nowMs) { syncLogReplReset(pMgr); - sWarn("vgId:%d, reset sync log repl mgr since stagnation. index:%" PRId64 ", peer:%" PRIx64, pNode->vgId, - index, pDestId->addr); + sWarn("vgId:%d, reset sync log repl since stagnation. index:%" PRId64 ", peer:%" PRIx64, pNode->vgId, index, + pDestId->addr); goto _out; } continue; } bool barrier = false; - if (syncLogReplReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplSendTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate sync log entry since %s. index:%" PRId64 ", dest:%" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); goto _out; @@ -708,7 +708,7 @@ int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNod ASSERT(pMgr->matchIndex == 0); if (pMsg->matchIndex < 0) { pMgr->restored = true; - sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 + sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); @@ -725,7 +725,7 @@ int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNod if (pMsg->success && pMsg->matchIndex == pMsg->lastSendIndex) { pMgr->matchIndex = pMsg->matchIndex; pMgr->restored = true; - sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 + sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); @@ -774,14 +774,14 @@ int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNod // attempt to replicate the raft log at index (void)syncLogReplReset(pMgr); - return syncLogReplReplicateProbe(pMgr, pNode, index); + return syncLogReplProbe(pMgr, pNode, index); } int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); if (pMsg->startTime != 0 && pMsg->startTime != pMgr->peerStartTime) { - sInfo("vgId:%d, reset sync log repl mgr in heartbeat. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64 "", + sInfo("vgId:%d, reset sync log repl in heartbeat. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64 "", pNode->vgId, pMsg->srcId.addr, pMsg->startTime, pMgr->peerStartTime); syncLogReplReset(pMgr); pMgr->peerStartTime = pMsg->startTime; @@ -794,8 +794,7 @@ int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncApp SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); if (pMsg->startTime != pMgr->peerStartTime) { - sInfo("vgId:%d, reset sync log repl mgr in appendlog reply. peer:%" PRIx64 ", start time:%" PRId64 - ", old:%" PRId64, + sInfo("vgId:%d, reset sync log repl in appendlog reply. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64, pNode->vgId, pMsg->srcId.addr, pMsg->startTime, pMgr->peerStartTime); syncLogReplReset(pMgr); pMgr->peerStartTime = pMsg->startTime; @@ -810,16 +809,16 @@ int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncApp return 0; } -int32_t syncLogReplReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplDoOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->restored) { - (void)syncLogReplReplicateAttempt(pMgr, pNode); + (void)syncLogReplAttempt(pMgr, pNode); } else { - (void)syncLogReplReplicateProbe(pMgr, pNode, pNode->pLogBuf->matchIndex); + (void)syncLogReplProbe(pMgr, pNode, pNode->pLogBuf->matchIndex); } return 0; } -int32_t syncLogReplReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { +int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { ASSERT(!pMgr->restored); ASSERT(pMgr->startIndex >= 0); int64_t retryMaxWaitMs = syncGetRetryMaxWaitMs(); @@ -834,7 +833,7 @@ int32_t syncLogReplReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncI SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; bool barrier = false; SyncTerm term = -1; - if (syncLogReplReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplSendTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index:%" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -857,7 +856,7 @@ int32_t syncLogReplReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncI return 0; } -int32_t syncLogReplReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { ASSERT(pMgr->restored); SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; @@ -879,7 +878,7 @@ int32_t syncLogReplReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; bool barrier = false; SyncTerm term = -1; - if (syncLogReplReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplSendTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index:%" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -917,10 +916,10 @@ int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, ASSERT(pMgr->restored == true); if (pMgr->startIndex <= pMsg->lastSendIndex && pMsg->lastSendIndex < pMgr->endIndex) { if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) { - int64_t firstSentMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs; - int64_t lastSentMs = pMgr->states[(pMgr->endIndex - 1) % pMgr->size].timeMs; - int64_t timeDiffMs = lastSentMs - firstSentMs; - if (timeDiffMs > 0 && timeDiffMs < ((int64_t)SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) { + int64_t firstMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs; + int64_t lastMs = pMgr->states[(pMgr->endIndex - 1) % pMgr->size].timeMs; + int64_t diffMs = lastMs - firstMs; + if (diffMs > 0 && diffMs < ((int64_t)SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) { pMgr->retryBackoff -= 1; } } @@ -932,7 +931,7 @@ int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, pMgr->startIndex = pMgr->matchIndex; } - return syncLogReplReplicateAttempt(pMgr, pNode); + return syncLogReplAttempt(pMgr, pNode); } SSyncLogReplMgr* syncLogReplCreate() { @@ -1127,8 +1126,8 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, return pEntry; } -int32_t syncLogReplReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier) { +int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, SRaftId* pDestId, + bool* pBarrier) { SSyncRaftEntry* pEntry = NULL; SRpcMsg msgOut = {0}; bool inBuf = false; @@ -1141,14 +1140,14 @@ int32_t syncLogReplReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncI if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId); if (pMgr) { - sInfo("vgId:%d, reset sync log repl mgr of peer:%" PRIx64 " since %s. index:%" PRId64, pNode->vgId, - pDestId->addr, terrstr(), index); + sInfo("vgId:%d, reset sync log repl of peer:%" PRIx64 " since %s. index:%" PRId64, pNode->vgId, pDestId->addr, + terrstr(), index); (void)syncLogReplReset(pMgr); } } goto _err; } - *pBarrier = syncLogIsReplicationBarrier(pEntry); + *pBarrier = syncLogReplBarrier(pEntry); prevLogTerm = syncLogReplGetPrevLogTerm(pMgr, pNode, index); if (prevLogTerm < 0) { diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 43d2bc839be87cc6343065b72e71402f20b445e7..8ac9a860e3e4b1f368fda48fc27b41d6c89f6ae9 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -74,7 +74,7 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) { continue; } SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; - (void)syncLogReplReplicateOnce(pMgr, pNode); + (void)syncLogReplDoOnce(pMgr, pNode); } return 0; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index db4e3a475984d60067ac4456dd10f311b8056e8b..dc3ff3e6de14b632a2f4a76927dfbae823e1652a 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -100,6 +100,8 @@ int32_t walNextValidMsg(SWalReader *pReader) { return -1; } +int64_t walReaderGetCurrentVer(const SWalReader *pReader) { return pReader->curVersion; } + static int64_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int64_t ver) { int64_t ret = 0; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index f811d2f20316e2aab87cea030ead08613b19d77c..288ea6052b580909198788f528871633048e19f1 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -187,7 +187,7 @@ static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType st } static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - int64_t ival = (int64_t)atoi(value); + int64_t ival = (int64_t)atoll(value); if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 9349e7f176992366056f47c6f8fed59980cee282..002d60579379d9f513779729a103c8cfd8bcf2ae 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -321,7 +321,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_ALREADY_DEPLOYED, "Snode already deploye TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_NOT_DEPLOYED, "Snode not deployed") // vnode -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode moved to another dnode or was deleted") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_EXIST, "Vnode not exist") TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_EXIST, "Vnode already exist") diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 631bcb443ebecdc89ab3338d5dd5254964232093..a49ff0cd5b26b4527886afebdf03ea1ffd82aa1f 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -218,7 +218,7 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem int32_t queueNum = taosGetQueueNumber(pool->qset); int32_t curWorkerNum = taosArrayGetSize(pool->workers); int32_t dstWorkerNum = ceil(queueNum * pool->ratio); - if (dstWorkerNum < 1) dstWorkerNum = 1; + if (dstWorkerNum < 2) dstWorkerNum = 2; // spawn a thread to process queue while (curWorkerNum < dstWorkerNum) { @@ -248,7 +248,8 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem } taosThreadAttrDestroy(&thAttr); - uInfo("worker:%s:%d is launched, total:%d", pool->name, worker->id, (int32_t)taosArrayGetSize(pool->workers)); + int32_t numOfThreads = taosArrayGetSize(pool->workers); + uInfo("worker:%s:%d is launched, total:%d, expect:%d", pool->name, worker->id, numOfThreads, dstWorkerNum); curWorkerNum++; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4d22bfe5c566568d80885cf7099d2f1145d9fdae..dda4ec3e84c585af785447f5b2c154cf67199c48 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -89,12 +89,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py -# ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py @@ -785,6 +785,7 @@ ,,y,script,./test.sh -f tsim/insert/query_multi_file.sim ,,y,script,./test.sh -f tsim/insert/tcp.sim ,,y,script,./test.sh -f tsim/insert/update0.sim +,,y,script,./test.sh -f tsim/insert/delete0.sim ,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim ,,y,script,./test.sh -f tsim/insert/update2.sim ,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim diff --git a/tests/script/tsim/insert/delete0.sim b/tests/script/tsim/insert/delete0.sim new file mode 100644 index 0000000000000000000000000000000000000000..56538536439bbf3d7e9937353ee65acaea34fb43 --- /dev/null +++ b/tests/script/tsim/insert/delete0.sim @@ -0,0 +1,161 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create database with different precision +sql create database d0 keep 365 +sql create database d1 keep 365 precision 'ms' +sql create database d2 keep 365 precision 'us' +sql create database d3 keep 365 precision 'ns' + +sql select * from information_schema.ins_databases +if $rows != 6 then + return -1 +endi + +print $data00 $data01 $data02 + + +sql create table if not exists d0.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d1.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d2.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d3.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d0.ntb (ts timestamp, c1 int, c2 float, c3 double) +sql create table if not exists d1.ntb (ts timestamp, c1 int, c2 float, c3 double) +sql create table if not exists d2.ntb (ts timestamp, c1 int, c2 float, c3 double) +sql create table if not exists d3.ntb (ts timestamp, c1 int, c2 float, c3 double) + +sql create table d0.ct1 using d0.stb tags(1000) +sql create table d1.ct1 using d1.stb tags(1000) +sql create table d2.ct1 using d2.stb tags(1000) +sql create table d3.ct1 using d3.stb tags(1000) +sql create table d0.ct2 using d0.stb tags(1000) +sql create table d1.ct2 using d1.stb tags(1000) +sql create table d2.ct2 using d2.stb tags(1000) +sql create table d3.ct2 using d3.stb tags(1000) + + +sql insert into d0.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d1.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d3.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d0.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d1.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d3.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d0.ntb values(now+0s, 10, 2.0, 3.0) +sql insert into d1.ntb values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ntb values(now+0s, 10, 2.0, 3.0) +sql insert into d3.ntb values(now+0s, 10, 2.0, 3.0) + + +print =============== query data from super table +sql select count(*) from d0.stb +if $data00 != 2 then + return -1 +endi +sql select count(*) from d1.stb +if $data00 != 2 then + return -1 +endi +sql select count(*) from d2.stb +if $data00 != 2 then + return -1 +endi +sql select count(*) from d3.stb +if $data00 != 2 then + return -1 +endi + +print =============== delete from child table +sql delete from d0.ct1 where ts < now() +sql delete from d1.ct1 where ts < now() +sql delete from d2.ct1 where ts < now() +sql delete from d3.ct1 where ts < now() + + +print =============== query data from super table +sql select count(*) from d0.stb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d1.stb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d2.stb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d3.stb +if $data00 != 1 then + return -1 +endi +print =============== query data from normal table +sql select count(*) from d0.ntb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d1.ntb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d2.ntb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d3.ntb +if $data00 != 1 then + return -1 +endi + +print =============== delete from super table +sql delete from d0.stb where ts < now() +sql delete from d1.stb where ts < now() +sql delete from d2.stb where ts < now() +sql delete from d3.stb where ts < now() + +print =============== query data from super table +sql select count(*) from d0.stb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d1.stb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d2.stb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d3.stb +if $data00 != 0 then + return -1 +endi + +print =============== delete from normal table +sql delete from d0.ntb where ts < now() +sql delete from d1.ntb where ts < now() +sql delete from d2.ntb where ts < now() +sql delete from d3.ntb where ts < now() + +print =============== query data from normal table +sql select count(*) from d0.ntb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d1.ntb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d2.ntb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d3.ntb +if $data00 != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/last_cache.sim b/tests/script/tsim/parser/last_cache.sim index 9a41a9f5aa6de119eddfc7cf31eeddccacd1fe5e..3f1a29d928eed421ee23aab79444495ecbeed6f2 100644 --- a/tests/script/tsim/parser/last_cache.sim +++ b/tests/script/tsim/parser/last_cache.sim @@ -54,6 +54,7 @@ sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL ) run tsim/parser/last_cache_query.sim +sql flush database $db system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tsim/parser/limit1.sim b/tests/script/tsim/parser/limit1.sim index d1a75f3ba9603ff2d97a029258dc91f0fa097d43..bae5eba7a4395099ad50105b6259cc69a7b5947e 100644 --- a/tests/script/tsim/parser/limit1.sim +++ b/tests/script/tsim/parser/limit1.sim @@ -55,6 +55,7 @@ print ====== tables created run tsim/parser/limit1_tb.sim run tsim/parser/limit1_stb.sim +sql flush database $db print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index e69875d69f20b01f91ea2b90fd912c84b88d2455..15ca6bf7c924bb7cffba8f0607cb0a5c1fea95cb 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -37,7 +37,7 @@ if $loop_count == 20 then endi if $rows != 4 then - print =====rows=$rows + print =====rows=$rows, expect 4 goto loop0 endi @@ -53,7 +53,7 @@ if $data02 != 2 then endi if $data03 != 5 then - print =====data03=$data03 + print =====data03=$data03, expect:5 goto loop0 endi diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index c5fbf41b66fcba4088e6480c721d4a6e21811d7a..0abe56ab3c3d7d09b6f441aed813445be0d2df92 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -114,6 +114,7 @@ run tsim/insert/basic1.sim run tsim/insert/commit-merge0.sim run tsim/insert/basic0.sim run tsim/insert/update0.sim +run tsim/insert/delete0.sim run tsim/insert/backquote.sim run tsim/insert/null.sim run tsim/catalog/alterInCurrent.sim diff --git a/tests/script/tsim/valgrind/checkUdf.sim b/tests/script/tsim/valgrind/checkUdf.sim index dc703e155d3bacf2ed17648c0020192f5bf77eab..e316c104ed68bcc32bc4f84fa3ee803b5a91ae1a 100644 --- a/tests/script/tsim/valgrind/checkUdf.sim +++ b/tests/script/tsim/valgrind/checkUdf.sim @@ -29,10 +29,10 @@ sql select udf1(f) from t; if $rows != 2 then return -1 endi -if $data00 != 88 then +if $data00 != 1 then return -1 endi -if $data10 != 88 then +if $data10 != 1 then return -1 endi @@ -51,10 +51,10 @@ sql select udf1(f1, f2) from t2; if $rows != 2 then return -1 endi -if $data00 != 88 then +if $data00 != 1 then return -1 endi -if $data10 != 88 then +if $data10 != 1 then return -1 endi @@ -72,10 +72,10 @@ print $rows , $data00 , $data10 , $data20 , $data30 if $rows != 4 then return -1 endi -if $data00 != 88 then +if $data00 != 1 then return -1 endi -if $data10 != 88 then +if $data10 != 1 then return -1 endi @@ -114,10 +114,10 @@ print $rows , $data00 , $data01 if $rows != 1 then return -1 endi -if $data00 != 176.000000000 then +if $data00 != 2.000000000 then return -1 endi -if $data01 != 152.420471066 then +if $data01 != 1.732050808 then return -1 endi diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index da77078208d5f54eb8b4d09ab5e489bb9eb2f1ff..78020cb9586e6c59f7c0f84e5aeacdbb596b421c 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -191,20 +191,20 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,1) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(0,4,1.000000000) - tdSql.checkData(0,5,88) + tdSql.checkData(0,5,1) tdSql.checkData(0,6,"binary1") - tdSql.checkData(0,7,88) + tdSql.checkData(0,7,1) tdSql.checkData(3,0,3) - tdSql.checkData(3,1,88) + tdSql.checkData(3,1,1) tdSql.checkData(3,2,33333) - tdSql.checkData(3,3,88) + tdSql.checkData(3,3,1) tdSql.checkData(3,4,33.000000000) - tdSql.checkData(3,5,88) + tdSql.checkData(3,5,1) tdSql.checkData(3,6,"binary1") - tdSql.checkData(3,7,88) + tdSql.checkData(3,7,1) tdSql.checkData(11,0,None) tdSql.checkData(11,1,None) @@ -213,7 +213,7 @@ class TDTestCase: tdSql.checkData(11,4,None) tdSql.checkData(11,5,None) tdSql.checkData(11,6,"binary1") - tdSql.checkData(11,7,88) + tdSql.checkData(11,7,1) tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") tdSql.checkData(0,0,None) @@ -226,13 +226,13 @@ class TDTestCase: tdSql.checkData(0,7,None) tdSql.checkData(20,0,8) - tdSql.checkData(20,1,88) + tdSql.checkData(20,1,1) tdSql.checkData(20,2,88888) - tdSql.checkData(20,3,88) + tdSql.checkData(20,3,1) tdSql.checkData(20,4,888) - tdSql.checkData(20,5,88) + tdSql.checkData(20,5,1) tdSql.checkData(20,6,88) - tdSql.checkData(20,7,88) + tdSql.checkData(20,7,1) # aggregate functions @@ -375,14 +375,14 @@ class TDTestCase: tdSql.checkRows(25) tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,8) tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") tdSql.checkRows(13) - tdSql.checkData(0,0,88) + tdSql.checkData(0,0,1) tdSql.checkData(0,1,8) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,7) # bug fix for crash @@ -401,9 +401,9 @@ class TDTestCase: tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") tdSql.checkRows(3) tdSql.checkData(0,0,9) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,-99.990000000) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) @@ -412,20 +412,20 @@ class TDTestCase: tdSql.checkData(1,1,10) tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - tdSql.checkData(0,0,88) - tdSql.checkData(0,1,88) - tdSql.checkData(1,0,88) - tdSql.checkData(1,1,88) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,1) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,0) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(1,0,1) - tdSql.checkData(1,1,88) + tdSql.checkData(1,1,1) tdSql.checkData(1,2,10) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,16.881943016) diff --git a/tests/system-test/0-others/udf_cfg2.py b/tests/system-test/0-others/udf_cfg2.py index cc6da81847126d91e2aed8c14f11ebfbc0729b01..b535b4f626cdf92de1a406185d9c1af807ea6c7c 100644 --- a/tests/system-test/0-others/udf_cfg2.py +++ b/tests/system-test/0-others/udf_cfg2.py @@ -193,20 +193,20 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,1) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(0,4,1.000000000) - tdSql.checkData(0,5,88) + tdSql.checkData(0,5,1) tdSql.checkData(0,6,"binary1") - tdSql.checkData(0,7,88) + tdSql.checkData(0,7,1) tdSql.checkData(3,0,3) - tdSql.checkData(3,1,88) + tdSql.checkData(3,1,1) tdSql.checkData(3,2,33333) - tdSql.checkData(3,3,88) + tdSql.checkData(3,3,1) tdSql.checkData(3,4,33.000000000) - tdSql.checkData(3,5,88) + tdSql.checkData(3,5,1) tdSql.checkData(3,6,"binary1") - tdSql.checkData(3,7,88) + tdSql.checkData(3,7,1) tdSql.checkData(11,0,None) tdSql.checkData(11,1,None) @@ -215,7 +215,7 @@ class TDTestCase: tdSql.checkData(11,4,None) tdSql.checkData(11,5,None) tdSql.checkData(11,6,"binary1") - tdSql.checkData(11,7,88) + tdSql.checkData(11,7,1) tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") tdSql.checkData(0,0,None) @@ -228,13 +228,13 @@ class TDTestCase: tdSql.checkData(0,7,None) tdSql.checkData(20,0,8) - tdSql.checkData(20,1,88) + tdSql.checkData(20,1,1) tdSql.checkData(20,2,88888) - tdSql.checkData(20,3,88) + tdSql.checkData(20,3,1) tdSql.checkData(20,4,888) - tdSql.checkData(20,5,88) + tdSql.checkData(20,5,1) tdSql.checkData(20,6,88) - tdSql.checkData(20,7,88) + tdSql.checkData(20,7,1) # aggregate functions @@ -377,14 +377,14 @@ class TDTestCase: tdSql.checkRows(25) tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,8) tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") tdSql.checkRows(13) - tdSql.checkData(0,0,88) + tdSql.checkData(0,0,1) tdSql.checkData(0,1,8) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,7) # bug fix for crash @@ -403,9 +403,9 @@ class TDTestCase: tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") tdSql.checkRows(3) tdSql.checkData(0,0,9) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,-99.990000000) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) @@ -414,20 +414,20 @@ class TDTestCase: tdSql.checkData(1,1,10) tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - tdSql.checkData(0,0,88) - tdSql.checkData(0,1,88) - tdSql.checkData(1,0,88) - tdSql.checkData(1,1,88) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,1) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,0) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(1,0,1) - tdSql.checkData(1,1,88) + tdSql.checkData(1,1,1) tdSql.checkData(1,2,10) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,16.881943016) diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index d35688c8dae3c419084e2f0414c5fb9031a4bea3..f467e802ac5d5f8adb15ef6b705d273e744270a3 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -193,20 +193,20 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,1) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(0,4,1.000000000) - tdSql.checkData(0,5,88) + tdSql.checkData(0,5,1) tdSql.checkData(0,6,"binary1") - tdSql.checkData(0,7,88) + tdSql.checkData(0,7,1) tdSql.checkData(3,0,3) - tdSql.checkData(3,1,88) + tdSql.checkData(3,1,1) tdSql.checkData(3,2,33333) - tdSql.checkData(3,3,88) + tdSql.checkData(3,3,1) tdSql.checkData(3,4,33.000000000) - tdSql.checkData(3,5,88) + tdSql.checkData(3,5,1) tdSql.checkData(3,6,"binary1") - tdSql.checkData(3,7,88) + tdSql.checkData(3,7,1) tdSql.checkData(11,0,None) tdSql.checkData(11,1,None) @@ -215,7 +215,7 @@ class TDTestCase: tdSql.checkData(11,4,None) tdSql.checkData(11,5,None) tdSql.checkData(11,6,"binary1") - tdSql.checkData(11,7,88) + tdSql.checkData(11,7,1) tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") tdSql.checkData(0,0,None) @@ -228,13 +228,13 @@ class TDTestCase: tdSql.checkData(0,7,None) tdSql.checkData(20,0,8) - tdSql.checkData(20,1,88) + tdSql.checkData(20,1,1) tdSql.checkData(20,2,88888) - tdSql.checkData(20,3,88) + tdSql.checkData(20,3,1) tdSql.checkData(20,4,888) - tdSql.checkData(20,5,88) + tdSql.checkData(20,5,1) tdSql.checkData(20,6,88) - tdSql.checkData(20,7,88) + tdSql.checkData(20,7,1) # aggregate functions @@ -377,14 +377,14 @@ class TDTestCase: tdSql.checkRows(25) tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,8) tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") tdSql.checkRows(13) - tdSql.checkData(0,0,88) + tdSql.checkData(0,0,1) tdSql.checkData(0,1,8) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,7) # bug fix for crash @@ -403,9 +403,9 @@ class TDTestCase: tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") tdSql.checkRows(3) tdSql.checkData(0,0,9) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,-99.990000000) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) @@ -414,20 +414,20 @@ class TDTestCase: tdSql.checkData(1,1,10) tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - tdSql.checkData(0,0,88) - tdSql.checkData(0,1,88) - tdSql.checkData(1,0,88) - tdSql.checkData(1,1,88) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,1) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,0) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(1,0,1) - tdSql.checkData(1,1,88) + tdSql.checkData(1,1,1) tdSql.checkData(1,2,10) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,16.881943016) diff --git a/tests/system-test/0-others/udf_restart_taosd.py b/tests/system-test/0-others/udf_restart_taosd.py index a0f70ccd49be9884ad1c0eeb5b7ee91c67ea4bee..61b6a4ea684f6150834cfc1e7c6ce1dc03a8b91b 100644 --- a/tests/system-test/0-others/udf_restart_taosd.py +++ b/tests/system-test/0-others/udf_restart_taosd.py @@ -190,20 +190,20 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,1) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(0,4,1.000000000) - tdSql.checkData(0,5,88) + tdSql.checkData(0,5,1) tdSql.checkData(0,6,"binary1") - tdSql.checkData(0,7,88) + tdSql.checkData(0,7,1) tdSql.checkData(3,0,3) - tdSql.checkData(3,1,88) + tdSql.checkData(3,1,1) tdSql.checkData(3,2,33333) - tdSql.checkData(3,3,88) + tdSql.checkData(3,3,1) tdSql.checkData(3,4,33.000000000) - tdSql.checkData(3,5,88) + tdSql.checkData(3,5,1) tdSql.checkData(3,6,"binary1") - tdSql.checkData(3,7,88) + tdSql.checkData(3,7,1) tdSql.checkData(11,0,None) tdSql.checkData(11,1,None) @@ -212,7 +212,7 @@ class TDTestCase: tdSql.checkData(11,4,None) tdSql.checkData(11,5,None) tdSql.checkData(11,6,"binary1") - tdSql.checkData(11,7,88) + tdSql.checkData(11,7,1) tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") tdSql.checkData(0,0,None) @@ -225,13 +225,13 @@ class TDTestCase: tdSql.checkData(0,7,None) tdSql.checkData(20,0,8) - tdSql.checkData(20,1,88) + tdSql.checkData(20,1,1) tdSql.checkData(20,2,88888) - tdSql.checkData(20,3,88) + tdSql.checkData(20,3,1) tdSql.checkData(20,4,888) - tdSql.checkData(20,5,88) + tdSql.checkData(20,5,1) tdSql.checkData(20,6,88) - tdSql.checkData(20,7,88) + tdSql.checkData(20,7,1) # aggregate functions @@ -374,14 +374,14 @@ class TDTestCase: tdSql.checkRows(25) tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,8) tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") tdSql.checkRows(13) - tdSql.checkData(0,0,88) + tdSql.checkData(0,0,1) tdSql.checkData(0,1,8) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,7) # bug fix for crash @@ -400,9 +400,9 @@ class TDTestCase: tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") tdSql.checkRows(3) tdSql.checkData(0,0,9) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,-99.990000000) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) @@ -411,20 +411,20 @@ class TDTestCase: tdSql.checkData(1,1,10) tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - tdSql.checkData(0,0,88) - tdSql.checkData(0,1,88) - tdSql.checkData(1,0,88) - tdSql.checkData(1,1,88) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,1) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,0) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(1,0,1) - tdSql.checkData(1,1,88) + tdSql.checkData(1,1,1) tdSql.checkData(1,2,10) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,16.881943016) @@ -468,12 +468,12 @@ class TDTestCase: tdSql.checkData(1,0,1) tdSql.checkData(1,1,1) tdSql.checkData(1,2,1.110000000) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select c1,c6,udf1(c1,c6) from stb1 order by ts") tdSql.checkData(1,0,8) tdSql.checkData(1,1,88.880000000) - tdSql.checkData(1,2,88) + tdSql.checkData(1,2,1) tdSql.query("select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null order by ts;") tdSql.checkRows(22) diff --git a/tests/system-test/2-query/To_unixtimestamp.py b/tests/system-test/2-query/To_unixtimestamp.py index 8ee2007450328b2edbb02822a68fa4ac9d9ca9a7..424ebff6c5764dec9a2395eaa2254acfb47056c5 100644 --- a/tests/system-test/2-query/To_unixtimestamp.py +++ b/tests/system-test/2-query/To_unixtimestamp.py @@ -26,7 +26,7 @@ class TDTestCase: 'c1':'int', 'c2':'float', 'c3':'binary(20)' - + } # structure of tag self.tag_dict = { @@ -60,7 +60,7 @@ class TDTestCase: if tb_type == 'ntb' or tb_type == 'ctb': tdSql.checkRows(len(values_list)) elif tb_type == 'stb': - tdSql.checkRows(len(self.values_list)*tb_num) + tdSql.checkRows(len(self.values_list)*tb_num) for time in ['2020-01-32T08:00:00','2020-13-32T08:00:00','acd']: tdSql.query(f"select to_unixtimestamp('{time}') from {tbname}") if tb_type == 'ntb' or tb_type == 'ctb': @@ -74,7 +74,7 @@ class TDTestCase: if tb_type == 'ntb' or tb_type == 'ctb': tdSql.checkRows(len(values_list)) elif tb_type == 'stb': - tdSql.checkRows(len(values_list)*tb_num) + tdSql.checkRows(len(values_list)*tb_num) for time in self.error_param: tdSql.error(f"select to_unixtimestamp({time}) from {tbname}") def timestamp_change_check_ntb(self): @@ -95,9 +95,20 @@ class TDTestCase: self.data_check(f'{self.stbname}_{i}',self.values_list,'ctb') self.data_check(self.stbname,self.values_list,'stb',self.tbnum) tdSql.execute(f'drop database {self.dbname}') + def timestamp_change_return_type(self): + tdSql.query(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 0);") + tdSql.checkEqual(tdSql.queryResult[0][0], 0) + tdSql.query(f"select to_unixtimestamp('1970-01-01 00:00:00', 1);") + tdSql.checkData(0, 0, '1970-01-01 00:00:00') + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 2);") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1.5);") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 'abc');") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', true);") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1, 3);") def run(self): # sourcery skip: extract-duplicate-method self.timestamp_change_check_ntb() self.timestamp_change_check_stb() + self.timestamp_change_return_type() def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 74b552dcc8bdbeafdab701beaea252efa1fb0ca8..ddf3f2534dd42d80b4b9a7da647748a8193902f0 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -2382,6 +2382,14 @@ class TDTestCase: tdSql.error(f"select interp('abcd') from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") tdSql.error(f"select interp('中文字符') from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + # invalid pseudo column usage + tdSql.error(f"select interp(_irowts) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(_isfilled) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(c0) from {dbname}.{tbname} where _isfilled = true range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(c0) from {dbname}.{tbname} where _irowts > 0 range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + + + tdLog.printNoPrefix("==========step13:stable cases") diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py index b8da02b9a6da6bfadc93d301d135464a1c7566f1..ba6ab53fc7c2503bfcdcd1b79474dc22f2717211 100644 --- a/tests/system-test/2-query/max.py +++ b/tests/system-test/2-query/max.py @@ -20,8 +20,8 @@ class TDTestCase: intData = [] floatData = [] tdSql.execute(f'''create table {dbname}.stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned, - col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''') - tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags('beijing')") + col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(t0 tinyint, t1 float, loc nchar(20))''') + tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags(5, 5.5, 'beijing')") for i in range(self.rowNum): tdSql.execute(f"insert into {dbname}.stb_1 values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) @@ -55,13 +55,20 @@ class TDTestCase: tdSql.checkData(0, 1, np.max(intData)) tdSql.query(f"select ts, min(col9) from {dbname}.stb") - tdSql.checkRows(1) + tdSql.checkRows(1) tdSql.checkData(0, 1, np.min(floatData)) tdSql.query(f"select ts, min(col9) from {dbname}.stb_1") - tdSql.checkRows(1) + tdSql.checkRows(1) tdSql.checkData(0, 1, np.min(floatData)) + # check tags + tdSql.query(f"select max(t0) from {dbname}.stb") + tdSql.checkData(0,0,5) + + tdSql.query(f"select max(t1) from {dbname}.stb") + tdSql.checkData(0,0,5.5) + def max_check_ntb_base(self, dbname="db"): tdSql.prepare() intData = [] diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py index 4b8c8c86294cd05357af1f3c289dd17437f0a168..aa9c8d25d0364635b11ba6c867cad67f2085c08e 100644 --- a/tests/system-test/7-tmq/tmqDelete-1ctb.py +++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py @@ -238,7 +238,7 @@ class TDTestCase: if self.snapshot == 0: consumerId = 2 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) * 2 elif self.snapshot == 1: consumerId = 3 expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4)) @@ -324,7 +324,7 @@ class TDTestCase: if self.snapshot == 0: consumerId = 4 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) * 2 elif self.snapshot == 1: consumerId = 5 expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4)) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 8300e2e1e3a7b39233b673f20113501202fdf760..52cb524e3bfb98875e4f93a20bf21fa185674a29 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -243,8 +243,8 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { SShellArgs *pArgs = &shell.args; for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0 || - strcmp(argv[i], "/?") == 0) { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 + || strcmp(argv[i], "-?") == 0 || strcmp(argv[i], "/?") == 0) { shellParseSingleOpt('?', NULL); return 0; } @@ -260,8 +260,10 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { return -1; } - if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' || key[1] == 'a' || key[1] == 'c' || key[1] == 's' || - key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N' + if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' + || key[1] == 'a' || key[1] == 'c' || key[1] == 's' + || key[1] == 'f' || key[1] == 'd' || key[1] == 'w' + || key[1] == 'n' || key[1] == 'l' || key[1] == 'N' #ifdef WEBSOCKET || key[1] == 'E' || key[1] == 'T' #endif @@ -277,10 +279,12 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { } shellParseSingleOpt(key[1], val); i++; - } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C' || key[1] == 'r' || key[1] == 'k' || key[1] == 't' || - key[1] == 'V' || key[1] == '?' || key[1] == 1 + } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C' + || key[1] == 'r' || key[1] == 'k' + || key[1] == 't' || key[1] == 'V' + || key[1] == '?' || key[1] == 1 #ifdef WEBSOCKET - || key[1] == 'R' + ||key[1] == 'R' #endif ) { shellParseSingleOpt(key[1], NULL); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 01ca2efaba93b445d3c11420284fea00ab8b135f..910b067d4e0bce5e289c136de29470591a446f2b 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -317,6 +317,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i quotationStr[0] = '\"'; quotationStr[1] = 0; + int n; char buf[TSDB_MAX_BYTES_PER_ROW]; switch (field->type) { case TSDB_DATA_TYPE_BOOL: @@ -347,11 +348,24 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i taosFprintfFile(pFile, "%" PRIu64, *((uint64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: - taosFprintfFile(pFile, "%e", GET_FLOAT_VAL(val)); + if (tsEnableScience) { + taosFprintfFile(pFile, "%e", GET_FLOAT_VAL(val)); + } else { + taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); + } break; case TSDB_DATA_TYPE_DOUBLE: - snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", 23, GET_DOUBLE_VAL(val)); - taosFprintfFile(pFile, "%s", buf); + if (tsEnableScience) { + snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9e", 23, GET_DOUBLE_VAL(val)); + taosFprintfFile(pFile, "%s", buf); + } else { + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val)); + if (n > TMAX(25, length)) { + taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val)); + } else { + taosFprintfFile(pFile, "%s", buf); + } + } break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: @@ -507,6 +521,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t return; } + int n; char buf[TSDB_MAX_BYTES_PER_ROW]; switch (field->type) { case TSDB_DATA_TYPE_BOOL: @@ -537,11 +552,24 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t printf("%*" PRIu64, width, *((uint64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: - printf("%*e", width, GET_FLOAT_VAL(val)); + if (tsEnableScience) { + printf("%*e", width, GET_FLOAT_VAL(val)); + } else { + printf("%*.5f", width, GET_FLOAT_VAL(val)); + } break; case TSDB_DATA_TYPE_DOUBLE: - snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%.9e", GET_DOUBLE_VAL(val)); - printf("%*s", width, buf); + if (tsEnableScience) { + snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%.9e", GET_DOUBLE_VAL(val)); + printf("%*s", width, buf); + } else { + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", width, GET_DOUBLE_VAL(val)); + if (n > TMAX(25, width)) { + printf("%*.15e", width, GET_DOUBLE_VAL(val)); + } else { + printf("%s", buf); + } + } break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index 1d81ce4b2f8c47ef3f8e5344c42ae5afa3b96bcb..d8920cb4c3d79aacb441bf9a1f512f4a5a4f62b6 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -24,7 +24,7 @@ int shell_conn_ws_server(bool first) { ((dsnLen-SHELL_WS_DSN_MASK) > SHELL_WS_DSN_BUFF)? SHELL_WS_DSN_BUFF:(dsnLen-SHELL_WS_DSN_MASK), "%s", shell.args.dsn); - fprintf(stdout, "trying to connect %s*** ", cuttedDsn); + fprintf(stdout, "trying to connect %s****** ", cuttedDsn); fflush(stdout); for (int i = 0; i < shell.args.timeout; i++) { shell.ws_conn = ws_connect_with_dsn(shell.args.dsn);