diff --git a/.gitmodules b/.gitmodules
index 7c84eac8a4ee7529005855bc836387561c49ae2d..156226d54486c17e64b9c514e47e3a7dc3fe6942 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -4,3 +4,9 @@
[submodule "src/connector/grafanaplugin"]
path = src/connector/grafanaplugin
url = https://github.com/taosdata/grafanaplugin
+[submodule "tests/examples/rust"]
+ path = tests/examples/rust
+ url = https://github.com/songtianyi/tdengine-rust-bindings.git
+[submodule "src/connector/hivemq-tdengine-extension"]
+ path = src/connector/hivemq-tdengine-extension
+ url = https://github.com/huskar-t/hivemq-tdengine-extension.git
\ No newline at end of file
diff --git a/Jenkinsfile b/Jenkinsfile
index ea50d6ef5a13fed2114868f8a36cd0acace78dd5..dc7836c3daacaa457f721f9278687b99770fc394 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -27,6 +27,7 @@ pipeline {
cd debug
cmake .. > /dev/null
make > /dev/null
+ make install > /dev/null
cd ${WKC}/tests
#./test-all.sh smoke
./test-all.sh pytest
@@ -79,7 +80,20 @@ pipeline {
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests/pytest
- ./crash_gen.sh -a -p -t 4 -s 2000
+ '''
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WKC}/tests/pytest
+ ./crash_gen.sh -a -p -t 4 -s 2000
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WKC}/tests/pytest
+ ./handle_crash_gen_val_log.sh
+ '''
+ }
+ sh '''
date
cd ${WKC}/tests
./test-all.sh b2
@@ -124,14 +138,33 @@ pipeline {
sh'''
cd ${WORKSPACE}
git checkout develop
- cd tests/gotest
- bash batchtest.sh
- cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
- mvn clean package assembly:single >/dev/null
- java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
- cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
- python3 PythonChecker.py
'''
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WORKSPACE}/tests/gotest
+ bash batchtest.sh
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
+ python3 PythonChecker.py
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
+ mvn clean package assembly:single >/dev/null
+ java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
+ dotnet run
+ '''
+ }
+
}
}
@@ -139,5 +172,82 @@ pipeline {
}
}
-
+ post {
+ success {
+ emailext (
+ subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
+ body: '''
+
+
+
+
+
+
+
+
+ 构建信息
+
|
+
+
+
+
+
+ - 构建名称>>分支:${PROJECT_NAME}
+ - 构建结果: Successful
+ - 构建编号:${BUILD_NUMBER}
+ - 触发用户:${CAUSE}
+ - 变更概要:${CHANGES}
+ - 构建地址:${BUILD_URL}
+ - 构建日志:${BUILD_URL}console
+ - 变更集:${JELLY_SCRIPT}
+
+
+ |
+
+
+
+ ''',
+ to: "yqliu@taosdata.com,pxiao@taosdata.com",
+ from: "support@taosdata.com"
+ )
+ }
+ failure {
+ emailext (
+ subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
+ body: '''
+
+
+
+
+
+
+
+
+ 构建信息
+
|
+
+
+
+
+
+ - 构建名称>>分支:${PROJECT_NAME}
+ - 构建结果: Successful
+ - 构建编号:${BUILD_NUMBER}
+ - 触发用户:${CAUSE}
+ - 变更概要:${CHANGES}
+ - 构建地址:${BUILD_URL}
+ - 构建日志:${BUILD_URL}console
+ - 变更集:${JELLY_SCRIPT}
+
+
+ |
+
+
+
+ ''',
+ to: "yqliu@taosdata.com,pxiao@taosdata.com",
+ from: "support@taosdata.com"
+ )
+ }
+ }
}
\ No newline at end of file
diff --git a/cmake/install.inc b/cmake/install.inc
index dfca758b9362c96bec0ce45aa385d54a4e75a9e5..9bbcc2cf40520a9ab5e7aa4d19f46b47b6b42192 100755
--- a/cmake/install.inc
+++ b/cmake/install.inc
@@ -13,6 +13,7 @@ ELSEIF (TD_WINDOWS)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/nodejs DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector)
+ INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/C\# DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)
diff --git a/cmake/version.inc b/cmake/version.inc
index a248f76f48ede5f1f483943c08fce6604756b6d4..741f76da43b18600122b413627d01a925ee1fb20 100644
--- a/cmake/version.inc
+++ b/cmake/version.inc
@@ -4,7 +4,7 @@ PROJECT(TDengine)
IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER})
ELSE ()
- SET(TD_VER_NUMBER "2.0.6.0")
+ SET(TD_VER_NUMBER "2.0.7.0")
ENDIF ()
IF (DEFINED VERCOMPATIBLE)
diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md
index 44b3ad46712019870be6fefb5234611f55f6e03e..79388a2edb9404a0f7b31b9182eb5ce2cb0d52be 100644
--- a/documentation/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation/webdocs/markdowndocs/administrator-ch.md
@@ -87,6 +87,7 @@ TDengine系统后台服务由taosd提供,可以在配置文件taos.cfg里修
- httpPort: RESTful服务使用的端口号,所有的HTTP请求(TCP)都需要向该接口发起查询/写入请求。
- dataDir: 数据文件目录,所有的数据文件都将写入该目录。默认值:/var/lib/taos。
- logDir:日志文件目录,客户端和服务器的运行日志文件将写入该目录。默认值:/var/log/taos。
+- tempDir:临时文件目录,客户端和服务器的临时文件(主要是查询时用于保存中间结果的问题)将写入该目录。 默认值:Linux下为 /tmp/,Windows下为环境变量 tmp 或 temp 指向的目录。
- arbitrator:系统中裁决器的end point, 缺省值为空。
- role:dnode的可选角色。0-any; 既可作为mnode,也可分配vnode;1-mgmt;只能作为mnode,不能分配vnode;2-dnode;不能作为mnode,只能分配vnode
- debugFlag:运行日志开关。131(输出错误和警告日志),135( 输出错误、警告和调试日志),143( 输出错误、警告、调试和跟踪日志)。默认值:131或135(不同模块有不同的默认值)。
diff --git a/documentation20/webdocs/markdowndocs/Documentation-ch.md b/documentation20/webdocs/markdowndocs/Documentation-ch.md
index 077a0431383da8067b6bd33c4b7218627b7adbd0..f1f2d58f0562b831395e5177eb34ec3534aaa715 100644
--- a/documentation20/webdocs/markdowndocs/Documentation-ch.md
+++ b/documentation20/webdocs/markdowndocs/Documentation-ch.md
@@ -34,7 +34,8 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台,专
- [SQL写入](https://www.taosdata.com/cn/documentation20/insert/#SQL写入):使用SQL insert命令向一张或多张表写入单条或多条记录
- [Telegraf写入](https://www.taosdata.com/cn/documentation20/insert/#Telegraf直接写入):配置Telegraf, 不用任何代码,将采集数据直接写入
- [Prometheus写入](https://www.taosdata.com/cn/documentation20/insert/#Prometheus直接写入):配置Prometheus, 不用任何代码,将数据直接写入
-- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入):配置EMQ X,不用任何代码,就可将MQTT数据直接写入
+- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入):配置EMQ X,不用任何代码,就可将 MQTT 数据直接写入
+- [HiveMQ Broker](https://www.taosdata.com/cn/documentation20/insert/#HiveMQ-Broker直接写入):通过 HiveMQ Extension,不用任何代码,就可将 MQTT 数据直接写入
## [高效查询数据](https://www.taosdata.com/cn/documentation20/queries)
diff --git a/documentation20/webdocs/markdowndocs/Evaluation-ch.md b/documentation20/webdocs/markdowndocs/Evaluation-ch.md
index 9e7e0ec6aa8b60bfa9dbe603a45a265a1d1eba00..a92f97a8d9dfc9a47d5554daa076f4d2f0774c92 100644
--- a/documentation20/webdocs/markdowndocs/Evaluation-ch.md
+++ b/documentation20/webdocs/markdowndocs/Evaluation-ch.md
@@ -10,7 +10,7 @@ TDengine的模块之一是时序数据库。但除此之外,为减少研发的
* __硬件或云服务成本降至1/5__:由于超强性能,计算资源不到通用大数据方案的1/5;通过列式存储和先进的压缩算法,存储空间不到通用数据库的1/10。
* __全栈时序数据处理引擎__:将数据库、消息队列、缓存、流式计算等功能融为一体,应用无需再集成Kafka/Redis/HBase/Spark/HDFS等软件,大幅降低应用开发和维护的复杂度成本。
* __强大的分析功能__:无论是十年前还是一秒钟前的数据,指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell, Python, R, Matlab随时进行。
-* __与第三方工具无缝连接__:不用一行代码,即可与Telegraf, Grafana, EMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。
+* __与第三方工具无缝连接__:不用一行代码,即可与Telegraf, Grafana, EMQ, HiveMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。
* __零运维成本、零学习成本__:安装集群简单快捷,无需分库分表,实时备份。类似标准SQL,支持RESTful, 支持Python/Java/C/C++/C#/Go/Node.js, 与MySQL相似,零学习成本。
采用TDengine,可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是,因充分利用了物联网时序数据的特点,它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。
diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md
index 4082d72f112e8ff62d2e8b2c8b15391dd6f39d8a..760ebae4fc3b7ddd609d1bfb5689f51b05fc7cb7 100644
--- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md
+++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md
@@ -90,7 +90,7 @@ TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMic
```mysql
ALTER DATABASE db_name REPLICA 2;
```
- REPLICA参数是指修改数据库副本数,取值范围[1, 3]。在集群中使用,副本数必须小于dnode的数目。
+ REPLICA参数是指修改数据库副本数,取值范围[1, 3]。在集群中使用,副本数必须小于或等于dnode的数目。
```mysql
ALTER DATABASE db_name KEEP 365;
@@ -844,7 +844,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
- **PERCENTILE**
```mysql
- SELECT PERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause];
+ SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause];
```
功能说明:统计表中某列的值百分比分位数。
返回结果数据类型: 双精度浮点数Double。
@@ -1016,9 +1016,9 @@ SELECT AVG(current),MAX(current),LEASTSQUARES(current, start_val, step_val), PER
```
## TAOS SQL 边界限制
-- 数据库名最大长度为33
-- 表名最大长度为193,每行数据最大长度16k个字符
-- 列名最大长度为65,最多允许1024列,最少需要2列,第一列必须是时间戳
+- 数据库名最大长度为32
+- 表名最大长度为192,每行数据最大长度16k个字符
+- 列名最大长度为64,最多允许1024列,最少需要2列,第一列必须是时间戳
- 标签最多允许128个,可以0个,标签总长度不超过16k个字符
- SQL语句最大长度65480个字符,但可通过系统配置参数maxSQLLength修改,最长可配置为1M
- 库的数目,超级表的数目、表的数目,系统不做限制,仅受系统资源限制
diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md
index 4b274e05e6e392879fc887ec1137968270a8e4b8..36466d2b7ea29307b5c17ad21b0db47a98598fa4 100644
--- a/documentation20/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation20/webdocs/markdowndocs/administrator-ch.md
@@ -35,7 +35,7 @@ TDengine相对于通用数据库,有超高的压缩比,在绝大多数场景
Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable
```
-示例:1000万台智能电表,每台电表每15分钟采集一次数据,每次采集的数据128字节,那么一年的原始数据量是:10000000\*128\*24\*60/15*365 = 44851T。TDengine大概需要消耗44851/5=8970T, 8.9P空间。
+示例:1000万台智能电表,每台电表每15分钟采集一次数据,每次采集的数据128字节,那么一年的原始数据量是:10000000\*128\*24\*60/15*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。
用户可以通过参数keep,设置数据在磁盘中的最大保存时长。为进一步减少存储成本,TDengine还提供多级存储,最冷的数据可以存放在最廉价的存储介质上,应用的访问不用做任何调整,只是读取速度降低了。
@@ -253,7 +253,7 @@ ALTER USER PASS <'password'>;
修改用户密码, 为避免被转换为小写,密码需要用单引号引用,单引号为英文半角
```
-ALTER USER PRIVILEDGE ;
+ALTER USER PRIVILEGE ;
```
修改用户权限为:super/write/read,不需要添加单引号
diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md
index d4705ccb05c092d8da38072368a167466bd78968..c9bfa30830fe7b2f3cd1364b589326255560ad83 100644
--- a/documentation20/webdocs/markdowndocs/architecture-ch.md
+++ b/documentation20/webdocs/markdowndocs/architecture-ch.md
@@ -4,16 +4,99 @@
### 物联网典型场景
在典型的物联网、车联网、运维监测场景中,往往有多种不同类型的数据采集设备,采集一个到多个不同的物理量。而同一种采集设备类型,往往又有多个具体的采集设备分布在不同的地点。大数据处理系统就是要将各种采集的数据汇总,然后进行计算和分析。对于同一类设备,其采集的数据都是很规则的。以智能电表为例,假设每个智能电表采集电流、电压、相位三个量,其采集的数据类似如下的表格:
-| Device ID | Time Stamp | current | voltage | phase | location | groupId |
-| :-------: | :-----------: | :-----: | :-----: | :---: | :--------------: | :-----: |
-| d1001 | 1538548685000 | 10.3 | 219 | 0.31 | Beijing.Chaoyang | 2 |
-| d1002 | 1538548684000 | 10.2 | 220 | 0.23 | Beijing.Chaoyang | 3 |
-| d1003 | 1538548686500 | 11.5 | 221 | 0.35 | Beijing.Haidian | 3 |
-| d1004 | 1538548685500 | 13.4 | 223 | 0.29 | Beijing.Haidian | 2 |
-| d1001 | 1538548695000 | 12.6 | 218 | 0.33 | Beijing.Chaoyang | 2 |
-| d1004 | 1538548696600 | 11.8 | 221 | 0.28 | Beijing.Haidian | 2 |
-| d1002 | 1538548696650 | 10.3 | 218 | 0.25 | Beijing.Chaoyang | 3 |
-| d1001 | 1538548696800 | 12.3 | 221 | 0.31 | Beijing.Chaoyang | 2 |
+
+
+ 设备ID |
+ 时间戳 |
+ 采集量 |
+ 标签 |
+
+
+
+Device ID |
+Time Stamp |
+current |
+voltage |
+phase |
+location |
+groupId |
+
+
+
+
+d1001 |
+1538548685000 |
+10.3 |
+219 |
+0.31 |
+Beijing.Chaoyang |
+2 |
+
+
+d1002 |
+1538548684000 |
+10.2 |
+220 |
+0.23 |
+Beijing.Chaoyang |
+3 |
+
+
+d1003 |
+1538548686500 |
+11.5 |
+221 |
+0.35 |
+Beijing.Haidian |
+3 |
+
+
+d1004 |
+1538548685500 |
+13.4 |
+223 |
+0.29 |
+Beijing.Haidian |
+2 |
+
+
+d1001 |
+1538548695000 |
+12.6 |
+218 |
+0.33 |
+Beijing.Chaoyang |
+2 |
+
+
+d1004 |
+1538548696600 |
+11.8 |
+221 |
+0.28 |
+Beijing.Haidian |
+2 |
+
+
+d1002 |
+1538548696650 |
+10.3 |
+218 |
+0.25 |
+Beijing.Chaoyang |
+3 |
+
+
+d1001 |
+1538548696800 |
+12.3 |
+221 |
+0.31 |
+Beijing.Chaoyang |
+2 |
+
+
+
表1:智能电表数据示例
@@ -221,7 +304,7 @@ TDengine采用时间驱动缓存管理策略(First-In-First-Out,FIFO),
TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中,可以更加快速地响应用户针对最近一条或一批数据的查询分析,整体上提供更快的数据库查询响应能力。从这个意义上来说,**可通过设置合适的配置参数将TDengine作为数据缓存来使用,而不需要再部署Redis或其他额外的缓存系统**,可有效地简化系统架构,降低运维的成本。需要注意的是,TDengine重启以后系统的缓存将被清空,之前缓存的数据均会被批量写入磁盘,缓存的数据将不会像专门的Key-value缓存系统再将之前缓存的数据重新加载到缓存中。
-每个vnode有自己独立的内存,而且由多个固定大小的内存块组成,不同vnode之间完全隔离。数据写入时,类似于日志的写法,数据被顺序追加写入内存,但每个vnode维护有自己的skip list,便于迅速查找。当一半以上的内存块写满时,启动落盘操作,而且后续写的操作在新的内存块进行。这样,一个vnode里有一半内存块是保留有最近的数据的,以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定,内存块的大小由配置参数cache决定。
+每个vnode有自己独立的内存,而且由多个固定大小的内存块组成,不同vnode之间完全隔离。数据写入时,类似于日志的写法,数据被顺序追加写入内存,但每个vnode维护有自己的skip list,便于迅速查找。当三分之一以上的内存块写满时,启动落盘操作,而且后续写的操作在新的内存块进行。这样,一个vnode里有三分之一内存块是保留有最近的数据的,以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定,内存块的大小由配置参数cache决定。
### 持久化存储
TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久化存储。当vnode中缓存的数据达到一定规模时,为了不阻塞后续数据的写入,TDengine也会拉起落盘线程将缓存的数据写入持久化存储。TDengine在数据落盘时会打开新的数据库日志文件,在落盘成功后则会删除老的数据库日志文件,避免日志文件无限制的增长。
diff --git a/documentation20/webdocs/markdowndocs/connector-ch.md b/documentation20/webdocs/markdowndocs/connector-ch.md
index 0e29b324871e61d184fde17710c863f06b60bdcc..c5a955f43fcc30e70d8aac3919433c2e767f76ba 100644
--- a/documentation20/webdocs/markdowndocs/connector-ch.md
+++ b/documentation20/webdocs/markdowndocs/connector-ch.md
@@ -616,6 +616,43 @@ HTTP请求URL采用`sqlutc`时,返回结果集的时间戳将采用UTC时间
- httpEnableCompress: 是否支持压缩,默认不支持,目前TDengine仅支持gzip压缩格式
- httpDebugFlag: 日志开关,131:仅错误和报警信息,135:调试信息,143:非常详细的调试信息,默认131
+## CSharp Connector
+
+在Windows系统上,C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作,后续版本将提供ORM(dapper)框架驱动。
+
+#### 安装TDengine客户端
+
+C#连接器需要使用`libtaos.so`和`taos.h`。因此,在使用C#连接器之前,需在程序运行的Windows环境安装TDengine的Windows客户端,以便获得相关驱动文件。
+
+安装完成后,在文件夹`C:/TDengine/examples/C#`中,将会看到两个文件
+
+- TDengineDriver.cs 调用taos.dll文件的Native C方法
+- TDengineTest.cs 参考程序示例
+
+在文件夹`C:\Windows\System32`,将会看到`taos.dll`文件
+
+#### 使用方法
+
+- 将C#接口文件TDengineDriver.cs加入到应用程序所在.NET项目中
+- 参考TDengineTest.cs来定义数据库连接参数,及执行数据插入、查询等操作的方法
+- 因为C#接口需要用到`taos.dll`文件,用户可以将`taos.dll`文件加入.NET解决方案中
+
+#### 注意事项
+
+- `taos.dll`文件使用x64平台编译,所以.NET项目在生成.exe文件时,“解决方案”/“项目”的“平台”请均选择“x64”。
+- 此.NET接口目前已经在Visual Studio 2013/2015/2017中验证过,其它VS版本尚待验证。
+
+#### 第三方驱动
+
+Maikebing.Data.Taos是一个TDengine的ADO.Net提供器,支持linux,windows。该开发包由热心贡献者`麦壳饼@@maikebing`提供,具体请参考
+
+```
+//接口下载
+https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos
+//用法说明
+https://www.taosdata.com/blog/2020/11/02/1901.html
+```
+
## Go Connector
diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md
index 80deb889ef0eee8e9b47f86b5e58a76c6c070d5b..757b6d9929b68a90e7bc558b553233f24b09ba17 100644
--- a/documentation20/webdocs/markdowndocs/faq-ch.md
+++ b/documentation20/webdocs/markdowndocs/faq-ch.md
@@ -38,9 +38,9 @@
6. 检查防火墙设置,确认TCP/UDP 端口6030-6042 是打开的
-7. 对于Linux上的JDBC(ODBC, Python, Go等接口类似)连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里
+7. 对于Linux上的JDBC(ODBC, Python, Go等接口类似)连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里
-8. 对于windows上的JDBC, ODBC, Python, Go等连接,确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
+8. 对于windows上的JDBC, ODBC, Python, Go等连接,确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
9. 如果仍不能排除连接故障,请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
检查UDP端口连接是否工作:`nc -vuz {hostIP} {port} `
diff --git a/documentation20/webdocs/markdowndocs/insert-ch.md b/documentation20/webdocs/markdowndocs/insert-ch.md
index fa53cbd62b17169c0f54877a62da8c48ac21edcf..77ba596d4ec6de76a39b878e57ea255b081fea45 100644
--- a/documentation20/webdocs/markdowndocs/insert-ch.md
+++ b/documentation20/webdocs/markdowndocs/insert-ch.md
@@ -1,6 +1,6 @@
# 高效写入数据
-TDengine支持多种接口写入数据,包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, CSV文件等,后续还将提供Kafka, OPC等接口。数据可以单条插入,也可以批量插入,可以插入一个数据采集点的数据,也可以同时插入多个数据采集点的数据。支持多线程插入,支持时间乱序数据插入,也支持历史数据插入。
+TDengine支持多种接口写入数据,包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, HiveMQ Broker, CSV文件等,后续还将提供Kafka, OPC等接口。数据可以单条插入,也可以批量插入,可以插入一个数据采集点的数据,也可以同时插入多个数据采集点的数据。支持多线程插入,支持时间乱序数据插入,也支持历史数据插入。
## SQL写入
@@ -218,7 +218,15 @@ use telegraf;
select * from cpu;
```
-## EMQ X Broker直接写入
-MQTT是一流行的物联网数据传输协议,[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件,无需任何代码,只需要在EMQ Dashboard里使用“规则”做简单配置,即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDEngine,也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。
+
+MQTT是一流行的物联网数据传输协议,TDengine 可以很方便的接入 MQTT Broker 接受的数据并写入到 TDengine。
+
+## EMQ Broker 直接写入
+
+[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件,无需任何代码,只需要在EMQ Dashboard里使用“规则”做简单配置,即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine,也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。
+
+## HiveMQ Broker 直接写入
+
+[HiveMQ](https://www.hivemq.com/) 是一个提供免费个人版和企业版的 MQTT 代理,主要用于企业和新兴的机器到机器M2M通讯和内部传输,满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 [HiveMQ extension - TDengine 说明文档](https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md)。
diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg
index 974b2b05c12f9fd5a482ef0072a9dacc2d1f09dc..ca88bca3c863dcfd4a95497fc572499dc93f31f9 100644
--- a/packaging/cfg/taos.cfg
+++ b/packaging/cfg/taos.cfg
@@ -20,6 +20,9 @@
# data file's directory
# dataDir /var/lib/taos
+# temporary file's directory
+# tempDir /tmp/
+
# the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only
# arbitrator arbitrator_hostname:6042
@@ -256,3 +259,5 @@
# maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden
# maxBinaryDisplayWidth 30
+# enable/disable telemetry reporting
+# telemetryReporting 1
\ No newline at end of file
diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh
index 450c6a8f551fe7dd8dff5e53f5dad639aec4656e..edc7de96923266729d1985cb43abfa3003f9178e 100755
--- a/packaging/deb/makedeb.sh
+++ b/packaging/deb/makedeb.sh
@@ -48,6 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_pat
cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin
+cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver
@@ -58,7 +59,7 @@ cp -r ${top_dir}/src/connector/grafanaplugin ${pkg_dir}${install_home_pat
cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector
cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector
cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector
-cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.* ${pkg_dir}${install_home_path}/connector
+cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.* ${pkg_dir}${install_home_path}/connector ||:
cp -r ${compile_dir}/../packaging/deb/DEBIAN ${pkg_dir}/
chmod 755 ${pkg_dir}/DEBIAN/*
diff --git a/packaging/release.sh b/packaging/release.sh
index 7542a5b4cafb69d5cee16bddfc9a5651eb717b92..68f947ccab3ef18a1b351b91a58db64a8f465c8e 100755
--- a/packaging/release.sh
+++ b/packaging/release.sh
@@ -156,9 +156,15 @@ build_time=$(date +"%F %R")
# get commint id from git
gitinfo=$(git rev-parse --verify HEAD)
-enterprise_dir="${top_dir}/../enterprise"
-cd ${enterprise_dir}
-gitinfoOfInternal=$(git rev-parse --verify HEAD)
+
+if [[ "$verMode" == "cluster" ]]; then
+ enterprise_dir="${top_dir}/../enterprise"
+ cd ${enterprise_dir}
+ gitinfoOfInternal=$(git rev-parse --verify HEAD)
+else
+ gitinfoOfInternal=NULL
+fi
+
cd ${curr_dir}
# 2. cmake executable file
@@ -193,23 +199,35 @@ cd ${curr_dir}
# 3. Call the corresponding script for packaging
if [ "$osType" != "Darwin" ]; then
if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]] && [[ "$dbName" == "taos" ]]; then
- echo "====do deb package for the ubuntu system===="
- output_dir="${top_dir}/debs"
- if [ -d ${output_dir} ]; then
- ${csudo} rm -rf ${output_dir}
+ ret='0'
+ command -v dpkg >/dev/null 2>&1 || { ret='1'; }
+ if [ "$ret" -eq 0 ]; then
+ echo "====do deb package for the ubuntu system===="
+ output_dir="${top_dir}/debs"
+ if [ -d ${output_dir} ]; then
+ ${csudo} rm -rf ${output_dir}
+ fi
+ ${csudo} mkdir -p ${output_dir}
+ cd ${script_dir}/deb
+ ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
+ else
+ echo "==========dpkg command not exist, so not release deb package!!!"
fi
- ${csudo} mkdir -p ${output_dir}
- cd ${script_dir}/deb
- ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
-
- echo "====do rpm package for the centos system===="
- output_dir="${top_dir}/rpms"
- if [ -d ${output_dir} ]; then
- ${csudo} rm -rf ${output_dir}
+
+ ret='0'
+ command -v rpmbuild >/dev/null 2>&1 || { ret='1'; }
+ if [ "$ret" -eq 0 ]; then
+ echo "====do rpm package for the centos system===="
+ output_dir="${top_dir}/rpms"
+ if [ -d ${output_dir} ]; then
+ ${csudo} rm -rf ${output_dir}
+ fi
+ ${csudo} mkdir -p ${output_dir}
+ cd ${script_dir}/rpm
+ ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
+ else
+ echo "==========rpmbuild command not exist, so not release rpm package!!!"
fi
- ${csudo} mkdir -p ${output_dir}
- cd ${script_dir}/rpm
- ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
fi
echo "====do tar.gz package for all systems===="
diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec
index 4e40263dc4ebaf6b566d20890ecd97f64e160340..8c23ab802d54088dafc4b4c6a5d1b6241c881f2e 100644
--- a/packaging/rpm/tdengine.spec
+++ b/packaging/rpm/tdengine.spec
@@ -2,7 +2,7 @@
%define cfg_install_dir /etc/taos
%define __strip /bin/true
-Name: TDengine
+Name: tdengine
Version: %{_version}
Release: 3%{?dist}
Summary: tdengine from taosdata
@@ -58,6 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/scri
cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin
+cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver
cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include
cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include
@@ -65,7 +66,7 @@ cp -r %{_compiledir}/../src/connector/grafanaplugin %{buildroot}%{homepath}/conn
cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector
cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector
cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector
-cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.* %{buildroot}%{homepath}/connector
+cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.* %{buildroot}%{homepath}/connector ||:
cp -r %{_compiledir}/../tests/examples/* %{buildroot}%{homepath}/examples
#Scripts executed before installation
@@ -134,6 +135,7 @@ if [ $1 -eq 0 ];then
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
+ #${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${cfg_link_dir}/* || :
${csudo} rm -f ${inc_link_dir}/taos.h || :
${csudo} rm -f ${inc_link_dir}/taoserror.h || :
diff --git a/packaging/tools/get_client.sh b/packaging/tools/get_client.sh
new file mode 100755
index 0000000000000000000000000000000000000000..0d34ecb311fb4b941d6f6773d1c3c921a9bd9886
--- /dev/null
+++ b/packaging/tools/get_client.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+
+log_dir=$1
+result_file=$2
+
+if [ ! -n "$1" ];then
+ echo "Pleas input the director of taosdlog."
+ echo "usage: ./get_client.sh "
+ exit 1
+else
+ log_dir=$1
+fi
+
+if [ ! -n "$2" ];then
+ result_file=clientInfo.txt
+else
+ result_file=$2
+fi
+
+grep "new TCP connection" ${log_dir}/taosdlog.* | sed -e "s/0x.* from / /"|sed -e "s/,.*$//"|sed -e "s/:[0-9]*$//"|sort -r|uniq -f 2|sort -k 3 -r|uniq -f 2 > ${result_file}
diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh
index eff70d8035af0291f6dc7040ec13632fec4fa3be..831012851ad70d05080bfae161c0b925d5215ae9 100755
--- a/packaging/tools/make_install.sh
+++ b/packaging/tools/make_install.sh
@@ -278,11 +278,11 @@ function install_service_on_sysvinit() {
# Install taosd service
if ((${os_type}==1)); then
- ${csudo} cp -f ${script_dir}/../deb/init.d/taosd ${install_main_dir}/init.d
- ${csudo} cp ${script_dir}/../deb/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
+ ${csudo} cp -f ${script_dir}/../deb/taosd ${install_main_dir}/init.d
+ ${csudo} cp ${script_dir}/../deb/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
elif ((${os_type}==2)); then
- ${csudo} cp -f ${script_dir}/../rpm/init.d/taosd ${install_main_dir}/init.d
- ${csudo} cp ${script_dir}/../rpm/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
+ ${csudo} cp -f ${script_dir}/../rpm/taosd ${install_main_dir}/init.d
+ ${csudo} cp ${script_dir}/../rpm/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
fi
#restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start"
diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh
index e17c678f263cb6b7a0ccbc32250265b9bc5cbd0e..ee79a560407c650de8c511964f611cf8cacfd2d5 100755
--- a/packaging/tools/makeclient.sh
+++ b/packaging/tools/makeclient.sh
@@ -45,7 +45,7 @@ if [ "$osType" != "Darwin" ]; then
strip ${build_dir}/bin/taos
bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
else
- bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
+ bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh"
fi
lib_files="${build_dir}/lib/libtaos.so.${version}"
else
@@ -110,7 +110,7 @@ mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
if [ "$osType" != "Darwin" ]; then
- cp ${build_dir}/lib/*.jar ${install_dir}/connector
+ cp ${build_dir}/lib/*.jar ${install_dir}/connector ||:
fi
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
cp -r ${connector_dir}/python ${install_dir}/connector/
diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh
index faa5a03f52b4c9b56981a6b1c0918e543262b3bb..fdb3e0e5cc8c7add9c8ea8cdc9224964b3c80153 100755
--- a/packaging/tools/makeclient_power.sh
+++ b/packaging/tools/makeclient_power.sh
@@ -76,8 +76,10 @@ if [ "$osType" != "Darwin" ]; then
else
cp ${build_dir}/bin/taos ${install_dir}/bin/power
cp ${script_dir}/remove_power.sh ${install_dir}/bin
- cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo
+ cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo
+ cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump
cp ${script_dir}/set_core.sh ${install_dir}/bin
+ cp ${script_dir}/get_client.sh ${install_dir}/bin
fi
else
cp ${bin_files} ${install_dir}/bin
@@ -135,7 +137,7 @@ mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
if [ "$osType" != "Darwin" ]; then
- cp ${build_dir}/lib/*.jar ${install_dir}/connector
+ cp ${build_dir}/lib/*.jar ${install_dir}/connector ||:
fi
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
cp -r ${connector_dir}/python ${install_dir}/connector/
diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh
index 75b45b544e0a4abbf709cc4c5b3a3b55dc315f0f..5ae5cbbcdc00c1fd862501a6eb0dccd3ac0fa007 100755
--- a/packaging/tools/makepkg.sh
+++ b/packaging/tools/makepkg.sh
@@ -36,7 +36,7 @@ if [ "$pagMode" == "lite" ]; then
strip ${build_dir}/bin/taos
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh"
else
- bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
+ bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh"
fi
lib_files="${build_dir}/lib/libtaos.so.${version}"
@@ -124,7 +124,7 @@ cp ${lib_files} ${install_dir}/driver
connector_dir="${code_dir}/connector"
mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
- cp ${build_dir}/lib/*.jar ${install_dir}/connector
+ cp ${build_dir}/lib/*.jar ${install_dir}/connector ||:
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
cp -r ${connector_dir}/python ${install_dir}/connector/
cp -r ${connector_dir}/go ${install_dir}/connector
diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh
index 2c02b99787c6d5ad6234de2319bf78b0b09d7e8a..13849484695f98a5fc206d88ec6a5de0cc930a9b 100755
--- a/packaging/tools/makepkg_power.sh
+++ b/packaging/tools/makepkg_power.sh
@@ -77,8 +77,10 @@ else
cp ${build_dir}/bin/taosd ${install_dir}/bin/powerd
cp ${script_dir}/remove_power.sh ${install_dir}/bin
cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo
+ cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump
cp ${build_dir}/bin/tarbitrator ${install_dir}/bin
cp ${script_dir}/set_core.sh ${install_dir}/bin
+ cp ${script_dir}/get_client.sh ${install_dir}/bin
fi
chmod a+x ${install_dir}/bin/* || :
@@ -156,7 +158,7 @@ cp ${lib_files} ${install_dir}/driver
connector_dir="${code_dir}/connector"
mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
- cp ${build_dir}/lib/*.jar ${install_dir}/connector
+ cp ${build_dir}/lib/*.jar ${install_dir}/connector ||:
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
cp -r ${connector_dir}/python ${install_dir}/connector/
cp -r ${connector_dir}/go ${install_dir}/connector
diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh
index d91daaa5c44488e34dea7ec2ddec0863699446f2..00705fad778c065eddbd7cb65a2f9c5583a9997b 100755
--- a/packaging/tools/post.sh
+++ b/packaging/tools/post.sh
@@ -81,8 +81,10 @@ function install_lib() {
${csudo} ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1
${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
- ${csudo} ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
- ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
+ if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
+ ${csudo} ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
+ ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
+ fi
}
function install_bin() {
@@ -121,8 +123,11 @@ function install_config() {
echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join${NC}"
echo
echo -e -n "${GREEN}OR leave it blank to build one${NC}:"
- read firstEp
- while true; do
+ #read firstEp
+ if exec < /dev/tty; then
+ read firstEp;
+ fi
+ while true; do
if [ ! -z "$firstEp" ]; then
# check the format of the firstEp
#if [[ $firstEp == $FQDN_PATTERN ]]; then
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index b0d8d8983bab473b8711d19805cea0fef2f10ee7..1738ff7ec8e412b51403efcb2874b179726b13f9 100644
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -1,6 +1,6 @@
name: tdengine
base: core18
-version: '2.0.5.1'
+version: '2.0.7.0'
icon: snap/gui/t-dengine.svg
summary: an open-source big data platform designed and optimized for IoT.
description: |
@@ -72,7 +72,7 @@ parts:
- usr/bin/taosd
- usr/bin/taos
- usr/bin/taosdemo
- - usr/lib/libtaos.so.2.0.5.1
+ - usr/lib/libtaos.so.2.0.6.0
- usr/lib/libtaos.so.1
- usr/lib/libtaos.so
diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c
index 0e9bb85b25defd169fea8711d3e0b40304500de4..df78f4fe270d34d17e436e80322f6c81b68d0d2c 100644
--- a/src/balance/src/balance.c
+++ b/src/balance/src/balance.c
@@ -490,7 +490,7 @@ static bool balanceMontiorDropping() {
if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
if (pDnode->lastAccess + tsOfflineThreshold > tsAccessSquence) continue;
- if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) continue;
+ if (dnodeIsMasterEp(pDnode->dnodeEp)) continue;
if (mnodeGetDnodesNum() <= 1) continue;
mLInfo("dnode:%d, set to removing state for it offline:%d seconds", pDnode->dnodeId,
@@ -571,8 +571,8 @@ static void balanceCheckDnodeAccess() {
if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) {
pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
- mInfo("dnode:%d, set to offline state, access seq:%d, last seq:%d", pDnode->dnodeId, tsAccessSquence,
- pDnode->lastAccess);
+ mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence,
+ pDnode->lastAccess, pDnode->status);
balanceSetVgroupOffline(pDnode);
}
}
diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h
index 5baa66a9e0229f35c431cea7a0d2dbb9e2ffb0e2..ce67344b03a015419e485aa28e54575c2cf60045 100644
--- a/src/client/inc/tscLocalMerge.h
+++ b/src/client/inc/tscLocalMerge.h
@@ -62,11 +62,11 @@ typedef struct SLocalReducer {
bool hasUnprocessedRow;
tOrderDescriptor * pDesc;
SColumnModel * resColModel;
+ SColumnModel* finalModel;
tExtMemBuffer ** pExtMemBuffer; // disk-based buffer
SFillInfo* pFillInfo; // interpolation support structure
- char * pFinalRes; // result data after interpo
- tFilePage * discardData;
- SResultInfo * pResInfo;
+ char* pFinalRes; // result data after interpo
+ tFilePage* discardData;
bool discard;
int32_t offset; // limit offset value
bool orderPrjOnSTable; // projection query on stable
@@ -75,7 +75,8 @@ typedef struct SLocalReducer {
typedef struct SRetrieveSupport {
tExtMemBuffer ** pExtMemBuffer; // for build loser tree
tOrderDescriptor *pOrderDescriptor;
- SColumnModel * pFinalColModel; // colModel for final result
+ SColumnModel* pFinalColModel; // colModel for final result
+ SColumnModel* pFFColModel;
int32_t subqueryIndex; // index of current vnode in vnode list
SSqlObj * pParentSql;
tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to
@@ -97,7 +98,7 @@ int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tF
* create local reducer to launch the second-stage reduce process at client site
*/
void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
- SColumnModel *finalModel, SSqlObj* pSql);
+ SColumnModel *finalModel, SColumnModel *pFFModel, SSqlObj* pSql);
void tscDestroyLocalReducer(SSqlObj *pSql);
diff --git a/src/client/inc/tscSubquery.h b/src/client/inc/tscSubquery.h
index bc01de110345e4c90cf5c15d3d7f6b010cb7308d..3226f705283fc4b5af5f1d8c6e3a736300bd2fa6 100644
--- a/src/client/inc/tscSubquery.h
+++ b/src/client/inc/tscSubquery.h
@@ -23,7 +23,7 @@ extern "C" {
#include "tscUtil.h"
#include "tsclient.h"
-void tscFetchDatablockFromSubquery(SSqlObj* pSql);
+void tscFetchDatablockForSubquery(SSqlObj* pSql);
void tscSetupOutputColumnIndex(SSqlObj* pSql);
void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code);
@@ -41,6 +41,8 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql);
void tscBuildResFromSubqueries(SSqlObj *pSql);
TAOS_ROW doSetResultRowData(SSqlObj *pSql, bool finalResult);
+char *getArithemicInputSrc(void *param, const char *name, int32_t colId);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h
index d86e1aa0fb38f0c7d0da8035352ce51fc318b6b3..223fb5d226690f480f6d8196180ec61b3f2fbd92 100644
--- a/src/client/inc/tscUtil.h
+++ b/src/client/inc/tscUtil.h
@@ -75,6 +75,7 @@ typedef struct SJoinSupporter {
SArray* exprList;
SFieldInfo fieldsInfo;
STagCond tagCond;
+ SSqlGroupbyExpr groupInfo; // group by info
struct STSBuf* pTSBuf; // the TSBuf struct that holds the compressed timestamp array
FILE* f; // temporary file in order to create TSBuf
char path[PATH_MAX]; // temporary file path, todo dynamic allocate memory
@@ -86,8 +87,8 @@ typedef struct SJoinSupporter {
} SJoinSupporter;
typedef struct SVgroupTableInfo {
- SCMVgroupInfo vgInfo;
- SArray* itemList; //SArray
+ SVgroupInfo vgInfo;
+ SArray* itemList; //SArray
} SVgroupTableInfo;
static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) {
@@ -124,6 +125,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t
*/
bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo);
bool tscIsTWAQuery(SQueryInfo* pQueryInfo);
+bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo);
bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo *pQueryInfo, int32_t tableIndex);
bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex);
@@ -157,7 +159,7 @@ SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t ind
TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index);
void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo);
-void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo);
+void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo);
int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index);
void tscFieldInfoClear(SFieldInfo* pFieldInfo);
@@ -166,15 +168,15 @@ static FORCE_INLINE int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQue
int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2);
-void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex);
+void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes);
int32_t tscGetResRowLength(SArray* pExprList);
SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
- int16_t size, int16_t interSize, bool isTagCol);
+ int16_t size, int16_t resColId, int16_t interSize, bool isTagCol);
SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
- int16_t size, int16_t interSize, bool isTagCol);
+ int16_t size, int16_t resColId, int16_t interSize, bool isTagCol);
SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type,
int16_t size);
@@ -225,8 +227,9 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo);
void tscClearSubqueryInfo(SSqlCmd* pCmd);
void tscFreeVgroupTableInfo(SArray* pVgroupTables);
-SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables);
+SArray* tscVgroupTableInfoClone(SArray* pVgroupTables);
void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index);
+void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo);
int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex);
int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo);
@@ -237,7 +240,7 @@ void tscDoQuery(SSqlObj* pSql);
SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *pInfo);
void* tscVgroupInfoClear(SVgroupsInfo *pInfo);
-void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src);
+void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src);
/**
* The create object function must be successful expect for the out of memory issue.
*
@@ -265,6 +268,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t sub
void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex);
int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid);
+int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId);
void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex);
diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h
index 67942ad42a0942756efe18e44eff711df59ba1d9..f6dc45398f35c38598f3f3132b2f6f5601a4ed68 100644
--- a/src/client/inc/tschemautil.h
+++ b/src/client/inc/tschemautil.h
@@ -77,7 +77,7 @@ SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex);
* @param colId
* @return
*/
-SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId);
+SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId);
/**
* check if the schema is valid or not, including following aspects:
@@ -107,9 +107,6 @@ SSchema tscGetTbnameColumnSchema();
*/
STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size);
-//todo tags value as well as the table id structure needs refactor
-char *tsGetTagsValue(STableMeta *pMeta);
-
#ifdef __cplusplus
}
#endif
diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index fa215db2702f63aff4c704f8393255a94987d681..6b3d97d6f9e5ffb6168938d3bc3e0a7a26dcbf33 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -30,6 +30,7 @@ extern "C" {
#include "tsqlfunction.h"
#include "tutil.h"
#include "tcache.h"
+#include "tref.h"
#include "qExecutor.h"
#include "qSqlparser.h"
@@ -89,12 +90,12 @@ typedef struct STableComInfo {
int32_t rowSize;
} STableComInfo;
-typedef struct SCMCorVgroupInfo {
- int32_t version;
- int8_t inUse;
- int8_t numOfEps;
- SEpAddr1 epAddr[TSDB_MAX_REPLICA];
-} SCMCorVgroupInfo;
+typedef struct SCorVgroupInfo {
+ int32_t version;
+ int8_t inUse;
+ int8_t numOfEps;
+ SEpAddr1 epAddr[TSDB_MAX_REPLICA];
+} SCorVgroupInfo;
typedef struct STableMeta {
STableComInfo tableInfo;
@@ -102,8 +103,8 @@ typedef struct STableMeta {
int16_t sversion;
int16_t tversion;
char sTableId[TSDB_TABLE_FNAME_LEN];
- SCMVgroupInfo vgroupInfo;
- SCMCorVgroupInfo corVgroupInfo;
+ SVgroupInfo vgroupInfo;
+ SCorVgroupInfo corVgroupInfo;
STableId id;
SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
} STableMeta;
@@ -127,7 +128,7 @@ typedef struct STableMetaInfo {
typedef struct SSqlExpr {
char aliasName[TSDB_COL_NAME_LEN]; // as aliasName
SColIndex colInfo;
- int64_t uid; // refactor use the pointer
+ uint64_t uid; // refactor use the pointer
int16_t functionId; // function id in aAgg array
int16_t resType; // return value type
int16_t resBytes; // length of return value
@@ -135,6 +136,7 @@ typedef struct SSqlExpr {
int16_t numOfParams; // argument value of each function
tVariant param[3]; // parameters are not more than 3
int32_t offset; // sub result column value of arithmetic expression.
+ int16_t resColId; // result column id
} SSqlExpr;
typedef struct SColumnIndex {
@@ -250,6 +252,7 @@ typedef struct SQueryInfo {
int64_t clauseLimit; // limit for current sub clause
int64_t prjOffset; // offset value in the original sql expression, only applied at client side
int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
+ int16_t resColumnId; // result column id
} SQueryInfo;
typedef struct {
@@ -329,6 +332,7 @@ typedef struct STscObj {
char writeAuth : 1;
char superAuth : 1;
uint32_t connId;
+ uint64_t rid; // ref ID returned by taosAddRef
struct SSqlObj * pHb;
struct SSqlObj * sqlList;
struct SSqlStream *streamList;
@@ -338,16 +342,16 @@ typedef struct STscObj {
} STscObj;
typedef struct SSubqueryState {
- int32_t numOfRemain; // the number of remain unfinished subquery
- int32_t numOfSub; // the number of total sub-queries
- uint64_t numOfRetrievedRows; // total number of points in this query
+ int32_t numOfRemain; // the number of remain unfinished subquery
+ int32_t numOfSub; // the number of total sub-queries
+ uint64_t numOfRetrievedRows; // total number of points in this query
} SSubqueryState;
typedef struct SSqlObj {
void *signature;
pthread_t owner; // owner of sql object, by which it is executed
STscObj *pTscObj;
- void *pRpcCtx;
+ int64_t rpcRid;
void (*fp)();
void (*fetchFp)();
void *param;
@@ -430,14 +434,6 @@ void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache);
*/
void tscFreeSqlResult(SSqlObj *pSql);
-/**
- * only free part of resources allocated during query.
- * TODO remove it later
- * Note: this function is multi-thread safe.
- * @param pObj
- */
-void tscPartiallyFreeSqlObj(SSqlObj *pSql);
-
/**
* free sql object, release allocated resource
* @param pObj
@@ -446,7 +442,7 @@ void tscFreeSqlObj(SSqlObj *pSql);
void tscFreeRegisteredSqlObj(void *pSql);
void tscFreeTableMetaHelper(void *pTableMeta);
-void tscCloseTscObj(STscObj *pObj);
+void tscCloseTscObj(void *pObj);
// todo move to taos? or create a new file: taos_internal.h
TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
@@ -468,17 +464,16 @@ int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* s
int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo);
-static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) {
+static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex, int32_t offset) {
SInternalField* pInfo = (SInternalField*) TARRAY_GET_ELEM(pFieldInfo->internalField, columnIndex);
- assert(pInfo->pSqlExpr != NULL);
- int32_t type = pInfo->pSqlExpr->resType;
- int32_t bytes = pInfo->pSqlExpr->resBytes;
+ int32_t type = pInfo->field.type;
+ int32_t bytes = pInfo->field.bytes;
- char* pData = pRes->data + (int32_t)(pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row);
+ char* pData = pRes->data + (int32_t)(offset * pRes->numOfRows + bytes * pRes->row);
// user defined constant value output columns
- if (TSDB_COL_IS_UD_COL(pInfo->pSqlExpr->colInfo.flag)) {
+ if (pInfo->pSqlExpr != NULL && TSDB_COL_IS_UD_COL(pInfo->pSqlExpr->colInfo.flag)) {
if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) {
pData = pInfo->pSqlExpr->param[1].pz;
pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen;
@@ -516,13 +511,14 @@ extern void * tscQhandle;
extern int tscKeepConn[];
extern int tsInsertHeadSize;
extern int tscNumOfThreads;
+extern int tscRefId;
extern SRpcCorEpSet tscMgmtEpSet;
extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
-int32_t tscCompareTidTags(const void* p1, const void* p2);
void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables);
+int16_t getNewResColId(SQueryInfo* pQueryInfo);
#ifdef __cplusplus
}
diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index c996bb2a76ff505fca4e09cd8763b324e5c4cb8d..e9e8214c4c15588b126b0768745db084e6e7dc3b 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -176,7 +176,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
}
if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
- tscFetchDatablockFromSubquery(pSql);
+ tscFetchDatablockForSubquery(pSql);
} else {
tscProcessSql(pSql);
}
@@ -226,7 +226,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) {
// handle the sub queries of join query
if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
- tscFetchDatablockFromSubquery(pSql);
+ tscFetchDatablockForSubquery(pSql);
} else if (pRes->completed) {
if(pCmd->command == TSDB_SQL_FETCH || (pCmd->command >= TSDB_SQL_SERV_STATUS && pCmd->command <= TSDB_SQL_CURRENT_USER)) {
if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes.
@@ -351,7 +351,7 @@ void tscProcessFetchRow(SSchedMsg *pMsg) {
SInternalField* pSup = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i);
if (pSup->pSqlExpr != NULL) {
- tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i);
+ tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i, 0);
} else {
// todo add
}
@@ -405,7 +405,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
SSqlRes *pRes = &pSql->res;
pRes->code = code;
- const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta";
+ SSqlObj *sub = (SSqlObj*) res;
+ const char* msg = (sub->cmd.command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta";
if (code != TSDB_CODE_SUCCESS) {
tscError("%p get %s failed, code:%s", pSql, msg, tstrerror(code));
goto _error;
@@ -427,8 +428,12 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
} else {
assert(code == TSDB_CODE_SUCCESS);
}
-
- assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pSql->param != NULL);
+
+ // param already freed by other routine and pSql in tscCache when ctrl + c
+ if (atomic_load_ptr(&pSql->param) == NULL) {
+ return;
+ }
+ assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0));
SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param;
SSqlObj * pParObj = trs->pParentSql;
@@ -437,6 +442,20 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex &&
pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL);
+ // tscProcessSql can add error into async res
+ tscProcessSql(pSql);
+ return;
+ } else if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) {
+ tscDebug("%p update table meta in local cache, continue to process sql and send corresponding tid_tag query", pSql);
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+ code = tscGetTableMeta(pSql, pTableMetaInfo);
+ if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+ return;
+ } else {
+ assert(code == TSDB_CODE_SUCCESS);
+ }
+
+ assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0));
// tscProcessSql can add error into async res
tscProcessSql(pSql);
return;
@@ -461,7 +480,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
tscResetSqlCmdObj(pCmd, false);
code = tsParseSql(pSql, true);
-
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return;
} else if (code != TSDB_CODE_SUCCESS) {
diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index 12d3b7dfd38e09b30aed6b8e66e56e7eead61034..35dc94f37bccf46ba95cee119faef3921a63705e 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -28,6 +28,7 @@
#include "tscompression.h"
#include "tsqlfunction.h"
#include "tutil.h"
+#include "ttype.h"
#define GET_INPUT_CHAR(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes))
#define GET_INPUT_CHAR_INDEX(x, y) (GET_INPUT_CHAR(x) + (y) * (x)->inputBytes)
@@ -99,7 +100,7 @@ typedef struct SSumInfo {
// the attribute of hasResult is not needed since the num attribute would server as this purpose
typedef struct SAvgInfo {
double sum;
- int64_t num; // num servers as the hasResult attribute in other struct
+ int64_t num;
} SAvgInfo;
typedef struct SStddevInfo {
@@ -167,7 +168,13 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) {
*type = (int16_t)dataType;
*bytes = (int16_t)dataBytes;
- *interBytes = *bytes + sizeof(SResultInfo);
+
+ if (functionId == TSDB_FUNC_INTERP) {
+ *interBytes = sizeof(SInterpInfoDetail);
+ } else {
+ *interBytes = 0;
+ }
+
return TSDB_CODE_SUCCESS;
}
@@ -175,21 +182,21 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
if (functionId == TSDB_FUNC_TID_TAG) { // todo use struct
*type = TSDB_DATA_TYPE_BINARY;
*bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE);
- *interBytes = *bytes;
+ *interBytes = 0;
return TSDB_CODE_SUCCESS;
}
if (functionId == TSDB_FUNC_COUNT) {
*type = TSDB_DATA_TYPE_BIGINT;
*bytes = sizeof(int64_t);
- *interBytes = *bytes;
+ *interBytes = 0;
return TSDB_CODE_SUCCESS;
}
if (functionId == TSDB_FUNC_ARITHM) {
*type = TSDB_DATA_TYPE_DOUBLE;
*bytes = sizeof(double);
- *interBytes = *bytes;
+ *interBytes = 0;
return TSDB_CODE_SUCCESS;
}
@@ -298,7 +305,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
} else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) {
*type = (int16_t)dataType;
*bytes = (int16_t)dataBytes;
- *interBytes = dataBytes + sizeof(SResultInfo);
+ *interBytes = dataBytes;
} else if (functionId == TSDB_FUNC_SPREAD) {
*type = (int16_t)TSDB_DATA_TYPE_DOUBLE;
*bytes = sizeof(double);
@@ -310,7 +317,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
} else if (functionId == TSDB_FUNC_LEASTSQR) {
*type = TSDB_DATA_TYPE_BINARY;
*bytes = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE; // string
- *interBytes = *bytes + sizeof(SResultInfo);
+ *interBytes = *bytes;
} else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) {
*type = TSDB_DATA_TYPE_BINARY;
*bytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo));
@@ -334,28 +341,20 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
return TSDB_CODE_SUCCESS;
}
-void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf) {
- assert(pResInfo->interResultBuf == NULL);
-
- pResInfo->bufLen = size;
- pResInfo->superTableQ = superTable;
- pResInfo->interResultBuf = buf;
-}
-
// set the query flag to denote that query is completed
static void no_next_step(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->complete = true;
}
static bool function_setup(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pResInfo->initialized) {
return false;
}
memset(pCtx->aOutputBuf, 0, (size_t)pCtx->outputBytes);
- initResultInfo(pResInfo);
+ initResultInfo(pResInfo, pCtx->interBufBytes);
return true;
}
@@ -367,7 +366,7 @@ static bool function_setup(SQLFunctionCtx *pCtx) {
* @param pCtx
*/
static void function_finalizer(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pResInfo->hasResult != DATA_SET_FLAG) {
if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
@@ -431,7 +430,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) {
*((int64_t *)pCtx->aOutputBuf) += 1;
// do not need it actually
- SResultInfo *pInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
pInfo->hasResult = DATA_SET_FLAG;
}
@@ -592,8 +591,8 @@ static void sum_function(SQLFunctionCtx *pCtx) {
do_sum(pCtx);
// keep the result data in output buffer, not in the intermediate buffer
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) {
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) {
// set the flag for super table query
SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf;
pSum->hasResult = DATA_SET_FLAG;
@@ -604,8 +603,8 @@ static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) {
do_sum_f(pCtx, index);
// keep the result data in output buffer, not in the intermediate buffer
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) {
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) {
SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf;
pSum->hasResult = DATA_SET_FLAG;
}
@@ -615,8 +614,7 @@ static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
GET_TRUE_DATA_TYPE();
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ assert(pCtx->stableQuery);
for (int32_t i = 0; i < pCtx->size; ++i) {
char * input = GET_INPUT_CHAR_INDEX(pCtx, i);
@@ -661,7 +659,7 @@ static void sum_func_second_merge(SQLFunctionCtx *pCtx) {
int32_t notNullElems = sum_merge_impl(pCtx);
SET_VAL(pCtx, notNullElems, 1);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (notNullElems > 0) {
pResInfo->hasResult = DATA_SET_FLAG;
@@ -755,9 +753,9 @@ static void avg_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
// NOTE: keep the intermediate result into the interResultBuf
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
- SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf;
+ SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo);
double * pVal = &pAvgInfo->sum;
if (pCtx->preAggVals.isSet) {
@@ -800,8 +798,8 @@ static void avg_function(SQLFunctionCtx *pCtx) {
}
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo));
}
}
@@ -814,9 +812,9 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) {
SET_VAL(pCtx, 1, 1);
// NOTE: keep the intermediate result into the interResultBuf
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
- SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf;
+ SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo);
if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) {
pAvgInfo->sum += GET_INT8_VAL(pData);
@@ -839,16 +837,16 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) {
pResInfo->hasResult = DATA_SET_FLAG;
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo));
}
}
static void avg_func_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ assert(pCtx->stableQuery);
- SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf;
+ SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo);
char * input = GET_INPUT_CHAR(pCtx);
for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) {
@@ -864,12 +862,12 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) {
// if the data set hasResult is not set, the result is null
if (pAvgInfo->num > 0) {
pResInfo->hasResult = DATA_SET_FLAG;
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo));
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo));
}
}
static void avg_func_second_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
double *sum = (double*) pCtx->aOutputBuf;
char * input = GET_INPUT_CHAR(pCtx);
@@ -883,7 +881,7 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) {
*sum += pInput->sum;
// keep the number of data into the temp buffer
- *(int64_t *)pResInfo->interResultBuf += pInput->num;
+ *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo) += pInput->num;
}
}
@@ -891,21 +889,21 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) {
* the average value is calculated in finalize routine, since current routine does not know the exact number of points
*/
static void avg_finalizer(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pCtx->currentStage == SECONDARY_STAGE_MERGE) {
assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY);
- if (GET_INT64_VAL(pResInfo->interResultBuf) <= 0) {
+ if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) {
setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
return; // empty table
}
- *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)pResInfo->interResultBuf;
+ *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo);
} else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY
assert(pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE);
- SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf;
+ SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo);
if (pAvgInfo->num == 0) { // all data are NULL or empty table
setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
@@ -1116,11 +1114,11 @@ static void min_function(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, notNullElems, 1);
if (notNullElems > 0) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
// set the flag for super table query
- if (pResInfo->superTableQ) {
+ if (pCtx->stableQuery) {
*(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG;
}
}
@@ -1133,11 +1131,11 @@ static void max_function(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, notNullElems, 1);
if (notNullElems > 0) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
// set the flag for super table query
- if (pResInfo->superTableQ) {
+ if (pCtx->stableQuery) {
*(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG;
}
}
@@ -1148,8 +1146,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp
GET_TRUE_DATA_TYPE();
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ assert(pCtx->stableQuery);
for (int32_t i = 0; i < pCtx->size; ++i) {
char *input = GET_INPUT_CHAR_INDEX(pCtx, i);
@@ -1210,7 +1207,7 @@ static void min_func_merge(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, notNullElems, 1);
- if (notNullElems > 0) { // for super table query, SResultInfo is not used
+ if (notNullElems > 0) { // for super table query, SResultRowCellInfo is not used
char *flag = pCtx->aOutputBuf + pCtx->inputBytes;
*flag = DATA_SET_FLAG;
}
@@ -1221,7 +1218,7 @@ static void min_func_second_merge(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, notNullElems, 1);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (notNullElems > 0) {
pResInfo->hasResult = DATA_SET_FLAG;
}
@@ -1242,7 +1239,7 @@ static void max_func_second_merge(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, numOfElem, 1);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (numOfElem > 0) {
pResInfo->hasResult = DATA_SET_FLAG;
}
@@ -1297,8 +1294,8 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) {
SET_VAL(pCtx, 1, 1);
minMax_function_f(pCtx, index, 0);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) {
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) {
char *flag = pCtx->aOutputBuf + pCtx->inputBytes;
*flag = DATA_SET_FLAG;
}
@@ -1313,8 +1310,8 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) {
SET_VAL(pCtx, 1, 1);
minMax_function_f(pCtx, index, 1);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) {
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) {
char *flag = pCtx->aOutputBuf + pCtx->inputBytes;
*flag = DATA_SET_FLAG;
}
@@ -1330,7 +1327,7 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) {
static void stddev_function(SQLFunctionCtx *pCtx) {
// the second stage to calculate standard deviation
- SStddevInfo *pStd = GET_RES_INFO(pCtx)->interResultBuf;
+ SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
if (pStd->stage == 0) { // the first stage is to calculate average value
avg_function(pCtx);
@@ -1381,8 +1378,8 @@ static void stddev_function(SQLFunctionCtx *pCtx) {
static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) {
// the second stage to calculate standard deviation
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SStddevInfo *pStd = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo);
/* the first stage is to calculate average value */
if (pStd->stage == 0) {
@@ -1433,8 +1430,8 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) {
* the stddevInfo and the average info struct share the same buffer area
* And the position of each element in their struct is exactly the same matched
*/
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SStddevInfo *pStd = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo);
if (pStd->stage == 0) {
/*
@@ -1449,7 +1446,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) {
pResInfo->initialized = true; // set it initialized to avoid re-initialization
// save average value into tmpBuf, for second stage scan
- SAvgInfo *pAvg = pResInfo->interResultBuf;
+ SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo);
pStd->avg = GET_DOUBLE_VAL(pCtx->aOutputBuf);
assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum));
@@ -1459,7 +1456,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) {
}
static void stddev_finalizer(SQLFunctionCtx *pCtx) {
- SStddevInfo *pStd = (SStddevInfo *)GET_RES_INFO(pCtx)->interResultBuf;
+ SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
if (pStd->num <= 0) {
setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
@@ -1505,7 +1502,7 @@ static void first_function(SQLFunctionCtx *pCtx) {
TSKEY k = pCtx->ptsList[i];
DO_UPDATE_TAG_COLUMNS(pCtx, k);
- SResultInfo *pInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
pInfo->hasResult = DATA_SET_FLAG;
pInfo->complete = true;
@@ -1532,7 +1529,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) {
TSKEY ts = pCtx->ptsList[index];
DO_UPDATE_TAG_COLUMNS(pCtx, ts);
- SResultInfo *pInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
pInfo->hasResult = DATA_SET_FLAG;
pInfo->complete = true; // get the first not-null data, completed
}
@@ -1576,7 +1573,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) {
first_data_assign_impl(pCtx, data, i);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
notNullElems++;
@@ -1604,8 +1601,7 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) {
static void first_dist_func_merge(SQLFunctionCtx *pCtx) {
char *pData = GET_INPUT_CHAR(pCtx);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pCtx->size == 1 && pResInfo->superTableQ);
+ assert(pCtx->size == 1 && pCtx->stableQuery);
SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes);
if (pInput->hasResult != DATA_SET_FLAG) {
@@ -1620,8 +1616,8 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) {
}
static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) {
- assert(pCtx->resultInfo->superTableQ);
-
+ assert(pCtx->stableQuery);
+
char * pData = GET_INPUT_CHAR(pCtx);
SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes);
if (pInput->hasResult != DATA_SET_FLAG) {
@@ -1668,7 +1664,7 @@ static void last_function(SQLFunctionCtx *pCtx) {
TSKEY ts = pCtx->ptsList[i];
DO_UPDATE_TAG_COLUMNS(pCtx, ts);
- SResultInfo *pInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
pInfo->hasResult = DATA_SET_FLAG;
pInfo->complete = true; // set query completed on this column
@@ -1691,7 +1687,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) {
TSKEY ts = pCtx->ptsList[index];
DO_UPDATE_TAG_COLUMNS(pCtx, ts);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
pResInfo->complete = true; // set query completed
}
@@ -1740,7 +1736,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) {
last_data_assign_impl(pCtx, data, i);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
notNullElems++;
@@ -1776,8 +1772,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) {
static void last_dist_func_merge(SQLFunctionCtx *pCtx) {
char *pData = GET_INPUT_CHAR(pCtx);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pCtx->size == 1 && pResInfo->superTableQ);
+ assert(pCtx->size == 1 && pCtx->stableQuery);
// the input data is null
SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes);
@@ -1833,11 +1828,11 @@ static void last_row_function(SQLFunctionCtx *pCtx) {
// assign the last element in current data block
assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
// set the result to final result buffer in case of super table query
- if (pResInfo->superTableQ) {
+ if (pCtx->stableQuery) {
SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes);
pInfo1->ts = pCtx->ptsList[pCtx->size - 1];
pInfo1->hasResult = DATA_SET_FLAG;
@@ -1852,7 +1847,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) {
static void last_row_finalizer(SQLFunctionCtx *pCtx) {
// do nothing at the first stage
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pResInfo->hasResult != DATA_SET_FLAG) {
if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
@@ -2044,8 +2039,8 @@ static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) {
static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); }
static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- STopBotInfo *pRes = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo);
tValuePair **tvp = pRes->res;
@@ -2123,7 +2118,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) {
}
}
- taosTFree(pData);
+ tfree(pData);
}
/*
@@ -2135,18 +2130,18 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) {
* top/bottom use the intermediate result buffer to keep the intermediate result
*/
static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
// only the first_stage_merge is directly written data into final output buffer
- if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) {
+ if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) {
return (STopBotInfo*) pCtx->aOutputBuf;
} else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer
- return pResInfo->interResultBuf;
+ return GET_ROWCELL_INTERBUF(pResInfo);
}
}
bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pResInfo == NULL) {
return true;
}
@@ -2252,7 +2247,7 @@ static void top_function(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, notNullElems, 1);
if (notNullElems > 0) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
}
}
@@ -2270,7 +2265,7 @@ static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) {
do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL,
0);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
}
@@ -2285,8 +2280,7 @@ static void top_func_merge(SQLFunctionCtx *pCtx) {
// remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary
buildTopBotStruct(pInput, pCtx);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1);
+ assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1);
STopBotInfo *pOutput = getTopBotOutputInfo(pCtx);
@@ -2314,7 +2308,7 @@ static void top_func_second_merge(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, pInput->num, pOutput->num);
if (pOutput->num > 0) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
}
}
@@ -2343,7 +2337,7 @@ static void bottom_function(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, notNullElems, 1);
if (notNullElems > 0) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
}
}
@@ -2359,7 +2353,7 @@ static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) {
do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo,
NULL, 0);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
}
@@ -2374,8 +2368,7 @@ static void bottom_func_merge(SQLFunctionCtx *pCtx) {
// remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary
buildTopBotStruct(pInput, pCtx);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1);
+ assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1);
STopBotInfo *pOutput = getTopBotOutputInfo(pCtx);
@@ -2403,16 +2396,16 @@ static void bottom_func_second_merge(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, pInput->num, pOutput->num);
if (pOutput->num > 0) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
}
}
static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
// data in temporary list is less than the required number of results, not enough qualified number of results
- STopBotInfo *pRes = pResInfo->interResultBuf;
+ STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo);
if (pRes->num == 0) { // no result
assert(pResInfo->hasResult != DATA_SET_FLAG);
// TODO:
@@ -2443,8 +2436,8 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
}
// in the first round, get the min-max value of all involved data
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SPercentileInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX);
SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX);
pInfo->numOfElems = 0;
@@ -2455,8 +2448,8 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
static void percentile_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SPercentileInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
// the first stage, only acquire the min/max value
if (pInfo->stage == 0) {
@@ -2487,28 +2480,8 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
continue;
}
- // TODO extract functions
double v = 0;
- switch (pCtx->inputType) {
- case TSDB_DATA_TYPE_TINYINT:
- v = GET_INT8_VAL(data);
- break;
- case TSDB_DATA_TYPE_SMALLINT:
- v = GET_INT16_VAL(data);
- break;
- case TSDB_DATA_TYPE_BIGINT:
- v = (double)(GET_INT64_VAL(data));
- break;
- case TSDB_DATA_TYPE_FLOAT:
- v = GET_FLOAT_VAL(data);
- break;
- case TSDB_DATA_TYPE_DOUBLE:
- v = GET_DOUBLE_VAL(data);
- break;
- default:
- v = GET_INT32_VAL(data);
- break;
- }
+ GET_TYPED_DATA(v, double, pCtx->inputType, data);
if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
SET_DOUBLE_VAL(&pInfo->minval, v);
@@ -2546,33 +2519,13 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
return;
}
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
-
- SPercentileInfo *pInfo = (SPercentileInfo *)pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SPercentileInfo *pInfo = (SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo);
if (pInfo->stage == 0) {
- // TODO extract functions
+
double v = 0;
- switch (pCtx->inputType) {
- case TSDB_DATA_TYPE_TINYINT:
- v = GET_INT8_VAL(pData);
- break;
- case TSDB_DATA_TYPE_SMALLINT:
- v = GET_INT16_VAL(pData);
- break;
- case TSDB_DATA_TYPE_BIGINT:
- v = (double)(GET_INT64_VAL(pData));
- break;
- case TSDB_DATA_TYPE_FLOAT:
- v = GET_FLOAT_VAL(pData);
- break;
- case TSDB_DATA_TYPE_DOUBLE:
- v = GET_DOUBLE_VAL(pData);
- break;
- default:
- v = GET_INT32_VAL(pData);
- break;
- }
+ GET_TYPED_DATA(v, double, pCtx->inputType, pData);
if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
SET_DOUBLE_VAL(&pInfo->minval, v);
@@ -2595,8 +2548,8 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
static void percentile_finalizer(SQLFunctionCtx *pCtx) {
double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey;
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- tMemBucket * pMemBucket = ((SPercentileInfo *)pResInfo->interResultBuf)->pMemBucket;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ tMemBucket * pMemBucket = ((SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo))->pMemBucket;
if (pMemBucket->total > 0) { // check for null
*(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v);
@@ -2609,8 +2562,8 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) {
}
static void percentile_next_step(SQLFunctionCtx *pCtx) {
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SPercentileInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
if (pInfo->stage == 0) {
// all data are null, set it completed
@@ -2627,12 +2580,12 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) {
//////////////////////////////////////////////////////////////////////////////////
static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
- if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) {
+ if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) {
return (SAPercentileInfo*) pCtx->aOutputBuf;
} else {
- return pResInfo->interResultBuf;
+ return GET_ROWCELL_INTERBUF(pResInfo);
}
}
@@ -2651,7 +2604,7 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx) {
static void apercentile_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
SAPercentileInfo *pInfo = getAPerctInfo(pCtx);
for (int32_t i = 0; i < pCtx->size; ++i) {
@@ -2661,29 +2614,9 @@ static void apercentile_function(SQLFunctionCtx *pCtx) {
}
notNullElems += 1;
+
double v = 0;
-
- switch (pCtx->inputType) {
- case TSDB_DATA_TYPE_TINYINT:
- v = GET_INT8_VAL(data);
- break;
- case TSDB_DATA_TYPE_SMALLINT:
- v = GET_INT16_VAL(data);
- break;
- case TSDB_DATA_TYPE_BIGINT:
- v = (double)(GET_INT64_VAL(data));
- break;
- case TSDB_DATA_TYPE_FLOAT:
- v = GET_FLOAT_VAL(data);
- break;
- case TSDB_DATA_TYPE_DOUBLE:
- v = GET_DOUBLE_VAL(data);
- break;
- default:
- v = GET_INT32_VAL(data);
- break;
- }
-
+ GET_TYPED_DATA(v, double, pCtx->inputType, data);
tHistogramAdd(&pInfo->pHisto, v);
}
@@ -2704,30 +2637,11 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
return;
}
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SAPercentileInfo *pInfo = getAPerctInfo(pCtx); // pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SAPercentileInfo *pInfo = getAPerctInfo(pCtx);
double v = 0;
- switch (pCtx->inputType) {
- case TSDB_DATA_TYPE_TINYINT:
- v = GET_INT8_VAL(pData);
- break;
- case TSDB_DATA_TYPE_SMALLINT:
- v = GET_INT16_VAL(pData);
- break;
- case TSDB_DATA_TYPE_BIGINT:
- v = (double)(GET_INT64_VAL(pData));
- break;
- case TSDB_DATA_TYPE_FLOAT:
- v = GET_FLOAT_VAL(pData);
- break;
- case TSDB_DATA_TYPE_DOUBLE:
- v = GET_DOUBLE_VAL(pData);
- break;
- default:
- v = GET_INT32_VAL(pData);
- break;
- }
+ GET_TYPED_DATA(v, double, pCtx->inputType, pData);
tHistogramAdd(&pInfo->pHisto, v);
@@ -2736,8 +2650,8 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
}
static void apercentile_func_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ assert(pCtx->stableQuery);
SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx);
@@ -2781,20 +2695,21 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) {
}
SAPercentileInfo *pOutput = getAPerctInfo(pCtx);
- SHistogramInfo * pHisto = pOutput->pHisto;
+ SHistogramInfo *pHisto = pOutput->pHisto;
if (pHisto->numOfElems <= 0) {
memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1));
pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo));
} else {
+ //TODO(dengyihao): avoid memcpy
pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo));
-
SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN);
- tHistogramDestroy(&pOutput->pHisto);
- pOutput->pHisto = pRes;
+ memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN);
+ pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo));
+ tHistogramDestroy(&pRes);
}
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
SET_VAL(pCtx, 1, 1);
}
@@ -2802,8 +2717,8 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) {
static void apercentile_finalizer(SQLFunctionCtx *pCtx) {
double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i64Key : pCtx->param[0].dKey;
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SAPercentileInfo *pOutput = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo);
if (pCtx->currentStage == SECONDARY_STAGE_MERGE) {
if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null
@@ -2840,8 +2755,8 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) {
return false;
}
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SLeastsquareInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
// 2*3 matrix
pInfo->startVal = pCtx->param[0].dKey;
@@ -2867,8 +2782,8 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) {
}
static void leastsquares_function(SQLFunctionCtx *pCtx) {
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SLeastsquareInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
double(*param)[3] = pInfo->mat;
double x = pInfo->startVal;
@@ -2938,8 +2853,8 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) {
return;
}
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SLeastsquareInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
double(*param)[3] = pInfo->mat;
@@ -2988,8 +2903,8 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) {
static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
// no data in query
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SLeastsquareInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
if (pInfo->num == 0) {
if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
@@ -3054,7 +2969,7 @@ static void col_project_function(SQLFunctionCtx *pCtx) {
}
static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0
return;
}
@@ -3486,7 +3401,7 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) {
return false;
}
- SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf;
+ SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
// this is the server-side setup function in client-side, the secondary merge do not need this procedure
if (pCtx->currentStage == SECONDARY_STAGE_MERGE) {
@@ -3501,8 +3416,8 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) {
}
static void spread_function(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SSpreadInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
int32_t numOfElems = 0;
@@ -3568,8 +3483,8 @@ static void spread_function(SQLFunctionCtx *pCtx) {
}
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo));
}
}
@@ -3581,8 +3496,8 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) {
SET_VAL(pCtx, 1, 1);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SSpreadInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
double val = 0.0;
if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) {
@@ -3611,16 +3526,16 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) {
pResInfo->hasResult = DATA_SET_FLAG;
pInfo->hasResult = DATA_SET_FLAG;
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo));
}
}
void spread_func_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ assert(pCtx->stableQuery);
- SSpreadInfo *pResData = pResInfo->interResultBuf;
+ SSpreadInfo *pResData = GET_ROWCELL_INTERBUF(pResInfo);
int32_t notNullElems = 0;
for (int32_t i = 0; i < pCtx->size; ++i) {
@@ -3644,7 +3559,7 @@ void spread_func_merge(SQLFunctionCtx *pCtx) {
}
if (notNullElems > 0) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo));
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo));
pResInfo->hasResult = DATA_SET_FLAG;
}
}
@@ -3675,7 +3590,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) {
* here we do not check the input data types, because in case of metric query,
* the type of intermediate data is binary
*/
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pCtx->currentStage == SECONDARY_STAGE_MERGE) {
assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY);
@@ -3690,7 +3605,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) {
assert((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE) ||
(pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP));
- SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf;
+ SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
if (pInfo->hasResult != DATA_SET_FLAG) {
setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
return;
@@ -3714,8 +3629,8 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) {
return false;
}
- SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes;
- STwaInfo * pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes;
+ STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->lastKey = INT64_MIN;
pInfo->type = pCtx->inputType;
@@ -3754,8 +3669,8 @@ static void twa_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- STwaInfo * pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo);
int32_t i = 0;
@@ -3808,7 +3723,7 @@ static void twa_function(SQLFunctionCtx *pCtx) {
pResInfo->hasResult = DATA_SET_FLAG;
}
- if (pResInfo->superTableQ) {
+ if (pCtx->stableQuery) {
memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo));
}
@@ -3825,8 +3740,8 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
TSKEY *primaryKey = pCtx->ptsList;
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- STwaInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
if (pInfo->lastKey == INT64_MIN) {
pInfo->lastKey = pCtx->nStartQueryTimestamp;
@@ -3848,14 +3763,13 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
// pCtx->numOfIteratedElems += 1;
pResInfo->hasResult = DATA_SET_FLAG;
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(STwaInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo));
}
}
static void twa_func_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ assert(pCtx->stableQuery);
STwaInfo *pBuf = (STwaInfo *)pCtx->aOutputBuf;
char * indicator = pCtx->aInputElemBuf;
@@ -3895,16 +3809,16 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) {
*/
void twa_function_copy(SQLFunctionCtx *pCtx) {
assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
- memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes);
+ memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes);
pResInfo->hasResult = ((STwaInfo *)pCtx->aInputElemBuf)->hasResult;
}
void twa_function_finalizer(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
- STwaInfo *pInfo = (STwaInfo *)pResInfo->interResultBuf;
+ STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo);
assert(pInfo->EKey >= pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult);
if (pInfo->hasResult != DATA_SET_FLAG) {
@@ -3932,8 +3846,8 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) {
*/
static void interp_function(SQLFunctionCtx *pCtx) {
// at this point, the value is existed, return directly
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SInterpInfoDetail* pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SInterpInfoDetail* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
if (pCtx->size == 1) {
char *pData = GET_INPUT_CHAR(pCtx);
@@ -3987,7 +3901,7 @@ static void interp_function(SQLFunctionCtx *pCtx) {
if (isNull(data1, srcType) || isNull(data2, srcType)) {
setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes);
} else {
- taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point);
+ taosGetLinearInterpolationVal(pCtx->outputType, &point1, &point2, &point);
}
} else if (srcType == TSDB_DATA_TYPE_FLOAT) {
point1.val = data1;
@@ -3996,7 +3910,7 @@ static void interp_function(SQLFunctionCtx *pCtx) {
if (isNull(data1, srcType) || isNull(data2, srcType)) {
setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes);
} else {
- taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point);
+ taosGetLinearInterpolationVal(pCtx->outputType, &point1, &point2, &point);
}
} else {
@@ -4008,7 +3922,6 @@ static void interp_function(SQLFunctionCtx *pCtx) {
}
}
}
-
}
SET_VAL(pCtx, pCtx->size, 1);
@@ -4019,8 +3932,8 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) {
return false; // not initialized since it has been initialized
}
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- STSCompInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->pTSBuf = tsBufCreate(false, pCtx->order);
pInfo->pTSBuf->tsOrder = pCtx->order;
@@ -4028,8 +3941,8 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) {
}
static void ts_comp_function(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- STSBuf * pTSbuf = ((STSCompInfo *)(pResInfo->interResultBuf))->pTSBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf;
const char *input = GET_INPUT_CHAR(pCtx);
@@ -4053,8 +3966,8 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) {
return;
}
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- STSCompInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
STSBuf *pTSbuf = pInfo->pTSBuf;
@@ -4065,9 +3978,9 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) {
}
static void ts_comp_finalize(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
- STSCompInfo *pInfo = pResInfo->interResultBuf;
+ STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
STSBuf * pTSbuf = pInfo->pTSBuf;
tsBufFlush(pTSbuf);
@@ -4116,8 +4029,8 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) {
return false;
}
- SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes;
- SRateInfo * pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes;
+ SRateInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->CorrectionValue = 0;
pInfo->firstKey = INT64_MIN;
@@ -4136,8 +4049,8 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) {
static void rate_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
TSKEY *primaryKey = pCtx->ptsList;
tscDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull);
@@ -4152,22 +4065,7 @@ static void rate_function(SQLFunctionCtx *pCtx) {
notNullElems++;
int64_t v = 0;
- switch (pCtx->inputType) {
- case TSDB_DATA_TYPE_TINYINT:
- v = (int64_t)GET_INT8_VAL(pData);
- break;
- case TSDB_DATA_TYPE_SMALLINT:
- v = (int64_t)GET_INT16_VAL(pData);
- break;
- case TSDB_DATA_TYPE_INT:
- v = (int64_t)GET_INT32_VAL(pData);
- break;
- case TSDB_DATA_TYPE_BIGINT:
- v = (int64_t)GET_INT64_VAL(pData);
- break;
- default:
- assert(0);
- }
+ GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData);
if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) {
pRateInfo->firstValue = v;
@@ -4200,8 +4098,8 @@ static void rate_function(SQLFunctionCtx *pCtx) {
}
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo));
}
}
@@ -4212,27 +4110,12 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) {
}
// NOTE: keep the intermediate result into the interResultBuf
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
TSKEY *primaryKey = pCtx->ptsList;
int64_t v = 0;
- switch (pCtx->inputType) {
- case TSDB_DATA_TYPE_TINYINT:
- v = (int64_t)GET_INT8_VAL(pData);
- break;
- case TSDB_DATA_TYPE_SMALLINT:
- v = (int64_t)GET_INT16_VAL(pData);
- break;
- case TSDB_DATA_TYPE_INT:
- v = (int64_t)GET_INT32_VAL(pData);
- break;
- case TSDB_DATA_TYPE_BIGINT:
- v = (int64_t)GET_INT64_VAL(pData);
- break;
- default:
- assert(0);
- }
+ GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData);
if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) {
pRateInfo->firstValue = v;
@@ -4257,20 +4140,18 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) {
pResInfo->hasResult = DATA_SET_FLAG;
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo));
}
}
static void rate_func_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ assert(pCtx->stableQuery);
tscDebug("rate_func_merge() size:%d", pCtx->size);
- //SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
SRateInfo *pBuf = (SRateInfo *)pCtx->aOutputBuf;
char *indicator = pCtx->aInputElemBuf;
@@ -4303,8 +4184,8 @@ static void rate_func_merge(SQLFunctionCtx *pCtx) {
static void rate_func_copy(SQLFunctionCtx *pCtx) {
assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes);
pResInfo->hasResult = ((SRateInfo*)pCtx->aInputElemBuf)->hasResult;
SRateInfo* pRateInfo = (SRateInfo*)pCtx->aInputElemBuf;
@@ -4315,8 +4196,8 @@ static void rate_func_copy(SQLFunctionCtx *pCtx) {
static void rate_finalizer(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
tscDebug("%p isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d",
pCtx, pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult);
@@ -4341,8 +4222,8 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) {
static void irate_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
TSKEY *primaryKey = pCtx->ptsList;
tscDebug("%p irate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull);
@@ -4361,22 +4242,7 @@ static void irate_function(SQLFunctionCtx *pCtx) {
notNullElems++;
int64_t v = 0;
- switch (pCtx->inputType) {
- case TSDB_DATA_TYPE_TINYINT:
- v = (int64_t)GET_INT8_VAL(pData);
- break;
- case TSDB_DATA_TYPE_SMALLINT:
- v = (int64_t)GET_INT16_VAL(pData);
- break;
- case TSDB_DATA_TYPE_INT:
- v = (int64_t)GET_INT32_VAL(pData);
- break;
- case TSDB_DATA_TYPE_BIGINT:
- v = (int64_t)GET_INT64_VAL(pData);
- break;
- default:
- assert(0);
- }
+ GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData);
// TODO: calc once if only call this function once ????
if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->lastValue)) {
@@ -4404,8 +4270,8 @@ static void irate_function(SQLFunctionCtx *pCtx) {
}
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo));
}
}
@@ -4416,28 +4282,13 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) {
}
// NOTE: keep the intermediate result into the interResultBuf
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
TSKEY *primaryKey = pCtx->ptsList;
int64_t v = 0;
- switch (pCtx->inputType) {
- case TSDB_DATA_TYPE_TINYINT:
- v = (int64_t)GET_INT8_VAL(pData);
- break;
- case TSDB_DATA_TYPE_SMALLINT:
- v = (int64_t)GET_INT16_VAL(pData);
- break;
- case TSDB_DATA_TYPE_INT:
- v = (int64_t)GET_INT32_VAL(pData);
- break;
- case TSDB_DATA_TYPE_BIGINT:
- v = (int64_t)GET_INT64_VAL(pData);
- break;
- default:
- assert(0);
- }
-
+ GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData);
+
pRateInfo->firstKey = pRateInfo->lastKey;
pRateInfo->firstValue = pRateInfo->lastValue;
@@ -4453,16 +4304,16 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) {
pResInfo->hasResult = DATA_SET_FLAG;
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo));
}
}
static void do_sumrate_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ assert(pCtx->stableQuery);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
char * input = GET_INPUT_CHAR(pCtx);
for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) {
@@ -4486,7 +4337,7 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) {
if (DATA_SET_FLAG == pRateInfo->hasResult) {
pResInfo->hasResult = DATA_SET_FLAG;
SET_VAL(pCtx, pRateInfo->num, 1);
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo));
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo));
}
}
@@ -4501,10 +4352,10 @@ static void sumrate_func_second_merge(SQLFunctionCtx *pCtx) {
}
static void sumrate_finalizer(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
- tscDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pResInfo->superTableQ, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult);
+ tscDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pCtx->stableQuery, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult);
if (pRateInfo->hasResult != DATA_SET_FLAG) {
setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c
index b4c3f3549b1576b3ff83300d4f10d9e07778530d..310c4c665761212f6fc46b889a8a2c887b7213eb 100644
--- a/src/client/src/tscLocal.c
+++ b/src/client/src/tscLocal.c
@@ -49,82 +49,6 @@ typedef struct SCreateBuilder {
} SCreateBuilder;
static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength);
-static int32_t getToStringLength(const char *pData, int32_t length, int32_t type) {
- char buf[512] = {0};
-
- int32_t len = 0;
- int32_t MAX_BOOL_TYPE_LENGTH = 5; // max(strlen("true"), strlen("false"));
- switch (type) {
- case TSDB_DATA_TYPE_BINARY:
- return length;
- case TSDB_DATA_TYPE_NCHAR:
- return length;
- case TSDB_DATA_TYPE_DOUBLE: {
- double dv = 0;
- dv = GET_DOUBLE_VAL(pData);
- len = sprintf(buf, "%lf", dv);
- if (strncasecmp("nan", buf, 3) == 0) {
- len = 4;
- }
- } break;
- case TSDB_DATA_TYPE_FLOAT: {
- float fv = 0;
- fv = GET_FLOAT_VAL(pData);
- len = sprintf(buf, "%f", fv);
- if (strncasecmp("nan", buf, 3) == 0) {
- len = 4;
- }
- } break;
- case TSDB_DATA_TYPE_TIMESTAMP:
- case TSDB_DATA_TYPE_BIGINT:
- len = sprintf(buf, "%" PRId64, *(int64_t *)pData);
- break;
- case TSDB_DATA_TYPE_BOOL:
- len = MAX_BOOL_TYPE_LENGTH;
- break;
- default:
- len = sprintf(buf, "%d", *(int32_t *)pData);
- break;
- };
- return len;
-}
-
-/*
- * we need to convert all data into string, so we need to sprintf all kinds of
- * non-string data into string, and record its length to get the right
- * maximum length. The length may be less or greater than its original binary length:
- * For example:
- * length((short) 1) == 1, less than sizeof(short)
- * length((uint64_t) 123456789011) > 12, greater than sizsof(uint64_t)
- */
-static int32_t tscMaxLengthOfTagsFields(SSqlObj *pSql) {
- STableMeta *pMeta = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->pTableMeta;
-
- if (pMeta->tableType == TSDB_SUPER_TABLE || pMeta->tableType == TSDB_NORMAL_TABLE ||
- pMeta->tableType == TSDB_STREAM_TABLE) {
- return 0;
- }
-
- char * pTagValue = tsGetTagsValue(pMeta);
- SSchema *pTagsSchema = tscGetTableTagSchema(pMeta);
-
- int32_t len = getToStringLength(pTagValue, pTagsSchema[0].bytes, pTagsSchema[0].type);
-
- pTagValue += pTagsSchema[0].bytes;
- int32_t numOfTags = tscGetNumOfTags(pMeta);
-
- for (int32_t i = 1; i < numOfTags; ++i) {
- int32_t tLen = getToStringLength(pTagValue, pTagsSchema[i].bytes, pTagsSchema[i].type);
- if (len < tLen) {
- len = tLen;
- }
-
- pTagValue += pTagsSchema[i].bytes;
- }
-
- return len;
-}
-
static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
SSqlRes *pRes = &pSql->res;
@@ -186,8 +110,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
return 0;
}
- // the following is handle display tags value for meters created according to metric
- char *pTagValue = tsGetTagsValue(pMeta);
+ // the following is handle display tags for table created according to super table
for (int32_t i = numOfRows; i < totalNumOfRows; ++i) {
// field name
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
@@ -219,8 +142,6 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i;
const char *src = "TAG";
STR_WITH_MAXSIZE_TO_VARSTR(target, src, pField->bytes);
-
- pTagValue += pSchema[i].bytes;
}
return 0;
@@ -241,7 +162,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY,
- (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, (TSDB_COL_NAME_LEN - 1), false);
+ (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, -1000, (TSDB_COL_NAME_LEN - 1), false);
rowLen += ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE);
@@ -251,7 +172,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(typeColLength + VARSTR_HEADER_SIZE),
- typeColLength, false);
+ -1000, typeColLength, false);
rowLen += typeColLength + VARSTR_HEADER_SIZE;
@@ -261,7 +182,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t),
- sizeof(int32_t), false);
+ -1000, sizeof(int32_t), false);
rowLen += sizeof(int32_t);
@@ -271,7 +192,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (int16_t)(noteColLength + VARSTR_HEADER_SIZE),
- noteColLength, false);
+ -1000, noteColLength, false);
rowLen += noteColLength + VARSTR_HEADER_SIZE;
return rowLen;
@@ -286,10 +207,10 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) {
const int32_t TYPE_COLUMN_LENGTH = 16;
const int32_t NOTE_COLUMN_MIN_LENGTH = 8;
- int32_t noteFieldLen = tscMaxLengthOfTagsFields(pSql);
- if (noteFieldLen == 0) {
- noteFieldLen = NOTE_COLUMN_MIN_LENGTH;
- }
+ int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH;//tscMaxLengthOfTagsFields(pSql);
+// if (noteFieldLen == 0) {
+// noteFieldLen = NOTE_COLUMN_MIN_LENGTH;
+// }
int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen);
tscFieldInfoUpdateOffset(pQueryInfo);
@@ -486,8 +407,7 @@ static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const
}
SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
- pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY,
- f.bytes, f.bytes - VARSTR_HEADER_SIZE, false);
+ pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, f.bytes, -1000, f.bytes - VARSTR_HEADER_SIZE, false);
rowLen += f.bytes;
@@ -501,7 +421,7 @@ static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const
pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY,
- (int16_t)(ddlLen + VARSTR_HEADER_SIZE), ddlLen, false);
+ (int16_t)(ddlLen + VARSTR_HEADER_SIZE), -1000, ddlLen, false);
rowLen += ddlLen + VARSTR_HEADER_SIZE;
@@ -698,7 +618,11 @@ static int32_t tscRebuildDDLForNormalTable(SSqlObj *pSql, const char *tableName,
for (int32_t i = 0; i < numOfRows; ++i) {
uint8_t type = pSchema[i].type;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
- snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName,pSchema->bytes);
+ int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE;
+ if (type == TSDB_DATA_TYPE_NCHAR) {
+ bytes = bytes/TSDB_NCHAR_SIZE;
+ }
+ snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name, tDataTypeDesc[pSchema[i].type].aName, bytes);
} else {
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypeDesc[pSchema[i].type].aName);
}
@@ -721,7 +645,11 @@ static int32_t tscRebuildDDLForSuperTable(SSqlObj *pSql, const char *tableName,
for (int32_t i = 0; i < numOfRows; ++i) {
uint8_t type = pSchema[i].type;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
- snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result),"%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName,pSchema->bytes);
+ int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE;
+ if (type == TSDB_DATA_TYPE_NCHAR) {
+ bytes = bytes/TSDB_NCHAR_SIZE;
+ }
+ snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result),"%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName, bytes);
} else {
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypeDesc[type].aName);
}
@@ -731,7 +659,11 @@ static int32_t tscRebuildDDLForSuperTable(SSqlObj *pSql, const char *tableName,
for (int32_t i = numOfRows; i < totalRows; i++) {
uint8_t type = pSchema[i].type;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
- snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName,pSchema->bytes);
+ int32_t bytes = pSchema[i].bytes - VARSTR_HEADER_SIZE;
+ if (type == TSDB_DATA_TYPE_NCHAR) {
+ bytes = bytes/TSDB_NCHAR_SIZE;
+ }
+ snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName, bytes);
} else {
snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypeDesc[type].aName);
}
diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c
index 18d72e2d1e23b63abde7ff7159eae2dad993b548..b07c7ca66dde0a9b568436f997572829b81cb9fa 100644
--- a/src/client/src/tscLocalMerge.c
+++ b/src/client/src/tscLocalMerge.c
@@ -13,14 +13,15 @@
* along with this program. If not, see .
*/
+#include "tscLocalMerge.h"
+#include "tscSubquery.h"
#include "os.h"
+#include "qAst.h"
#include "tlosertree.h"
+#include "tscLog.h"
#include "tscUtil.h"
#include "tschemautil.h"
#include "tsclient.h"
-#include "tutil.h"
-#include "tscLog.h"
-#include "tscLocalMerge.h"
typedef struct SCompareParam {
SLocalDataSource **pLocalData;
@@ -29,6 +30,8 @@ typedef struct SCompareParam {
int32_t groupOrderType;
} SCompareParam;
+static void doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize);
+
int32_t treeComparator(const void *pLeft, const void *pRight, void *param) {
int32_t pLeftIdx = *(int32_t *)pLeft;
int32_t pRightIdx = *(int32_t *)pRight;
@@ -97,14 +100,14 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
pCtx->param[2].i64Key = pQueryInfo->order.order;
pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
pCtx->param[1].i64Key = pQueryInfo->order.orderColId;
+ } else if (functionId == TSDB_FUNC_APERCT) {
+ pCtx->param[0].i64Key = pExpr->param[0].i64Key;
+ pCtx->param[0].nType = pExpr->param[0].nType;
}
- SResultInfo *pResInfo = &pReducer->pResInfo[i];
- pResInfo->bufLen = pExpr->interBytes;
- pResInfo->interResultBuf = calloc(1, (size_t) pResInfo->bufLen);
-
- pCtx->resultInfo = &pReducer->pResInfo[i];
- pCtx->resultInfo->superTableQ = true;
+ pCtx->interBufBytes = pExpr->interBytes;
+ pCtx->resultInfo = calloc(1, pCtx->interBufBytes + sizeof(SResultRowCellInfo));
+ pCtx->stableQuery = true;
}
int16_t n = 0;
@@ -132,28 +135,41 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
}
static SFillColInfo* createFillColInfo(SQueryInfo* pQueryInfo) {
- int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
+ int32_t numOfCols = (int32_t)tscNumOfFields(pQueryInfo);
int32_t offset = 0;
SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo));
for(int32_t i = 0; i < numOfCols; ++i) {
- SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
-
- pFillCol[i].col.bytes = pExpr->resBytes;
- pFillCol[i].col.type = (int8_t)pExpr->resType;
- pFillCol[i].col.colId = pExpr->colInfo.colId;
- pFillCol[i].flag = pExpr->colInfo.flag;
- pFillCol[i].col.offset = offset;
- pFillCol[i].functionId = pExpr->functionId;
- pFillCol[i].fillVal.i = pQueryInfo->fillVal[i];
- offset += pExpr->resBytes;
+ SInternalField* pIField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i);
+
+ if (pIField->pArithExprInfo == NULL) {
+ SSqlExpr* pExpr = pIField->pSqlExpr;
+
+ pFillCol[i].col.bytes = pExpr->resBytes;
+ pFillCol[i].col.type = (int8_t)pExpr->resType;
+ pFillCol[i].col.colId = pExpr->colInfo.colId;
+ pFillCol[i].flag = pExpr->colInfo.flag;
+ pFillCol[i].col.offset = offset;
+ pFillCol[i].functionId = pExpr->functionId;
+ pFillCol[i].fillVal.i = pQueryInfo->fillVal[i];
+ } else {
+ pFillCol[i].col.bytes = pIField->field.bytes;
+ pFillCol[i].col.type = (int8_t)pIField->field.type;
+ pFillCol[i].col.colId = -100;
+ pFillCol[i].flag = TSDB_COL_NORMAL;
+ pFillCol[i].col.offset = offset;
+ pFillCol[i].functionId = -1;
+ pFillCol[i].fillVal.i = pQueryInfo->fillVal[i];
+ }
+
+ offset += pFillCol[i].col.bytes;
}
return pFillCol;
}
void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
- SColumnModel *finalmodel, SSqlObj* pSql) {
+ SColumnModel *finalmodel, SColumnModel *pFFModel, SSqlObj* pSql) {
SSqlCmd* pCmd = &pSql->cmd;
SSqlRes* pRes = &pSql->res;
@@ -227,7 +243,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (ds == NULL) {
tscError("%p failed to create merge structure", pSql);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
- taosTFree(pReducer);
+ tfree(pReducer);
return;
}
@@ -254,7 +270,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (ds->filePage.num == 0) { // no data in this flush, the index does not increase
tscDebug("%p flush data is empty, ignore %d flush record", pSql, idx);
- taosTFree(ds);
+ tfree(ds);
continue;
}
@@ -264,7 +280,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
// no data actually, no need to merge result.
if (idx == 0) {
- taosTFree(pReducer);
+ tfree(pReducer);
return;
}
@@ -272,7 +288,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
SCompareParam *param = malloc(sizeof(SCompareParam));
if (param == NULL) {
- taosTFree(pReducer);
+ tfree(pReducer);
return;
}
@@ -286,8 +302,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator);
if (pReducer->pLoserTree == NULL || pRes->code != 0) {
- taosTFree(param);
- taosTFree(pReducer);
+ tfree(param);
+ tfree(pReducer);
return;
}
@@ -330,22 +346,19 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL ||
/*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
- taosTFree(pReducer->pTempBuffer);
- taosTFree(pReducer->discardData);
- taosTFree(pReducer->pResultBuf);
- taosTFree(pReducer->pFinalRes);
- taosTFree(pReducer->prevRowOfInput);
- taosTFree(pReducer->pLoserTree);
- taosTFree(param);
- taosTFree(pReducer);
+ tfree(pReducer->pTempBuffer);
+ tfree(pReducer->discardData);
+ tfree(pReducer->pResultBuf);
+ tfree(pReducer->pFinalRes);
+ tfree(pReducer->prevRowOfInput);
+ tfree(pReducer->pLoserTree);
+ tfree(param);
+ tfree(pReducer);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
return;
}
- size_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo);
-
pReducer->pTempBuffer->num = 0;
- pReducer->pResInfo = calloc(numOfCols, sizeof(SResultInfo));
tscCreateResPointerInfo(pRes, pQueryInfo);
tscInitSqlContext(pCmd, pReducer, pDesc);
@@ -373,8 +386,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
SFillColInfo* pFillCol = createFillColInfo(pQueryInfo);
pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
- 4096, (int32_t)numOfCols, pQueryInfo->interval.sliding, pQueryInfo->interval.slidingUnit,
- tinfo.precision, pQueryInfo->fillType, pFillCol);
+ 4096, (int32_t)pQueryInfo->fieldsInfo.numOfOutput, pQueryInfo->interval.sliding, pQueryInfo->interval.slidingUnit,
+ tinfo.precision, pQueryInfo->fillType, pFillCol, pSql);
}
}
@@ -489,47 +502,41 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
tscDebug("%p waiting for delete procedure, status: %d", pSql, status);
}
- pLocalReducer->pFillInfo = taosDestoryFillInfo(pLocalReducer->pFillInfo);
+ pLocalReducer->pFillInfo = taosDestroyFillInfo(pLocalReducer->pFillInfo);
if (pLocalReducer->pCtx != NULL) {
- for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
+ int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo);
+ for (int32_t i = 0; i < numOfExprs; ++i) {
SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i];
tVariantDestroy(&pCtx->tag);
+ tfree(pCtx->resultInfo);
+
if (pCtx->tagInfo.pTagCtxList != NULL) {
- taosTFree(pCtx->tagInfo.pTagCtxList);
+ tfree(pCtx->tagInfo.pTagCtxList);
}
}
- taosTFree(pLocalReducer->pCtx);
+ tfree(pLocalReducer->pCtx);
}
- taosTFree(pLocalReducer->prevRowOfInput);
-
- taosTFree(pLocalReducer->pTempBuffer);
- taosTFree(pLocalReducer->pResultBuf);
+ tfree(pLocalReducer->prevRowOfInput);
- if (pLocalReducer->pResInfo != NULL) {
- size_t num = tscSqlExprNumOfExprs(pQueryInfo);
- for (int32_t i = 0; i < num; ++i) {
- taosTFree(pLocalReducer->pResInfo[i].interResultBuf);
- }
-
- taosTFree(pLocalReducer->pResInfo);
- }
+ tfree(pLocalReducer->pTempBuffer);
+ tfree(pLocalReducer->pResultBuf);
if (pLocalReducer->pLoserTree) {
- taosTFree(pLocalReducer->pLoserTree->param);
- taosTFree(pLocalReducer->pLoserTree);
+ tfree(pLocalReducer->pLoserTree->param);
+ tfree(pLocalReducer->pLoserTree);
}
- taosTFree(pLocalReducer->pFinalRes);
- taosTFree(pLocalReducer->discardData);
+ tfree(pLocalReducer->pFinalRes);
+ tfree(pLocalReducer->discardData);
tscLocalReducerEnvDestroy(pLocalReducer->pExtMemBuffer, pLocalReducer->pDesc, pLocalReducer->resColModel,
pLocalReducer->numOfVnode);
for (int32_t i = 0; i < pLocalReducer->numOfBuffer; ++i) {
- taosTFree(pLocalReducer->pLocalDataSrc[i]);
+ tfree(pLocalReducer->pLocalDataSrc[i]);
}
pLocalReducer->numOfBuffer = 0;
@@ -563,7 +570,8 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm
if (numOfGroupByCols > 0) {
if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) {
- int32_t startCols = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols;
+ int32_t numOfInternalOutput = (int32_t) tscSqlExprNumOfExprs(pQueryInfo);
+ int32_t startCols = numOfInternalOutput - pQueryInfo->groupbyExpr.numOfGroupCols;
// the last "pQueryInfo->groupbyExpr.numOfGroupCols" columns are order-by columns
for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) {
@@ -596,7 +604,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm
}
*pOrderDesc = tOrderDesCreate(orderColIndexList, numOfGroupByCols, pModel, pQueryInfo->order.order);
- taosTFree(orderColIndexList);
+ tfree(orderColIndexList);
if (*pOrderDesc == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -682,6 +690,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
pSchema[i].bytes = pExpr->resBytes;
pSchema[i].type = (int8_t)pExpr->resType;
+ tstrncpy(pSchema[i].name, pExpr->aliasName, tListLen(pSchema[i].name));
+
rlen += pExpr->resBytes;
}
@@ -707,7 +717,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
- taosTFree(pSchema);
+ tfree(pSchema);
return pRes->code;
}
@@ -744,8 +754,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
}
*pFinalModel = createColumnModel(pSchema, (int32_t)size, capacity);
- taosTFree(pSchema);
+ tfree(pSchema);
return TSDB_CODE_SUCCESS;
}
@@ -764,7 +774,7 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe
pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]);
}
- taosTFree(pMemBuffer);
+ tfree(pMemBuffer);
}
/**
@@ -924,7 +934,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
}
while (1) {
- int64_t newRows = taosGenerateDataBlock(pFillInfo, pResPages, pLocalReducer->resColModel->capacity);
+ int64_t newRows = taosFillResultDataBlock(pFillInfo, pResPages, pLocalReducer->resColModel->capacity);
if (pQueryInfo->limit.offset < newRows) {
newRows -= pQueryInfo->limit.offset;
@@ -953,7 +963,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
}
// all output in current group are completed
- int32_t totalRemainRows = (int32_t)getFilledNumOfRes(pFillInfo, actualETime, pLocalReducer->resColModel->capacity);
+ int32_t totalRemainRows = (int32_t)getNumOfResWithFill(pFillInfo, actualETime, pLocalReducer->resColModel->capacity);
if (totalRemainRows <= 0) {
break;
}
@@ -974,10 +984,11 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pFillInfo);
}
+ int32_t offset = 0;
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
- int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i);
memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, (size_t)(pField->bytes * pRes->numOfRows));
+ offset += pField->bytes;
}
pRes->numOfRowsGroup += pRes->numOfRows;
@@ -986,10 +997,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
pBeforeFillData->num = 0;
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
- taosTFree(pResPages[i]);
+ tfree(pResPages[i]);
}
- taosTFree(pResPages);
+ tfree(pResPages);
}
static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) {
@@ -1072,7 +1083,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx)
continue;
}
- SResultInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
+ SResultRowCellInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
if (maxOutput < pResInfo->numOfRes) {
maxOutput = pResInfo->numOfRes;
}
@@ -1230,6 +1241,10 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur
tColModelCompact(pModel, pResBuf, pModel->capacity);
+ if (tscIsSecondStageQuery(pQueryInfo)) {
+ doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalReducer->finalRowSize);
+ }
+
#ifdef _DEBUG_VIEW
printf("final result before interpo:\n");
// tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->num, pResBuf->num);
@@ -1253,10 +1268,11 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur
return true;
}
-void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { // reset output buffer to the beginning
- for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
- pLocalReducer->pCtx[i].aOutputBuf =
- pLocalReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pLocalReducer->resColModel->capacity;
+void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {// reset output buffer to the beginning
+ size_t t = tscSqlExprNumOfExprs(pQueryInfo);
+ for (int32_t i = 0; i < t; ++i) {
+ SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
+ pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + pExpr->offset * pLocalReducer->resColModel->capacity;
}
memset(pLocalReducer->pResultBuf, 0, pLocalReducer->nResultBufSize + sizeof(tFilePage));
@@ -1301,7 +1317,7 @@ static bool doBuildFilledResultForGroup(SSqlObj *pSql) {
int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pFillInfo->numOfRows - 1));
// the first column must be the timestamp column
- int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
+ int32_t rows = (int32_t) getNumOfResWithFill(pFillInfo, etime, pLocalReducer->resColModel->capacity);
if (rows > 0) { // do fill gap
doFillResult(pSql, pLocalReducer, false);
}
@@ -1330,7 +1346,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
((pRes->numOfRowsGroup < pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) || (pQueryInfo->limit.limit < 0))) {
int64_t etime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey : pQueryInfo->window.skey;
- int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
+ int32_t rows = (int32_t)getNumOfResWithFill(pFillInfo, etime, pLocalReducer->resColModel->capacity);
if (rows > 0) {
doFillResult(pSql, pLocalReducer, true);
}
@@ -1501,8 +1517,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
if (pLocalReducer->discard && sameGroup) {
pLocalReducer->hasUnprocessedRow = false;
tmpBuffer->num = 0;
- } else {
- // current row does not belongs to the previous group, so it is not be handled yet.
+ } else { // current row does not belongs to the previous group, so it is not be handled yet.
pLocalReducer->hasUnprocessedRow = true;
}
@@ -1596,3 +1611,44 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen)
pRes->pLocalReducer->pResultBuf->num = numOfRes;
pRes->data = pRes->pLocalReducer->pResultBuf->data;
}
+
+void doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) {
+ char* pbuf = calloc(1, pOutput->num * rowSize);
+
+ size_t size = tscNumOfFields(pQueryInfo);
+ SArithmeticSupport arithSup = {0};
+
+ // todo refactor
+ arithSup.offset = 0;
+ arithSup.numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo);
+ arithSup.exprList = pQueryInfo->exprList;
+ arithSup.data = calloc(arithSup.numOfCols, POINTER_BYTES);
+
+ for(int32_t k = 0; k < arithSup.numOfCols; ++k) {
+ SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k);
+ arithSup.data[k] = (pOutput->data + pOutput->num* pExpr->offset);
+ }
+
+ int32_t offset = 0;
+
+ for (int i = 0; i < size; ++i) {
+ SInternalField* pSup = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i);
+
+ // calculate the result from several other columns
+ if (pSup->pArithExprInfo != NULL) {
+ arithSup.pArithExpr = pSup->pArithExprInfo;
+ tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithemicInputSrc);
+ } else {
+ SSqlExpr* pExpr = pSup->pSqlExpr;
+ memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, pExpr->resBytes * pOutput->num);
+ }
+
+ offset += pSup->field.bytes;
+ }
+
+ assert(finalRowSize <= rowSize);
+ memcpy(pOutput->data, pbuf, pOutput->num * finalRowSize);
+
+ tfree(pbuf);
+ tfree(arithSup.data);
+}
\ No newline at end of file
diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index ac34d26d2fe747773a550dd76ac7e967a592199e..a44a158f93095c752f6aa203eb35e58e1d7ba2a0 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -702,7 +702,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st
}
int32_t code = TSDB_CODE_TSC_INVALID_SQL;
- char * tmpTokenBuf = calloc(1, 4096); // used for deleting Escape character: \\, \', \"
+ char * tmpTokenBuf = calloc(1, 16*1024); // used for deleting Escape character: \\, \', \"
if (NULL == tmpTokenBuf) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
@@ -1148,6 +1148,10 @@ int tsParseInsertSql(SSqlObj *pSql) {
index = 0;
sToken = tStrGetToken(str, &index, false, 0, NULL);
+ if (sToken.type != TK_STRING && sToken.type != TK_ID) {
+ code = tscInvalidSQLErrMsg(pCmd->payload, "file path is required following keyword FILE", sToken.z);
+ goto _error;
+ }
str += index;
if (sToken.n == 0) {
code = tscInvalidSQLErrMsg(pCmd->payload, "file path is required following keyword FILE", sToken.z);
@@ -1309,7 +1313,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) {
if ((!pCmd->parseFinished) && (!initial)) {
tscDebug("%p resume to parse sql: %s", pSql, pCmd->curSql);
}
-
+
ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE);
if (TSDB_CODE_SUCCESS != ret) {
return ret;
@@ -1406,7 +1410,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
assert(taos_errno(pSql) == code);
taos_free_result(pSql);
- taosTFree(pSupporter);
+ tfree(pSupporter);
fclose(fp);
pParentSql->res.code = code;
@@ -1445,7 +1449,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
char *tokenBuf = calloc(1, 4096);
- while ((readLen = taosGetline(&line, &n, fp)) != -1) {
+ while ((readLen = tgetline(&line, &n, fp)) != -1) {
if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
line[--readLen] = 0;
}
@@ -1470,7 +1474,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
}
}
- taosTFree(tokenBuf);
+ tfree(tokenBuf);
free(line);
if (count > 0) {
@@ -1483,7 +1487,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
} else {
taos_free_result(pSql);
- taosTFree(pSupporter);
+ tfree(pSupporter);
fclose(fp);
pParentSql->fp = pParentSql->fetchFp;
@@ -1513,7 +1517,7 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) {
pSql->res.code = TAOS_SYSTEM_ERROR(errno);
tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code));
- taosTFree(pSupporter)
+ tfree(pSupporter)
tscQueueAsyncRes(pSql);
return;
diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c
index eb6843b0e4fd844c9d13da7f55dca47008f62a79..acc5acd786bfe00036e539a104da04af1485a263 100644
--- a/src/client/src/tscProfile.c
+++ b/src/client/src/tscProfile.c
@@ -222,7 +222,7 @@ void tscKillStream(STscObj *pObj, uint32_t killId) {
}
int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
- SCMHeartBeatMsg *pHeartbeat = pMsg;
+ SHeartBeatMsg *pHeartbeat = pMsg;
int allocedQueriesNum = pHeartbeat->numOfQueries;
int allocedStreamsNum = pHeartbeat->numOfStreams;
@@ -277,7 +277,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
}
int32_t msgLen = pHeartbeat->numOfQueries * sizeof(SQueryDesc) + pHeartbeat->numOfStreams * sizeof(SStreamDesc) +
- sizeof(SCMHeartBeatMsg);
+ sizeof(SHeartBeatMsg);
pHeartbeat->connId = htonl(pObj->connId);
pHeartbeat->numOfQueries = htonl(pHeartbeat->numOfQueries);
pHeartbeat->numOfStreams = htonl(pHeartbeat->numOfStreams);
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 815af79d8f610c2660934c32a30700ad872b57ba..b55326bbd37afb1255dfe71d253b084826983f72 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -16,6 +16,7 @@
#define _BSD_SOURCE
#define _XOPEN_SOURCE 500
#define _DEFAULT_SOURCE
+#define _GNU_SOURCE
#include "os.h"
#include "qAst.h"
@@ -51,26 +52,29 @@ typedef struct SConvertFunc {
int32_t originFuncId;
int32_t execFuncId;
} SConvertFunc;
-static SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t colIndex, int32_t tableIndex);
+
+static SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex);
static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo);
static char* getAccountId(SSqlObj* pSql);
-static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name);
+static bool has(SArray* pFieldList, int32_t startIdx, const char* name);
static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken);
static bool hasSpecifyDB(SStrToken* pTableName);
-static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd);
-static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSqlCmd* pCmd);
+static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd);
+static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd);
static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len);
static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength);
-static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName);
static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult);
static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes,
int8_t type, char* fieldName, SSqlExpr* pSqlExpr);
-static int32_t changeFunctionID(int32_t optr, int16_t* functionId);
+
+static int32_t convertFunctionId(int32_t optr, int16_t* functionId);
+static uint8_t convertOptr(SStrToken *pToken);
+
static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery);
static bool validateIpAddress(const char* ip, size_t size);
@@ -78,11 +82,11 @@ static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQuer
static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery);
static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo);
-static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd);
+static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd);
-static int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
+static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
-static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
+static int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem);
@@ -114,7 +118,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString);
static int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex);
static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo);
static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
-static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate);
+static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate);
static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex);
@@ -124,6 +128,49 @@ static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo);
static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index);
static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid);
+int16_t getNewResColId(SQueryInfo* pQueryInfo) {
+ return pQueryInfo->resColumnId--;
+}
+
+static uint8_t convertOptr(SStrToken *pToken) {
+ switch (pToken->type) {
+ case TK_LT:
+ return TSDB_RELATION_LESS;
+ case TK_LE:
+ return TSDB_RELATION_LESS_EQUAL;
+ case TK_GT:
+ return TSDB_RELATION_GREATER;
+ case TK_GE:
+ return TSDB_RELATION_GREATER_EQUAL;
+ case TK_NE:
+ return TSDB_RELATION_NOT_EQUAL;
+ case TK_AND:
+ return TSDB_RELATION_AND;
+ case TK_OR:
+ return TSDB_RELATION_OR;
+ case TK_EQ:
+ return TSDB_RELATION_EQUAL;
+ case TK_PLUS:
+ return TSDB_BINARY_OP_ADD;
+ case TK_MINUS:
+ return TSDB_BINARY_OP_SUBTRACT;
+ case TK_STAR:
+ return TSDB_BINARY_OP_MULTIPLY;
+ case TK_SLASH:
+ case TK_DIVIDE:
+ return TSDB_BINARY_OP_DIVIDE;
+ case TK_REM:
+ return TSDB_BINARY_OP_REMAINDER;
+ case TK_LIKE:
+ return TSDB_RELATION_LIKE;
+ case TK_ISNULL:
+ return TSDB_RELATION_ISNULL;
+ case TK_NOTNULL:
+ return TSDB_RELATION_NOTNULL;
+ default: { return 0; }
+ }
+}
+
/*
* Used during parsing query sql. Since the query sql usually small in length, error position
* is not needed in the final error message.
@@ -417,7 +464,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
char* pMsg = pCmd->payload;
- SCMCfgDnodeMsg* pCfg = (SCMCfgDnodeMsg*)pMsg;
+ SCfgDnodeMsg* pCfg = (SCfgDnodeMsg*)pMsg;
pDCL->a[0].n = strdequote(pDCL->a[0].z);
strncpy(pCfg->ep, pDCL->a[0].z, pDCL->a[0].n);
@@ -616,14 +663,20 @@ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) {
return false;
}
-int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
+int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
const char* msg1 = "invalid query expression";
const char* msg2 = "interval cannot be less than 10 ms";
+ const char* msg3 = "sliding cannot be used without interval";
+
+ SSqlCmd* pCmd = &pSql->cmd;
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
if (pQuerySql->interval.type == 0 || pQuerySql->interval.n == 0) {
+ if (pQuerySql->sliding.n > 0) {
+ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
+ }
return TSDB_CODE_SUCCESS;
}
@@ -656,7 +709,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
return TSDB_CODE_TSC_INVALID_SQL;
}
- if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
+ if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@@ -709,7 +762,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
return TSDB_CODE_TSC_INVALID_SQL;
}
- if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
+ if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@@ -767,13 +820,15 @@ int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQue
return TSDB_CODE_SUCCESS;
}
-int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
+int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
const char* msg0 = "sliding value too small";
const char* msg1 = "sliding value no larger than the interval value";
const char* msg2 = "sliding value can not less than 1% of interval value";
- const char* msg3 = "does not support sliding when interval is natual month/year";
+ const char* msg3 = "does not support sliding when interval is natural month/year";
+// const char* msg4 = "sliding not support yet in ordinary query";
const static int32_t INTERVAL_SLIDING_FACTOR = 100;
+ SSqlCmd* pCmd = &pSql->cmd;
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
@@ -806,6 +861,10 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
+// if (pQueryInfo->interval.sliding != pQueryInfo->interval.interval && pSql->pStream == NULL) {
+// return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
+// }
+
return TSDB_CODE_SUCCESS;
}
@@ -852,7 +911,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableNa
return TSDB_CODE_SUCCESS;
}
-static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
+static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) {
assert(pFieldList != NULL);
const char* msg = "illegal number of columns";
@@ -864,20 +923,22 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
const char* msg6 = "invalid column name";
// number of fields no less than 2
- if (pFieldList->nField <= 1 || pFieldList->nField > TSDB_MAX_COLUMNS) {
+ size_t numOfCols = taosArrayGetSize(pFieldList);
+ if (numOfCols <= 1 || numOfCols > TSDB_MAX_COLUMNS) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
return false;
}
// first column must be timestamp
- if (pFieldList->p[0].type != TSDB_DATA_TYPE_TIMESTAMP) {
+ TAOS_FIELD* pField = taosArrayGet(pFieldList, 0);
+ if (pField->type != TSDB_DATA_TYPE_TIMESTAMP) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
return false;
}
int32_t nLen = 0;
- for (int32_t i = 0; i < pFieldList->nField; ++i) {
- TAOS_FIELD* pField = &pFieldList->p[i];
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ pField = taosArrayGet(pFieldList, i);
if (pField->bytes == 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
@@ -901,7 +962,7 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
}
// field name must be unique
- if (has(pFieldList, i + 1, pFieldList->p[i].name) == true) {
+ if (has(pFieldList, i + 1, pField->name) == true) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
return false;
}
@@ -918,7 +979,7 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
return true;
}
-static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSqlCmd* pCmd) {
+static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd) {
assert(pTagsList != NULL);
const char* msg1 = "invalid number of tag columns";
@@ -930,18 +991,21 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq
const char* msg7 = "invalid binary/nchar tag length";
// number of fields at least 1
- if (pTagsList->nField < 1 || pTagsList->nField > TSDB_MAX_TAGS) {
+ size_t numOfTags = taosArrayGetSize(pTagsList);
+ if (numOfTags < 1 || numOfTags > TSDB_MAX_TAGS) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
return false;
}
int32_t nLen = 0;
- for (int32_t i = 0; i < pTagsList->nField; ++i) {
- if (pTagsList->p[i].bytes == 0) {
+ for (int32_t i = 0; i < numOfTags; ++i) {
+ TAOS_FIELD* p = taosArrayGet(pTagsList, i);
+ if (p->bytes == 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
return false;
}
- nLen += pTagsList->p[i].bytes;
+
+ nLen += p->bytes;
}
// max tag row length must be less than TSDB_MAX_TAGS_LEN
@@ -951,37 +1015,41 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq
}
// field name must be unique
- for (int32_t i = 0; i < pTagsList->nField; ++i) {
- if (has(pFieldList, 0, pTagsList->p[i].name) == true) {
+ for (int32_t i = 0; i < numOfTags; ++i) {
+ TAOS_FIELD* p = taosArrayGet(pTagsList, i);
+
+ if (has(pFieldList, 0, p->name) == true) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
return false;
}
}
/* timestamp in tag is not allowed */
- for (int32_t i = 0; i < pTagsList->nField; ++i) {
- if (pTagsList->p[i].type == TSDB_DATA_TYPE_TIMESTAMP) {
+ for (int32_t i = 0; i < numOfTags; ++i) {
+ TAOS_FIELD* p = taosArrayGet(pTagsList, i);
+
+ if (p->type == TSDB_DATA_TYPE_TIMESTAMP) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
return false;
}
- if (pTagsList->p[i].type < TSDB_DATA_TYPE_BOOL || pTagsList->p[i].type > TSDB_DATA_TYPE_NCHAR) {
+ if (p->type < TSDB_DATA_TYPE_BOOL || p->type > TSDB_DATA_TYPE_NCHAR) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
return false;
}
- if ((pTagsList->p[i].type == TSDB_DATA_TYPE_BINARY && pTagsList->p[i].bytes <= 0) ||
- (pTagsList->p[i].type == TSDB_DATA_TYPE_NCHAR && pTagsList->p[i].bytes <= 0)) {
+ if ((p->type == TSDB_DATA_TYPE_BINARY && p->bytes <= 0) ||
+ (p->type == TSDB_DATA_TYPE_NCHAR && p->bytes <= 0)) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
return false;
}
- if (validateColumnName(pTagsList->p[i].name) != TSDB_CODE_SUCCESS) {
+ if (validateColumnName(p->name) != TSDB_CODE_SUCCESS) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
return false;
}
- if (has(pTagsList, i + 1, pTagsList->p[i].name) == true) {
+ if (has(pTagsList, i + 1, p->name) == true) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
return false;
}
@@ -1128,9 +1196,10 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) {
}
/* is contained in pFieldList or not */
-static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name) {
- for (int32_t j = startIdx; j < pFieldList->nField; ++j) {
- TAOS_FIELD* field = pFieldList->p + j;
+static bool has(SArray* pFieldList, int32_t startIdx, const char* name) {
+ size_t numOfCols = taosArrayGetSize(pFieldList);
+ for (int32_t j = startIdx; j < numOfCols; ++j) {
+ TAOS_FIELD* field = taosArrayGet(pFieldList, j);
if (strncasecmp(name, field->name, sizeof(field->name) - 1) == 0) return true;
}
@@ -1210,8 +1279,9 @@ static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex*
SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
tscColumnListInsert(pQueryInfo->colList, &tsCol);
}
+
static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) {
- const char* msg1 = "invalid column name, or illegal column type";
+ const char* msg1 = "invalid column name, illegal column type, or columns in arithmetic expression from two tables";
const char* msg2 = "invalid arithmetic expression in select clause";
const char* msg3 = "tag columns can not be used in arithmetic expression";
const char* msg4 = "columns from different table mixed up in arithmetic expression";
@@ -1241,7 +1311,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
SColumnIndex index = {.tableIndex = tableIndex};
SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double),
- sizeof(double), false);
+ -1000, sizeof(double), false);
char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z;
size_t len = MIN(sizeof(pExpr->aliasName), pItem->pNode->token.n + 1);
@@ -1257,6 +1327,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
+ // check for if there is a tag in the arithmetic express
size_t numOfNode = taosArrayGetSize(colList);
for(int32_t k = 0; k < numOfNode; ++k) {
SColIndex* pIndex = taosArrayGet(colList, k);
@@ -1282,9 +1353,9 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
char* c = tbufGetData(&bw, false);
// set the serialized binary string as the parameter of arithmetic expression
- addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, (int32_t)len, index.tableIndex);
-
+ addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, (int32_t)len);
insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr);
+
// add ts column
tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
@@ -1316,6 +1387,10 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
pArithExprInfo->interBytes = sizeof(double);
pArithExprInfo->type = TSDB_DATA_TYPE_DOUBLE;
+ pArithExprInfo->base.functionId = TSDB_FUNC_ARITHM;
+ pArithExprInfo->base.numOfParams = 1;
+ pArithExprInfo->base.resColId = getNewResColId(pQueryInfo);
+
int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo, NULL, &pArithExprInfo->uid);
if (ret != TSDB_CODE_SUCCESS) {
tExprTreeDestroy(&pArithExprInfo->pExpr, NULL);
@@ -1324,14 +1399,30 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
pInfo->pArithExprInfo = pArithExprInfo;
}
+
+ SBufferWriter bw = tbufInitWriter(NULL, false);
+
+ TRY(0) {
+ exprTreeToBinary(&bw, pInfo->pArithExprInfo->pExpr);
+ } CATCH(code) {
+ tbufCloseWriter(&bw);
+ UNUSED(code);
+ // TODO: other error handling
+ } END_TRY
+
+ SSqlFuncMsg* pFuncMsg = &pInfo->pArithExprInfo->base;
+ pFuncMsg->arg[0].argBytes = (int16_t) tbufTell(&bw);
+ pFuncMsg->arg[0].argValue.pz = tbufGetData(&bw, true);
+ pFuncMsg->arg[0].argType = TSDB_DATA_TYPE_BINARY;
+
+// tbufCloseWriter(&bw); // TODO there is a memory leak
}
return TSDB_CODE_SUCCESS;
}
-
static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) {
- SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex);
+ SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, pIndex->columnIndex, pIndex->tableIndex);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex);
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
@@ -1476,7 +1567,7 @@ int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnLi
return TSDB_CODE_SUCCESS;
}
-SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t colIndex, int32_t tableIndex) {
+SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tableIndex) {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
int32_t numOfCols = tscGetNumOfColumns(pTableMeta);
@@ -1488,20 +1579,22 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c
if (functionId == TSDB_FUNC_TAGPRJ) {
index.columnIndex = colIndex - tscGetNumOfColumns(pTableMeta);
-
tscColumnListInsert(pTableMetaInfo->tagColList, &index);
} else {
index.columnIndex = colIndex;
}
-
- return tscSqlExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes,
- pSchema->bytes, functionId == TSDB_FUNC_TAGPRJ);
+
+ int16_t colId = getNewResColId(pQueryInfo);
+ return tscSqlExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, colId, pSchema->bytes,
+ (functionId == TSDB_FUNC_TAGPRJ));
}
SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId,
SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) {
+ int16_t colId = getNewResColId(pQueryInfo);
+
SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type,
- pColSchema->bytes, pColSchema->bytes, TSDB_COL_IS_TAG(flag));
+ pColSchema->bytes, colId, pColSchema->bytes, TSDB_COL_IS_TAG(flag));
tstrncpy(pExpr->aliasName, pColSchema->name, sizeof(pExpr->aliasName));
SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex);
@@ -1537,7 +1630,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum
}
for (int32_t j = 0; j < numOfTotalColumns; ++j) {
- SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos + j, j, pIndex->tableIndex);
+ SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, j, pIndex->tableIndex);
tstrncpy(pExpr->aliasName, pSchema[j].name, sizeof(pExpr->aliasName));
pIndex->columnIndex = j;
@@ -1625,16 +1718,15 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
}
static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc,
- char* aliasName, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) {
+ const char* name, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) {
const char* msg1 = "not support column types";
int16_t type = 0;
int16_t bytes = 0;
- char columnName[TSDB_COL_NAME_LEN] = {0};
int32_t functionID = cvtFunc.execFuncId;
if (functionID == TSDB_FUNC_SPREAD) {
- int32_t t1 = pSchema[pColIndex->columnIndex].type;
+ int32_t t1 = pSchema->type;
if (t1 == TSDB_DATA_TYPE_BINARY || t1 == TSDB_DATA_TYPE_NCHAR || t1 == TSDB_DATA_TYPE_BOOL) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
return -1;
@@ -1643,18 +1735,12 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
bytes = tDataTypeDesc[type].nSize;
}
} else {
- type = pSchema[pColIndex->columnIndex].type;
- bytes = pSchema[pColIndex->columnIndex].bytes;
+ type = pSchema->type;
+ bytes = pSchema->bytes;
}
- if (aliasName != NULL) {
- tstrncpy(columnName, aliasName, sizeof(columnName));
- } else {
- getRevisedName(columnName, cvtFunc.originFuncId, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name);
- }
-
- SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false);
- tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName));
+ SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, getNewResColId(pQueryInfo), bytes, false);
+ tstrncpy(pExpr->aliasName, name, tListLen(pExpr->aliasName));
if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != functionID) {
pExpr->colInfo.flag |= TSDB_COL_NULL;
@@ -1674,7 +1760,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
// if it is not in the final result, do not add it
SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex);
if (finalResult) {
- insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, columnName, pExpr);
+ insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, pExpr->aliasName, pExpr);
} else {
tscColumnListInsert(pQueryInfo->colList, &(ids.ids[0]));
}
@@ -1682,6 +1768,23 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
return TSDB_CODE_SUCCESS;
}
+void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken) {
+ if (pItem->aliasName != NULL) {
+ tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN);
+ } else {
+ char uname[TSDB_COL_NAME_LEN] = {0};
+ int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN);
+ tstrncpy(uname, pToken->z, len);
+
+ int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1;
+ char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1] = {0};
+
+ snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname);
+
+ tstrncpy(name, tmp, TSDB_COL_NAME_LEN);
+ }
+}
+
int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult) {
STableMetaInfo* pTableMetaInfo = NULL;
int32_t optr = pItem->pNode->nSQLOptr;
@@ -1704,7 +1807,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
}
int16_t functionID = 0;
- if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) {
+ if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@@ -1730,7 +1833,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX};
int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize;
- pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size, false);
+ pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false);
} else if (sqlOptr == TK_INTEGER) { // select count(1) from table1
char buf[8] = {0};
int64_t val = -1;
@@ -1742,7 +1845,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
if (val == 1) {
index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX};
int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize;
- pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size, false);
+ pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false);
} else {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
@@ -1762,12 +1865,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
}
int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize;
- pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size, isTag);
+ pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, isTag);
}
} else { // count(*) is equalled to count(primary_timestamp_key)
index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX};
int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize;
- pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size, false);
+ pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false);
}
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
@@ -1840,7 +1943,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
int32_t intermediateResSize = 0;
int16_t functionID = 0;
- if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) {
+ if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@@ -1854,7 +1957,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
colIndex += 1;
SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0};
SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE,
- TSDB_KEYSIZE, false);
+ getNewResColId(pQueryInfo), TSDB_KEYSIZE, false);
SColumnList ids = getColumnList(1, 0, 0);
insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].aName, pExpr);
@@ -1865,7 +1968,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
- SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, resultType, resultSize, resultSize, false);
+ SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, resultType, resultSize, getNewResColId(pQueryInfo), resultSize, false);
if (optr == TK_LEASTSQUARES) {
/* set the leastsquares parameters */
@@ -1874,14 +1977,14 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
return TSDB_CODE_TSC_INVALID_SQL;
}
- addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, DOUBLE_BYTES, 0);
+ addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, DOUBLE_BYTES);
memset(val, 0, tListLen(val));
if (tVariantDump(&pParamElem[2].pNode->val, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) {
return TSDB_CODE_TSC_INVALID_SQL;
}
- addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0);
+ addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
}
SColumnList ids = {0};
@@ -1911,7 +2014,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
bool requireAllFields = (pItem->pNode->pParam == NULL);
int16_t functionID = 0;
- if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) {
+ if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
}
@@ -1939,8 +2042,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
SColumnIndex index = COLUMN_INDEX_INITIALIZER;
- if (pParamElem->pNode->nSQLOptr == TK_ALL) {
- // select table.*
+ if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.*
SStrToken tmpToken = pParamElem->pNode->colInfo;
if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
@@ -1950,9 +2052,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
+ char name[TSDB_COL_NAME_LEN] = {0};
for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) {
index.columnIndex = j;
- if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex++, &index, finalResult) != 0) {
+ SStrToken t = {.z = pSchema[j].name, .n = (uint32_t)strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)};
+ setResultColName(name, pItem, cvtFunc.originFuncId, &t);
+
+ if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, finalResult) != 0) {
return TSDB_CODE_TSC_INVALID_SQL;
}
}
@@ -1963,14 +2069,18 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
}
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
- SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
// functions can not be applied to tags
if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
- if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex + i, &index, finalResult) != 0) {
+ char name[TSDB_COL_NAME_LEN] = {0};
+
+ SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex);
+ setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo);
+
+ if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex + i, &index, finalResult) != 0) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@@ -2007,7 +2117,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) {
SColumnIndex index = {.tableIndex = j, .columnIndex = i};
- if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex, &index, finalResult) != 0) {
+
+ char name[TSDB_COL_NAME_LEN] = {0};
+ SStrToken t = {.z = pSchema[i].name, .n = (uint32_t)strnlen(pSchema[i].name, TSDB_COL_NAME_LEN)};
+ setResultColName(name, pItem, cvtFunc.originFuncId, &t);
+
+ if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@@ -2088,14 +2203,14 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
* for dp = 100, it is max,
*/
int16_t functionId = 0;
- if (changeFunctionID(optr, &functionId) != TSDB_CODE_SUCCESS) {
+ if (convertFunctionId(optr, &functionId) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
colIndex += 1; // the first column is ts
- pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, resultSize, false);
- addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0);
+ pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pQueryInfo), resultSize, false);
+ addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
} else {
tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT, true);
@@ -2105,15 +2220,15 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
}
int16_t functionId = 0;
- if (changeFunctionID(optr, &functionId) != TSDB_CODE_SUCCESS) {
+ if (convertFunctionId(optr, &functionId) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
// todo REFACTOR
// set the first column ts for top/bottom query
SColumnIndex index1 = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX};
- pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE,
- TSDB_KEYSIZE, false);
+ pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pQueryInfo),
+ TSDB_KEYSIZE, false);
tstrncpy(pExpr->aliasName, aAggs[TSDB_FUNC_TS].aName, sizeof(pExpr->aliasName));
const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX;
@@ -2123,8 +2238,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
colIndex += 1; // the first column is ts
- pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, resultSize, false);
- addExprParams(pExpr, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0);
+ pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pQueryInfo), resultSize, false);
+ addExprParams(pExpr, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t));
}
memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName));
@@ -2240,10 +2355,6 @@ void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLengt
}
}
-void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName) {
- snprintf(resultFieldName, maxLen, "%s(%s)", aAggs[functionId].aName, columnName);
-}
-
static bool isTablenameToken(SStrToken* token) {
SStrToken tmpToken = *token;
SStrToken tableToken = {0};
@@ -2368,7 +2479,7 @@ int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo*
return doGetColumnIndexByName(pCmd, &tmpToken, pQueryInfo, pIndex);
}
-int32_t changeFunctionID(int32_t optr, int16_t* functionId) {
+int32_t convertFunctionId(int32_t optr, int16_t* functionId) {
switch (optr) {
case TK_COUNT:
*functionId = TSDB_FUNC_COUNT;
@@ -2612,7 +2723,7 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) {
}
}
- tscFieldInfoUpdateOffsetForInterResult(pQueryInfo);
+ tscFieldInfoUpdateOffset(pQueryInfo);
return TSDB_CODE_SUCCESS;
}
@@ -2686,7 +2797,10 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo)
static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) {
int32_t startIdx = 0;
-
+
+ size_t numOfExpr = tscSqlExprNumOfExprs(pQueryInfo);
+ assert(numOfExpr > 0);
+
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx);
int32_t functionID = pExpr->functionId;
@@ -2725,10 +2839,11 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) {
return true;
}
-int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd) {
+int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd) {
const char* msg1 = "too many columns in group by clause";
const char* msg2 = "invalid column name in group by clause";
-// const char* msg3 = "group by columns must belong to one table";
+ const char* msg3 = "columns from one table allowed as group by columns";
+ const char* msg4 = "join query does not support group by";
const char* msg7 = "not support group by expression";
const char* msg8 = "not allowed column type for group by";
const char* msg9 = "tags not allowed for table query";
@@ -2744,19 +2859,26 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES);
}
- pQueryInfo->groupbyExpr.numOfGroupCols = pList->nExpr;
- if (pList->nExpr > TSDB_MAX_TAGS) {
+ pQueryInfo->groupbyExpr.numOfGroupCols = (int16_t)taosArrayGetSize(pList);
+ if (pQueryInfo->groupbyExpr.numOfGroupCols > TSDB_MAX_TAGS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
+ if (pQueryInfo->numOfTables > 1) {
+ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
+ }
+
STableMeta* pTableMeta = NULL;
SSchema* pSchema = NULL;
SSchema s = tscGetTbnameColumnSchema();
int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL;
-
- for (int32_t i = 0; i < pList->nExpr; ++i) {
- tVariant* pVar = &pList->a[i].pVar;
+
+ size_t num = taosArrayGetSize(pList);
+ for (int32_t i = 0; i < num; ++i) {
+ tVariantListItem * pItem = taosArrayGet(pList, i);
+ tVariant* pVar = &pItem->pVar;
+
SStrToken token = {pVar->nLen, pVar->nType, pVar->pz};
SColumnIndex index = COLUMN_INDEX_INITIALIZER;
@@ -2764,7 +2886,11 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
- tableIndex = index.tableIndex;
+ if (tableIndex == COLUMN_INDEX_INITIAL_VAL) {
+ tableIndex = index.tableIndex;
+ } else if (tableIndex != index.tableIndex) {
+ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
+ }
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
pTableMeta = pTableMetaInfo->pTableMeta;
@@ -2803,7 +2929,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
tscColumnListInsert(pTableMetaInfo->tagColList, &index);
} else {
// check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by
- if (pSchema->type > TSDB_DATA_TYPE_BINARY) {
+ if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8);
}
@@ -2813,7 +2939,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
taosArrayPush(pGroupExpr->columnInfo, &colIndex);
pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC;
- if (i == 0 && pList->nExpr > 1) {
+ if (i == 0 && num > 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
}
}
@@ -2825,7 +2951,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo) {
if (QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
- tscFieldInfoUpdateOffsetForInterResult(pQueryInfo);
+ tscFieldInfoUpdateOffset(pQueryInfo);
} else {
tscFieldInfoUpdateOffset(pQueryInfo);
}
@@ -3349,7 +3475,8 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) {
return TSDB_CODE_SUCCESS;
}
-static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) {
+static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList,
+ int32_t* type, uint64_t* uid) {
if (pExpr->nSQLOptr == TK_ID) {
if (*type == NON_ARITHMEIC_EXPR) {
*type = NORMAL_ARITHMETIC;
@@ -3398,13 +3525,22 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer
}
// Not supported data type in arithmetic expression
+ uint64_t id = -1;
for(int32_t i = 0; i < inc; ++i) {
SSqlExpr* p1 = tscSqlExprGet(pQueryInfo, i + outputIndex);
int16_t t = p1->resType;
if (t == TSDB_DATA_TYPE_BINARY || t == TSDB_DATA_TYPE_NCHAR || t == TSDB_DATA_TYPE_BOOL || t == TSDB_DATA_TYPE_TIMESTAMP) {
return TSDB_CODE_TSC_INVALID_SQL;
}
+
+ if (i == 0) {
+ id = p1->uid;
+ } else if (id != p1->uid){
+ return TSDB_CODE_TSC_INVALID_SQL;
+ }
}
+
+ *uid = id;
}
return TSDB_CODE_SUCCESS;
@@ -3416,13 +3552,16 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI
}
tSQLExpr* pLeft = pExpr->pLeft;
+ uint64_t uidLeft = 0;
+ uint64_t uidRight = 0;
+
if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) {
int32_t ret = validateArithmeticSQLExpr(pCmd, pLeft, pQueryInfo, pList, type);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
} else {
- int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type);
+ int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type, &uidLeft);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
@@ -3435,10 +3574,15 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI
return ret;
}
} else {
- int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type);
+ int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type, &uidRight);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
+
+ // the expression not from the same table, return error
+ if (uidLeft != uidRight && uidLeft != 0 && uidRight != 0) {
+ return TSDB_CODE_TSC_INVALID_SQL;
+ }
}
return TSDB_CODE_SUCCESS;
@@ -3900,7 +4044,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen);
if (ret != TSDB_CODE_SUCCESS) {
taosStringBuilderDestroy(&sb1);
- taosTFree(segments);
+ tfree(segments);
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
return ret;
@@ -3913,7 +4057,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
pQueryInfo->tagCond.tbnameCond.cond = strdup(str);
taosStringBuilderDestroy(&sb1);
- taosTFree(segments);
+ tfree(segments);
return TSDB_CODE_SUCCESS;
}
@@ -4309,8 +4453,8 @@ int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) {
}
int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) {
- tVariantList* pFillToken = pQuerySQL->fillType;
- tVariantListItem* pItem = &pFillToken->a[0];
+ SArray* pFillToken = pQuerySQL->fillType;
+ tVariantListItem* pItem = taosArrayGet(pFillToken, 0);
const int32_t START_INTERPO_COL_IDX = 1;
@@ -4322,7 +4466,7 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
- size_t size = tscSqlExprNumOfExprs(pQueryInfo);
+ size_t size = tscNumOfFields(pQueryInfo);
if (pQueryInfo->fillVal == NULL) {
pQueryInfo->fillVal = calloc(size, sizeof(int64_t));
@@ -4336,12 +4480,8 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
} else if (strncasecmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4) {
pQueryInfo->fillType = TSDB_FILL_NULL;
for (int32_t i = START_INTERPO_COL_IDX; i < size; ++i) {
- TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
- if (pFields->type == TSDB_DATA_TYPE_BINARY || pFields->type == TSDB_DATA_TYPE_NCHAR) {
- setVardataNull((char*) &pQueryInfo->fillVal[i], pFields->type);
- } else {
- setNull((char*)&pQueryInfo->fillVal[i], pFields->type, pFields->bytes);
- };
+ TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
+ setNull((char*)&pQueryInfo->fillVal[i], pField->type, pField->bytes);
}
} else if (strncasecmp(pItem->pVar.pz, "prev", 4) == 0 && pItem->pVar.nLen == 4) {
pQueryInfo->fillType = TSDB_FILL_PREV;
@@ -4350,12 +4490,13 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
} else if (strncasecmp(pItem->pVar.pz, "value", 5) == 0 && pItem->pVar.nLen == 5) {
pQueryInfo->fillType = TSDB_FILL_SET_VALUE;
- if (pFillToken->nExpr == 1) {
+ size_t num = taosArrayGetSize(pFillToken);
+ if (num == 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
int32_t startPos = 1;
- int32_t numOfFillVal = pFillToken->nExpr - 1;
+ int32_t numOfFillVal = (int32_t)(num - 1);
/* for point interpolation query, we do not have the timestamp column */
if (tscIsPointInterpQuery(pQueryInfo)) {
@@ -4365,35 +4506,36 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
numOfFillVal = (int32_t)size;
}
} else {
- numOfFillVal = (pFillToken->nExpr > (int32_t)size) ? (int32_t)size : pFillToken->nExpr;
+ numOfFillVal = (int16_t)((num > (int32_t)size) ? (int32_t)size : num);
}
int32_t j = 1;
for (int32_t i = startPos; i < numOfFillVal; ++i, ++j) {
- TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
+ TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
- if (pFields->type == TSDB_DATA_TYPE_BINARY || pFields->type == TSDB_DATA_TYPE_NCHAR) {
- setVardataNull((char*) &pQueryInfo->fillVal[i], pFields->type);
+ if (pField->type == TSDB_DATA_TYPE_BINARY || pField->type == TSDB_DATA_TYPE_NCHAR) {
+ setVardataNull((char*) &pQueryInfo->fillVal[i], pField->type);
continue;
}
- int32_t ret = tVariantDump(&pFillToken->a[j].pVar, (char*)&pQueryInfo->fillVal[i], pFields->type, true);
+ tVariant* p = taosArrayGet(pFillToken, j);
+ int32_t ret = tVariantDump(p, (char*)&pQueryInfo->fillVal[i], pField->type, true);
if (ret != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
}
- if ((pFillToken->nExpr < size) || ((pFillToken->nExpr - 1 < size) && (tscIsPointInterpQuery(pQueryInfo)))) {
- tVariantListItem* lastItem = &pFillToken->a[pFillToken->nExpr - 1];
+ if ((num < size) || ((num - 1 < size) && (tscIsPointInterpQuery(pQueryInfo)))) {
+ tVariantListItem* lastItem = taosArrayGetLast(pFillToken);
for (int32_t i = numOfFillVal; i < size; ++i) {
- TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
+ TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
- if (pFields->type == TSDB_DATA_TYPE_BINARY || pFields->type == TSDB_DATA_TYPE_NCHAR) {
- setVardataNull((char*) &pQueryInfo->fillVal[i], pFields->type);
+ if (pField->type == TSDB_DATA_TYPE_BINARY || pField->type == TSDB_DATA_TYPE_NCHAR) {
+ setVardataNull((char*) &pQueryInfo->fillVal[i], pField->type);
} else {
- tVariantDump(&lastItem->pVar, (char*)&pQueryInfo->fillVal[i], pFields->type, true);
+ tVariantDump(&lastItem->pVar, (char*)&pQueryInfo->fillVal[i], pField->type, true);
}
}
}
@@ -4424,8 +4566,8 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) {
int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema* pSchema) {
const char* msg0 = "only support order by primary timestamp";
const char* msg1 = "invalid column name";
- const char* msg2 = "only support order by primary timestamp and queried column";
- const char* msg3 = "only support order by primary timestamp and first tag in groupby clause";
+ const char* msg2 = "only support order by primary timestamp or queried column";
+ const char* msg3 = "only support order by primary timestamp or first tag in groupby clause";
setDefaultOrderInfo(pQueryInfo);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
@@ -4434,7 +4576,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
return TSDB_CODE_SUCCESS;
}
- tVariantList* pSortorder = pQuerySql->pSortOrder;
+ SArray* pSortorder = pQuerySql->pSortOrder;
/*
* for table query, there is only one or none order option is allowed, which is the
@@ -4442,18 +4584,19 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
*
* for super table query, the order option must be less than 3.
*/
+ size_t size = taosArrayGetSize(pSortorder);
if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
- if (pSortorder->nExpr > 1) {
+ if (size > 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0);
}
} else {
- if (pSortorder->nExpr > 2) {
+ if (size > 2) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
}
// handle the first part of order by
- tVariant* pVar = &pSortorder->a[0].pVar;
+ tVariant* pVar = taosArrayGet(pSortorder, 0);
// e.g., order by 1 asc, return directly with out further check.
if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) {
@@ -4496,10 +4639,13 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
assert(!(orderByTags && orderByTS));
}
- if (pSortorder->nExpr == 1) {
+ size_t s = taosArrayGetSize(pSortorder);
+ if (s == 1) {
if (orderByTags) {
pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
- pQueryInfo->groupbyExpr.orderType = pQuerySql->pSortOrder->a[0].sortOrder;
+
+ tVariantListItem* p1 = taosArrayGet(pQuerySql->pSortOrder, 0);
+ pQueryInfo->groupbyExpr.orderType = p1->sortOrder;
} else if (isTopBottomQuery(pQueryInfo)) {
/* order of top/bottom query in interval is not valid */
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0);
@@ -4510,11 +4656,14 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
- pQueryInfo->order.order = pQuerySql->pSortOrder->a[0].sortOrder;
+ tVariantListItem* p1 = taosArrayGet(pQuerySql->pSortOrder, 0);
+ pQueryInfo->order.order = p1->sortOrder;
pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId;
return TSDB_CODE_SUCCESS;
} else {
- pQueryInfo->order.order = pSortorder->a[0].sortOrder;
+ tVariantListItem* p1 = taosArrayGet(pQuerySql->pSortOrder, 0);
+
+ pQueryInfo->order.order = p1->sortOrder;
pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
// orderby ts query on super table
@@ -4524,16 +4673,18 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
}
}
- if (pSortorder->nExpr == 2) {
+ if (s == 2) {
+ tVariantListItem *pItem = taosArrayGet(pQuerySql->pSortOrder, 0);
if (orderByTags) {
pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
- pQueryInfo->groupbyExpr.orderType = pQuerySql->pSortOrder->a[0].sortOrder;
+ pQueryInfo->groupbyExpr.orderType = pItem->sortOrder;
} else {
- pQueryInfo->order.order = pSortorder->a[0].sortOrder;
+ pQueryInfo->order.order = pItem->sortOrder;
pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
}
- tVariant* pVar2 = &pSortorder->a[1].pVar;
+ pItem = taosArrayGet(pQuerySql->pSortOrder, 1);
+ tVariant* pVar2 = &pItem->pVar;
SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz};
if (getColumnIndexByName(pCmd, &cname, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
@@ -4542,7 +4693,8 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
} else {
- pQueryInfo->order.order = pSortorder->a[1].sortOrder;
+ tVariantListItem* p1 = taosArrayGet(pSortorder, 1);
+ pQueryInfo->order.order = p1->sortOrder;
pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
}
}
@@ -4566,12 +4718,14 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
- pQueryInfo->order.order = pQuerySql->pSortOrder->a[0].sortOrder;
+ tVariantListItem* pItem = taosArrayGet(pQuerySql->pSortOrder, 0);
+ pQueryInfo->order.order = pItem->sortOrder;
pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId;
return TSDB_CODE_SUCCESS;
}
- pQueryInfo->order.order = pQuerySql->pSortOrder->a[0].sortOrder;
+ tVariantListItem* pItem = taosArrayGet(pQuerySql->pSortOrder, 0);
+ pQueryInfo->order.order = pItem->sortOrder;
}
return TSDB_CODE_SUCCESS;
@@ -4639,27 +4793,28 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) {
- tFieldList* pFieldList = pAlterSQL->pAddColumns;
- if (pFieldList->nField > 1) {
+ SArray* pFieldList = pAlterSQL->pAddColumns;
+ if (taosArrayGetSize(pFieldList) > 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
}
- if (!validateOneTags(pCmd, &pFieldList->p[0])) {
+ TAOS_FIELD* p = taosArrayGet(pFieldList, 0);
+ if (!validateOneTags(pCmd, p)) {
return TSDB_CODE_TSC_INVALID_SQL;
}
- tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]);
+ tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
} else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) {
if (tscGetNumOfTags(pTableMeta) == 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
}
// numOfTags == 1
- if (pAlterSQL->varList->nExpr > 1) {
+ if (taosArrayGetSize(pAlterSQL->varList) > 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8);
}
- tVariantListItem* pItem = &pAlterSQL->varList->a[0];
+ tVariantListItem* pItem = taosArrayGet(pAlterSQL->varList, 0);
if (pItem->pVar.nLen >= TSDB_COL_NAME_LEN) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
}
@@ -4684,13 +4839,13 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
} else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) {
- tVariantList* pVarList = pAlterSQL->varList;
- if (pVarList->nExpr > 2) {
+ SArray* pVarList = pAlterSQL->varList;
+ if (taosArrayGetSize(pVarList) > 2) {
return TSDB_CODE_TSC_INVALID_SQL;
}
- tVariantListItem* pSrcItem = &pAlterSQL->varList->a[0];
- tVariantListItem* pDstItem = &pAlterSQL->varList->a[1];
+ tVariantListItem* pSrcItem = taosArrayGet(pAlterSQL->varList, 0);
+ tVariantListItem* pDstItem = taosArrayGet(pAlterSQL->varList, 1);
if (pSrcItem->pVar.nLen >= TSDB_COL_NAME_LEN || pDstItem->pVar.nLen >= TSDB_COL_NAME_LEN) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
@@ -4713,13 +4868,17 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg19);
}
+ tVariantListItem* pItem = taosArrayGet(pVarList, 0);
+
char name[TSDB_COL_NAME_LEN] = {0};
- strncpy(name, pVarList->a[0].pVar.pz, pVarList->a[0].pVar.nLen);
+ strncpy(name, pItem->pVar.pz, pItem->pVar.nLen);
TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
+ pItem = taosArrayGet(pVarList, 1);
memset(name, 0, tListLen(name));
- strncpy(name, pVarList->a[1].pVar.pz, pVarList->a[1].pVar.nLen);
+
+ strncpy(name, pItem->pVar.pz, pItem->pVar.nLen);
f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
} else if (pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) {
@@ -4727,12 +4886,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
// the following is used to handle tags value for table created according to super table
pCmd->command = TSDB_SQL_UPDATE_TAGS_VAL;
- tVariantList* pVarList = pAlterSQL->varList;
- tVariant* pTagName = &pVarList->a[0].pVar;
+ SArray* pVarList = pAlterSQL->varList;
+ tVariantListItem* item = taosArrayGet(pVarList, 0);
int16_t numOfTags = tscGetNumOfTags(pTableMeta);
SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER;
- SStrToken name = {.type = TK_STRING, .z = pTagName->pz, .n = pTagName->nLen};
+ SStrToken name = {.type = TK_STRING, .z = item->pVar.pz, .n = item->pVar.nLen};
if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@@ -4741,8 +4900,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg12);
}
+ tVariantListItem* pItem = taosArrayGet(pVarList, 1);
SSchema* pTagsSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex);
- if (tVariantDump(&pVarList->a[1].pVar, pAlterSQL->tagData.data, pTagsSchema->type, true) != TSDB_CODE_SUCCESS) {
+ if (tVariantDump(&pItem->pVar, pAlterSQL->tagData.data, pTagsSchema->type, true) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg13);
}
@@ -4787,7 +4947,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
// copy the tag value to msg body
- tVariantDump(&pVarList->a[1].pVar, pUpdateMsg->data + schemaLen, pTagsSchema->type, true);
+ pItem = taosArrayGet(pVarList, 1);
+ tVariantDump(&pItem->pVar, pUpdateMsg->data + schemaLen, pTagsSchema->type, true);
int32_t len = 0;
if (pTagsSchema->type != TSDB_DATA_TYPE_BINARY && pTagsSchema->type != TSDB_DATA_TYPE_NCHAR) {
@@ -4802,27 +4963,29 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
pUpdateMsg->head.contLen = htonl(total);
} else if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN) {
- tFieldList* pFieldList = pAlterSQL->pAddColumns;
- if (pFieldList->nField > 1) {
+ SArray* pFieldList = pAlterSQL->pAddColumns;
+ if (taosArrayGetSize(pFieldList) > 1) {
const char* msg = "only support add one column";
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
- if (!validateOneColumn(pCmd, &pFieldList->p[0])) {
+ TAOS_FIELD* p = taosArrayGet(pFieldList, 0);
+ if (!validateOneColumn(pCmd, p)) {
return TSDB_CODE_TSC_INVALID_SQL;
}
- tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]);
+ tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
} else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) {
if (tscGetNumOfColumns(pTableMeta) == TSDB_MIN_COLUMNS) { //
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg15);
}
- if (pAlterSQL->varList->nExpr > 1) {
+ size_t size = taosArrayGetSize(pAlterSQL->varList);
+ if (size > 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg16);
}
- tVariantListItem* pItem = &pAlterSQL->varList->a[0];
+ tVariantListItem* pItem = taosArrayGet(pAlterSQL->varList, 0);
SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER;
SStrToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen};
@@ -5186,28 +5349,35 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn
return TSDB_CODE_SUCCESS;
}
-static int32_t setKeepOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
+static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
const char* msg = "invalid number of options";
pMsg->daysToKeep = htonl(-1);
pMsg->daysToKeep1 = htonl(-1);
pMsg->daysToKeep2 = htonl(-1);
- tVariantList* pKeep = pCreateDb->keep;
+ SArray* pKeep = pCreateDb->keep;
if (pKeep != NULL) {
- switch (pKeep->nExpr) {
- case 1:
- pMsg->daysToKeep = htonl((int32_t)pKeep->a[0].pVar.i64Key);
+ size_t s = taosArrayGetSize(pKeep);
+ tVariantListItem* p0 = taosArrayGet(pKeep, 0);
+ switch (s) {
+ case 1: {
+ pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64Key);
+ }
break;
case 2: {
- pMsg->daysToKeep = htonl((int32_t)pKeep->a[0].pVar.i64Key);
- pMsg->daysToKeep1 = htonl((int32_t)pKeep->a[1].pVar.i64Key);
+ tVariantListItem* p1 = taosArrayGet(pKeep, 1);
+ pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64Key);
+ pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64Key);
break;
}
case 3: {
- pMsg->daysToKeep = htonl((int32_t)pKeep->a[0].pVar.i64Key);
- pMsg->daysToKeep1 = htonl((int32_t)pKeep->a[1].pVar.i64Key);
- pMsg->daysToKeep2 = htonl((int32_t)pKeep->a[2].pVar.i64Key);
+ tVariantListItem* p1 = taosArrayGet(pKeep, 1);
+ tVariantListItem* p2 = taosArrayGet(pKeep, 2);
+
+ pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64Key);
+ pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64Key);
+ pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64Key);
break;
}
default: { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); }
@@ -5217,7 +5387,7 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo*
return TSDB_CODE_SUCCESS;
}
-static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) {
+static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) {
const char* msg = "invalid time precision";
pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default
@@ -5241,7 +5411,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBIn
return TSDB_CODE_SUCCESS;
}
-static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
+static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
pMsg->maxTables = htonl(-1); // max tables can not be set anymore
pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize);
pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks);
@@ -5255,10 +5425,11 @@ static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
pMsg->replications = pCreateDb->replica;
pMsg->quorum = pCreateDb->quorum;
pMsg->ignoreExist = pCreateDb->ignoreExists;
+ pMsg->update = pCreateDb->update;
}
int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) {
- SCMCreateDbMsg* pMsg = (SCMCreateDbMsg*)(pCmd->payload);
+ SCreateDbMsg* pMsg = (SCreateDbMsg *)(pCmd->payload);
setCreateDBOption(pMsg, pCreateDbSql);
if (setKeepOption(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) {
@@ -5281,21 +5452,27 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau
if (pParentQueryInfo->groupbyExpr.numOfGroupCols > 0) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, subClauseIndex);
+ SSqlExpr* pExpr = NULL;
+
size_t size = taosArrayGetSize(pQueryInfo->exprList);
-
- SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, (int32_t)size - 1);
+ if (size > 0) {
+ pExpr = tscSqlExprGet(pQueryInfo, (int32_t)size - 1);
+ }
- if (pExpr->functionId != TSDB_FUNC_TAG) {
- STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
- int16_t columnInfo = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
- SColumnIndex index = {.tableIndex = 0, .columnIndex = columnInfo};
- SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
+ if (pExpr == NULL || pExpr->functionId != TSDB_FUNC_TAG) {
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pParentQueryInfo, tableIndex);
- int16_t type = pSchema[index.columnIndex].type;
- int16_t bytes = pSchema[index.columnIndex].bytes;
- char* name = pSchema[index.columnIndex].name;
-
- pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true);
+ int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
+
+ SSchema* pTagSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, colId);
+ int16_t colIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, colId);
+ SColumnIndex index = {.tableIndex = 0, .columnIndex = colIndex};
+
+ char* name = pTagSchema->name;
+ int16_t type = pTagSchema->type;
+ int16_t bytes = pTagSchema->bytes;
+
+ pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, getNewResColId(pQueryInfo), bytes, true);
pExpr->colInfo.flag = TSDB_COL_TAG;
// NOTE: tag column does not add to source column list
@@ -5598,7 +5775,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo
if (TSDB_COL_IS_TAG(pColIndex->flag)) {
SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex};
- SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true);
+ SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, getNewResColId(pQueryInfo), bytes, true);
memset(pExpr->aliasName, 0, sizeof(pExpr->aliasName));
tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName));
@@ -5761,7 +5938,7 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
SColumnIndex ind = {0};
SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT,
- tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false);
+ tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, getNewResColId(pQueryInfo), tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false);
const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name;
tstrncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName));
@@ -5770,7 +5947,7 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
}
// can only perform the parameters based on the macro definitation
-int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) {
+int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) {
char msg[512] = {0};
if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) {
@@ -5887,8 +6064,8 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p
SCreateTableSQL* pCreateTable = pInfo->pCreateTableInfo;
- tFieldList* pFieldList = pCreateTable->colInfo.pColumns;
- tFieldList* pTagList = pCreateTable->colInfo.pTagColumns;
+ SArray* pFieldList = pCreateTable->colInfo.pColumns;
+ SArray* pTagList = pCreateTable->colInfo.pTagColumns;
assert(pFieldList != NULL);
@@ -5910,18 +6087,23 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p
}
int32_t col = 0;
- for (; col < pFieldList->nField; ++col) {
- tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[col]);
+ size_t numOfFields = taosArrayGetSize(pFieldList);
+
+ for (; col < numOfFields; ++col) {
+ TAOS_FIELD* p = taosArrayGet(pFieldList, col);
+ tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
}
- pCmd->numOfCols = (int16_t)pFieldList->nField;
+ pCmd->numOfCols = (int16_t)numOfFields;
if (pTagList != NULL) { // create super table[optional]
- for (int32_t i = 0; i < pTagList->nField; ++i) {
- tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pTagList->p[i]);
+ size_t numOfTags = taosArrayGetSize(pTagList);
+ for (int32_t i = 0; i < numOfTags; ++i) {
+ TAOS_FIELD* p = taosArrayGet(pTagList, i);
+ tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
}
- pCmd->count = pTagList->nField;
+ pCmd->count =(int32_t) numOfTags;
}
return TSDB_CODE_SUCCESS;
@@ -5962,14 +6144,15 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
// get meter meta from mnode
tstrncpy(pCreateTable->usingInfo.tagdata.name, pStableMeterMetaInfo->name, sizeof(pCreateTable->usingInfo.tagdata.name));
- tVariantList* pList = pInfo->pCreateTableInfo->usingInfo.pTagVals;
+ SArray* pList = pInfo->pCreateTableInfo->usingInfo.pTagVals;
code = tscGetTableMeta(pSql, pStableMeterMetaInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
- if (tscGetNumOfTags(pStableMeterMetaInfo->pTableMeta) != pList->nExpr) {
+ size_t size = taosArrayGetSize(pList);
+ if (tscGetNumOfTags(pStableMeterMetaInfo->pTableMeta) != size) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
}
@@ -5983,18 +6166,19 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
}
int32_t ret = TSDB_CODE_SUCCESS;
- for (int32_t i = 0; i < pList->nExpr; ++i) {
+ for (int32_t i = 0; i < size; ++i) {
SSchema* pSchema = &pTagSchema[i];
-
+ tVariantListItem* pItem = taosArrayGet(pList, i);
+
char tagVal[TSDB_MAX_TAGS_LEN];
if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
- if (pList->a[i].pVar.nLen > pSchema->bytes) {
+ if (pItem->pVar.nLen > pSchema->bytes) {
tdDestroyKVRowBuilder(&kvRowBuilder);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
}
- ret = tVariantDump(&(pList->a[i].pVar), tagVal, pSchema->type, true);
+ ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true);
// check again after the convert since it may be converted from binary to nchar.
if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
@@ -6061,13 +6245,13 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
- tVariantList* pSrcMeterName = pInfo->pCreateTableInfo->pSelect->from;
- if (pSrcMeterName == NULL || pSrcMeterName->nExpr == 0) {
+ SArray* pSrcMeterName = pInfo->pCreateTableInfo->pSelect->from;
+ if (pSrcMeterName == NULL || taosArrayGetSize(pSrcMeterName) == 0) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
- tVariant* pVar = &pSrcMeterName->a[0].pVar;
- SStrToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING};
+ tVariantListItem* p1 = taosArrayGet(pSrcMeterName, 0);
+ SStrToken srcToken = {.z = p1->pVar.pz, .n = p1->pVar.nLen, .type = TK_STRING};
if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
@@ -6094,7 +6278,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
}
// set interval value
- if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
+ if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
} else {
if ((pQueryInfo->interval.interval > 0) &&
@@ -6132,7 +6316,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
- tVariantListItem* pItem = &pQuerySql->fillType->a[0];
+ tVariantListItem* pItem = taosArrayGet(pQuerySql->fillType, 0);
if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) {
if (!((strncmp(pItem->pVar.pz, "none", 4) == 0 && pItem->pVar.nLen == 4) ||
(strncmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4))) {
@@ -6147,7 +6331,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
}
int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
- assert(pQuerySql != NULL && (pQuerySql->from == NULL || pQuerySql->from->nExpr > 0));
+ assert(pQuerySql != NULL && (pQuerySql->from == NULL || taosArrayGetSize(pQuerySql->from) > 0));
const char* msg0 = "invalid table name";
const char* msg2 = "point interpolation query needs timestamp";
@@ -6189,19 +6373,21 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
return doLocalQueryProcess(pCmd, pQueryInfo, pQuerySql);
}
- if (pQuerySql->from->nExpr > TSDB_MAX_JOIN_TABLE_NUM * 2) {
+ size_t fromSize = taosArrayGetSize(pQuerySql->from);
+ if (fromSize > TSDB_MAX_JOIN_TABLE_NUM * 2) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
}
pQueryInfo->command = TSDB_SQL_SELECT;
- if (pQuerySql->from->nExpr > 4) {
+ if (fromSize > 4) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10);
}
// set all query tables, which are maybe more than one.
- for (int32_t i = 0; i < pQuerySql->from->nExpr; ) {
- tVariant* pTableItem = &pQuerySql->from->a[i].pVar;
+ for (int32_t i = 0; i < fromSize; ) {
+ tVariantListItem* item = taosArrayGet(pQuerySql->from, i);
+ tVariant* pTableItem = &item->pVar;
if (pTableItem->nType != TSDB_DATA_TYPE_BINARY) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0);
@@ -6226,21 +6412,21 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
return code;
}
- tVariant* pTableItem1 = &pQuerySql->from->a[i + 1].pVar;
- if (pTableItem1->nType != TSDB_DATA_TYPE_BINARY) {
+ tVariantListItem* p1 = taosArrayGet(pQuerySql->from, i);
+ if (p1->pVar.nType != TSDB_DATA_TYPE_BINARY) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11);
}
- SStrToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING};
+ SStrToken aliasName = {.z = p1->pVar.pz, .n = p1->pVar.nLen, .type = TK_STRING};
if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11);
}
// has no table alias name
- if (memcmp(pTableItem->pz, pTableItem1->pz, pTableItem1->nLen) == 0) {
+ if (memcmp(pTableItem->pz, p1->pVar.pz, p1->pVar.nLen) == 0) {
extractTableName(pTableMetaInfo1->name, pTableMetaInfo1->aliasName);
} else {
- tstrncpy(pTableMetaInfo1->aliasName, pTableItem1->pz, sizeof(pTableMetaInfo1->aliasName));
+ tstrncpy(pTableMetaInfo1->aliasName, p1->pVar.pz, sizeof(pTableMetaInfo1->aliasName));
}
code = tscGetTableMeta(pSql, pTableMetaInfo1);
@@ -6251,7 +6437,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
i += 2;
}
- assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr / 2);
+ assert(pQueryInfo->numOfTables == taosArrayGetSize(pQuerySql->from) / 2);
bool isSTable = false;
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
@@ -6285,12 +6471,12 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000;
}
} else { // set the time rang
- if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed.
+ if (taosArrayGetSize(pQuerySql->from) > 2) { // it is a join query, no wher clause is not allowed.
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query ");
}
}
- int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2);
+ int32_t joinQuery = (pQuerySql->from != NULL && taosArrayGetSize(pQuerySql->from) > 2);
if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
@@ -6302,7 +6488,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
}
// set interval value
- if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
+ if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
} else {
if ((pQueryInfo->interval.interval > 0) &&
@@ -6424,6 +6610,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
if (strcmp((*pExpr)->pSchema->name, p1->aliasName) == 0) {
(*pExpr)->pSchema->type = (uint8_t)p1->resType;
(*pExpr)->pSchema->bytes = p1->resBytes;
+ (*pExpr)->pSchema->colId = p1->resColId;
if (uid != NULL) {
*uid = p1->uid;
@@ -6473,7 +6660,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
(*pExpr)->_node.pRight = pRight;
SStrToken t = {.type = pSqlExpr->nSQLOptr};
- (*pExpr)->_node.optr = getBinaryExprOptr(&t);
+ (*pExpr)->_node.optr = convertOptr(&t);
assert((*pExpr)->_node.optr != 0);
diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c
index ac740555af649c8c20dc1fe51cc7ea48592e064c..fcc93ffadc4f7dc62b79fffe245947799a770b40 100644
--- a/src/client/src/tscSchemaUtil.c
+++ b/src/client/src/tscSchemaUtil.c
@@ -118,7 +118,7 @@ SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex)
}
// TODO for large number of columns, employ the binary search method
-SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
+SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
STableComInfo tinfo = tscGetTableInfo(pTableMeta);
for(int32_t i = 0; i < tinfo.numOfColumns + tinfo.numOfTags; ++i) {
@@ -130,17 +130,7 @@ SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
return NULL;
}
-struct SSchema tscGetTbnameColumnSchema() {
- struct SSchema s = {
- .colId = TSDB_TBNAME_COLUMN_INDEX,
- .type = TSDB_DATA_TYPE_BINARY,
- .bytes = TSDB_TABLE_NAME_LEN
- };
-
- strcpy(s.name, TSQL_TBNAME_L);
- return s;
-}
-static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo *vgroupInfo) {
+static void tscInitCorVgroupInfo(SCorVgroupInfo *corVgroupInfo, SVgroupInfo *vgroupInfo) {
corVgroupInfo->version = 0;
corVgroupInfo->inUse = 0;
corVgroupInfo->numOfEps = vgroupInfo->numOfEps;
@@ -166,7 +156,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
pTableMeta->id.tid = pTableMetaMsg->tid;
pTableMeta->id.uid = pTableMetaMsg->uid;
- SCMVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo;
+ SVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo;
pVgroupInfo->numOfEps = pTableMetaMsg->vgroup.numOfEps;
pVgroupInfo->vgId = pTableMetaMsg->vgroup.vgId;
@@ -197,28 +187,6 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
return pTableMeta;
}
-/**
- * the TableMeta data format in memory is as follows:
- *
- * +--------------------+
- * |STableMeta Body data| sizeof(STableMeta)
- * +--------------------+
- * |Schema data | numOfTotalColumns * sizeof(SSchema)
- * +--------------------+
- * |Tags data | tag_col_1.bytes + tag_col_2.bytes + ....
- * +--------------------+
- *
- * @param pTableMeta
- * @return
- */
-char* tsGetTagsValue(STableMeta* pTableMeta) {
- int32_t offset = 0;
-// int32_t numOfTotalCols = pTableMeta->numOfColumns + pTableMeta->numOfTags;
-// uint32_t offset = sizeof(STableMeta) + numOfTotalCols * sizeof(SSchema);
-
- return ((char*)pTableMeta + offset);
-}
-
// todo refactor
UNUSED_FUNC static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) {
for (int32_t i = 0; i < num; ++i) {
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index 581271f8455f655451a0e9291e131a7eb8ac2dec..e8b6cb284ee40a3b2200e8d54913dca4ca158309 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -48,7 +48,7 @@ static int32_t getWaitingTimeInterval(int32_t count) {
return initial * (2<<(count - 2));
}
-static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) {
+static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) {
assert(pSql != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0);
SRpcEpSet* pEpSet = &pSql->epSet;
@@ -100,7 +100,7 @@ void tscUpdateMgmtEpSet(SRpcEpSet *pEpSet) {
tscMgmtEpSet.epSet = *pEpSet;
taosCorEndWrite(&tscMgmtEpSet.version);
}
-static void tscDumpEpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) {
+static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) {
if (pVgroupInfo == NULL) { return;}
taosCorBeginRead(&pVgroupInfo->version);
int8_t inUse = pVgroupInfo->inUse;
@@ -117,14 +117,14 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) {
SSqlCmd *pCmd = &pObj->cmd;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;}
- SCMCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo;
+ SCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo;
taosCorBeginWrite(&pVgroupInfo->version);
tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse);
pVgroupInfo->inUse = pEpSet->inUse;
pVgroupInfo->numOfEps = pEpSet->numOfEps;
for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) {
- taosTFree(pVgroupInfo->epAddr[i].fqdn);
+ tfree(pVgroupInfo->epAddr[i].fqdn);
pVgroupInfo->epAddr[i].fqdn = strndup(pEpSet->fqdn[i], tListLen(pEpSet->fqdn[i]));
pVgroupInfo->epAddr[i].port = pEpSet->port[i];
}
@@ -150,7 +150,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
if (pObj == NULL) return;
if (pObj != pObj->signature) {
- tscError("heart beat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature);
+ tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature);
return;
}
@@ -158,12 +158,12 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
SSqlRes *pRes = &pSql->res;
if (code == 0) {
- SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp;
- SRpcEpSet * epSet = &pRsp->epSet;
+ SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp;
+ SRpcEpSet * epSet = &pRsp->epSet;
if (epSet->numOfEps > 0) {
tscEpSetHtons(epSet);
tscUpdateMgmtEpSet(epSet);
- }
+ }
pSql->pTscObj->connId = htonl(pRsp->connId);
@@ -175,44 +175,44 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId));
}
} else {
- tscDebug("heartbeat failed, code:%s", tstrerror(code));
+ tscDebug("%p heartbeat failed, code:%s", pObj->pHb, tstrerror(code));
}
if (pObj->pHb != NULL) {
int32_t waitingDuring = tsShellActivityTimer * 500;
- tscDebug("%p start heartbeat in %dms", pSql, waitingDuring);
+ tscDebug("%p send heartbeat in %dms", pSql, waitingDuring);
- taosTmrReset(tscProcessActivityTimer, waitingDuring, pObj, tscTmr, &pObj->pTimer);
+ taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer);
} else {
tscDebug("%p start to close tscObj:%p, not send heartbeat again", pSql, pObj);
}
}
void tscProcessActivityTimer(void *handle, void *tmrId) {
- STscObj *pObj = (STscObj *)handle;
- if (pObj == NULL || pObj->signature != pObj) {
- return;
- }
+ int64_t rid = (int64_t) handle;
+ STscObj *pObj = taosAcquireRef(tscRefId, rid);
+ if (pObj == NULL) return;
SSqlObj* pHB = pObj->pHb;
- if (pObj->pTimer != tmrId || pHB == NULL) {
- return;
- }
void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE));
if (p == NULL) {
tscWarn("%p HB object has been released already", pHB);
+ taosReleaseRef(tscRefId, pObj->rid);
return;
}
assert(*pHB->self == pHB);
+ pHB->retry = 0;
int32_t code = tscProcessSql(pHB);
taosCacheRelease(tscObjCache, (void**) &p, false);
if (code != TSDB_CODE_SUCCESS) {
tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code));
}
+
+ taosReleaseRef(tscRefId, rid);
}
int tscSendMsgToServer(SSqlObj *pSql) {
@@ -237,15 +237,11 @@ int tscSendMsgToServer(SSqlObj *pSql) {
.pCont = pMsg,
.contLen = pSql->cmd.payloadLen,
.ahandle = pSql,
- .handle = &pSql->pRpcCtx,
+ .handle = NULL,
.code = 0
};
- // NOTE: the rpc context should be acquired before sending data to server.
- // Otherwise, the pSql object may have been released already during the response function, which is
- // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely
- // cause crash.
- rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg);
+ rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid);
return TSDB_CODE_SUCCESS;
}
@@ -265,7 +261,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
SSqlCmd *pCmd = &pSql->cmd;
assert(*pSql->self == pSql);
- pSql->pRpcCtx = NULL;
+ pSql->rpcRid = -1;
if (pObj->signature != pObj) {
tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
@@ -361,7 +357,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen);
}
} else {
- pRes->pRsp = NULL;
+ tfree(pRes->pRsp);
}
/*
@@ -549,11 +545,28 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);
int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo));
-
- size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
+
+ size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
int32_t exprSize = (int32_t)(sizeof(SSqlFuncMsg) * numOfExprs);
-
- return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + 4096;
+
+ int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0;
+
+ int32_t tableSerialize = 0;
+ STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+ if (pTableMetaInfo->pVgroupTables != NULL) {
+ size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables);
+
+ int32_t totalTables = 0;
+ for (int32_t i = 0; i < numOfGroups; ++i) {
+ SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i);
+ totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList);
+ }
+
+ tableSerialize = totalTables * sizeof(STableIdInfo);
+ }
+
+ return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize +
+ tableSerialize + 4096;
}
static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) {
@@ -563,7 +576,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) {
- SCMVgroupInfo* pVgroupInfo = NULL;
+ SVgroupInfo* pVgroupInfo = NULL;
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
int32_t index = pTableMetaInfo->vgroupIndex;
assert(index >= 0);
@@ -685,7 +698,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->queryType = htonl(pQueryInfo->type);
size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo);
- pQueryMsg->numOfOutput = htons((int16_t)numOfOutput);
+ pQueryMsg->numOfOutput = htons((int16_t)numOfOutput); // this is the stage one output column number
// set column list ids
size_t numOfCols = taosArrayGetSize(pQueryInfo->colList);
@@ -747,12 +760,15 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_TSC_INVALID_SQL;
}
+ assert(pExpr->resColId < 0);
+
pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId);
pSqlFuncExpr->colInfo.colIndex = htons(pExpr->colInfo.colIndex);
pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag);
pSqlFuncExpr->functionId = htons(pExpr->functionId);
pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams);
+ pSqlFuncExpr->resColId = htons(pExpr->resColId);
pMsg += sizeof(SSqlFuncMsg);
for (int32_t j = 0; j < pExpr->numOfParams; ++j) {
@@ -770,7 +786,73 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pSqlFuncExpr = (SSqlFuncMsg *)pMsg;
}
-
+
+ if(tscIsSecondStageQuery(pQueryInfo)) {
+ size_t output = tscNumOfFields(pQueryInfo);
+ pQueryMsg->secondStageOutput = htonl((int32_t) output);
+
+ SSqlFuncMsg *pSqlFuncExpr1 = (SSqlFuncMsg *)pMsg;
+
+ for (int32_t i = 0; i < output; ++i) {
+ SInternalField* pField = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i);
+ SSqlExpr *pExpr = pField->pSqlExpr;
+ if (pExpr != NULL) {
+ if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) {
+ tscError("%p table schema is not matched with parsed sql", pSql);
+ return TSDB_CODE_TSC_INVALID_SQL;
+ }
+
+ pSqlFuncExpr1->colInfo.colId = htons(pExpr->colInfo.colId);
+ pSqlFuncExpr1->colInfo.colIndex = htons(pExpr->colInfo.colIndex);
+ pSqlFuncExpr1->colInfo.flag = htons(pExpr->colInfo.flag);
+
+ pSqlFuncExpr1->functionId = htons(pExpr->functionId);
+ pSqlFuncExpr1->numOfParams = htons(pExpr->numOfParams);
+ pMsg += sizeof(SSqlFuncMsg);
+
+ for (int32_t j = 0; j < pExpr->numOfParams; ++j) {
+ // todo add log
+ pSqlFuncExpr1->arg[j].argType = htons((uint16_t)pExpr->param[j].nType);
+ pSqlFuncExpr1->arg[j].argBytes = htons(pExpr->param[j].nLen);
+
+ if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) {
+ memcpy(pMsg, pExpr->param[j].pz, pExpr->param[j].nLen);
+ pMsg += pExpr->param[j].nLen;
+ } else {
+ pSqlFuncExpr1->arg[j].argValue.i64 = htobe64(pExpr->param[j].i64Key);
+ }
+ }
+
+ pSqlFuncExpr1 = (SSqlFuncMsg *)pMsg;
+ } else {
+ assert(pField->pArithExprInfo != NULL);
+ SExprInfo* pExprInfo = pField->pArithExprInfo;
+
+ pSqlFuncExpr1->colInfo.colId = htons(pExprInfo->base.colInfo.colId);
+ pSqlFuncExpr1->functionId = htons(pExprInfo->base.functionId);
+ pSqlFuncExpr1->numOfParams = htons(pExprInfo->base.numOfParams);
+ pMsg += sizeof(SSqlFuncMsg);
+
+ for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) {
+ // todo add log
+ pSqlFuncExpr1->arg[j].argType = htons((uint16_t)pExprInfo->base.arg[j].argType);
+ pSqlFuncExpr1->arg[j].argBytes = htons(pExprInfo->base.arg[j].argBytes);
+
+ if (pExprInfo->base.arg[j].argType == TSDB_DATA_TYPE_BINARY) {
+ memcpy(pMsg, pExprInfo->base.arg[j].argValue.pz, pExprInfo->base.arg[j].argBytes);
+ pMsg += pExprInfo->base.arg[j].argBytes;
+ } else {
+ pSqlFuncExpr1->arg[j].argValue.i64 = htobe64(pExprInfo->base.arg[j].argValue.i64);
+ }
+ }
+
+ pSqlFuncExpr1 = (SSqlFuncMsg *)pMsg;
+ }
+ }
+ } else {
+ pQueryMsg->secondStageOutput = 0;
+ }
+
// serialize the table info (sid, uid, tags)
pMsg = doSerializeTableInfo(pQueryMsg, pSql, pMsg);
@@ -797,7 +879,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
}
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
- for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
+ for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) {
*((int64_t *)pMsg) = htobe64(pQueryInfo->fillVal[i]);
pMsg += sizeof(pQueryInfo->fillVal[0]);
}
@@ -857,42 +939,25 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
// compressed ts block
pQueryMsg->tsOffset = htonl((int32_t)(pMsg - pCmd->payload));
- int32_t tsLen = 0;
- int32_t numOfBlocks = 0;
if (pQueryInfo->tsBuf != NULL) {
- int32_t vnodeId = htonl(pQueryMsg->head.vgId);
- STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, vnodeId);
- assert(QUERY_IS_JOIN_QUERY(pQueryInfo->type) && pBlockInfo != NULL); // this query should not be sent
-
- // todo refactor
- if (fseek(pQueryInfo->tsBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
- int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f));
- tscError("%p: fseek failed: %s", pSql, tstrerror(code));
- return code;
- }
-
- size_t s = fread(pMsg, 1, pBlockInfo->compLen, pQueryInfo->tsBuf->f);
- if (s != pBlockInfo->compLen) {
- int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f));
- tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
+ // note: here used the index instead of actual vnode id.
+ int32_t vnodeIndex = pTableMetaInfo->vgroupIndex;
+ int32_t code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsLen, &pQueryMsg->tsNumOfBlocks);
+ if (code != TSDB_CODE_SUCCESS) {
return code;
}
- pMsg += pBlockInfo->compLen;
- tsLen = pBlockInfo->compLen;
- numOfBlocks = pBlockInfo->numOfBlocks;
- }
+ pMsg += pQueryMsg->tsLen;
- pQueryMsg->tsLen = htonl(tsLen);
- pQueryMsg->tsNumOfBlocks = htonl(numOfBlocks);
- if (pQueryInfo->tsBuf != NULL) {
pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder);
+ pQueryMsg->tsLen = htonl(pQueryMsg->tsLen);
+ pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks);
}
int32_t msgLen = (int32_t)(pMsg - pCmd->payload);
- tscDebug("%p msg built success,len:%d bytes", pSql, msgLen);
+ tscDebug("%p msg built success, len:%d bytes", pSql, msgLen);
pCmd->payloadLen = msgLen;
pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY;
@@ -904,10 +969,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMCreateDbMsg);
+ pCmd->payloadLen = sizeof(SCreateDbMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DB;
- SCMCreateDbMsg *pCreateDbMsg = (SCMCreateDbMsg*)pCmd->payload;
+ SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload;
assert(pCmd->numOfClause == 1);
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
@@ -918,13 +983,13 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMCreateDnodeMsg);
+ pCmd->payloadLen = sizeof(SCreateDnodeMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMCreateDnodeMsg *pCreate = (SCMCreateDnodeMsg *)pCmd->payload;
+ SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload;
strncpy(pCreate->ep, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n);
pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DNODE;
@@ -934,13 +999,13 @@ int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMCreateAcctMsg);
+ pCmd->payloadLen = sizeof(SCreateAcctMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMCreateAcctMsg *pAlterMsg = (SCMCreateAcctMsg *)pCmd->payload;
+ SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload;
SStrToken *pName = &pInfo->pDCLInfo->user.user;
SStrToken *pPwd = &pInfo->pDCLInfo->user.passwd;
@@ -979,14 +1044,14 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMCreateUserMsg);
+ pCmd->payloadLen = sizeof(SCreateUserMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMCreateUserMsg *pAlterMsg = (SCMCreateUserMsg*)pCmd->payload;
+ SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload;
SUserInfo *pUser = &pInfo->pDCLInfo->user;
strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n);
@@ -1011,21 +1076,21 @@ int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMCfgDnodeMsg);
+ pCmd->payloadLen = sizeof(SCfgDnodeMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_CONFIG_DNODE;
return TSDB_CODE_SUCCESS;
}
int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMDropDbMsg);
+ pCmd->payloadLen = sizeof(SDropDbMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMDropDbMsg *pDropDbMsg = (SCMDropDbMsg*)pCmd->payload;
+ SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
tstrncpy(pDropDbMsg->db, pTableMetaInfo->name, sizeof(pDropDbMsg->db));
@@ -1055,13 +1120,13 @@ int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMDropDnodeMsg);
+ pCmd->payloadLen = sizeof(SDropDnodeMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload;
+ SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
tstrncpy(pDrop->ep, pTableMetaInfo->name, sizeof(pDrop->ep));
pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE;
@@ -1071,7 +1136,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMDropUserMsg);
+ pCmd->payloadLen = sizeof(SDropUserMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_USER;
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
@@ -1079,7 +1144,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload;
+ SDropUserMsg * pDropMsg = (SDropUserMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user));
@@ -1088,7 +1153,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMDropUserMsg);
+ pCmd->payloadLen = sizeof(SDropUserMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_ACCT;
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
@@ -1096,7 +1161,7 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload;
+ SDropUserMsg * pDropMsg = (SDropUserMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user));
@@ -1105,14 +1170,14 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMUseDbMsg);
+ pCmd->payloadLen = sizeof(SUseDbMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMUseDbMsg *pUseDbMsg = (SCMUseDbMsg*)pCmd->payload;
+ SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
strcpy(pUseDbMsg->db, pTableMetaInfo->name);
pCmd->msgType = TSDB_MSG_TYPE_CM_USE_DB;
@@ -1124,14 +1189,14 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
STscObj *pObj = pSql->pTscObj;
SSqlCmd *pCmd = &pSql->cmd;
pCmd->msgType = TSDB_MSG_TYPE_CM_SHOW;
- pCmd->payloadLen = sizeof(SCMShowMsg) + 100;
+ pCmd->payloadLen = sizeof(SShowMsg) + 100;
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMShowMsg *pShowMsg = (SCMShowMsg*)pCmd->payload;
+ SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
size_t nameLen = strlen(pTableMetaInfo->name);
@@ -1158,13 +1223,13 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pShowMsg->payloadLen = htons(pEpAddr->n);
}
- pCmd->payloadLen = sizeof(SCMShowMsg) + pShowMsg->payloadLen;
+ pCmd->payloadLen = sizeof(SShowMsg) + pShowMsg->payloadLen;
return TSDB_CODE_SUCCESS;
}
int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMKillQueryMsg);
+ pCmd->payloadLen = sizeof(SKillQueryMsg);
switch (pCmd->command) {
case TSDB_SQL_KILL_QUERY:
@@ -1276,8 +1341,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
- return minMsgSize() + sizeof(SCMAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) +
- TSDB_EXTRA_PAYLOAD_SIZE;
+ return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE;
}
int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
@@ -1296,7 +1360,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMAlterTableMsg *pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload;
+ SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload;
tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pAlterTableMsg->db);
strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name);
@@ -1345,10 +1409,10 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) {
int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMAlterDbMsg);
+ pCmd->payloadLen = sizeof(SAlterDbMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_ALTER_DB;
- SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload;
+ SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
tstrncpy(pAlterDbMsg->db, pTableMetaInfo->name, sizeof(pAlterDbMsg->db));
@@ -1473,14 +1537,14 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
STscObj *pObj = pSql->pTscObj;
SSqlCmd *pCmd = &pSql->cmd;
pCmd->msgType = TSDB_MSG_TYPE_CM_CONNECT;
- pCmd->payloadLen = sizeof(SCMConnectMsg);
+ pCmd->payloadLen = sizeof(SConnectMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMConnectMsg *pConnect = (SCMConnectMsg*)pCmd->payload;
+ SConnectMsg *pConnect = (SConnectMsg*)pCmd->payload;
// TODO refactor full_name
char *db; // ugly code to move the space
@@ -1502,11 +1566,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
- SCMTableInfoMsg* pInfoMsg = (SCMTableInfoMsg *)pCmd->payload;
+ STableInfoMsg *pInfoMsg = (STableInfoMsg *)pCmd->payload;
strcpy(pInfoMsg->tableId, pTableMetaInfo->name);
pInfoMsg->createFlag = htons(pSql->cmd.autoCreated ? 1 : 0);
- char* pMsg = (char*)pInfoMsg + sizeof(SCMTableInfoMsg);
+ char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg);
size_t len = htonl(pCmd->tagData.dataLen);
if (pSql->cmd.autoCreated) {
@@ -1525,7 +1589,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
/**
* multi table meta req pkg format:
- * | SMgmtHead | SCMMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ......
+ * | SMgmtHead | SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ......
* no used 4B
**/
int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
@@ -1543,16 +1607,16 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SMgmtHead *pMgmt = (SMgmtHead *)(pCmd->payload + tsRpcHeadSize);
memset(pMgmt->db, 0, TSDB_TABLE_FNAME_LEN); // server don't need the db
- SCMMultiTableInfoMsg *pInfoMsg = (SCMMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead));
+ SMultiTableInfoMsg *pInfoMsg = (SMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead));
pInfoMsg->numOfTables = htonl((int32_t)pCmd->count);
if (pCmd->payloadLen > 0) {
memcpy(pInfoMsg->tableIds, tmpData, pCmd->payloadLen);
}
- taosTFree(tmpData);
+ tfree(tmpData);
- pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SCMMultiTableInfoMsg);
+ pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SMultiTableInfoMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_TABLES_META;
assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize);
@@ -1597,12 +1661,12 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
char* pMsg = pCmd->payload;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-
- SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pMsg;
+
+ SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg;
pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables);
- pMsg += sizeof(SCMSTableVgroupMsg);
-
- for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
+ pMsg += sizeof(SSTableVgroupMsg);
+
+ for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i);
size_t size = sizeof(pTableMetaInfo->name);
tstrncpy(pMsg, pTableMetaInfo->name, size);
@@ -1635,14 +1699,17 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
numOfStreams++;
}
- int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SCMHeartBeatMsg) + 100;
+ int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100;
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) {
pthread_mutex_unlock(&pObj->mutex);
- tscError("%p failed to malloc for heartbeat msg", pSql);
+ tscError("%p failed to create heartbeat msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload;
+ // TODO the expired hb and client can not be identified by server till now.
+ SHeartBeatMsg *pHeartbeat = (SHeartBeatMsg *)pCmd->payload;
+ tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer));
+
pHeartbeat->numOfQueries = numOfQueries;
pHeartbeat->numOfStreams = numOfStreams;
@@ -1716,8 +1783,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name,
strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000);
-
- // todo handle out of memory case
+
if (pTableMetaInfo->pTableMeta == NULL) {
free(pTableMeta);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -1731,7 +1797,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
/**
* multi table meta rsp pkg format:
- * | STaosRsp | ieType | SCMMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2
+ * | STaosRsp | ieType | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2
* |...... 1B 1B 4B
**/
int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
@@ -1748,9 +1814,9 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
rsp++;
- SCMMultiTableInfoMsg *pInfo = (SCMMultiTableInfoMsg *)rsp;
+ SMultiTableInfoMsg *pInfo = (SMultiTableInfoMsg *)rsp;
totalNum = htonl(pInfo->numOfTables);
- rsp += sizeof(SCMMultiTableInfoMsg);
+ rsp += sizeof(SMultiTableInfoMsg);
for (i = 0; i < totalNum; i++) {
SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp;
@@ -1842,10 +1908,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
SSqlRes* pRes = &pSql->res;
// NOTE: the order of several table must be preserved.
- SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pRes->pRsp;
+ SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp;
pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables);
- char* pMsg = pRes->pRsp + sizeof(SCMSTableVgroupRspMsg);
-
+ char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg);
+
// master sqlObj locates in param
SSqlObj* parent = pSql->param;
assert(parent != NULL);
@@ -1857,18 +1923,18 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
SVgroupsMsg * pVgroupMsg = (SVgroupsMsg *) pMsg;
pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups);
- size_t size = sizeof(SCMVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg);
+ size_t size = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg);
- size_t vgroupsz = sizeof(SCMVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo);
+ size_t vgroupsz = sizeof(SVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo);
pInfo->vgroupList = calloc(1, vgroupsz);
assert(pInfo->vgroupList != NULL);
pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups;
for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) {
//just init, no need to lock
- SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j];
+ SVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j];
- SCMVgroupMsg *vmsg = &pVgroupMsg->vgroups[j];
+ SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j];
pVgroups->vgId = htonl(vmsg->vgId);
pVgroups->numOfEps = vmsg->numOfEps;
@@ -1890,10 +1956,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
* current process do not use the cache at all
*/
int tscProcessShowRsp(SSqlObj *pSql) {
- STableMetaMsg * pMetaMsg;
- SCMShowRsp *pShow;
- SSchema * pSchema;
- char key[20];
+ STableMetaMsg *pMetaMsg;
+ SShowRsp * pShow;
+ SSchema * pSchema;
+ char key[20];
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
@@ -1902,7 +1968,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
- pShow = (SCMShowRsp *)pRes->pRsp;
+ pShow = (SShowRsp *)pRes->pRsp;
pShow->qhandle = htobe64(pShow->qhandle);
pRes->qhandle = pShow->qhandle;
@@ -1949,13 +2015,13 @@ int tscProcessShowRsp(SSqlObj *pSql) {
SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index,
- pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes, false);
+ pTableSchema[i].type, pTableSchema[i].bytes, getNewResColId(pQueryInfo), pTableSchema[i].bytes, false);
}
pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput;
tscFieldInfoUpdateOffset(pQueryInfo);
- taosTFree(pTableMeta);
+ tfree(pTableMeta);
return 0;
}
@@ -1980,7 +2046,7 @@ static void createHBObj(STscObj* pObj) {
pSql->cmd.command = pQueryInfo->command;
if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) {
- taosTFree(pSql);
+ tfree(pSql);
return;
}
@@ -1995,11 +2061,12 @@ static void createHBObj(STscObj* pObj) {
}
int tscProcessConnectRsp(SSqlObj *pSql) {
- char temp[TSDB_TABLE_FNAME_LEN * 2];
STscObj *pObj = pSql->pTscObj;
SSqlRes *pRes = &pSql->res;
- SCMConnectRsp *pConnect = (SCMConnectRsp *)pRes->pRsp;
+ char temp[TSDB_TABLE_FNAME_LEN * 2] = {0};
+
+ SConnectRsp *pConnect = (SConnectRsp *)pRes->pRsp;
tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId)); // copy acctId from response
int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db);
@@ -2017,7 +2084,9 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
pObj->connId = htonl(pConnect->connId);
createHBObj(pObj);
- taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer);
+
+ //launch a timer to send heartbeat to maintain the connection and send status to mnode
+ taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pObj->rid, tscTmr, &pObj->pTimer);
return 0;
}
diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index 8cac9b3398a0bc4569b52acc7858bbc1e50de542..a7b859b294da689406911cb027d75b3c91149ccc 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -28,7 +28,6 @@
#include "tutil.h"
#include "ttimer.h"
#include "tscProfile.h"
-#include "ttimer.h"
static bool validImpl(const char* str, size_t maxsize) {
if (str == NULL) {
@@ -161,6 +160,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
registerSqlObj(pSql);
tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
+ pObj->rid = taosAddRef(tscRefId, pObj);
return pSql;
}
@@ -278,9 +278,9 @@ void taos_close(TAOS *taos) {
SSqlObj* pHb = pObj->pHb;
if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
- if (pHb->pRpcCtx != NULL) { // wait for rsp from dnode
- rpcCancelRequest(pHb->pRpcCtx);
- pHb->pRpcCtx = NULL;
+ if (pHb->rpcRid > 0) { // wait for rsp from dnode
+ rpcCancelRequest(pHb->rpcRid);
+ pHb->rpcRid = -1;
}
tscDebug("%p HB is freed", pHb);
@@ -296,7 +296,8 @@ void taos_close(TAOS *taos) {
}
tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
- tscCloseTscObj(pObj);
+
+ taosRemoveRef(tscRefId, pObj->rid);
}
void waitForQueryRsp(void *param, TAOS_RES *tres, int code) {
@@ -480,7 +481,7 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) {
assert(0);
for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
- tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i);
+ tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i, 0);
}
*rows = pRes->tsrow;
@@ -564,7 +565,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
pRes->rspType = 0;
pSql->subState.numOfSub = 0;
- taosTFree(pSql->pSubs);
+ tfree(pSql->pSubs);
assert(pSql->fp == NULL);
@@ -746,9 +747,9 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
assert(pSubObj->self == (SSqlObj**) p);
pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
- if (pSubObj->pRpcCtx != NULL) {
- rpcCancelRequest(pSubObj->pRpcCtx);
- pSubObj->pRpcCtx = NULL;
+ if (pSubObj->rpcRid > 0) {
+ rpcCancelRequest(pSubObj->rpcRid);
+ pSubObj->rpcRid = -1;
}
tscQueueAsyncRes(pSubObj);
@@ -773,7 +774,7 @@ void taos_stop_query(TAOS_RES *res) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
- assert(pSql->pRpcCtx == NULL);
+ assert(pSql->rpcRid <= 0);
tscKillSTableQuery(pSql);
} else {
if (pSql->cmd.command < TSDB_SQL_LOCAL) {
@@ -782,9 +783,9 @@ void taos_stop_query(TAOS_RES *res) {
* reset and freed in the processMsgFromServer function, and causes the invalid
* write problem for rpcCancelRequest.
*/
- if (pSql->pRpcCtx != NULL) {
- rpcCancelRequest(pSql->pRpcCtx);
- pSql->pRpcCtx = NULL;
+ if (pSql->rpcRid > 0) {
+ rpcCancelRequest(pSql->rpcRid);
+ pSql->rpcRid = -1;
}
tscQueueAsyncRes(pSql);
@@ -892,7 +893,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
if (sqlLen > tsMaxSQLStringLen) {
tscError("%p sql too long", pSql);
pRes->code = TSDB_CODE_TSC_INVALID_SQL;
- taosTFree(pSql);
+ tfree(pSql);
return pRes->code;
}
@@ -901,7 +902,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscError("%p failed to malloc sql string buffer", pSql);
tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(pSql), pObj);
- taosTFree(pSql);
+ tfree(pSql);
return pRes->code;
}
diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c
index 0f67911bbea992503979e5de019e4e10d3bf3c14..68c3bcae165050863cc4bf9c92a1510581531c3a 100644
--- a/src/client/src/tscStream.c
+++ b/src/client/src/tscStream.c
@@ -273,7 +273,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false);
tscFreeSqlResult(pSql);
- taosTFree(pSql->pSubs);
+ tfree(pSql->pSubs);
pSql->subState.numOfSub = 0;
pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList);
tscSetNextLaunchTimer(pStream, pSql);
@@ -617,6 +617,6 @@ void taos_close_stream(TAOS_STREAM *handle) {
pStream->pSql = NULL;
taos_free_result(pSql);
- taosTFree(pStream);
+ tfree(pStream);
}
}
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 794b7a068b4ede0e8a5ea5fd1f22a664a8d1ca3c..bc522d4007e24a01ed1bf61846f924c912cf3387 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -32,11 +32,26 @@ typedef struct SInsertSupporter {
static void freeJoinSubqueryObj(SSqlObj* pSql);
static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql);
-static bool tsCompare(int32_t order, int64_t left, int64_t right) {
+static int32_t tsCompare(int32_t order, int64_t left, int64_t right) {
+ if (left == right) {
+ return 0;
+ }
+
if (order == TSDB_ORDER_ASC) {
- return left < right;
+ return left < right? -1:1;
} else {
- return left > right;
+ return left > right? -1:1;
+ }
+}
+
+static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) {
+ while (tsBufNextPos(pTSBuf)) {
+ STSElem el1 = tsBufGetElem(pTSBuf);
+
+ int32_t res = tVariantCompare(el1.tag, tag1);
+ if (res != 0) { // it is a record with new tag
+ return;
+ }
}
}
@@ -51,10 +66,10 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
SLimitVal* pLimit = &pQueryInfo->limit;
int32_t order = pQueryInfo->order.order;
-
+
SQueryInfo* pSubQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[0]->cmd, 0);
SQueryInfo* pSubQueryInfo2 = tscGetQueryInfoDetail(&pSql->pSubs[1]->cmd, 0);
-
+
pSubQueryInfo1->tsBuf = output1;
pSubQueryInfo2->tsBuf = output2;
@@ -88,59 +103,74 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
int64_t numOfInput1 = 1;
int64_t numOfInput2 = 1;
- while (1) {
- STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf);
- STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf);
+ while(1) {
+ STSElem elem = tsBufGetElem(pSupporter1->pTSBuf);
-#ifdef _DEBUG_VIEW
- tscInfo("%" PRId64 ", tags:%"PRId64" \t %" PRId64 ", tags:%"PRId64, elem1.ts, elem1.tag.i64Key, elem2.ts, elem2.tag.i64Key);
-#endif
+ // no data in pSupporter1 anymore, jump out of loop
+ if (!tsBufIsValidElem(&elem)) {
+ break;
+ }
- int32_t res = tVariantCompare(elem1.tag, elem2.tag);
- if (res == -1 || (res == 0 && tsCompare(order, elem1.ts, elem2.ts))) {
- if (!tsBufNextPos(pSupporter1->pTSBuf)) {
- break;
- }
+ // find the data in supporter2 with the same tag value
+ STSElem e2 = tsBufFindElemStartPosByTag(pSupporter2->pTSBuf, elem.tag);
- numOfInput1++;
- } else if ((res > 0) || (res == 0 && tsCompare(order, elem2.ts, elem1.ts))) {
- if (!tsBufNextPos(pSupporter2->pTSBuf)) {
- break;
- }
+ /**
+ * there are elements in pSupporter2 with the same tag, continue
+ */
+ tVariant tag1 = {0};
+ tVariantAssign(&tag1, elem.tag);
- numOfInput2++;
- } else {
- /*
- * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the
- * final results which is acquired after the secondry merge of in the client.
- */
- if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
- if (win->skey > elem1.ts) {
- win->skey = elem1.ts;
- }
-
- if (win->ekey < elem1.ts) {
- win->ekey = elem1.ts;
+ if (tsBufIsValidElem(&e2)) {
+ while (1) {
+ STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf);
+ STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf);
+
+ // data with current are exhausted
+ if (!tsBufIsValidElem(&elem1) || tVariantCompare(elem1.tag, &tag1) != 0) {
+ break;
}
-
- tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts));
- tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts));
- } else {
- pLimit->offset -= 1;
- }
+ if (!tsBufIsValidElem(&elem2) || tVariantCompare(elem2.tag, &tag1) != 0) { // ignore all records with the same tag
+ skipRemainValue(pSupporter1->pTSBuf, &tag1);
+ break;
+ }
- if (!tsBufNextPos(pSupporter1->pTSBuf)) {
- break;
- }
+ /*
+ * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the
+ * final results which is acquired after the secondary merge of in the client.
+ */
+ int32_t re = tsCompare(order, elem1.ts, elem2.ts);
+ if (re < 0) {
+ tsBufNextPos(pSupporter1->pTSBuf);
+ numOfInput1++;
+ } else if (re > 0) {
+ tsBufNextPos(pSupporter2->pTSBuf);
+ numOfInput2++;
+ } else {
+ if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
+ if (win->skey > elem1.ts) {
+ win->skey = elem1.ts;
+ }
+
+ if (win->ekey < elem1.ts) {
+ win->ekey = elem1.ts;
+ }
+
+ tsBufAppend(output1, elem1.id, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts));
+ tsBufAppend(output2, elem2.id, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts));
+ } else {
+ pLimit->offset -= 1;//offset apply to projection?
+ }
- numOfInput1++;
+ tsBufNextPos(pSupporter1->pTSBuf);
+ numOfInput1++;
- if (!tsBufNextPos(pSupporter2->pTSBuf)) {
- break;
+ tsBufNextPos(pSupporter2->pTSBuf);
+ numOfInput2++;
+ }
}
-
- numOfInput2++;
+ } else { // no data in pSupporter2, ignore current data in pSupporter2
+ skipRemainValue(pSupporter1->pTSBuf, &tag1);
}
}
@@ -162,8 +192,9 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
TSKEY et = taosGetTimestampUs();
tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks "
- "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elasped time:%"PRId64" us", pSql, numOfInput1, numOfInput2, output1->numOfTotal,
- output1->numOfVnodes, win->skey, win->ekey, tsBufGetNumOfVnodes(output1), et - st);
+ "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elapsed time:%" PRId64 " us",
+ pSql, numOfInput1, numOfInput2, output1->numOfTotal, output1->numOfGroups, win->skey, win->ekey,
+ tsBufGetNumOfGroup(output1), et - st);
return output1->numOfTotal;
}
@@ -224,7 +255,7 @@ static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) {
pSupporter->pVgroupTables = NULL;
}
- taosTFree(pSupporter->pIdTagList);
+ tfree(pSupporter->pIdTagList);
tscTagCondRelease(&pSupporter->tagCond);
free(pSupporter);
}
@@ -248,6 +279,68 @@ static UNUSED_FUNC bool needSecondaryQuery(SQueryInfo* pQueryInfo) {
return false;
}
+static void filterVgroupTables(SQueryInfo* pQueryInfo, SArray* pVgroupTables) {
+ int32_t num = 0;
+ int32_t* list = NULL;
+ tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list);
+
+ // The virtual node, of which all tables are disqualified after the timestamp intersection,
+ // is removed to avoid next stage query.
+ // TODO: If tables from some vnodes are not qualified for next stage query, discard them.
+ for (int32_t k = 0; k < taosArrayGetSize(pVgroupTables);) {
+ SVgroupTableInfo* p = taosArrayGet(pVgroupTables, k);
+
+ bool found = false;
+ for (int32_t f = 0; f < num; ++f) {
+ if (p->vgInfo.vgId == list[f]) {
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ tscRemoveVgroupTableGroup(pVgroupTables, k);
+ } else {
+ k++;
+ }
+ }
+
+ assert(taosArrayGetSize(pVgroupTables) > 0);
+ TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
+
+ tfree(list);
+}
+
+static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupTables) {
+ int32_t num = 0;
+ int32_t* list = NULL;
+ tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list);
+
+ size_t numOfGroups = taosArrayGetSize(pVgroupTables);
+
+ SArray* pNew = taosArrayInit(num, sizeof(SVgroupTableInfo));
+
+ SVgroupTableInfo info;
+ for (int32_t i = 0; i < num; ++i) {
+ int32_t vnodeId = list[i];
+
+ for (int32_t j = 0; j < numOfGroups; ++j) {
+ SVgroupTableInfo* p1 = taosArrayGet(pVgroupTables, j);
+ if (p1->vgInfo.vgId == vnodeId) {
+ tscVgroupTableCopy(&info, p1);
+ break;
+ }
+ }
+
+ taosArrayPush(pNew, &info);
+ }
+
+ tfree(list);
+ TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
+
+ return pNew;
+}
+
/*
* launch secondary stage query to fetch the result that contains timestamp in set
*/
@@ -320,26 +413,27 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
pQueryInfo->colList = pSupporter->colList;
pQueryInfo->exprList = pSupporter->exprList;
pQueryInfo->fieldsInfo = pSupporter->fieldsInfo;
+ pQueryInfo->groupbyExpr = pSupporter->groupInfo;
- pSupporter->exprList = NULL;
- pSupporter->colList = NULL;
- memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo));
-
- SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
- assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
+ assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1);
- tscFieldInfoUpdateOffset(pNewQueryInfo);
+ tscFieldInfoUpdateOffset(pQueryInfo);
- STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0);
+ STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables;
+
+ pSupporter->exprList = NULL;
+ pSupporter->colList = NULL;
pSupporter->pVgroupTables = NULL;
+ memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo));
+ memset(&pSupporter->groupInfo, 0, sizeof(SSqlGroupbyExpr));
/*
* When handling the projection query, the offset value will be modified for table-table join, which is changed
* during the timestamp intersection.
*/
pSupporter->limit = pQueryInfo->limit;
- pNewQueryInfo->limit = pSupporter->limit;
+ pQueryInfo->limit = pSupporter->limit;
SColumnIndex index = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
SSchema* s = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, 0);
@@ -354,7 +448,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
tscAddSpecialColumnForSelect(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL);
tscPrintSelectClause(pNew, 0);
- tscFieldInfoUpdateOffset(pNewQueryInfo);
+ tscFieldInfoUpdateOffset(pQueryInfo);
pExpr = tscSqlExprGet(pQueryInfo, 0);
}
@@ -369,39 +463,21 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
pExpr->numOfParams = 1;
}
- int32_t num = 0;
- int32_t *list = NULL;
- tsBufGetVnodeIdList(pNewQueryInfo->tsBuf, &num, &list);
-
- if (pTableMetaInfo->pVgroupTables != NULL) {
- for(int32_t k = 0; k < taosArrayGetSize(pTableMetaInfo->pVgroupTables);) {
- SVgroupTableInfo* p = taosArrayGet(pTableMetaInfo->pVgroupTables, k);
-
- bool found = false;
- for(int32_t f = 0; f < num; ++f) {
- if (p->vgInfo.vgId == list[f]) {
- found = true;
- break;
- }
- }
-
- if (!found) {
- tscRemoveVgroupTableGroup(pTableMetaInfo->pVgroupTables, k);
- } else {
- k++;
- }
+ if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+ assert(pTableMetaInfo->pVgroupTables != NULL);
+ if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
+ SArray* p = buildVgroupTableByResult(pQueryInfo, pTableMetaInfo->pVgroupTables);
+ tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables);
+ pTableMetaInfo->pVgroupTables = p;
+ } else {
+ filterVgroupTables(pQueryInfo, pTableMetaInfo->pVgroupTables);
}
-
- assert(taosArrayGetSize(pTableMetaInfo->pVgroupTables) > 0);
- TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
}
- taosTFree(list);
-
- size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList);
+ size_t numOfCols = taosArrayGetSize(pQueryInfo->colList);
tscDebug("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s",
- pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, taosArrayGetSize(pNewQueryInfo->exprList),
- numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name);
+ pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, taosArrayGetSize(pQueryInfo->exprList),
+ numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name);
}
//prepare the subqueries object failed, abort
@@ -447,7 +523,7 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) {
assert(pSqlObj->subState.numOfRemain > 0);
if (atomic_sub_fetch_32(&pSqlObj->subState.numOfRemain, 1) <= 0) {
- tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code);
+ tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code));
freeJoinSubqueryObj(pSqlObj);
}
}
@@ -460,18 +536,36 @@ static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) {
}
-int32_t tscCompareTidTags(const void* p1, const void* p2) {
- const STidTags* t1 = (const STidTags*) varDataVal(p1);
- const STidTags* t2 = (const STidTags*) varDataVal(p2);
+int32_t tidTagsCompar(const void* p1, const void* p2) {
+ const STidTags* t1 = (const STidTags*) (p1);
+ const STidTags* t2 = (const STidTags*) (p2);
if (t1->vgId != t2->vgId) {
return (t1->vgId > t2->vgId) ? 1 : -1;
}
- if (t1->tid != t2->tid) {
- return (t1->tid > t2->tid) ? 1 : -1;
+ tstr* tag1 = (tstr*) t1->tag;
+ tstr* tag2 = (tstr*) t2->tag;
+
+ if (tag1->len != tag2->len) {
+ return (tag1->len > tag2->len)? 1: -1;
}
- return 0;
+
+ return strncmp(tag1->data, tag2->data, tag1->len);
+}
+
+int32_t tagValCompar(const void* p1, const void* p2) {
+ const STidTags* t1 = (const STidTags*) varDataVal(p1);
+ const STidTags* t2 = (const STidTags*) varDataVal(p2);
+
+ tstr* tag1 = (tstr*) t1->tag;
+ tstr* tag2 = (tstr*) t2->tag;
+
+ if (tag1->len != tag2->len) {
+ return (tag1->len > tag2->len)? 1: -1;
+ }
+
+ return memcmp(tag1->data, tag2->data, tag1->len);
}
void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) {
@@ -489,7 +583,7 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr
SVgroupTableInfo info = {{0}};
for (int32_t m = 0; m < pvg->numOfVgroups; ++m) {
if (tt->vgId == pvg->vgroups[m].vgId) {
- tscSCMVgroupInfoCopy(&info.vgInfo, &pvg->vgroups[m]);
+ tscSVgroupInfoCopy(&info.vgInfo, &pvg->vgroups[m]);
break;
}
}
@@ -497,17 +591,29 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr
vgTables = taosArrayInit(4, sizeof(STableIdInfo));
info.itemList = vgTables;
+
+ if (taosArrayGetSize(result) > 0) {
+ SVgroupTableInfo* prevGroup = taosArrayGet(result, taosArrayGetSize(result) - 1);
+ tscDebug("%p vgId:%d, tables:%"PRId64, pSql, prevGroup->vgInfo.vgId, taosArrayGetSize(prevGroup->itemList));
+ }
+
taosArrayPush(result, &info);
}
- tscDebug("%p tid:%d, uid:%"PRIu64",vgId:%d added for vnode query", pSql, tt->tid, tt->uid, tt->vgId)
STableIdInfo item = {.uid = tt->uid, .tid = tt->tid, .key = INT64_MIN};
taosArrayPush(vgTables, &item);
+
+ tscTrace("%p tid:%d, uid:%"PRIu64",vgId:%d added", pSql, tt->tid, tt->uid, tt->vgId);
prev = tt;
}
pTableMetaInfo->pVgroupTables = result;
pTableMetaInfo->vgroupIndex = 0;
+
+ if (taosArrayGetSize(result) > 0) {
+ SVgroupTableInfo* g = taosArrayGet(result, taosArrayGetSize(result) - 1);
+ tscDebug("%p vgId:%d, tables:%"PRId64, pSql, g->vgInfo.vgId, taosArrayGetSize(g->itemList));
+ }
}
static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) {
@@ -582,22 +688,24 @@ static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSq
}
static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray** s1, SArray** s2) {
- tscDebug("%p all subqueries retrieve complete, do tags match", pParentSql);
-
SJoinSupporter* p1 = pParentSql->pSubs[0]->param;
SJoinSupporter* p2 = pParentSql->pSubs[1]->param;
- qsort(p1->pIdTagList, p1->num, p1->tagSize, tscCompareTidTags);
- qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags);
+ tscDebug("%p all subquery retrieve complete, do tags match, %d, %d", pParentSql, p1->num, p2->num);
+
+ // sort according to the tag value
+ qsort(p1->pIdTagList, p1->num, p1->tagSize, tagValCompar);
+ qsort(p2->pIdTagList, p2->num, p2->tagSize, tagValCompar);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
- SSchema* pColSchema = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
+ SSchema* pColSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
// int16_t for padding
- *s1 = taosArrayInit(p1->num, p1->tagSize - sizeof(int16_t));
- *s2 = taosArrayInit(p2->num, p2->tagSize - sizeof(int16_t));
+ int32_t size = p1->tagSize - sizeof(int16_t);
+ *s1 = taosArrayInit(p1->num, size);
+ *s2 = taosArrayInit(p2->num, size);
if (!(checkForDuplicateTagVal(pColSchema, p1, pParentSql) && checkForDuplicateTagVal(pColSchema, p2, pParentSql))) {
return TSDB_CODE_QRY_DUP_JOIN_KEY;
@@ -625,6 +733,27 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar
}
}
+ // reorganize the tid-tag value according to both the vgroup id and tag values
+ // sort according to the tag value
+ size_t t1 = taosArrayGetSize(*s1);
+ size_t t2 = taosArrayGetSize(*s2);
+
+ qsort((*s1)->pData, t1, size, tidTagsCompar);
+ qsort((*s2)->pData, t2, size, tidTagsCompar);
+
+#if 0
+ for(int32_t k = 0; k < t1; ++k) {
+ STidTags* p = (*s1)->pData + size * k;
+ printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data);
+ }
+
+ for(int32_t k = 0; k < t1; ++k) {
+ STidTags* p = (*s2)->pData + size * k;
+ printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data);
+ }
+#endif
+
+ tscDebug("%p tags match complete, result: %"PRId64", %"PRId64, pParentSql, t1, t2);
return TSDB_CODE_SUCCESS;
}
@@ -724,11 +853,15 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
}
if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) { // no results,return.
+ assert(pParentSql->fp != tscJoinQueryCallback);
+
tscDebug("%p tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", pParentSql);
freeJoinSubqueryObj(pParentSql);
// set no result command
pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
+ assert(pParentSql->fp != tscJoinQueryCallback);
+
(*pParentSql->fp)(pParentSql->param, pParentSql, 0);
} else {
// proceed to for ts_comp query
@@ -744,10 +877,10 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo2, s2);
SSqlObj* psub1 = pParentSql->pSubs[0];
- ((SJoinSupporter*)psub1->param)->pVgroupTables = tscCloneVgroupTableInfo(pTableMetaInfo1->pVgroupTables);
+ ((SJoinSupporter*)psub1->param)->pVgroupTables = tscVgroupTableInfoClone(pTableMetaInfo1->pVgroupTables);
SSqlObj* psub2 = pParentSql->pSubs[1];
- ((SJoinSupporter*)psub2->param)->pVgroupTables = tscCloneVgroupTableInfo(pTableMetaInfo2->pVgroupTables);
+ ((SJoinSupporter*)psub2->param)->pVgroupTables = tscVgroupTableInfoClone(pTableMetaInfo2->pVgroupTables);
pParentSql->subState.numOfSub = 2;
pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub;
@@ -928,6 +1061,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
}
}
+ assert(pState->numOfRemain > 0);
if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pState->numOfRemain, pState->numOfSub);
return;
@@ -941,6 +1075,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
}
// update the records for each subquery in parent sql object.
+ bool stableQuery = tscIsTwoStageSTableQuery(pQueryInfo, 0);
for (int32_t i = 0; i < pState->numOfSub; ++i) {
if (pParentSql->pSubs[i] == NULL) {
tscDebug("%p %p sub:%d not retrieve data", pParentSql, NULL, i);
@@ -954,7 +1089,10 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
pRes1->numOfRows, pRes1->numOfTotal);
assert(pRes1->row < pRes1->numOfRows);
} else {
- pRes1->numOfClauseTotal += pRes1->numOfRows;
+ if (!stableQuery) {
+ pRes1->numOfClauseTotal += pRes1->numOfRows;
+ }
+
tscDebug("%p sub:%p index:%d numOfRows:%"PRId64" total:%"PRId64, pParentSql, pParentSql->pSubs[i], i,
pRes1->numOfRows, pRes1->numOfTotal);
}
@@ -964,7 +1102,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
tscBuildResFromSubqueries(pParentSql);
}
-void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
+void tscFetchDatablockForSubquery(SSqlObj* pSql) {
assert(pSql->subState.numOfSub >= 1);
int32_t numOfFetch = 0;
@@ -1026,11 +1164,22 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
if (numOfFetch <= 0) {
bool tryNextVnode = false;
- SSqlObj* pp = pSql->pSubs[0];
- SQueryInfo* pi = tscGetQueryInfoDetail(&pp->cmd, 0);
+ bool orderedPrjQuery = false;
+ for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
+ SSqlObj* pSub = pSql->pSubs[i];
+ if (pSub == NULL) {
+ continue;
+ }
+
+ SQueryInfo* p = tscGetQueryInfoDetail(&pSub->cmd, 0);
+ orderedPrjQuery = tscNonOrderedProjectionQueryOnSTable(p, 0);
+ if (orderedPrjQuery) {
+ break;
+ }
+ }
// get the number of subquery that need to retrieve the next vnode.
- if (tscNonOrderedProjectionQueryOnSTable(pi, 0)) {
+ if (orderedPrjQuery) {
for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
SSqlObj* pSub = pSql->pSubs[i];
if (pSub != NULL && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) {
@@ -1134,7 +1283,6 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) {
SSqlCmd* pCmd = &pSql->cmd;
SSqlRes* pRes = &pSql->res;
- tscDebug("%p all subquery response, retrieve data for subclause:%d", pSql, pCmd->clauseIndex);
// the column transfer support struct has been built
if (pRes->pColumnIndex != NULL) {
@@ -1230,21 +1378,23 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
return;
}
- // wait for the other subqueries response from vnode
- if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) {
- return;
+
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+
+ // In case of consequence query from other vnode, do not wait for other query response here.
+ if (!(pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0))) {
+ if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) {
+ return;
+ }
}
tscSetupOutputColumnIndex(pParentSql);
- STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
/**
* if the query is a continue query (vgroupIndex > 0 for projection query) for next vnode, do the retrieval of
* data instead of returning to its invoker
*/
if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
-// pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub; // reset the record value
-
pSql->fp = joinRetrieveFinalResCallback; // continue retrieve data
pSql->cmd.command = TSDB_SQL_FETCH;
tscProcessSql(pSql);
@@ -1313,6 +1463,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
+ pSupporter->groupInfo = pNewQueryInfo->groupbyExpr;
+ memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr));
+
pNew->cmd.numOfCols = 0;
pNewQueryInfo->interval.interval = 0;
pSupporter->limit = pNewQueryInfo->limit;
@@ -1333,17 +1486,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
assert(pTagCond->joinInfo.hasJoin);
int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid);
- SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
-
- // get the tag colId column index
- int32_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta);
- SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
- for(int32_t i = 0; i < numOfTags; ++i) {
- if (pSchema[i].colId == tagColId) {
- colIndex.columnIndex = i;
- break;
- }
- }
+ SSchema* s = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
+
+ colIndex.columnIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, tagColId);
int16_t bytes = 0;
int16_t type = 0;
@@ -1424,6 +1569,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) {
tscDebug("%p start subquery, total:%d", pSql, pQueryInfo->numOfTables);
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
+
SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, i);
if (pSupporter == NULL) { // failed to create support struct, abort current query
@@ -1478,8 +1624,8 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) {
SRetrieveSupport* pSupport = pSub->param;
- taosTFree(pSupport->localBuffer);
- taosTFree(pSupport);
+ tfree(pSupport->localBuffer);
+ tfree(pSupport);
taos_free_result(pSub);
}
@@ -1496,9 +1642,9 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
}
tExtMemBuffer ** pMemoryBuf = NULL;
- tOrderDescriptor *pDesc = NULL;
- SColumnModel * pModel = NULL;
-
+ tOrderDescriptor *pDesc = NULL;
+ SColumnModel *pModel = NULL;
+
pRes->qhandle = 0x1; // hack the qhandle check
const uint32_t nBufferSize = (1u << 16); // 64KB
@@ -1520,7 +1666,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
if (ret != 0) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscQueueAsyncRes(pSql);
- taosTFree(pMemoryBuf);
+ tfree(pMemoryBuf);
return ret;
}
@@ -1529,7 +1675,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
tscDebug("%p retrieved query data from %d vnode(s)", pSql, pState->numOfSub);
if (pSql->pSubs == NULL) {
- taosTFree(pSql->pSubs);
+ tfree(pSql->pSubs);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pState->numOfSub);
@@ -1554,19 +1700,19 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
trs->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage));
if (trs->localBuffer == NULL) {
tscError("%p failed to malloc buffer for local buffer, orderOfSub:%d, reason:%s", pSql, i, strerror(errno));
- taosTFree(trs);
+ tfree(trs);
break;
}
trs->subqueryIndex = i;
trs->pParentSql = pSql;
trs->pFinalColModel = pModel;
-
+
SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL);
if (pNew == NULL) {
tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno));
- taosTFree(trs->localBuffer);
- taosTFree(trs);
+ tfree(trs->localBuffer);
+ tfree(trs);
break;
}
@@ -1616,12 +1762,8 @@ static void tscFreeRetrieveSup(SSqlObj *pSql) {
}
tscDebug("%p start to free subquery supp obj:%p", pSql, trsupport);
-// int32_t index = trsupport->subqueryIndex;
-// SSqlObj *pParentSql = trsupport->pParentSql;
-
-// assert(pSql == pParentSql->pSubs[index]);
- taosTFree(trsupport->localBuffer);
- taosTFree(trsupport);
+ tfree(trsupport->localBuffer);
+ tfree(trsupport);
}
static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows);
@@ -1645,8 +1787,8 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in
SSqlObj *pParentSql = trsupport->pParentSql;
int32_t subqueryIndex = trsupport->subqueryIndex;
- STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
- SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
+ STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
+ SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]);
@@ -1810,7 +1952,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0);
tscClearInterpInfo(pPQueryInfo);
- tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, trsupport->pFinalColModel, pParentSql);
+ tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, trsupport->pFinalColModel, trsupport->pFFColModel, pParentSql);
tscDebug("%p build loser tree completed", pParentSql);
pParentSql->res.precision = pSql->res.precision;
@@ -1848,7 +1990,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0);
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
- SCMVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
+ SVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
@@ -1959,7 +2101,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1);
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
- SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex];
+ SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex];
// stable query killed or other subquery failed, all query stopped
if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
@@ -2021,7 +2163,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
pParentObj->res.code = pSql->res.code;
}
- taosTFree(pSupporter);
+ tfree(pSupporter);
if (atomic_sub_fetch_32(&pParentObj->subState.numOfRemain, 1) > 0) {
return;
@@ -2165,7 +2307,8 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) {
numOfRes = (int32_t)(MIN(numOfRes, remain));
}
- if (numOfRes == 0) {
+ if (numOfRes == 0) { // no result any more, free all subquery objects
+ freeJoinSubqueryObj(pSql);
return;
}
@@ -2223,7 +2366,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
- pRes->numOfCols = (int32_t)numOfExprs;
+ pRes->numOfCols = (int16_t)numOfExprs;
pRes->tsrow = calloc(numOfExprs, POINTER_BYTES);
pRes->buffer = calloc(numOfExprs, POINTER_BYTES);
@@ -2271,7 +2414,7 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF
}
}
-static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) {
+char *getArithemicInputSrc(void *param, const char *name, int32_t colId) {
SArithmeticSupport *pSupport = (SArithmeticSupport *) param;
int32_t index = -1;
@@ -2295,55 +2438,29 @@ TAOS_ROW doSetResultRowData(SSqlObj *pSql, bool finalResult) {
assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows);
if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker
- taosTFree(pRes->tsrow);
+ tfree(pRes->tsrow);
return pRes->tsrow;
}
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
size_t size = tscNumOfFields(pQueryInfo);
+ int32_t offset = 0;
+
for (int i = 0; i < size; ++i) {
- SInternalField* pSup = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i);
- if (pSup->pSqlExpr != NULL) {
- tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i);
- }
+ tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i, offset);
+ TAOS_FIELD *pField = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i);
+
+ offset += pField->bytes;
// primary key column cannot be null in interval query, no need to check
if (i == 0 && pQueryInfo->interval.interval > 0) {
continue;
}
- TAOS_FIELD *pField = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i);
if (pRes->tsrow[i] != NULL && pField->type == TSDB_DATA_TYPE_NCHAR) {
transferNcharData(pSql, i, pField);
}
-
- // calculate the result from several other columns
- if (pSup->pArithExprInfo != NULL) {
- if (pRes->pArithSup == NULL) {
- pRes->pArithSup = (SArithmeticSupport*)calloc(1, sizeof(SArithmeticSupport));
- }
-
- pRes->pArithSup->offset = 0;
- pRes->pArithSup->pArithExpr = pSup->pArithExprInfo;
- pRes->pArithSup->numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
- pRes->pArithSup->exprList = pQueryInfo->exprList;
- pRes->pArithSup->data = calloc(pRes->pArithSup->numOfCols, POINTER_BYTES);
-
- if (pRes->buffer[i] == NULL) {
- TAOS_FIELD* field = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
- pRes->buffer[i] = malloc(field->bytes);
- }
-
- for(int32_t k = 0; k < pRes->pArithSup->numOfCols; ++k) {
- SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k);
- pRes->pArithSup->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes;
- }
-
- tExprTreeCalcTraverse(pRes->pArithSup->pArithExpr->pExpr, 1, pRes->buffer[i], pRes->pArithSup,
- TSDB_ORDER_ASC, getArithemicInputSrc);
- pRes->tsrow[i] = (unsigned char*)pRes->buffer[i];
- }
}
pRes->row++; // index increase one-step
diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c
index 1895591e4ad278c9d54439c98d9e1139a5114d04..685b6b4e3f30b00dfc58cec6d3eb77cb6d548eb8 100644
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@@ -35,6 +35,7 @@ void * tscTmr;
void * tscQhandle;
void * tscCheckDiskUsageTmr;
int tsInsertHeadSize;
+int tscRefId = -1;
int tscNumOfThreads;
@@ -78,7 +79,7 @@ int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeCon
void taos_init_imp(void) {
- char temp[128];
+ char temp[128] = {0};
errno = TSDB_CODE_SUCCESS;
srand(taosGetTimestampSec());
@@ -102,8 +103,8 @@ void taos_init_imp(void) {
taosReadGlobalCfg();
taosCheckGlobalCfg();
- taosPrintGlobalCfg();
+ rpcInit();
tscDebug("starting to initialize TAOS client ...");
tscDebug("Local End Point is:%s", tsLocalEp);
}
@@ -145,29 +146,45 @@ void taos_init_imp(void) {
tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj");
}
+ tscRefId = taosOpenRef(200, tscCloseTscObj);
+
+ // in other language APIs, taos_cleanup is not available yet.
+ // So, to make sure taos_cleanup will be invoked to clean up the allocated
+ // resource to suppress the valgrind warning.
+ atexit(taos_cleanup);
tscDebug("client is initialized successfully");
}
void taos_init() { pthread_once(&tscinit, taos_init_imp); }
-void taos_cleanup() {
- if (tscMetaCache != NULL) {
- taosCacheCleanup(tscMetaCache);
- tscMetaCache = NULL;
+// this function may be called by user or system, or by both simultaneously.
+void taos_cleanup(void) {
+ tscDebug("start to cleanup client environment");
- taosCacheCleanup(tscObjCache);
- tscObjCache = NULL;
+ void* m = tscMetaCache;
+ if (m != NULL && atomic_val_compare_exchange_ptr(&tscMetaCache, m, 0) == m) {
+ taosCacheCleanup(m);
}
-
- if (tscQhandle != NULL) {
- taosCleanUpScheduler(tscQhandle);
- tscQhandle = NULL;
+
+ m = tscObjCache;
+ if (m != NULL && atomic_val_compare_exchange_ptr(&tscObjCache, m, 0) == m) {
+ taosCacheCleanup(m);
+ }
+
+ m = tscQhandle;
+ if (m != NULL && atomic_val_compare_exchange_ptr(&tscQhandle, m, 0) == m) {
+ taosCleanUpScheduler(m);
}
+ taosCloseRef(tscRefId);
taosCleanupKeywordsTable();
taosCloseLog();
-
- taosTmrCleanUp(tscTmr);
+ if (tscEmbedded == 0) rpcCleanup();
+
+ m = tscTmr;
+ if (m != NULL && atomic_val_compare_exchange_ptr(&tscTmr, m, 0) == m) {
+ taosTmrCleanUp(m);
+ }
}
static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index 07bd9d1b07b6190f6ab443bba2ea31ad8fc042de..27824fc1ffe01615a55b78056817733ac4c400c8 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -71,7 +71,8 @@ void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) {
}
bool tscQueryTags(SQueryInfo* pQueryInfo) {
- for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
+ int32_t numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo);
+ for (int32_t i = 0; i < numOfCols; ++i) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
int32_t functId = pExpr->functionId;
@@ -201,13 +202,9 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) {
bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) {
size_t size = tscSqlExprNumOfExprs(pQueryInfo);
-
for (int32_t i = 0; i < size; ++i) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
assert(pExpr != NULL);
-// if (pExpr == NULL) {
-// return false;
-// }
int32_t functionId = pExpr->functionId;
if (functionId == TSDB_FUNC_TAG) {
@@ -222,6 +219,24 @@ bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) {
return true;
}
+bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo) {
+ size_t numOfOutput = tscNumOfFields(pQueryInfo);
+ size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
+
+ if (numOfOutput == numOfExprs) {
+ return false;
+ }
+
+ for(int32_t i = 0; i < numOfOutput; ++i) {
+ SExprInfo* pExprInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i)->pArithExprInfo;
+ if (pExprInfo != NULL) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
bool tscIsTWAQuery(SQueryInfo* pQueryInfo) {
size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
for (int32_t i = 0; i < numOfExprs; ++i) {
@@ -245,7 +260,7 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) {
}
pQueryInfo->fillType = TSDB_FILL_NONE;
- taosTFree(pQueryInfo->fillVal);
+ tfree(pQueryInfo->fillVal);
}
int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
@@ -259,7 +274,7 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
// not enough memory
if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) {
- taosTFree(pRes->tsrow);
+ tfree(pRes->tsrow);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
return pRes->code;
}
@@ -271,24 +286,24 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
static void tscDestroyResPointerInfo(SSqlRes* pRes) {
if (pRes->buffer != NULL) { // free all buffers containing the multibyte string
for (int i = 0; i < pRes->numOfCols; i++) {
- taosTFree(pRes->buffer[i]);
+ tfree(pRes->buffer[i]);
}
pRes->numOfCols = 0;
}
- taosTFree(pRes->pRsp);
+ tfree(pRes->pRsp);
- taosTFree(pRes->tsrow);
- taosTFree(pRes->length);
- taosTFree(pRes->buffer);
+ tfree(pRes->tsrow);
+ tfree(pRes->length);
+ tfree(pRes->buffer);
- taosTFree(pRes->pGroupRec);
- taosTFree(pRes->pColumnIndex);
+ tfree(pRes->pGroupRec);
+ tfree(pRes->pColumnIndex);
if (pRes->pArithSup != NULL) {
- taosTFree(pRes->pArithSup->data);
- taosTFree(pRes->pArithSup);
+ tfree(pRes->pArithSup->data);
+ tfree(pRes->pArithSup);
}
pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free
@@ -305,11 +320,11 @@ static void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) {
freeQueryInfoImpl(pQueryInfo);
clearAllTableMetaInfo(pQueryInfo, (const char*)addr, removeFromCache);
- taosTFree(pQueryInfo);
+ tfree(pQueryInfo);
}
pCmd->numOfClause = 0;
- taosTFree(pCmd->pQueryInfo);
+ tfree(pCmd->pQueryInfo);
}
void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) {
@@ -338,34 +353,6 @@ void tscFreeSqlResult(SSqlObj* pSql) {
memset(&pSql->res, 0, sizeof(SSqlRes));
}
-void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
- if (pSql == NULL || pSql->signature != pSql) {
- return;
- }
-
- SSqlCmd* pCmd = &pSql->cmd;
- int32_t cmd = pCmd->command;
- if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
- cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
- tscRemoveFromSqlList(pSql);
- }
-
- // pSql->sqlstr will be used by tscBuildQueryStreamDesc
-// if (pObj->signature == pObj) {
- //pthread_mutex_lock(&pObj->mutex);
- taosTFree(pSql->sqlstr);
- //pthread_mutex_unlock(&pObj->mutex);
-// }
-
- tscFreeSqlResult(pSql);
-
- taosTFree(pSql->pSubs);
- pSql->subState.numOfSub = 0;
- pSql->self = 0;
-
- tscResetSqlCmdObj(pCmd, false);
-}
-
static void tscFreeSubobj(SSqlObj* pSql) {
if (pSql->subState.numOfSub == 0) {
return;
@@ -404,7 +391,7 @@ void tscFreeRegisteredSqlObj(void *pSql) {
tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref);
if (ref == 0) {
tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj);
- tscCloseTscObj(pTscObj);
+ taosRemoveRef(tscRefId, pTscObj->rid);
}
}
@@ -415,14 +402,14 @@ void tscFreeTableMetaHelper(void *pTableMeta) {
assert(numOfEps >= 0 && numOfEps <= TSDB_MAX_REPLICA);
for(int32_t i = 0; i < numOfEps; ++i) {
- taosTFree(p->vgroupInfo.epAddr[i].fqdn);
+ tfree(p->vgroupInfo.epAddr[i].fqdn);
}
int32_t numOfEps1 = p->corVgroupInfo.numOfEps;
assert(numOfEps1 >= 0 && numOfEps1 <= TSDB_MAX_REPLICA);
for(int32_t i = 0; i < numOfEps1; ++i) {
- taosTFree(p->corVgroupInfo.epAddr[i].fqdn);
+ tfree(p->corVgroupInfo.epAddr[i].fqdn);
}
}
@@ -434,22 +421,32 @@ void tscFreeSqlObj(SSqlObj* pSql) {
tscDebug("%p start to free sqlObj", pSql);
pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
+
tscFreeSubobj(pSql);
- tscPartiallyFreeSqlObj(pSql);
+ SSqlCmd* pCmd = &pSql->cmd;
+ int32_t cmd = pCmd->command;
+ if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
+ cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
+ tscRemoveFromSqlList(pSql);
+ }
pSql->signature = NULL;
pSql->fp = NULL;
-
- SSqlCmd* pCmd = &pSql->cmd;
+ tfree(pSql->sqlstr);
+
+ tfree(pSql->pSubs);
+ pSql->subState.numOfSub = 0;
+ pSql->self = 0;
+
+ tscFreeSqlResult(pSql);
+ tscResetSqlCmdObj(pCmd, false);
memset(pCmd->payload, 0, (size_t)pCmd->allocSize);
- taosTFree(pCmd->payload);
+ tfree(pCmd->payload);
pCmd->allocSize = 0;
- taosTFree(pSql->sqlstr);
tsem_destroy(&pSql->rspSem);
-
free(pSql);
}
@@ -458,15 +455,15 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
return;
}
- taosTFree(pDataBlock->pData);
- taosTFree(pDataBlock->params);
+ tfree(pDataBlock->pData);
+ tfree(pDataBlock->params);
// free the refcount for metermeta
if (pDataBlock->pTableMeta != NULL) {
taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false);
}
- taosTFree(pDataBlock);
+ tfree(pDataBlock);
}
SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes,
@@ -741,7 +738,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
taosHashCleanup(pVnodeDataBlockHashList);
tscDestroyBlockArrayList(pVnodeDataBlockList);
- taosTFree(dataBuf->pData);
+ tfree(dataBuf->pData);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
@@ -786,8 +783,8 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
}
// TODO: all subqueries should be freed correctly before close this connection.
-void tscCloseTscObj(STscObj* pObj) {
- assert(pObj != NULL);
+void tscCloseTscObj(void *param) {
+ STscObj *pObj = param;
pObj->signature = NULL;
taosTmrStopA(&(pObj->pTimer));
@@ -801,7 +798,7 @@ void tscCloseTscObj(STscObj* pObj) {
pthread_mutex_destroy(&pObj->mutex);
tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p);
- taosTFree(pObj);
+ tfree(pObj);
}
bool tscIsInsertData(char* sqlstr) {
@@ -876,28 +873,11 @@ void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) {
SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
pExpr->offset = 0;
-
- for (int32_t i = 1; i < numOfExprs; ++i) {
- SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprList, i - 1);
- SSqlExpr* p = taosArrayGetP(pQueryInfo->exprList, i);
-
- p->offset = prev->offset + prev->resBytes;
- }
-}
-void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) {
- if (tscSqlExprNumOfExprs(pQueryInfo) == 0) {
- return;
- }
-
- SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0);
- pExpr->offset = 0;
-
- size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
for (int32_t i = 1; i < numOfExprs; ++i) {
SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprList, i - 1);
SSqlExpr* p = taosArrayGetP(pQueryInfo->exprList, i);
-
+
p->offset = prev->offset + prev->resBytes;
}
}
@@ -965,18 +945,26 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) {
if (pInfo->pArithExprInfo != NULL) {
tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL);
- taosTFree(pInfo->pArithExprInfo);
+
+ SSqlFuncMsg* pFuncMsg = &pInfo->pArithExprInfo->base;
+ for(int32_t j = 0; j < pFuncMsg->numOfParams; ++j) {
+ if (pFuncMsg->arg[j].argType == TSDB_DATA_TYPE_BINARY) {
+ tfree(pFuncMsg->arg[j].argValue.pz);
+ }
+ }
+
+ tfree(pInfo->pArithExprInfo);
}
}
taosArrayDestroy(pFieldInfo->internalField);
- taosTFree(pFieldInfo->final);
+ tfree(pFieldInfo->final);
memset(pFieldInfo, 0, sizeof(SFieldInfo));
}
static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
- int16_t size, int16_t interSize, int32_t colType) {
+ int16_t size, int16_t resColId, int16_t interSize, int32_t colType) {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex);
SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr));
@@ -1009,8 +997,9 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol
pExpr->resType = type;
pExpr->resBytes = size;
+ pExpr->resColId = resColId;
pExpr->interBytes = interSize;
-
+
if (pTableMetaInfo->pTableMeta) {
pExpr->uid = pTableMetaInfo->pTableMeta->id.uid;
}
@@ -1019,20 +1008,20 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol
}
SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
- int16_t size, int16_t interSize, bool isTagCol) {
+ int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) {
int32_t num = (int32_t)taosArrayGetSize(pQueryInfo->exprList);
if (index == num) {
- return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol);
+ return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol);
}
- SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol);
+ SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol);
taosArrayInsert(pQueryInfo->exprList, index, &pExpr);
return pExpr;
}
SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
- int16_t size, int16_t interSize, bool isTagCol) {
- SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol);
+ int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) {
+ SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol);
taosArrayPush(pQueryInfo->exprList, &pExpr);
return pExpr;
}
@@ -1060,16 +1049,14 @@ size_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo) {
return taosArrayGetSize(pQueryInfo->exprList);
}
-void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex) {
- if (pExpr == NULL || argument == NULL || bytes == 0) {
- return;
- }
+void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes) {
+ assert (pExpr != NULL || argument != NULL || bytes != 0);
// set parameter value
// transfer to tVariant from byte data/no ascii data
tVariantCreateFromBinary(&pExpr->param[pExpr->numOfParams], argument, bytes, type);
-
pExpr->numOfParams += 1;
+
assert(pExpr->numOfParams <= 3);
}
@@ -1086,7 +1073,7 @@ void* sqlExprDestroy(SSqlExpr* pExpr) {
tVariantDestroy(&pExpr->param[i]);
}
- taosTFree(pExpr);
+ tfree(pExpr);
return NULL;
}
@@ -1186,11 +1173,11 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) {
static void destroyFilterInfo(SColumnFilterInfo* pFilterInfo, int32_t numOfFilters) {
for(int32_t i = 0; i < numOfFilters; ++i) {
if (pFilterInfo[i].filterstr) {
- taosTFree(pFilterInfo[i].pz);
+ tfree(pFilterInfo[i].pz);
}
}
- taosTFree(pFilterInfo);
+ tfree(pFilterInfo);
}
SColumn* tscColumnClone(const SColumn* src) {
@@ -1256,8 +1243,7 @@ void tscColumnListDestroy(SArray* pColumnList) {
*
*/
static int32_t validateQuoteToken(SStrToken* pToken) {
- strdequote(pToken->z);
- pToken->n = (uint32_t)strtrim(pToken->z);
+ tscDequoteAndTrimToken(pToken);
int32_t k = tSQLGetToken(pToken->z, &pToken->type);
@@ -1272,8 +1258,6 @@ static int32_t validateQuoteToken(SStrToken* pToken) {
}
void tscDequoteAndTrimToken(SStrToken* pToken) {
- assert(pToken->type == TK_STRING);
-
uint32_t first = 0, last = pToken->n;
// trim leading spaces
@@ -1385,7 +1369,8 @@ int32_t tscValidateName(SStrToken* pToken) {
} else {
pStr[firstPartLen] = TS_PATH_DELIMITER[0];
memmove(&pStr[firstPartLen + 1], pToken->z, pToken->n);
- pStr[firstPartLen + sizeof(TS_PATH_DELIMITER[0]) + pToken->n] = 0;
+ uint32_t offset = (uint32_t)(pToken->z - (pStr + firstPartLen + 1));
+ memset(pToken->z + pToken->n - offset, ' ', offset);
}
pToken->n += (firstPartLen + sizeof(TS_PATH_DELIMITER[0]));
pToken->z = pStr;
@@ -1478,7 +1463,7 @@ void tscTagCondRelease(STagCond* pTagCond) {
size_t s = taosArrayGetSize(pTagCond->pCond);
for (int32_t i = 0; i < s; ++i) {
SCond* p = taosArrayGet(pTagCond->pCond, i);
- taosTFree(p->cond);
+ tfree(p->cond);
}
taosArrayDestroy(pTagCond->pCond);
@@ -1624,6 +1609,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) {
pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES);
pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES);
pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX;
+ pQueryInfo->resColumnId= -1000;
}
int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) {
@@ -1665,11 +1651,12 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) {
if (pQueryInfo->groupbyExpr.columnInfo != NULL) {
taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo);
pQueryInfo->groupbyExpr.columnInfo = NULL;
+ pQueryInfo->groupbyExpr.numOfGroupCols = 0;
}
pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf);
- taosTFree(pQueryInfo->fillVal);
+ tfree(pQueryInfo->fillVal);
}
void tscClearSubqueryInfo(SSqlCmd* pCmd) {
@@ -1689,7 +1676,7 @@ void tscFreeVgroupTableInfo(SArray* pVgroupTables) {
SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i);
for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
- taosTFree(pInfo->vgInfo.epAddr[j].fqdn);
+ tfree(pInfo->vgInfo.epAddr[j].fqdn);
}
taosArrayDestroy(pInfo->itemList);
@@ -1706,14 +1693,25 @@ void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) {
SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTable, index);
for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
- taosTFree(pInfo->vgInfo.epAddr[j].fqdn);
+ tfree(pInfo->vgInfo.epAddr[j].fqdn);
}
taosArrayDestroy(pInfo->itemList);
taosArrayRemove(pVgroupTable, index);
}
-SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) {
+void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) {
+ memset(info, 0, sizeof(SVgroupTableInfo));
+
+ info->vgInfo = pInfo->vgInfo;
+ for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
+ info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn);
+ }
+
+ info->itemList = taosArrayClone(pInfo->itemList);
+}
+
+SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) {
if (pVgroupTables == NULL) {
return NULL;
}
@@ -1724,14 +1722,8 @@ SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) {
SVgroupTableInfo info;
for (size_t i = 0; i < num; i++) {
SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i);
- memset(&info, 0, sizeof(SVgroupTableInfo));
-
- info.vgInfo = pInfo->vgInfo;
- for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
- info.vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn);
- }
+ tscVgroupTableCopy(&info, pInfo);
- info.itemList = taosArrayClone(pInfo->itemList);
taosArrayPush(pa, &info);
}
@@ -1739,7 +1731,7 @@ SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) {
}
void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) {
- tscDebug("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables);
+ tscDebug("%p unref %d tables in the tableMeta cache", address, pQueryInfo->numOfTables);
for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
@@ -1749,7 +1741,7 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool rem
free(pTableMetaInfo);
}
- taosTFree(pQueryInfo->pTableMetaInfo);
+ tfree(pQueryInfo->pTableMetaInfo);
}
STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta,
@@ -1779,6 +1771,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList);
}
+ // TODO handle malloc failure
pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES);
if (pTableMetaInfo->tagColList == NULL) {
return NULL;
@@ -1788,7 +1781,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1);
}
- pTableMetaInfo->pVgroupTables = tscCloneVgroupTableInfo(pVgroupTables);
+ pTableMetaInfo->pVgroupTables = tscVgroupTableInfoClone(pVgroupTables);
pQueryInfo->numOfTables += 1;
return pTableMetaInfo;
@@ -2155,6 +2148,21 @@ int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) {
}
}
+int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId) {
+ int32_t numOfTags = tscGetNumOfTags(pTableMeta);
+
+ SSchema* pSchema = tscGetTableTagSchema(pTableMeta);
+ for(int32_t i = 0; i < numOfTags; ++i) {
+ if (pSchema[i].colId == colId) {
+ return i;
+ }
+ }
+
+ // can not reach here
+ assert(0);
+ return INT16_MIN;
+}
+
bool tscIsUpdateQuery(SSqlObj* pSql) {
if (pSql == NULL || pSql->signature != pSql) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
@@ -2332,7 +2340,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) {
pRes->numOfTotal = num;
- taosTFree(pSql->pSubs);
+ tfree(pSql->pSubs);
pSql->subState.numOfSub = 0;
pSql->fp = fp;
@@ -2424,7 +2432,7 @@ SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) {
return NULL;
}
- size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups;
+ size_t size = sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupList->numOfVgroups;
SVgroupsInfo* pNew = calloc(1, size);
if (pNew == NULL) {
return NULL;
@@ -2433,9 +2441,9 @@ SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) {
pNew->numOfVgroups = vgroupList->numOfVgroups;
for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) {
- SCMVgroupInfo* pNewVInfo = &pNew->vgroups[i];
+ SVgroupInfo* pNewVInfo = &pNew->vgroups[i];
- SCMVgroupInfo* pvInfo = &vgroupList->vgroups[i];
+ SVgroupInfo* pvInfo = &vgroupList->vgroups[i];
pNewVInfo->vgId = pvInfo->vgId;
pNewVInfo->numOfEps = pvInfo->numOfEps;
@@ -2454,21 +2462,22 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) {
}
for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) {
- SCMVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i];
+ SVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i];
for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) {
- taosTFree(pVgroupInfo->epAddr[j].fqdn);
+ tfree(pVgroupInfo->epAddr[j].fqdn);
}
}
- taosTFree(vgroupList);
+ tfree(vgroupList);
return NULL;
}
-void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src) {
+void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src) {
dst->vgId = src->vgId;
dst->numOfEps = src->numOfEps;
for(int32_t i = 0; i < dst->numOfEps; ++i) {
+ tfree(dst->epAddr[i].fqdn);
dst->epAddr[i].port = src->epAddr[i].port;
dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn);
}
diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h
index cc4afeb3f850785ffb18a972961668f91061c6a4..8d4949d9b4364fd1e8c70cbb883aa56468724108 100644
--- a/src/common/inc/tdataformat.h
+++ b/src/common/inc/tdataformat.h
@@ -80,7 +80,7 @@ typedef struct {
#define schemaFLen(s) ((s)->flen)
#define schemaVLen(s) ((s)->vlen)
#define schemaColAt(s, i) ((s)->columns + i)
-#define tdFreeSchema(s) taosTFree((s))
+#define tdFreeSchema(s) tfree((s))
STSchema *tdDupSchema(STSchema *pSchema);
int tdEncodeSchema(void **buf, STSchema *pSchema);
@@ -119,6 +119,33 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes);
STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
+// ----------------- Semantic timestamp key definition
+typedef uint64_t TKEY;
+
+#define TKEY_INVALID UINT64_MAX
+#define TKEY_NULL TKEY_INVALID
+#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
+#define TKEY_DELETE_FLAG (((TKEY)1) << 62)
+#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG))
+
+#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
+#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0)
+#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG)
+#define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
+#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
+
+static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) {
+ TSKEY key1 = tdGetKey(*(TKEY *)tkey1);
+ TSKEY key2 = tdGetKey(*(TKEY *)tkey2);
+
+ if (key1 < key2) {
+ return -1;
+ } else if (key1 > key2) {
+ return 1;
+ } else {
+ return 0;
+ }
+}
// ----------------- Data row structure
/* A data row, the format is like below:
@@ -129,6 +156,8 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
* +----------+----------+---------------------------------+---------------------------------+
* | len | sversion | First part | Second part |
* +----------+----------+---------------------------------+---------------------------------+
+ *
+ * NOTE: timestamp in this row structure is TKEY instead of TSKEY
*/
typedef void *SDataRow;
@@ -137,11 +166,13 @@ typedef void *SDataRow;
#define dataRowLen(r) (*(uint16_t *)(r))
#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))
#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
-#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
+#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r)))
+#define dataRowKey(r) tdGetKey(dataRowTKey(r))
#define dataRowSetLen(r, l) (dataRowLen(r) = (l))
#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v))
#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
+#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r))
SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
void tdFreeDataRow(SDataRow row);
@@ -154,16 +185,18 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i
int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row));
- switch (type) {
- case TSDB_DATA_TYPE_BINARY:
- case TSDB_DATA_TYPE_NCHAR:
- *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
- memcpy(ptr, value, varDataTLen(value));
- dataRowLen(row) += varDataTLen(value);
- break;
- default:
+ if (IS_VAR_DATA_TYPE(type)) {
+ *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
+ memcpy(ptr, value, varDataTLen(value));
+ dataRowLen(row) += varDataTLen(value);
+ } else {
+ if (offset == 0) {
+ ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP);
+ TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
+ memcpy(POINTER_SHIFT(row, toffset), (void *)(&tvalue), TYPE_BYTES[type]);
+ } else {
memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
- break;
+ }
}
return 0;
@@ -171,12 +204,10 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i
// NOTE: offset here including the header size
static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
- switch (type) {
- case TSDB_DATA_TYPE_BINARY:
- case TSDB_DATA_TYPE_NCHAR:
- return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
- default:
- return POINTER_SHIFT(row, offset);
+ if (IS_VAR_DATA_TYPE(type)) {
+ return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
+ } else {
+ return POINTER_SHIFT(row, offset);
}
}
@@ -196,7 +227,6 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints);
-void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows);
void dataColSetOffset(SDataCol *pCol, int nEle);
bool isNEleNull(SDataCol *pCol, int nEle);
@@ -204,28 +234,20 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
// Get the data pointer from a column-wised data
static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
- switch (pCol->type) {
- case TSDB_DATA_TYPE_BINARY:
- case TSDB_DATA_TYPE_NCHAR:
- return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
- break;
-
- default:
- return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
- break;
+ if (IS_VAR_DATA_TYPE(pCol->type)) {
+ return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
+ } else {
+ return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
}
}
static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
ASSERT(rows > 0);
- switch (pDataCol->type) {
- case TSDB_DATA_TYPE_BINARY:
- case TSDB_DATA_TYPE_NCHAR:
- return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
- break;
- default:
- return TYPE_BYTES[pDataCol->type] * rows;
+ if (IS_VAR_DATA_TYPE(pDataCol->type)) {
+ return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
+ } else {
+ return TYPE_BYTES[pDataCol->type] * rows;
}
}
@@ -243,9 +265,14 @@ typedef struct {
} SDataCols;
#define keyCol(pCols) (&((pCols)->cols[0])) // Key column
-#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)]
-#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
-#define dataColsKeyLast(pCols) ((pCols->numOfRows == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
+#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]
+#define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx))
+#define dataColsTKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0))
+#define dataColsKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0))
+#define dataColsTKeyLast(pCols) \
+ (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1))
+#define dataColsKeyLast(pCols) \
+ (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
void tdResetDataCols(SDataCols *pCols);
@@ -253,10 +280,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
void tdFreeDataCols(SDataCols *pCols);
void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
-void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!!
int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
-void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
- int limit2, int tRows);
// ----------------- K-V data row structure
/*
@@ -284,7 +308,7 @@ typedef struct {
#define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r))
#define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset)
#define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i))
-#define kvRowFree(r) taosTFree(r)
+#define kvRowFree(r) tfree(r)
#define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r))
SKVRow tdKVRowDup(SKVRow row);
diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h
index a30a94e35a3fd19bee78b38790af0fd12e44eb56..d3bd29a016a274338c1f6713770f4fa783d00605 100644
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@@ -44,6 +44,7 @@ extern int32_t tsMaxShellConns;
extern int32_t tsShellActivityTimer;
extern uint32_t tsMaxTmrCtrl;
extern float tsNumOfThreadsPerCore;
+extern int32_t tsNumOfCommitThreads;
extern float tsRatioOfQueryThreads; // todo remove it
extern int8_t tsDaylight;
extern char tsTimezone[];
@@ -51,6 +52,7 @@ extern char tsLocale[];
extern char tsCharset[]; // default encode string
extern int32_t tsEnableCoreFile;
extern int32_t tsCompressMsgSize;
+extern char tsTempDir[];
//query buffer management
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing
@@ -87,6 +89,7 @@ extern int16_t tsWAL;
extern int32_t tsFsyncPeriod;
extern int32_t tsReplications;
extern int32_t tsQuorum;
+extern int32_t tsUpdate;
// balance
extern int32_t tsEnableBalance;
diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h
index 6b73d98b81fa8cbcce8d322f566d8b6709396f1c..6c48ca72f3b5ea010c662faa09211afd40152615 100644
--- a/src/common/inc/tname.h
+++ b/src/common/inc/tname.h
@@ -35,6 +35,6 @@ bool tscValidateTableNameLength(size_t len);
SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters);
-// int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision);
+SSchema tscGetTbnameColumnSchema();
#endif // TDENGINE_NAME_H
diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c
index 28289b051e4ef32e1b1e22847df584238857002b..f21205479396ba606a1212f30350df2e0b3f59b5 100644
--- a/src/common/src/tdataformat.c
+++ b/src/common/src/tdataformat.c
@@ -18,6 +18,9 @@
#include "tcoding.h"
#include "wchar.h"
+static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
+ int limit2, int tRows);
+
/**
* Duplicate the schema and return a new object
*/
@@ -94,7 +97,7 @@ int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {
void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) {
if (pBuilder) {
- taosTFree(pBuilder->columns);
+ tfree(pBuilder->columns);
}
}
@@ -202,7 +205,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE;
pDataCol->len = 0;
- if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
+ if (IS_VAR_DATA_TYPE(pDataCol->type)) {
pDataCol->dataOff = (VarDataOffsetT *)(*pBuf);
pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints);
pDataCol->spaceSize = pDataCol->bytes * maxPoints;
@@ -215,60 +218,29 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
}
}
+// value from timestamp should be TKEY here instead of TSKEY
void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) {
ASSERT(pCol != NULL && value != NULL);
- switch (pCol->type) {
- case TSDB_DATA_TYPE_BINARY:
- case TSDB_DATA_TYPE_NCHAR:
- // set offset
- pCol->dataOff[numOfRows] = pCol->len;
- // Copy data
- memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
- // Update the length
- pCol->len += varDataTLen(value);
- break;
- default:
- ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
- memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
- pCol->len += pCol->bytes;
- break;
- }
-}
-
-void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows) {
- int pointsLeft = numOfRows - pointsToPop;
-
- ASSERT(pointsLeft > 0);
-
- if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) {
- ASSERT(pCol->len > 0);
- VarDataOffsetT toffset = pCol->dataOff[pointsToPop];
- pCol->len = pCol->len - toffset;
- ASSERT(pCol->len > 0);
- memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len);
- dataColSetOffset(pCol, pointsLeft);
+ if (IS_VAR_DATA_TYPE(pCol->type)) {
+ // set offset
+ pCol->dataOff[numOfRows] = pCol->len;
+ // Copy data
+ memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
+ // Update the length
+ pCol->len += varDataTLen(value);
} else {
ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
- pCol->len = TYPE_BYTES[pCol->type] * pointsLeft;
- memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len);
+ memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
+ pCol->len += pCol->bytes;
}
}
bool isNEleNull(SDataCol *pCol, int nEle) {
- switch (pCol->type) {
- case TSDB_DATA_TYPE_BINARY:
- case TSDB_DATA_TYPE_NCHAR:
- for (int i = 0; i < nEle; i++) {
- if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
- }
- return true;
- default:
- for (int i = 0; i < nEle; i++) {
- if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
- }
- return true;
+ for (int i = 0; i < nEle; i++) {
+ if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
}
+ return true;
}
void dataColSetNullAt(SDataCol *pCol, int index) {
@@ -367,8 +339,8 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
void tdFreeDataCols(SDataCols *pCols) {
if (pCols) {
- taosTFree(pCols->buf);
- taosTFree(pCols->cols);
+ tfree(pCols->buf);
+ tfree(pCols->cols);
free(pCols);
}
}
@@ -390,7 +362,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize;
pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf)));
- if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
+ if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
ASSERT(pDataCols->cols[i].dataOff != NULL);
pRet->cols[i].dataOff =
(int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf)));
@@ -400,7 +372,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
pRet->cols[i].len = pDataCols->cols[i].len;
if (pDataCols->cols[i].len > 0) {
memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
- if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
+ if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints);
}
}
@@ -420,58 +392,54 @@ void tdResetDataCols(SDataCols *pCols) {
}
void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) {
- ASSERT(dataColsKeyLast(pCols) < dataRowKey(row));
+ ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row));
int rcol = 0;
int dcol = 0;
- while (dcol < pCols->numOfCols) {
- SDataCol *pDataCol = &(pCols->cols[dcol]);
- if (rcol >= schemaNCols(pSchema)) {
- dataColSetNullAt(pDataCol, pCols->numOfRows);
- dcol++;
- continue;
+ if (dataRowDeleted(row)) {
+ for (; dcol < pCols->numOfCols; dcol++) {
+ SDataCol *pDataCol = &(pCols->cols[dcol]);
+ if (dcol == 0) {
+ dataColAppendVal(pDataCol, dataRowTuple(row), pCols->numOfRows, pCols->maxPoints);
+ } else {
+ dataColSetNullAt(pDataCol, pCols->numOfRows);
+ }
}
+ } else {
+ while (dcol < pCols->numOfCols) {
+ SDataCol *pDataCol = &(pCols->cols[dcol]);
+ if (rcol >= schemaNCols(pSchema)) {
+ dataColSetNullAt(pDataCol, pCols->numOfRows);
+ dcol++;
+ continue;
+ }
- STColumn *pRowCol = schemaColAt(pSchema, rcol);
- if (pRowCol->colId == pDataCol->colId) {
- void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset+TD_DATA_ROW_HEAD_SIZE);
- dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
- dcol++;
- rcol++;
- } else if (pRowCol->colId < pDataCol->colId) {
- rcol++;
- } else {
- dataColSetNullAt(pDataCol, pCols->numOfRows);
- dcol++;
+ STColumn *pRowCol = schemaColAt(pSchema, rcol);
+ if (pRowCol->colId == pDataCol->colId) {
+ void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE);
+ dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
+ dcol++;
+ rcol++;
+ } else if (pRowCol->colId < pDataCol->colId) {
+ rcol++;
+ } else {
+ dataColSetNullAt(pDataCol, pCols->numOfRows);
+ dcol++;
+ }
}
}
pCols->numOfRows++;
}
-// Pop pointsToPop points from the SDataCols
-void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
- int pointsLeft = pCols->numOfRows - pointsToPop;
- if (pointsLeft <= 0) {
- tdResetDataCols(pCols);
- return;
- }
-
- for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
- SDataCol *pCol = pCols->cols + iCol;
- dataColPopPoints(pCol, pointsToPop, pCols->numOfRows);
- }
- pCols->numOfRows = pointsLeft;
-}
-
int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
- ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
ASSERT(target->numOfCols == source->numOfCols);
SDataCols *pTarget = NULL;
if (dataColsKeyLast(target) < dataColsKeyFirst(source)) { // No overlap
+ ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
for (int i = 0; i < rowsToMerge; i++) {
for (int j = 0; j < source->numOfCols; j++) {
if (source->cols[j].len > 0) {
@@ -499,17 +467,23 @@ _err:
return -1;
}
-void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) {
+// src2 data has more priority than src1
+static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
+ int limit2, int tRows) {
tdResetDataCols(target);
ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);
while (target->numOfRows < tRows) {
if (*iter1 >= limit1 && *iter2 >= limit2) break;
- TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
- TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
+ TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1);
+ TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
+ TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2);
+ TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);
- if (key1 <= key2) {
+ ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
+
+ if (key1 < key2) {
for (int i = 0; i < src1->numOfCols; i++) {
ASSERT(target->cols[i].type == src1->cols[i].type);
if (src1->cols[i].len > 0) {
@@ -520,19 +494,23 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi
target->numOfRows++;
(*iter1)++;
- if (key1 == key2) (*iter2)++;
- } else {
- for (int i = 0; i < src2->numOfCols; i++) {
- ASSERT(target->cols[i].type == src2->cols[i].type);
- if (src2->cols[i].len > 0) {
- dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
- target->maxPoints);
+ } else if (key1 >= key2) {
+ if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
+ for (int i = 0; i < src2->numOfCols; i++) {
+ ASSERT(target->cols[i].type == src2->cols[i].type);
+ if (src2->cols[i].len > 0) {
+ dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
+ target->maxPoints);
+ }
}
+ target->numOfRows++;
}
- target->numOfRows++;
(*iter2)++;
+ if (key1 == key2) (*iter1)++;
}
+
+ ASSERT(target->numOfRows <= target->maxPoints);
}
}
@@ -691,8 +669,8 @@ int tdInitKVRowBuilder(SKVRowBuilder *pBuilder) {
}
void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder) {
- taosTFree(pBuilder->pColIdx);
- taosTFree(pBuilder->buf);
+ tfree(pBuilder->pColIdx);
+ tfree(pBuilder->buf);
}
void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) {
diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index 9b12a6586070cd888ede8d2179d92632bd3ac036..335e7c007cbbdcb194cbf6d385d438dfe556fdcb 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -51,6 +51,7 @@ int32_t tsMaxShellConns = 5000;
int32_t tsMaxConnections = 5000;
int32_t tsShellActivityTimer = 3; // second
float tsNumOfThreadsPerCore = 1.0f;
+int32_t tsNumOfCommitThreads = 1;
float tsRatioOfQueryThreads = 0.5f;
int8_t tsDaylight = 0;
char tsTimezone[TSDB_TIMEZONE_LEN] = {0};
@@ -58,6 +59,7 @@ char tsLocale[TSDB_LOCALE_LEN] = {0};
char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string
int32_t tsEnableCoreFile = 0;
int32_t tsMaxBinaryDisplayWidth = 30;
+char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/";
/*
* denote if the server needs to compress response message at the application layer to client, including query rsp,
@@ -119,6 +121,7 @@ int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL;
int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION;
int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION;
+int32_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION;
int32_t tsMaxVgroupsPerDb = 0;
int32_t tsMinTablePerVnode = TSDB_TABLES_STEP;
int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
@@ -222,6 +225,8 @@ int32_t (*monitorStartSystemFp)() = NULL;
void (*monitorStopSystemFp)() = NULL;
void (*monitorExecuteSQLFp)(char *sql) = NULL;
+char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"};
+
static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
void taosSetAllDebugFlag() {
@@ -461,6 +466,16 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
+ cfg.option = "numOfCommitThreads";
+ cfg.ptr = &tsNumOfCommitThreads;
+ cfg.valType = TAOS_CFG_VTYPE_INT32;
+ cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
+ cfg.minValue = 1;
+ cfg.maxValue = 100;
+ cfg.ptrLength = 0;
+ cfg.unitType = TAOS_CFG_UTYPE_NONE;
+ taosInitConfigOption(cfg);
+
cfg.option = "ratioOfQueryThreads";
cfg.ptr = &tsRatioOfQueryThreads;
cfg.valType = TAOS_CFG_VTYPE_FLOAT;
@@ -824,6 +839,16 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
+ cfg.option = "update";
+ cfg.ptr = &tsUpdate;
+ cfg.valType = TAOS_CFG_VTYPE_INT32;
+ cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+ cfg.minValue = TSDB_MIN_DB_UPDATE;
+ cfg.maxValue = TSDB_MAX_DB_UPDATE;
+ cfg.ptrLength = 0;
+ cfg.unitType = TAOS_CFG_UTYPE_NONE;
+ taosInitConfigOption(cfg);
+
cfg.option = "mqttHostName";
cfg.ptr = tsMqttHostName;
cfg.valType = TAOS_CFG_VTYPE_STRING;
@@ -1349,13 +1374,23 @@ static void doInitGlobalConfig(void) {
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
+
+ cfg.option = "tempDir";
+ cfg.ptr = tsTempDir;
+ cfg.valType = TAOS_CFG_VTYPE_STRING;
+ cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
+ cfg.minValue = 0;
+ cfg.maxValue = 0;
+ cfg.ptrLength = tListLen(tsTempDir);
+ cfg.unitType = TAOS_CFG_UTYPE_NONE;
+ taosInitConfigOption(cfg);
}
void taosInitGlobalCfg() {
pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig);
}
-bool taosCheckGlobalCfg() {
+int32_t taosCheckGlobalCfg() {
char fqdn[TSDB_FQDN_LEN];
uint16_t port;
@@ -1415,7 +1450,9 @@ bool taosCheckGlobalCfg() {
tsSyncPort = tsServerPort + TSDB_PORT_SYNC;
tsHttpPort = tsServerPort + TSDB_PORT_HTTP;
- return true;
+ taosPrintGlobalCfg();
+
+ return 0;
}
int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) {
diff --git a/src/common/src/tname.c b/src/common/src/tname.c
index 8879e9e7979be5b019b3048389105faa86ac9225..bea8c52ef21ea607bee5379ace879ef936ff60bb 100644
--- a/src/common/src/tname.c
+++ b/src/common/src/tname.c
@@ -188,3 +188,14 @@ void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) {
pToken->z = r;
}
}
+
+SSchema tscGetTbnameColumnSchema() {
+ struct SSchema s = {
+ .colId = TSDB_TBNAME_COLUMN_INDEX,
+ .type = TSDB_DATA_TYPE_BINARY,
+ .bytes = TSDB_TABLE_NAME_LEN
+ };
+
+ strcpy(s.name, TSQL_TBNAME_L);
+ return s;
+}
\ No newline at end of file
diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c
index 50554ce08e3fb659c1a5915c4c50b09f950324b4..f28481977f41bf550205d2669c6f90a016d2b7c4 100644
--- a/src/common/src/ttypes.c
+++ b/src/common/src/ttypes.c
@@ -355,32 +355,6 @@ bool isValidDataType(int32_t type) {
return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_NCHAR;
}
-//bool isNull(const char *val, int32_t type) {
-// switch (type) {
-// case TSDB_DATA_TYPE_BOOL:
-// return *(uint8_t *)val == TSDB_DATA_BOOL_NULL;
-// case TSDB_DATA_TYPE_TINYINT:
-// return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL;
-// case TSDB_DATA_TYPE_SMALLINT:
-// return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL;
-// case TSDB_DATA_TYPE_INT:
-// return *(uint32_t *)val == TSDB_DATA_INT_NULL;
-// case TSDB_DATA_TYPE_BIGINT:
-// case TSDB_DATA_TYPE_TIMESTAMP:
-// return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL;
-// case TSDB_DATA_TYPE_FLOAT:
-// return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL;
-// case TSDB_DATA_TYPE_DOUBLE:
-// return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL;
-// case TSDB_DATA_TYPE_NCHAR:
-// return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
-// case TSDB_DATA_TYPE_BINARY:
-// return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;
-// default:
-// return false;
-// };
-//}
-
void setVardataNull(char* val, int32_t type) {
if (type == TSDB_DATA_TYPE_BINARY) {
varDataSetLen(val, sizeof(int8_t));
@@ -433,14 +407,10 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) {
*(uint64_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_DOUBLE_NULL;
}
break;
- case TSDB_DATA_TYPE_NCHAR: // todo : without length?
- for (int32_t i = 0; i < numOfElems; ++i) {
- *(uint32_t *)(val + i * bytes) = TSDB_DATA_NCHAR_NULL;
- }
- break;
+ case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_BINARY:
for (int32_t i = 0; i < numOfElems; ++i) {
- *(uint8_t *)(val + i * bytes) = TSDB_DATA_BINARY_NULL;
+ setVardataNull(val + i * bytes, type);
}
break;
default: {
diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c
index 9eb9924932c7757c3c999eb5afadd8c719dc16bc..6dd065382265cb09e26f8d1b6f86c4b5b3f5110b 100644
--- a/src/common/src/tvariant.c
+++ b/src/common/src/tvariant.c
@@ -108,7 +108,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
break;
}
case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method
- pVar->pz = calloc(len, sizeof(char));
+ pVar->pz = calloc(len + 1, sizeof(char));
memcpy(pVar->pz, pz, len);
pVar->nLen = (int32_t)len;
break;
@@ -125,7 +125,7 @@ void tVariantDestroy(tVariant *pVar) {
if (pVar == NULL) return;
if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) {
- taosTFree(pVar->pz);
+ tfree(pVar->pz);
pVar->nLen = 0;
}
diff --git a/src/connector/C#/TDengineDriver.cs b/src/connector/C#/TDengineDriver.cs
new file mode 100644
index 0000000000000000000000000000000000000000..b6f143e1813d60c1ac4ae8356efdca4929c51345
--- /dev/null
+++ b/src/connector/C#/TDengineDriver.cs
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+
+namespace TDengineDriver
+{
+ enum TDengineDataType {
+ TSDB_DATA_TYPE_NULL = 0, // 1 bytes
+ TSDB_DATA_TYPE_BOOL = 1, // 1 bytes
+ TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes
+ TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes
+ TSDB_DATA_TYPE_INT = 4, // 4 bytes
+ TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes
+ TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes
+ TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes
+ TSDB_DATA_TYPE_BINARY = 8, // string
+ TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes
+ TSDB_DATA_TYPE_NCHAR = 10 // unicode string
+ }
+
+ enum TDengineInitOption
+ {
+ TSDB_OPTION_LOCALE = 0,
+ TSDB_OPTION_CHARSET = 1,
+ TSDB_OPTION_TIMEZONE = 2,
+ TDDB_OPTION_CONFIGDIR = 3,
+ TDDB_OPTION_SHELL_ACTIVITY_TIMER = 4
+ }
+
+ class TDengineMeta
+ {
+ public string name;
+ public short size;
+ public byte type;
+ public string TypeName()
+ {
+ switch ((TDengineDataType)type)
+ {
+ case TDengineDataType.TSDB_DATA_TYPE_BOOL:
+ return "BOOLEAN";
+ case TDengineDataType.TSDB_DATA_TYPE_TINYINT:
+ return "BYTE";
+ case TDengineDataType.TSDB_DATA_TYPE_SMALLINT:
+ return "SHORT";
+ case TDengineDataType.TSDB_DATA_TYPE_INT:
+ return "INT";
+ case TDengineDataType.TSDB_DATA_TYPE_BIGINT:
+ return "LONG";
+ case TDengineDataType.TSDB_DATA_TYPE_FLOAT:
+ return "FLOAT";
+ case TDengineDataType.TSDB_DATA_TYPE_DOUBLE:
+ return "DOUBLE";
+ case TDengineDataType.TSDB_DATA_TYPE_BINARY:
+ return "STRING";
+ case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP:
+ return "TIMESTAMP";
+ case TDengineDataType.TSDB_DATA_TYPE_NCHAR:
+ return "NCHAR";
+ default:
+ return "undefine";
+ }
+ }
+ }
+
+ class TDengine
+ {
+ public const int TSDB_CODE_SUCCESS = 0;
+
+ [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)]
+ static extern public void Init();
+
+ [DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)]
+ static extern public void Cleanup();
+
+ [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)]
+ static extern public void Options(int option, string value);
+
+ [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)]
+ static extern public IntPtr Connect(string ip, string user, string password, string db, short port);
+
+ [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)]
+ static extern private IntPtr taos_errstr(IntPtr res);
+ static public string Error(IntPtr res)
+ {
+ IntPtr errPtr = taos_errstr(res);
+ return Marshal.PtrToStringAnsi(errPtr);
+ }
+
+ [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)]
+ static extern public int ErrorNo(IntPtr res);
+
+ [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)]
+ static extern public IntPtr Query(IntPtr conn, string sqlstr);
+
+ [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)]
+ static extern public int AffectRows(IntPtr res);
+
+ [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)]
+ static extern public int FieldCount(IntPtr res);
+
+ [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)]
+ static extern private IntPtr taos_fetch_fields(IntPtr res);
+ static public List FetchFields(IntPtr res)
+ {
+ const int fieldSize = 68;
+
+ List metas = new List();
+ if (res == IntPtr.Zero)
+ {
+ return metas;
+ }
+
+ int fieldCount = FieldCount(res);
+ IntPtr fieldsPtr = taos_fetch_fields(res);
+
+ for (int i = 0; i < fieldCount; ++i)
+ {
+ int offset = i * fieldSize;
+
+ TDengineMeta meta = new TDengineMeta();
+ meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset);
+ meta.type = Marshal.ReadByte(fieldsPtr + offset + 65);
+ meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66);
+ metas.Add(meta);
+ }
+
+ return metas;
+ }
+
+ [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)]
+ static extern public IntPtr FetchRows(IntPtr res);
+
+ [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)]
+ static extern public IntPtr FreeResult(IntPtr res);
+
+ [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)]
+ static extern public int Close(IntPtr taos);
+ }
+}
\ No newline at end of file
diff --git a/src/connector/go b/src/connector/go
index 8d7bf743852897110cbdcc7c4322cd7a74d4167b..050667e5b4d0eafa5387e4283e713559b421203f 160000
--- a/src/connector/go
+++ b/src/connector/go
@@ -1 +1 @@
-Subproject commit 8d7bf743852897110cbdcc7c4322cd7a74d4167b
+Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f
diff --git a/src/connector/hivemq-tdengine-extension b/src/connector/hivemq-tdengine-extension
new file mode 160000
index 0000000000000000000000000000000000000000..b62a26ecc164a310104df57691691b237e091c89
--- /dev/null
+++ b/src/connector/hivemq-tdengine-extension
@@ -0,0 +1 @@
+Subproject commit b62a26ecc164a310104df57691691b237e091c89
diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml
index 3f6ebeff03f538e4963540e015c7fd4b310acc42..51db837c7b1149bfc5dca6d69a953ceb6b3eb898 100755
--- a/src/connector/jdbc/deploy-pom.xml
+++ b/src/connector/jdbc/deploy-pom.xml
@@ -5,14 +5,13 @@
com.taosdata.jdbc
taos-jdbcdriver
- 2.0.6
+ 2.0.10
jar
JDBCDriver
https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc
TDengine JDBC Driver
-
GNU AFFERO GENERAL PUBLIC LICENSE Version 3
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java
index ac0e4eb84aa0fbcc8162e68d94de00cb2f4e79f5..f93412ffecb0673f946f187f13472a0b4d6a140e 100644
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java
@@ -52,7 +52,6 @@ public class TSDBConnection implements Connection {
public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException {
this.dbMetaData = meta;
-
connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST),
Integer.parseInt(info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "0")),
info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), info.getProperty(TSDBDriver.PROPERTY_KEY_USER),
@@ -197,12 +196,14 @@ public class TSDBConnection implements Connection {
}
public SQLWarning getWarnings() throws SQLException {
- throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
+ //todo: implement getWarnings according to the warning messages returned from TDengine
+ return null;
+// throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
}
public void clearWarnings() throws SQLException {
// left blank to support HikariCP connection
- //todo: implement getWarnings according to the warning messages returned from TDengine
+ //todo: implement clearWarnings according to the warning messages returned from TDengine
}
public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException {
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java
index c1d9d2af8e5a5c24dcfed6039e3ce06530b95276..f4dee67adf03a3474f582dbe662a8c5e988e3fab 100644
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java
@@ -96,7 +96,7 @@ public class TSDBDatabaseMetaData implements java.sql.DatabaseMetaData {
}
public int getDriverMajorVersion() {
- return 0;
+ return 2;
}
public int getDriverMinorVersion() {
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
index 97d93fb0a18fa887465583bce1492c8305faaec5..63c42ca399b6098a5fa8c34d6fafeb51bdc8f588 100755
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
@@ -14,13 +14,9 @@
*****************************************************************************/
package com.taosdata.jdbc;
-
import java.io.*;
-
import java.sql.*;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
+import java.util.*;
import java.util.logging.Logger;
/**
@@ -44,76 +40,53 @@ import java.util.logging.Logger;
*/
public class TSDBDriver implements java.sql.Driver {
-
@Deprecated
private static final String URL_PREFIX1 = "jdbc:TSDB://";
private static final String URL_PREFIX = "jdbc:TAOS://";
- /**
- * Key used to retrieve the database value from the properties instance passed
- * to the driver.
- */
- public static final String PROPERTY_KEY_DBNAME = "dbname";
-
/**
* Key used to retrieve the host value from the properties instance passed to
* the driver.
*/
public static final String PROPERTY_KEY_HOST = "host";
- /**
- * Key used to retrieve the password value from the properties instance passed
- * to the driver.
- */
- public static final String PROPERTY_KEY_PASSWORD = "password";
-
/**
* Key used to retrieve the port number value from the properties instance
* passed to the driver.
*/
public static final String PROPERTY_KEY_PORT = "port";
-
+ /**
+ * Key used to retrieve the database value from the properties instance passed
+ * to the driver.
+ */
+ public static final String PROPERTY_KEY_DBNAME = "dbname";
/**
* Key used to retrieve the user value from the properties instance passed to
* the driver.
*/
public static final String PROPERTY_KEY_USER = "user";
-
-
+ /**
+ * Key used to retrieve the password value from the properties instance passed
+ * to the driver.
+ */
+ public static final String PROPERTY_KEY_PASSWORD = "password";
/**
* Key for the configuration file directory of TSDB client in properties instance
*/
public static final String PROPERTY_KEY_CONFIG_DIR = "cfgdir";
-
/**
* Key for the timezone used by the TSDB client in properties instance
*/
public static final String PROPERTY_KEY_TIME_ZONE = "timezone";
-
/**
* Key for the locale used by the TSDB client in properties instance
*/
public static final String PROPERTY_KEY_LOCALE = "locale";
-
-
/**
* Key for the char encoding used by the TSDB client in properties instance
*/
public static final String PROPERTY_KEY_CHARSET = "charset";
- public static final String PROPERTY_KEY_PROTOCOL = "protocol";
-
-
- /**
- * Index for port coming out of parseHostPortPair().
- */
- public final static int PORT_NUMBER_INDEX = 1;
-
- /**
- * Index for host coming out of parseHostPortPair().
- */
- public final static int HOST_NAME_INDEX = 0;
-
private TSDBDatabaseMetaData dbMetaData = null;
static {
@@ -169,9 +142,11 @@ public class TSDBDriver implements java.sql.Driver {
}
public Connection connect(String url, Properties info) throws SQLException {
- if (url == null) {
+ if (url == null)
throw new SQLException(TSDBConstants.WrapErrMsg("url is not set!"));
- }
+
+ if (!acceptsURL(url))
+ return null;
Properties props = null;
if ((props = parseURL(url, info)) == null) {
@@ -179,7 +154,10 @@ public class TSDBDriver implements java.sql.Driver {
}
//load taos.cfg start
- if (info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null && info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null) {
+ if ((info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null ||
+ info.getProperty(TSDBDriver.PROPERTY_KEY_HOST).isEmpty()) && (
+ info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null ||
+ info.getProperty(TSDBDriver.PROPERTY_KEY_PORT).isEmpty())) {
File cfgDir = loadConfigDir(info.getProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR));
File cfgFile = cfgDir.listFiles((dir, name) -> "taos.cfg".equalsIgnoreCase(name))[0];
List endpoints = loadConfigEndpoints(cfgFile);
@@ -190,7 +168,9 @@ public class TSDBDriver implements java.sql.Driver {
}
try {
- TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE), (String) props.get(PROPERTY_KEY_CHARSET),
+ TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR),
+ (String) props.get(PROPERTY_KEY_LOCALE),
+ (String) props.get(PROPERTY_KEY_CHARSET),
(String) props.get(PROPERTY_KEY_TIME_ZONE));
Connection newConn = new TSDBConnection(props, this.dbMetaData);
return newConn;
@@ -208,43 +188,15 @@ public class TSDBDriver implements java.sql.Driver {
}
/**
- * Parses hostPortPair in the form of [host][:port] into an array, with the
- * element of index HOST_NAME_INDEX being the host (or null if not specified),
- * and the element of index PORT_NUMBER_INDEX being the port (or null if not
- * specified).
- *
- * @param hostPortPair host and port in form of of [host][:port]
- * @return array containing host and port as Strings
- * @throws SQLException if a parse error occurs
+ * @param url the URL of the database
+ * @return true
if this driver understands the given URL;
+ * false
otherwise
+ * @throws SQLException if a database access error occurs or the url is {@code null}
*/
- protected static String[] parseHostPortPair(String hostPortPair) throws SQLException {
- String[] splitValues = new String[2];
-
- int portIndex = hostPortPair.indexOf(":");
-
- String hostname = null;
-
- if (portIndex != -1) {
- if ((portIndex + 1) < hostPortPair.length()) {
- String portAsString = hostPortPair.substring(portIndex + 1);
- hostname = hostPortPair.substring(0, portIndex);
-
- splitValues[HOST_NAME_INDEX] = hostname;
-
- splitValues[PORT_NUMBER_INDEX] = portAsString;
- } else {
- throw new SQLException(TSDBConstants.WrapErrMsg("port is not proper!"));
- }
- } else {
- splitValues[HOST_NAME_INDEX] = hostPortPair;
- splitValues[PORT_NUMBER_INDEX] = null;
- }
-
- return splitValues;
- }
-
public boolean acceptsURL(String url) throws SQLException {
- return (url != null && url.length() > 0 && url.trim().length() > 0) && url.startsWith(URL_PREFIX);
+ if (url == null)
+ throw new SQLException(TSDBConstants.WrapErrMsg("url is null"));
+ return (url != null && url.length() > 0 && url.trim().length() > 0) && (url.startsWith(URL_PREFIX) || url.startsWith(URL_PREFIX1));
}
public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException {
@@ -252,15 +204,17 @@ public class TSDBDriver implements java.sql.Driver {
info = new Properties();
}
- if ((url != null) && (url.startsWith(URL_PREFIX) || url.startsWith(URL_PREFIX1))) {
+ if (acceptsURL(url)) {
info = parseURL(url, info);
}
DriverPropertyInfo hostProp = new DriverPropertyInfo(PROPERTY_KEY_HOST, info.getProperty(PROPERTY_KEY_HOST));
- hostProp.required = true;
+ hostProp.required = false;
+ hostProp.description = "Hostname";
DriverPropertyInfo portProp = new DriverPropertyInfo(PROPERTY_KEY_PORT, info.getProperty(PROPERTY_KEY_PORT, TSDBConstants.DEFAULT_PORT));
portProp.required = false;
+ portProp.description = "Port";
DriverPropertyInfo dbProp = new DriverPropertyInfo(PROPERTY_KEY_DBNAME, info.getProperty(PROPERTY_KEY_DBNAME));
dbProp.required = false;
@@ -268,9 +222,11 @@ public class TSDBDriver implements java.sql.Driver {
DriverPropertyInfo userProp = new DriverPropertyInfo(PROPERTY_KEY_USER, info.getProperty(PROPERTY_KEY_USER));
userProp.required = true;
+ userProp.description = "User";
DriverPropertyInfo passwordProp = new DriverPropertyInfo(PROPERTY_KEY_PASSWORD, info.getProperty(PROPERTY_KEY_PASSWORD));
passwordProp.required = true;
+ passwordProp.description = "Password";
DriverPropertyInfo[] propertyInfo = new DriverPropertyInfo[5];
propertyInfo[0] = hostProp;
@@ -283,20 +239,68 @@ public class TSDBDriver implements java.sql.Driver {
}
/**
- * example: jdbc:TSDB://127.0.0.1:0/db?user=root&password=your_password
+ * example: jdbc:TAOS://127.0.0.1:0/db?user=root&password=your_password
*/
- public Properties parseURL(String url, Properties defaults) throws java.sql.SQLException {
+ public Properties parseURL(String url, Properties defaults) {
Properties urlProps = (defaults != null) ? defaults : new Properties();
- if (url == null) {
+ if (url == null || url.length() <= 0 || url.trim().length() <= 0)
return null;
- }
-
- if (!url.startsWith(URL_PREFIX) && !url.startsWith(URL_PREFIX1)) {
+ if (!url.startsWith(URL_PREFIX) && !url.startsWith(URL_PREFIX1))
return null;
- }
+ // parse properties
String urlForMeta = url;
+ int beginningOfSlashes = url.indexOf("//");
+ int index = url.indexOf("?");
+ if (index != -1) {
+ String paramString = url.substring(index + 1, url.length());
+ url = url.substring(0, index);
+ StringTokenizer queryParams = new StringTokenizer(paramString, "&");
+ while (queryParams.hasMoreElements()) {
+ String parameterValuePair = queryParams.nextToken();
+ int indexOfEqual = parameterValuePair.indexOf("=");
+ String parameter = null;
+ String value = null;
+ if (indexOfEqual != -1) {
+ parameter = parameterValuePair.substring(0, indexOfEqual);
+ if (indexOfEqual + 1 < parameterValuePair.length()) {
+ value = parameterValuePair.substring(indexOfEqual + 1);
+ }
+ }
+ if ((value != null && value.length() > 0) && (parameter != null && parameter.length() > 0)) {
+ urlProps.setProperty(parameter, value);
+ }
+ }
+ }
+ // parse Product Name
+ String dbProductName = url.substring(0, beginningOfSlashes);
+ dbProductName = dbProductName.substring(dbProductName.indexOf(":") + 1);
+ dbProductName = dbProductName.substring(0, dbProductName.indexOf(":"));
+ // parse dbname
+ url = url.substring(beginningOfSlashes + 2);
+ int indexOfSlash = url.indexOf("/");
+ if (indexOfSlash != -1) {
+ if (indexOfSlash + 1 < url.length()) {
+ urlProps.setProperty(TSDBDriver.PROPERTY_KEY_DBNAME, url.substring(indexOfSlash + 1));
+ }
+ url = url.substring(0, indexOfSlash);
+ }
+ // parse port
+ int indexOfColon = url.indexOf(":");
+ if (indexOfColon != -1) {
+ if (indexOfColon + 1 < url.length()) {
+ urlProps.setProperty(TSDBDriver.PROPERTY_KEY_PORT, url.substring(indexOfColon + 1));
+ }
+ url = url.substring(0, indexOfColon);
+ }
+ if (url != null && url.length() > 0 && url.trim().length() > 0) {
+ urlProps.setProperty(TSDBDriver.PROPERTY_KEY_HOST, url);
+ }
+ this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, urlProps.getProperty(TSDBDriver.PROPERTY_KEY_USER));
+
+ /*
+ String urlForMeta = url;
String dbProductName = url.substring(url.indexOf(":") + 1);
dbProductName = dbProductName.substring(0, dbProductName.indexOf(":"));
int beginningOfSlashes = url.indexOf("//");
@@ -345,11 +349,11 @@ public class TSDBDriver implements java.sql.Driver {
user = urlProps.getProperty(PROPERTY_KEY_USER).toString();
this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, user);
-
+*/
return urlProps;
}
- public void setPropertyValue(Properties property, String[] keyValuePair) {
+ private void setPropertyValue(Properties property, String[] keyValuePair) {
switch (keyValuePair[0].toLowerCase()) {
case PROPERTY_KEY_USER:
property.setProperty(PROPERTY_KEY_USER, keyValuePair[1]);
@@ -372,13 +376,12 @@ public class TSDBDriver implements java.sql.Driver {
}
}
-
public int getMajorVersion() {
- return 1;
+ return 2;
}
public int getMinorVersion() {
- return 1;
+ return 0;
}
public boolean jdbcCompliant() {
@@ -389,33 +392,4 @@ public class TSDBDriver implements java.sql.Driver {
return null;
}
- /**
- * Returns the host property
- *
- * @param props the java.util.Properties instance to retrieve the hostname from.
- * @return the host
- */
- public String host(Properties props) {
- return props.getProperty(PROPERTY_KEY_HOST, "localhost");
- }
-
- /**
- * Returns the port number property
- *
- * @param props the properties to get the port number from
- * @return the port number
- */
- public int port(Properties props) {
- return Integer.parseInt(props.getProperty(PROPERTY_KEY_PORT, TSDBConstants.DEFAULT_PORT));
- }
-
- /**
- * Returns the database property from props
- *
- * @param props the Properties to look for the database property.
- * @return the database name.
- */
- public String database(Properties props) {
- return props.getProperty(PROPERTY_KEY_DBNAME);
- }
}
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
index 0cd185de50d858aec78b60acc4055c1ae99a4cb5..edc160e323d2dc9c885b5404545a6d0689857a31 100755
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
@@ -111,8 +111,8 @@ public class TSDBJNIConnector {
* @throws SQLException
*/
public long executeQuery(String sql) throws SQLException {
- // close previous result set if the user forgets to invoke the
- // free method to close previous result set.
+ // close previous result set if the user forgets to invoke the
+ // free method to close previous result set.
if (!this.isResultsetClosed) {
freeResultSet(taosResultSetPointer);
}
@@ -122,23 +122,23 @@ public class TSDBJNIConnector {
pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos);
} catch (Exception e) {
e.printStackTrace();
- this.freeResultSet(pSql);
+ this.freeResultSetImp(this.taos, pSql);
throw new SQLException(TSDBConstants.WrapErrMsg("Unsupported encoding"));
}
-
+
int code = this.getErrCode(pSql);
if (code != 0) {
affectedRows = -1;
String msg = this.getErrMsg(pSql);
-
- this.freeResultSet(pSql);
+
+ this.freeResultSetImp(this.taos, pSql);
throw new SQLException(TSDBConstants.WrapErrMsg(msg), "", code);
}
// Try retrieving result set for the executed SQL using the current connection pointer.
taosResultSetPointer = this.getResultSetImp(this.taos, pSql);
isResultsetClosed = (taosResultSetPointer == TSDBConstants.JNI_NULL_POINTER);
-
+
return pSql;
}
@@ -171,11 +171,11 @@ public class TSDBJNIConnector {
}
private native long getResultSetImp(long connection, long pSql);
-
+
public boolean isUpdateQuery(long pSql) {
- return isUpdateQueryImp(this.taos, pSql) == 1? true:false;
+ return isUpdateQueryImp(this.taos, pSql) == 1 ? true : false;
}
-
+
private native long isUpdateQueryImp(long connection, long pSql);
/**
@@ -191,7 +191,7 @@ public class TSDBJNIConnector {
res = this.freeResultSetImp(this.taos, result);
taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER;
}
-
+
isResultsetClosed = true;
return res;
}
@@ -274,7 +274,7 @@ public class TSDBJNIConnector {
* Consume a subscription
*/
long consume(long subscription) {
- return this.consumeImp(subscription);
+ return this.consumeImp(subscription);
}
private native long consumeImp(long subscription);
diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java
index a0981063a5052f04b849b2187a78352a2c2560be..8adcdefb2974d5817793297091eee1c2f40a52b9 100644
--- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java
@@ -1,47 +1,210 @@
package com.taosdata.jdbc;
+import org.junit.BeforeClass;
import org.junit.Test;
-import java.sql.SQLException;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.sql.*;
import java.util.Properties;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
public class TSDBDriverTest {
+ private static final String[] validURLs = {
+ "jdbc:TAOS://localhost:0",
+ "jdbc:TAOS://localhost",
+ "jdbc:TAOS://localhost:6030/test",
+ "jdbc:TAOS://localhost:6030",
+ "jdbc:TAOS://localhost:6030/",
+ "jdbc:TSDB://localhost:6030",
+ "jdbc:TSDB://localhost:6030/",
+ "jdbc:TAOS://127.0.0.1:0/db?user=root&password=taosdata",
+ "jdbc:TAOS://:",
+ "jdbc:TAOS://:/",
+ "jdbc:TAOS://:/test",
+ "jdbc:TAOS://localhost:0/?user=root&password=taosdata"
+ };
+ private static boolean islibLoaded = false;
+ private static boolean isTaosdActived;
+
+ private Connection conn;
+
+ @BeforeClass
+ public static void before() {
+ String osName = System.getProperty("os.name").toLowerCase();
+ if (!osName.equals("linux"))
+ return;
+ // try to load taos lib
+ try {
+ System.loadLibrary("taos");
+ islibLoaded = true;
+ } catch (UnsatisfiedLinkError error) {
+ System.out.println("load tdengine lib failed.");
+ error.printStackTrace();
+ }
+ // check taosd is activated
+ try {
+ String[] cmd = {"/bin/bash", "-c", "ps -ef | grep taosd | grep -v \"grep\""};
+ Process exec = Runtime.getRuntime().exec(cmd);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(exec.getInputStream()));
+ int lineCnt = 0;
+ while (reader.readLine() != null) {
+ lineCnt++;
+ }
+ if (lineCnt > 0)
+ isTaosdActived = true;
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ try {
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testConnectWithJdbcURL() {
+ final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata";
+ try {
+ if (islibLoaded && isTaosdActived) {
+ conn = DriverManager.getConnection(url);
+ assertNotNull("failure - connection should not be null", conn);
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ fail("failure - should not throw Exception");
+ }
+ }
+
+ @Test
+ public void testConnectWithProperties() {
+ final String jdbcUrl = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata";
+ Properties connProps = new Properties();
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ try {
+ if (islibLoaded && isTaosdActived) {
+ conn = DriverManager.getConnection(jdbcUrl, connProps);
+ assertNotNull("failure - connection should not be null", conn);
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ fail("failure - should not throw Exception");
+ }
+ }
+
@Test
- public void urlParserTest() throws SQLException {
+ public void testConnectWithConfigFile() {
+ String jdbcUrl = "jdbc:TAOS://:/log?user=root&password=taosdata";
+ Properties connProps = new Properties();
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ try {
+ if (islibLoaded && isTaosdActived) {
+ conn = DriverManager.getConnection(jdbcUrl, connProps);
+ assertNotNull("failure - connection should not be null", conn);
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ fail("failure - should not throw Exception");
+ }
+ }
+
+ @Test(expected = SQLException.class)
+ public void testAcceptsURL() throws SQLException {
+ Driver driver = new TSDBDriver();
+ for (String url : validURLs) {
+ assertTrue("failure - acceptsURL(\" " + url + " \") should be true", driver.acceptsURL(url));
+ }
+ driver.acceptsURL(null);
+ fail("acceptsURL throws exception when parameter is null");
+ }
+
+ @Test
+ public void testParseURL() {
TSDBDriver driver = new TSDBDriver();
- String url = "jdbc:TSDB://127.0.0.1:0/db";
-
- Properties properties = new Properties();
- driver.parseURL(url, properties);
- assertEquals(properties.get("host"), "127.0.0.1");
- assertEquals(properties.get("port"), "0");
- assertEquals(properties.get("dbname"), "db");
- assertEquals(properties.get("user"), "root");
- assertEquals(properties.get("password"), "your_password");
-
- url = "jdbc:TSDB://127.0.0.1:0/log?charset=UTF-8";
- properties = new Properties();
- driver.parseURL(url, properties);
- assertEquals(properties.get("host"), "127.0.0.1");
- assertEquals(properties.get("port"), "0");
- assertEquals(properties.get("dbname"), "log");
- assertEquals(properties.get("charset"), "UTF-8");
-
- url = "jdbc:TSDB://127.0.0.1:0/";
- properties = new Properties();
- driver.parseURL(url, properties);
- assertEquals(properties.get("host"), "127.0.0.1");
- assertEquals(properties.get("port"), "0");
- assertEquals(properties.get("dbname"), null);
-
- url = "jdbc:TSDB://127.0.0.1:0/db";
- properties = new Properties();
- driver.parseURL(url, properties);
- assertEquals(properties.get("host"), "127.0.0.1");
- assertEquals(properties.get("port"), "0");
- assertEquals(properties.get("dbname"), "db");
+
+ String url = "jdbc:TAOS://127.0.0.1:0/db?user=root&password=taosdata&charset=UTF-8";
+ Properties config = new Properties();
+ Properties actual = driver.parseURL(url, config);
+ assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.get("host"));
+ assertEquals("failure - port should be 0", "0", actual.get("port"));
+ assertEquals("failure - dbname should be db", "db", actual.get("dbname"));
+ assertEquals("failure - user should be root", "root", actual.get("user"));
+ assertEquals("failure - password should be taosdata", "taosdata", actual.get("password"));
+ assertEquals("failure - charset should be UTF-8", "UTF-8", actual.get("charset"));
+
+ url = "jdbc:TAOS://127.0.0.1:0";
+ config = new Properties();
+ actual = driver.parseURL(url, config);
+ assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.getProperty("host"));
+ assertEquals("failure - port should be 0", "0", actual.get("port"));
+ assertNull("failure - dbname should be null", actual.get("dbname"));
+
+ url = "jdbc:TAOS://127.0.0.1:0/db";
+ config = new Properties();
+ actual = driver.parseURL(url, config);
+ assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.getProperty("host"));
+ assertEquals("failure - port should be 0", "0", actual.get("port"));
+ assertEquals("failure - dbname should be db", "db", actual.get("dbname"));
+
+ url = "jdbc:TAOS://:/?";
+ config = new Properties();
+ config.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+ config.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
+ actual = driver.parseURL(url, config);
+ assertEquals("failure - user should be root", "root", actual.getProperty("user"));
+ assertEquals("failure - password should be taosdata", "taosdata", actual.getProperty("password"));
+ assertNull("failure - host should be null", actual.getProperty("host"));
+ assertNull("failure - port should be null", actual.getProperty("port"));
+ assertNull("failure - dbname should be null", actual.getProperty("dbname"));
+ }
+
+ @Test
+ public void testGetPropertyInfo() throws SQLException {
+ Driver driver = new TSDBDriver();
+ final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata";
+ Properties connProps = new Properties();
+ DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, connProps);
+ for (DriverPropertyInfo info : propertyInfo) {
+ if (info.name.equals(TSDBDriver.PROPERTY_KEY_HOST))
+ assertEquals("failure - host should be localhost", "localhost", info.value);
+ if (info.name.equals(TSDBDriver.PROPERTY_KEY_PORT))
+ assertEquals("failure - port should be 6030", "6030", info.value);
+ if (info.name.equals(TSDBDriver.PROPERTY_KEY_DBNAME))
+ assertEquals("failure - dbname should be test", "log", info.value);
+ if (info.name.equals(TSDBDriver.PROPERTY_KEY_USER))
+ assertEquals("failure - user should be root", "root", info.value);
+ if (info.name.equals(TSDBDriver.PROPERTY_KEY_PASSWORD))
+ assertEquals("failure - password should be root", "taosdata", info.value);
+ }
+ }
+
+ @Test
+ public void testGetMajorVersion() {
+ assertEquals("failure - getMajorVersion should be 2", 2, new TSDBDriver().getMajorVersion());
+ }
+
+ @Test
+ public void testGetMinorVersion() {
+ assertEquals("failure - getMinorVersion should be 0", 0, new TSDBDriver().getMinorVersion());
+ }
+
+ @Test
+ public void testJdbcCompliant() {
+ assertFalse("failure - jdbcCompliant should be false", new TSDBDriver().jdbcCompliant());
}
+
+ @Test
+ public void testGetParentLogger() throws SQLFeatureNotSupportedException {
+ assertNull("failure - getParentLogger should be be null", new TSDBDriver().getParentLogger());
+ }
+
}
\ No newline at end of file
diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c
index 1a99a84b8ebfe8ed503213299646da39b5b1d27a..1be7552a892e25cf44317a99fe52ff57689ad338 100644
--- a/src/cq/src/cqMain.c
+++ b/src/cq/src/cqMain.c
@@ -39,16 +39,16 @@
#define cTrace(...) { if (cqDebugFlag & DEBUG_TRACE) { taosPrintLog("CQ ", cqDebugFlag, __VA_ARGS__); }}
typedef struct {
- int vgId;
+ int32_t vgId;
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char db[TSDB_DB_NAME_LEN];
FCqWrite cqWrite;
void *ahandle;
- int num; // number of continuous streams
+ int32_t num; // number of continuous streams
struct SCqObj *pHead;
void *dbConn;
- int master;
+ int32_t master;
void *tmrCtrl;
pthread_mutex_t mutex;
} SCqContext;
@@ -57,7 +57,7 @@ typedef struct SCqObj {
tmr_h tmrId;
uint64_t uid;
int32_t tid; // table ID
- int rowSize; // bytes of a row
+ int32_t rowSize; // bytes of a row
char * sqlStr; // SQL string
STSchema * pSchema; // pointer to schema array
void * pStream;
@@ -115,7 +115,7 @@ void cqClose(void *handle) {
SCqObj *pTemp = pObj;
pObj = pObj->next;
tdFreeSchema(pTemp->pSchema);
- taosTFree(pTemp->sqlStr);
+ tfree(pTemp->sqlStr);
free(pTemp);
}
@@ -175,7 +175,7 @@ void cqStop(void *handle) {
pthread_mutex_unlock(&pContext->mutex);
}
-void *cqCreate(void *handle, uint64_t uid, int tid, char *sqlStr, STSchema *pSchema) {
+void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) {
SCqContext *pContext = handle;
SCqObj *pObj = calloc(sizeof(SCqObj), 1);
@@ -237,7 +237,7 @@ void cqDrop(void *handle) {
pthread_mutex_unlock(&pContext->mutex);
}
-static void doCreateStream(void *param, TAOS_RES *result, int code) {
+static void doCreateStream(void *param, TAOS_RES *result, int32_t code) {
SCqObj* pObj = (SCqObj*)param;
SCqContext* pContext = pObj->pContext;
SSqlObj* pSql = (SSqlObj*)result;
@@ -288,7 +288,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr);
- int size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize;
+ int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize;
char *buffer = calloc(size, 1);
SWalHead *pHead = (SWalHead *)buffer;
@@ -334,7 +334,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
pHead->version = 0;
// write into vnode write queue
- pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ);
+ pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ, NULL);
free(buffer);
}
diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c
index 1daee644a7effc5fa50fbc340ad7df8e2b5d70b0..e1114fc024054f7a4af32ca1497a63f7da942147 100644
--- a/src/cq/test/cqtest.c
+++ b/src/cq/test/cqtest.c
@@ -24,7 +24,7 @@
int64_t ver = 0;
void *pCq = NULL;
-int writeToQueue(void *pVnode, void *data, int type) {
+int writeToQueue(void *pVnode, void *data, int type, void *pMsg) {
return 0;
}
diff --git a/src/dnode/inc/dnodeCfg.h b/src/dnode/inc/dnodeCfg.h
new file mode 100644
index 0000000000000000000000000000000000000000..35d889646021a079abe18c01a26fa5891718aaa0
--- /dev/null
+++ b/src/dnode/inc/dnodeCfg.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_DNODE_CFG_H
+#define TDENGINE_DNODE_CFG_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t dnodeInitCfg();
+void dnodeCleanupCfg();
+void dnodeUpdateCfg(SDnodeCfg *cfg);
+int32_t dnodeGetDnodeId();
+void dnodeGetCfg(int32_t *dnodeId, char *clusterId);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/dnode/inc/dnodeEps.h b/src/dnode/inc/dnodeEps.h
new file mode 100644
index 0000000000000000000000000000000000000000..2a203498c1f270c7e456694a4e1e195cbd9022cd
--- /dev/null
+++ b/src/dnode/inc/dnodeEps.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_DNODE_EP_H
+#define TDENGINE_DNODE_EP_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taosmsg.h"
+
+int32_t dnodeInitEps();
+void dnodeCleanupEps();
+void dnodeUpdateEps(SDnodeEps *eps);
+void dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port);
+bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/dnode/inc/dnodeMInfos.h b/src/dnode/inc/dnodeMInfos.h
new file mode 100644
index 0000000000000000000000000000000000000000..9c3c85c47e2dbcc11c5b5a80fbf091bd93855149
--- /dev/null
+++ b/src/dnode/inc/dnodeMInfos.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_DNODE_MINFOS_H
+#define TDENGINE_DNODE_MINFOS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taosmsg.h"
+
+int32_t dnodeInitMInfos();
+void dnodeCleanupMInfos();
+void dnodeUpdateMInfos(SMnodeInfos *minfos);
+void dnodeUpdateEpSetForPeer(SRpcEpSet *epSet);
+void dnodeGetMInfos(SMnodeInfos *minfos);
+bool dnodeIsMasterEp(char *ep);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/dnode/inc/dnodeMPeer.h b/src/dnode/inc/dnodeMPeer.h
index 9a48703110c389f10ac623ccaeaa85420a32817f..00221baa221a411d614c1fa1bb3dc4525de2ed71 100644
--- a/src/dnode/inc/dnodeMPeer.h
+++ b/src/dnode/inc/dnodeMPeer.h
@@ -20,11 +20,11 @@
extern "C" {
#endif
-int32_t dnodeInitMnodePeer();
-void dnodeCleanupMnodePeer();
-int32_t dnodeAllocateMnodePqueue();
-void dnodeFreeMnodePqueue();
-void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg);
+int32_t dnodeInitMPeer();
+void dnodeCleanupMPeer();
+int32_t dnodeAllocateMPeerQueue();
+void dnodeFreeMPeerQueue();
+void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg);
#ifdef __cplusplus
}
diff --git a/src/dnode/inc/dnodeMRead.h b/src/dnode/inc/dnodeMRead.h
index 4e93838b7998850e1bec79f017fdfbd28f286a11..8a8e71227ddf3cd7b7c77ef664ac3d4c935889db 100644
--- a/src/dnode/inc/dnodeMRead.h
+++ b/src/dnode/inc/dnodeMRead.h
@@ -20,11 +20,11 @@
extern "C" {
#endif
-int32_t dnodeInitMnodeRead();
-void dnodeCleanupMnodeRead();
-int32_t dnodeAllocateMnodeRqueue();
-void dnodeFreeMnodeRqueue();
-void dnodeDispatchToMnodeReadQueue(SRpcMsg *rpcMsg);
+int32_t dnodeInitMRead();
+void dnodeCleanupMRead();
+int32_t dnodeAllocMReadQueue();
+void dnodeFreeMReadQueue();
+void dnodeDispatchToMReadQueue(SRpcMsg *rpcMsg);
#ifdef __cplusplus
}
diff --git a/src/dnode/inc/dnodeMWrite.h b/src/dnode/inc/dnodeMWrite.h
index 498fea81c59329b4d30874d36d10182b6e3ae54f..6a3d41bc81a03465a2e097bc711c83a8d8af362e 100644
--- a/src/dnode/inc/dnodeMWrite.h
+++ b/src/dnode/inc/dnodeMWrite.h
@@ -20,11 +20,11 @@
extern "C" {
#endif
-int32_t dnodeInitMnodeWrite();
-void dnodeCleanupMnodeWrite();
-int32_t dnodeAllocateMnodeWqueue();
-void dnodeFreeMnodeWqueue();
-void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg);
+int32_t dnodeInitMWrite();
+void dnodeCleanupMWrite();
+int32_t dnodeAllocMWritequeue();
+void dnodeFreeMWritequeue();
+void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg);
#ifdef __cplusplus
}
diff --git a/src/dnode/inc/dnodeMgmt.h b/src/dnode/inc/dnodeMgmt.h
index e8f4a0823f3cc9846b85804d323e73a8cfc0476e..2038ef5286b32522b11409ba5a253b33228b984d 100644
--- a/src/dnode/inc/dnodeMgmt.h
+++ b/src/dnode/inc/dnodeMgmt.h
@@ -20,6 +20,8 @@
extern "C" {
#endif
+#include "trpc.h"
+
int32_t dnodeInitMgmt();
void dnodeCleanupMgmt();
int32_t dnodeInitMgmtTimer();
@@ -35,8 +37,8 @@ void* dnodeGetVnodeTsdb(void *pVnode);
void dnodeReleaseVnode(void *pVnode);
void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
-void dnodeGetMnodeEpSetForPeer(void *epSet);
-void dnodeGetMnodeEpSetForShell(void *epSet);
+void dnodeGetEpSetForPeer(SRpcEpSet *epSet);
+void dnodeGetEpSetForShell(SRpcEpSet *epSet);
#ifdef __cplusplus
}
diff --git a/src/dnode/inc/dnodeVRead.h b/src/dnode/inc/dnodeVRead.h
index a1035200475259bd91757f934d35a0dd5a69b1fe..b3c3df80b2e201ff9f4b40ebfcceafba57de1366 100644
--- a/src/dnode/inc/dnodeVRead.h
+++ b/src/dnode/inc/dnodeVRead.h
@@ -20,9 +20,11 @@
extern "C" {
#endif
-int32_t dnodeInitVnodeRead();
-void dnodeCleanupVnodeRead();
-void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg);
+int32_t dnodeInitVRead();
+void dnodeCleanupVRead();
+void dnodeDispatchToVReadQueue(SRpcMsg *pMsg);
+void * dnodeAllocVReadQueue(void *pVnode);
+void dnodeFreeVReadQueue(void *rqueue);
#ifdef __cplusplus
}
diff --git a/src/dnode/inc/dnodeVWrite.h b/src/dnode/inc/dnodeVWrite.h
index 7da701a8e270239ee2f48e090138f048cc81f880..759e9ca8a5599236d08228ddd87ed8d4f8c55dca 100644
--- a/src/dnode/inc/dnodeVWrite.h
+++ b/src/dnode/inc/dnodeVWrite.h
@@ -20,9 +20,12 @@
extern "C" {
#endif
-int32_t dnodeInitVnodeWrite();
-void dnodeCleanupVnodeWrite();
-void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg);
+int32_t dnodeInitVWrite();
+void dnodeCleanupVWrite();
+void dnodeDispatchToVWriteQueue(SRpcMsg *pMsg);
+void * dnodeAllocVWriteQueue(void *pVnode);
+void dnodeFreeVWriteQueue(void *pWqueue);
+void dnodeSendRpcVWriteRsp(void *pVnode, void *pWrite, int32_t code);
#ifdef __cplusplus
}
diff --git a/src/dnode/src/dnodeCfg.c b/src/dnode/src/dnodeCfg.c
new file mode 100644
index 0000000000000000000000000000000000000000..16d109a13a7600c5414b1f1edd175a8f15aa4cf0
--- /dev/null
+++ b/src/dnode/src/dnodeCfg.c
@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeCfg.h"
+
+static SDnodeCfg tsCfg = {0};
+static pthread_mutex_t tsCfgMutex;
+
+static int32_t dnodeReadCfg();
+static int32_t dnodeWriteCfg();
+static void dnodeResetCfg(SDnodeCfg *cfg);
+static void dnodePrintCfg(SDnodeCfg *cfg);
+
+int32_t dnodeInitCfg() {
+ pthread_mutex_init(&tsCfgMutex, NULL);
+ dnodeResetCfg(NULL);
+ int32_t ret = dnodeReadCfg();
+ if (ret == 0) {
+ dInfo("dnode cfg is initialized");
+ }
+ return ret;
+}
+
+void dnodeCleanupCfg() { pthread_mutex_destroy(&tsCfgMutex); }
+
+void dnodeUpdateCfg(SDnodeCfg *cfg) {
+ if (tsCfg.dnodeId != 0) return;
+ dnodeResetCfg(cfg);
+}
+
+int32_t dnodeGetDnodeId() {
+ int32_t dnodeId = 0;
+ pthread_mutex_lock(&tsCfgMutex);
+ dnodeId = tsCfg.dnodeId;
+ pthread_mutex_unlock(&tsCfgMutex);
+ return dnodeId;
+}
+
+void dnodeGetCfg(int32_t *dnodeId, char *clusterId) {
+ pthread_mutex_lock(&tsCfgMutex);
+ *dnodeId = tsCfg.dnodeId;
+ tstrncpy(clusterId, tsCfg.clusterId, TSDB_CLUSTER_ID_LEN);
+ pthread_mutex_unlock(&tsCfgMutex);
+}
+
+static void dnodeResetCfg(SDnodeCfg *cfg) {
+ if (cfg == NULL) return;
+ if (cfg->dnodeId == 0) return;
+
+ pthread_mutex_lock(&tsCfgMutex);
+ tsCfg.dnodeId = cfg->dnodeId;
+ tstrncpy(tsCfg.clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN);
+ dnodePrintCfg(cfg);
+ dnodeWriteCfg();
+ pthread_mutex_unlock(&tsCfgMutex);
+}
+
+static void dnodePrintCfg(SDnodeCfg *cfg) {
+ dInfo("dnodeId is set to %d, clusterId is set to %s", cfg->dnodeId, cfg->clusterId);
+}
+
+static int32_t dnodeReadCfg() {
+ int32_t len = 0;
+ int32_t maxLen = 200;
+ char * content = calloc(1, maxLen + 1);
+ cJSON * root = NULL;
+ FILE * fp = NULL;
+ SDnodeCfg cfg = {0};
+
+ char file[TSDB_FILENAME_LEN + 20] = {0};
+ sprintf(file, "%s/dnodeCfg.json", tsDnodeDir);
+
+ fp = fopen(file, "r");
+ if (!fp) {
+ dDebug("failed to read %s, file not exist", file);
+ goto PARSE_CFG_OVER;
+ }
+
+ len = fread(content, 1, maxLen, fp);
+ if (len <= 0) {
+ dError("failed to read %s, content is null", file);
+ goto PARSE_CFG_OVER;
+ }
+
+ content[len] = 0;
+ root = cJSON_Parse(content);
+ if (root == NULL) {
+ dError("failed to read %s, invalid json format", file);
+ goto PARSE_CFG_OVER;
+ }
+
+ cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId");
+ if (!dnodeId || dnodeId->type != cJSON_Number) {
+ dError("failed to read %s, dnodeId not found", file);
+ goto PARSE_CFG_OVER;
+ }
+ cfg.dnodeId = dnodeId->valueint;
+
+ cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
+ if (!clusterId || clusterId->type != cJSON_String) {
+ dError("failed to read %s, clusterId not found", file);
+ goto PARSE_CFG_OVER;
+ }
+ tstrncpy(cfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN);
+
+ dInfo("read file %s successed", file);
+
+PARSE_CFG_OVER:
+ if (content != NULL) free(content);
+ if (root != NULL) cJSON_Delete(root);
+ if (fp != NULL) fclose(fp);
+ terrno = 0;
+
+ dnodeResetCfg(&cfg);
+ return 0;
+}
+
+static int32_t dnodeWriteCfg() {
+ char file[TSDB_FILENAME_LEN + 20] = {0};
+ sprintf(file, "%s/dnodeCfg.json", tsDnodeDir);
+
+ FILE *fp = fopen(file, "w");
+ if (!fp) {
+ dError("failed to write %s, reason:%s", file, strerror(errno));
+ return -1;
+ }
+
+ int32_t len = 0;
+ int32_t maxLen = 200;
+ char * content = calloc(1, maxLen + 1);
+
+ len += snprintf(content + len, maxLen - len, "{\n");
+ len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", tsCfg.dnodeId);
+ len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%s\"\n", tsCfg.clusterId);
+ len += snprintf(content + len, maxLen - len, "}\n");
+
+ fwrite(content, 1, len, fp);
+ fflush(fp);
+ fclose(fp);
+ free(content);
+ terrno = 0;
+
+ dInfo("successed to write %s", file);
+ return 0;
+}
diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c
index 9b68fc1f6c7a4db266eeb36cc20459bc71870739..a9ee4ac649c2f4b2734ce7f0dd59004b08e2fb67 100644
--- a/src/dnode/src/dnodeCheck.c
+++ b/src/dnode/src/dnodeCheck.c
@@ -15,9 +15,7 @@
#define _DEFAULT_SOURCE
#include "os.h"
-#include "taosdef.h"
#include "tglobal.h"
-#include "mnode.h"
#include "dnodeInt.h"
#include "dnodeCheck.h"
@@ -30,8 +28,8 @@ typedef struct {
void (*stopFp)();
} SCheckItem;
-static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}};
-int64_t tsMinFreeMemSizeForStart = 0;
+static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}};
+int64_t tsMinFreeMemSizeForStart = 0;
static int bindTcpPort(int port) {
int serverSocket;
@@ -264,8 +262,6 @@ int32_t dnodeInitCheck() {
}
}
+ dInfo("dnode check is initialized");
return 0;
}
-
-
-
diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c
new file mode 100644
index 0000000000000000000000000000000000000000..9c90c391813f57024ecf090136dfa86f5e5f91e6
--- /dev/null
+++ b/src/dnode/src/dnodeEps.c
@@ -0,0 +1,283 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "hash.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeEps.h"
+
+static SDnodeEps *tsEps = NULL;
+static SHashObj * tsEpsHash = NULL;
+static pthread_mutex_t tsEpsMutex;
+
+static int32_t dnodeReadEps();
+static int32_t dnodeWriteEps();
+static void dnodeResetEps(SDnodeEps *eps);
+static void dnodePrintEps(SDnodeEps *eps);
+
+int32_t dnodeInitEps() {
+ pthread_mutex_init(&tsEpsMutex, NULL);
+ tsEpsHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
+ dnodeResetEps(NULL);
+ int32_t ret = dnodeReadEps();
+ if (ret == 0) {
+ dInfo("dnode eps is initialized");
+ }
+ return ret;
+}
+
+void dnodeCleanupEps() {
+ pthread_mutex_lock(&tsEpsMutex);
+ if (tsEps) {
+ free(tsEps);
+ tsEps = NULL;
+ }
+ if (tsEpsHash) {
+ taosHashCleanup(tsEpsHash);
+ tsEpsHash = NULL;
+ }
+ pthread_mutex_unlock(&tsEpsMutex);
+ pthread_mutex_destroy(&tsEpsMutex);
+}
+
+void dnodeUpdateEps(SDnodeEps *eps) {
+ if (eps == NULL) return;
+
+ eps->dnodeNum = htonl(eps->dnodeNum);
+ for (int32_t i = 0; i < eps->dnodeNum; ++i) {
+ eps->dnodeEps[i].dnodeId = htonl(eps->dnodeEps[i].dnodeId);
+ eps->dnodeEps[i].dnodePort = htons(eps->dnodeEps[i].dnodePort);
+ }
+
+ pthread_mutex_lock(&tsEpsMutex);
+ if (eps->dnodeNum != tsEps->dnodeNum) {
+ dnodeResetEps(eps);
+ dnodeWriteEps();
+ } else {
+ int32_t size = sizeof(SDnodeEps) + eps->dnodeNum * sizeof(SDnodeEp);
+ if (memcmp(eps, tsEps, size) != 0) {
+ dnodeResetEps(eps);
+ dnodeWriteEps();
+ }
+ }
+ pthread_mutex_unlock(&tsEpsMutex);
+}
+
+bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr) {
+ bool changed = false;
+ pthread_mutex_lock(&tsEpsMutex);
+ SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t));
+ if (ep != NULL) {
+ char epSaved[TSDB_EP_LEN + 1];
+ snprintf(epSaved, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort);
+ changed = strcmp(epstr, epSaved) != 0;
+ tstrncpy(epstr, epSaved, TSDB_EP_LEN);
+ }
+ pthread_mutex_unlock(&tsEpsMutex);
+ return changed;
+}
+
+void dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) {
+ pthread_mutex_lock(&tsEpsMutex);
+ SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t));
+ if (ep != NULL) {
+ if (port) *port = ep->dnodePort;
+ if (fqdn) tstrncpy(fqdn, ep->dnodeFqdn, TSDB_FQDN_LEN);
+ if (epstr) snprintf(epstr, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort);
+ }
+ pthread_mutex_unlock(&tsEpsMutex);
+}
+
+static void dnodeResetEps(SDnodeEps *eps) {
+ if (eps == NULL) {
+ int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp);
+ if (tsEps == NULL) {
+ tsEps = calloc(1, size);
+ } else {
+ tsEps->dnodeNum = 0;
+ }
+ } else {
+ assert(tsEps);
+
+ int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp) * eps->dnodeNum;
+ if (eps->dnodeNum > tsEps->dnodeNum) {
+ tsEps = realloc(tsEps, size);
+ }
+ memcpy(tsEps, eps, size);
+ dnodePrintEps(eps);
+ }
+
+ for (int32_t i = 0; i < tsEps->dnodeNum; ++i) {
+ SDnodeEp *ep = &tsEps->dnodeEps[i];
+ taosHashPut(tsEpsHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp));
+ }
+}
+
+static void dnodePrintEps(SDnodeEps *eps) {
+ dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum);
+ for (int32_t i = 0; i < eps->dnodeNum; i++) {
+ SDnodeEp *ep = &eps->dnodeEps[i];
+ dDebug("dnodeId:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort);
+ }
+}
+
+static int32_t dnodeReadEps() {
+ int32_t ret = -1;
+ int32_t len = 0;
+ int32_t maxLen = 30000;
+ char * content = calloc(1, maxLen + 1);
+ cJSON * root = NULL;
+ FILE * fp = NULL;
+ SDnodeEps *eps = NULL;
+
+ char file[TSDB_FILENAME_LEN + 20] = {0};
+ sprintf(file, "%s/dnodeEps.json", tsDnodeDir);
+
+ fp = fopen(file, "r");
+ if (!fp) {
+ dDebug("failed to read %s, file not exist", file);
+ goto PRASE_EPS_OVER;
+ }
+
+ len = fread(content, 1, maxLen, fp);
+ if (len <= 0) {
+ dError("failed to read %s, content is null", file);
+ goto PRASE_EPS_OVER;
+ }
+
+ content[len] = 0;
+ root = cJSON_Parse(content);
+ if (root == NULL) {
+ dError("failed to read %s, invalid json format", file);
+ goto PRASE_EPS_OVER;
+ }
+
+ cJSON *dnodeNum = cJSON_GetObjectItem(root, "dnodeNum");
+ if (!dnodeNum || dnodeNum->type != cJSON_Number) {
+ dError("failed to read %s, dnodeNum not found", file);
+ goto PRASE_EPS_OVER;
+ }
+
+ cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos");
+ if (!dnodeInfos || dnodeInfos->type != cJSON_Array) {
+ dError("failed to read %s, dnodeInfos not found", file);
+ goto PRASE_EPS_OVER;
+ }
+
+ int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos);
+ if (dnodeInfosSize != dnodeNum->valueint) {
+ dError("failed to read %s, dnodeInfos size:%d not matched dnodeNum:%d", file, dnodeInfosSize,
+ (int32_t)dnodeNum->valueint);
+ goto PRASE_EPS_OVER;
+ }
+
+ int32_t epsSize = sizeof(SDnodeEps) + dnodeInfosSize * sizeof(SDnodeEp);
+ eps = calloc(1, epsSize);
+ eps->dnodeNum = dnodeInfosSize;
+
+ for (int32_t i = 0; i < dnodeInfosSize; ++i) {
+ cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i);
+ if (dnodeInfo == NULL) break;
+
+ SDnodeEp *ep = &eps->dnodeEps[i];
+
+ cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId");
+ if (!dnodeId || dnodeId->type != cJSON_Number) {
+ dError("failed to read %s, dnodeId not found", file);
+ goto PRASE_EPS_OVER;
+ }
+ ep->dnodeId = dnodeId->valueint;
+
+ cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn");
+ if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
+ dError("failed to read %s, dnodeFqdn not found", file);
+ goto PRASE_EPS_OVER;
+ }
+ strncpy(ep->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
+
+ cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort");
+ if (!dnodePort || dnodePort->type != cJSON_Number) {
+ dError("failed to read %s, dnodePort not found", file);
+ goto PRASE_EPS_OVER;
+ }
+ ep->dnodePort = (uint16_t)dnodePort->valueint;
+ }
+
+ ret = 0;
+
+ dInfo("read file %s successed", file);
+ dnodePrintEps(eps);
+
+PRASE_EPS_OVER:
+ if (content != NULL) free(content);
+ if (root != NULL) cJSON_Delete(root);
+ if (fp != NULL) fclose(fp);
+ if (ret != 0) {
+ if (eps) free(eps);
+ eps = NULL;
+ }
+
+ dnodeResetEps(eps);
+ if (eps) free(eps);
+
+ dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort);
+
+ terrno = 0;
+ return 0;
+}
+
+static int32_t dnodeWriteEps() {
+ char file[TSDB_FILENAME_LEN + 20] = {0};
+ sprintf(file, "%s/dnodeEps.json", tsDnodeDir);
+
+ FILE *fp = fopen(file, "w");
+ if (!fp) {
+ dError("failed to write %s, reason:%s", file, strerror(errno));
+ return -1;
+ }
+
+ int32_t len = 0;
+ int32_t maxLen = 30000;
+ char * content = calloc(1, maxLen + 1);
+
+ len += snprintf(content + len, maxLen - len, "{\n");
+ len += snprintf(content + len, maxLen - len, " \"dnodeNum\": %d,\n", tsEps->dnodeNum);
+ len += snprintf(content + len, maxLen - len, " \"dnodeInfos\": [{\n");
+ for (int32_t i = 0; i < tsEps->dnodeNum; ++i) {
+ SDnodeEp *ep = &tsEps->dnodeEps[i];
+ len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", ep->dnodeId);
+ len += snprintf(content + len, maxLen - len, " \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn);
+ len += snprintf(content + len, maxLen - len, " \"dnodePort\": %u\n", ep->dnodePort);
+ if (i < tsEps->dnodeNum - 1) {
+ len += snprintf(content + len, maxLen - len, " },{\n");
+ } else {
+ len += snprintf(content + len, maxLen - len, " }]\n");
+ }
+ }
+ len += snprintf(content + len, maxLen - len, "}\n");
+
+ fwrite(content, 1, len, fp);
+ fflush(fp);
+ fclose(fp);
+ free(content);
+ terrno = 0;
+
+ dInfo("successed to write %s", file);
+ return 0;
+}
diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c
new file mode 100644
index 0000000000000000000000000000000000000000..cefe44aebe7f87803141ce3d75c45dca18463849
--- /dev/null
+++ b/src/dnode/src/dnodeMInfos.c
@@ -0,0 +1,288 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "mnode.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeMInfos.h"
+
+static SMnodeInfos tsMInfos;
+static SRpcEpSet tsMEpSet;
+static pthread_mutex_t tsMInfosMutex;
+
+static void dnodeResetMInfos(SMnodeInfos *minfos);
+static void dnodePrintMInfos(SMnodeInfos *minfos);
+static int32_t dnodeReadMInfos();
+static int32_t dnodeWriteMInfos();
+
+int32_t dnodeInitMInfos() {
+ pthread_mutex_init(&tsMInfosMutex, NULL);
+ dnodeResetMInfos(NULL);
+ int32_t ret = dnodeReadMInfos();
+ if (ret == 0) {
+ dInfo("dnode minfos is initialized");
+ }
+
+ return ret;
+}
+
+void dnodeCleanupMInfos() { pthread_mutex_destroy(&tsMInfosMutex); }
+
+void dnodeUpdateMInfos(SMnodeInfos *minfos) {
+ if (minfos->mnodeNum <= 0 || minfos->mnodeNum > 3) {
+ dError("invalid mnode infos, mnodeNum:%d", minfos->mnodeNum);
+ return;
+ }
+
+ for (int32_t i = 0; i < minfos->mnodeNum; ++i) {
+ SMnodeInfo *minfo = &minfos->mnodeInfos[i];
+ minfo->mnodeId = htonl(minfo->mnodeId);
+ if (minfo->mnodeId <= 0 || strlen(minfo->mnodeEp) <= 5) {
+ dError("invalid mnode info:%d, mnodeId:%d mnodeEp:%s", i, minfo->mnodeId, minfo->mnodeEp);
+ return;
+ }
+ }
+
+ pthread_mutex_lock(&tsMInfosMutex);
+ if (minfos->mnodeNum != tsMInfos.mnodeNum) {
+ dnodeResetMInfos(minfos);
+ dnodeWriteMInfos();
+ sdbUpdateAsync();
+ } else {
+ int32_t size = sizeof(SMnodeInfos);
+ if (memcmp(minfos, &tsMInfos, size) != 0) {
+ dnodeResetMInfos(minfos);
+ dnodeWriteMInfos();
+ sdbUpdateAsync();
+ }
+ }
+ pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+void dnodeUpdateEpSetForPeer(SRpcEpSet *ep) {
+ if (ep->numOfEps <= 0) {
+ dError("minfos is changed, but content is invalid, discard it");
+ return;
+ }
+
+ pthread_mutex_lock(&tsMInfosMutex);
+ dInfo("minfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse);
+ for (int i = 0; i < ep->numOfEps; ++i) {
+ ep->port[i] -= TSDB_PORT_DNODEDNODE;
+ dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]);
+ }
+ tsMEpSet = *ep;
+ pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+bool dnodeIsMasterEp(char *ep) {
+ pthread_mutex_lock(&tsMInfosMutex);
+ bool isMaster = strcmp(ep, tsMInfos.mnodeInfos[tsMEpSet.inUse].mnodeEp) == 0;
+ pthread_mutex_unlock(&tsMInfosMutex);
+
+ return isMaster;
+}
+
+void dnodeGetMInfos(SMnodeInfos *minfos) {
+ pthread_mutex_lock(&tsMInfosMutex);
+ memcpy(minfos, &tsMInfos, sizeof(SMnodeInfos));
+ for (int32_t i = 0; i < tsMInfos.mnodeNum; ++i) {
+ minfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId);
+ }
+ pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+void dnodeGetEpSetForPeer(SRpcEpSet *epSet) {
+ pthread_mutex_lock(&tsMInfosMutex);
+ *epSet = tsMEpSet;
+ for (int i = 0; i < epSet->numOfEps; ++i) {
+ epSet->port[i] += TSDB_PORT_DNODEDNODE;
+ }
+ pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+void dnodeGetEpSetForShell(SRpcEpSet *epSet) {
+ pthread_mutex_lock(&tsMInfosMutex);
+ *epSet = tsMEpSet;
+ pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+static void dnodePrintMInfos(SMnodeInfos *minfos) {
+ dInfo("print mnode infos, mnodeNum:%d inUse:%d", minfos->mnodeNum, minfos->inUse);
+ for (int32_t i = 0; i < minfos->mnodeNum; i++) {
+ dInfo("mnode index:%d, %s", minfos->mnodeInfos[i].mnodeId, minfos->mnodeInfos[i].mnodeEp);
+ }
+}
+
+static void dnodeResetMInfos(SMnodeInfos *minfos) {
+ if (minfos == NULL) {
+ tsMEpSet.numOfEps = 1;
+ taosGetFqdnPortFromEp(tsFirst, tsMEpSet.fqdn[0], &tsMEpSet.port[0]);
+
+ if (strcmp(tsSecond, tsFirst) != 0) {
+ tsMEpSet.numOfEps = 2;
+ taosGetFqdnPortFromEp(tsSecond, tsMEpSet.fqdn[1], &tsMEpSet.port[1]);
+ }
+ return;
+ }
+
+ if (minfos->mnodeNum == 0) return;
+
+ int32_t size = sizeof(SMnodeInfos);
+ memcpy(&tsMInfos, minfos, size);
+
+ tsMEpSet.inUse = tsMInfos.inUse;
+ tsMEpSet.numOfEps = tsMInfos.mnodeNum;
+ for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) {
+ taosGetFqdnPortFromEp(tsMInfos.mnodeInfos[i].mnodeEp, tsMEpSet.fqdn[i], &tsMEpSet.port[i]);
+ }
+
+ dnodePrintMInfos(minfos);
+}
+
+static int32_t dnodeReadMInfos() {
+ int32_t len = 0;
+ int32_t maxLen = 2000;
+ char * content = calloc(1, maxLen + 1);
+ cJSON * root = NULL;
+ FILE * fp = NULL;
+ SMnodeInfos minfos = {0};
+
+ char file[TSDB_FILENAME_LEN + 20] = {0};
+ sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir);
+
+ fp = fopen(file, "r");
+ if (!fp) {
+ dDebug("failed to read %s, file not exist", file);
+ goto PARSE_MINFOS_OVER;
+ }
+
+ len = fread(content, 1, maxLen, fp);
+ if (len <= 0) {
+ dError("failed to read %s, content is null", file);
+ goto PARSE_MINFOS_OVER;
+ }
+
+ content[len] = 0;
+ root = cJSON_Parse(content);
+ if (root == NULL) {
+ dError("failed to read %s, invalid json format", file);
+ goto PARSE_MINFOS_OVER;
+ }
+
+ cJSON *inUse = cJSON_GetObjectItem(root, "inUse");
+ if (!inUse || inUse->type != cJSON_Number) {
+ dError("failed to read mnodeEpSet.json, inUse not found");
+ goto PARSE_MINFOS_OVER;
+ }
+ tsMInfos.inUse = inUse->valueint;
+
+ cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum");
+ if (!nodeNum || nodeNum->type != cJSON_Number) {
+ dError("failed to read mnodeEpSet.json, nodeNum not found");
+ goto PARSE_MINFOS_OVER;
+ }
+ minfos.mnodeNum = nodeNum->valueint;
+
+ cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
+ if (!nodeInfos || nodeInfos->type != cJSON_Array) {
+ dError("failed to read mnodeEpSet.json, nodeInfos not found");
+ goto PARSE_MINFOS_OVER;
+ }
+
+ int size = cJSON_GetArraySize(nodeInfos);
+ if (size != minfos.mnodeNum) {
+ dError("failed to read mnodeEpSet.json, nodeInfos size not matched");
+ goto PARSE_MINFOS_OVER;
+ }
+
+ for (int i = 0; i < size; ++i) {
+ cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
+ if (nodeInfo == NULL) continue;
+
+ cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
+ if (!nodeId || nodeId->type != cJSON_Number) {
+ dError("failed to read mnodeEpSet.json, nodeId not found");
+ goto PARSE_MINFOS_OVER;
+ }
+ minfos.mnodeInfos[i].mnodeId = nodeId->valueint;
+
+ cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
+ if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
+ dError("failed to read mnodeEpSet.json, nodeName not found");
+ goto PARSE_MINFOS_OVER;
+ }
+ strncpy(minfos.mnodeInfos[i].mnodeEp, nodeEp->valuestring, TSDB_EP_LEN);
+ }
+
+ dInfo("read file %s successed", file);
+ dnodePrintMInfos(&minfos);
+
+PARSE_MINFOS_OVER:
+ if (content != NULL) free(content);
+ if (root != NULL) cJSON_Delete(root);
+ if (fp != NULL) fclose(fp);
+ terrno = 0;
+
+ for (int32_t i = 0; i < minfos.mnodeNum; ++i) {
+ SMnodeInfo *mInfo = &minfos.mnodeInfos[i];
+ dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL);
+ }
+ dnodeResetMInfos(&minfos);
+ return 0;
+}
+
+static int32_t dnodeWriteMInfos() {
+ char file[TSDB_FILENAME_LEN + 20] = {0};
+ sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir);
+
+ FILE *fp = fopen(file, "w");
+ if (!fp) {
+ dError("failed to write %s, reason:%s", file, strerror(errno));
+ return -1;
+ }
+
+ int32_t len = 0;
+ int32_t maxLen = 2000;
+ char * content = calloc(1, maxLen + 1);
+
+ len += snprintf(content + len, maxLen - len, "{\n");
+ len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsMInfos.inUse);
+ len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsMInfos.mnodeNum);
+ len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
+ for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) {
+ len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMInfos.mnodeInfos[i].mnodeId);
+ len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", tsMInfos.mnodeInfos[i].mnodeEp);
+ if (i < tsMInfos.mnodeNum - 1) {
+ len += snprintf(content + len, maxLen - len, " },{\n");
+ } else {
+ len += snprintf(content + len, maxLen - len, " }]\n");
+ }
+ }
+ len += snprintf(content + len, maxLen - len, "}\n");
+
+ fwrite(content, 1, len, fp);
+ fflush(fp);
+ fclose(fp);
+ free(content);
+ terrno = 0;
+
+ dInfo("successed to write %s", file);
+ return 0;
+}
diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c
index 8414d79a9815287dfdfae3af5cc123304745c56d..05b37bd3388c22046d2a0dd1827655c97c2e40ad 100644
--- a/src/dnode/src/dnodeMPeer.c
+++ b/src/dnode/src/dnodeMPeer.c
@@ -35,44 +35,44 @@ typedef struct {
typedef struct {
int32_t curNum;
int32_t maxNum;
- SMPeerWorker *peerWorker;
+ SMPeerWorker *worker;
} SMPeerWorkerPool;
-static SMPeerWorkerPool tsMPeerPool;
+static SMPeerWorkerPool tsMPeerWP;
static taos_qset tsMPeerQset;
static taos_queue tsMPeerQueue;
-static void *dnodeProcessMnodePeerQueue(void *param);
+static void *dnodeProcessMPeerQueue(void *param);
-int32_t dnodeInitMnodePeer() {
+int32_t dnodeInitMPeer() {
tsMPeerQset = taosOpenQset();
- tsMPeerPool.maxNum = 1;
- tsMPeerPool.curNum = 0;
- tsMPeerPool.peerWorker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerPool.maxNum);
+ tsMPeerWP.maxNum = 1;
+ tsMPeerWP.curNum = 0;
+ tsMPeerWP.worker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerWP.maxNum);
- if (tsMPeerPool.peerWorker == NULL) return -1;
- for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
- SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+ if (tsMPeerWP.worker == NULL) return -1;
+ for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
+ SMPeerWorker *pWorker = tsMPeerWP.worker + i;
pWorker->workerId = i;
dDebug("dnode mpeer worker:%d is created", i);
}
- dDebug("dnode mpeer is opened, workers:%d qset:%p", tsMPeerPool.maxNum, tsMPeerQset);
+ dDebug("dnode mpeer is initialized, workers:%d qset:%p", tsMPeerWP.maxNum, tsMPeerQset);
return 0;
}
-void dnodeCleanupMnodePeer() {
- for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
- SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+void dnodeCleanupMPeer() {
+ for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
+ SMPeerWorker *pWorker = tsMPeerWP.worker + i;
if (pWorker->thread) {
taosQsetThreadResume(tsMPeerQset);
}
dDebug("dnode mpeer worker:%d is closed", i);
}
- for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
- SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+ for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
+ SMPeerWorker *pWorker = tsMPeerWP.worker + i;
dDebug("dnode mpeer worker:%d start to join", i);
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
@@ -84,61 +84,60 @@ void dnodeCleanupMnodePeer() {
taosCloseQset(tsMPeerQset);
tsMPeerQset = NULL;
- taosTFree(tsMPeerPool.peerWorker);
+ tfree(tsMPeerWP.worker);
}
-int32_t dnodeAllocateMnodePqueue() {
+int32_t dnodeAllocateMPeerQueue() {
tsMPeerQueue = taosOpenQueue();
if (tsMPeerQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
taosAddIntoQset(tsMPeerQset, tsMPeerQueue, NULL);
- for (int32_t i = tsMPeerPool.curNum; i < tsMPeerPool.maxNum; ++i) {
- SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+ for (int32_t i = tsMPeerWP.curNum; i < tsMPeerWP.maxNum; ++i) {
+ SMPeerWorker *pWorker = tsMPeerWP.worker + i;
pWorker->workerId = i;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
- if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodePeerQueue, pWorker) != 0) {
+ if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMPeerQueue, pWorker) != 0) {
dError("failed to create thread to process mpeer queue, reason:%s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
- tsMPeerPool.curNum = i + 1;
- dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerPool.maxNum);
+ tsMPeerWP.curNum = i + 1;
+ dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerWP.maxNum);
}
dDebug("dnode mpeer queue:%p is allocated", tsMPeerQueue);
return TSDB_CODE_SUCCESS;
}
-void dnodeFreeMnodePqueue() {
+void dnodeFreeMPeerQueue() {
dDebug("dnode mpeer queue:%p is freed", tsMPeerQueue);
taosCloseQueue(tsMPeerQueue);
tsMPeerQueue = NULL;
}
-void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg) {
+void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg) {
if (!mnodeIsRunning() || tsMPeerQueue == NULL) {
dnodeSendRedirectMsg(pMsg, false);
- rpcFreeCont(pMsg->pCont);
- return;
+ } else {
+ SMnodeMsg *pPeer = mnodeCreateMsg(pMsg);
+ taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer);
}
- SMnodeMsg *pPeer = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
- mnodeCreateMsg(pPeer, pMsg);
- taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer);
+ rpcFreeCont(pMsg->pCont);
}
-static void dnodeFreeMnodePeerMsg(SMnodeMsg *pPeer) {
+static void dnodeFreeMPeerMsg(SMnodeMsg *pPeer) {
mnodeCleanupMsg(pPeer);
taosFreeQitem(pPeer);
}
-static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) {
+static void dnodeSendRpcMPeerRsp(SMnodeMsg *pPeer, int32_t code) {
if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
SRpcMsg rpcRsp = {
@@ -149,10 +148,10 @@ static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) {
};
rpcSendResponse(&rpcRsp);
- dnodeFreeMnodePeerMsg(pPeer);
+ dnodeFreeMPeerMsg(pPeer);
}
-static void *dnodeProcessMnodePeerQueue(void *param) {
+static void *dnodeProcessMPeerQueue(void *param) {
SMnodeMsg *pPeerMsg;
int32_t type;
void * unUsed;
@@ -165,7 +164,7 @@ static void *dnodeProcessMnodePeerQueue(void *param) {
dDebug("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]);
int32_t code = mnodeProcessPeerReq(pPeerMsg);
- dnodeSendRpcMnodePeerRsp(pPeerMsg, code);
+ dnodeSendRpcMPeerRsp(pPeerMsg, code);
}
return NULL;
diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c
index fdcbb5889f766ddebdb3f1e56ccffa0b5b129552..c14c7a81586d306ff1423e8dba91e0844293c407 100644
--- a/src/dnode/src/dnodeMRead.c
+++ b/src/dnode/src/dnodeMRead.c
@@ -35,46 +35,46 @@ typedef struct {
typedef struct {
int32_t curNum;
int32_t maxNum;
- SMReadWorker *readWorker;
+ SMReadWorker *worker;
} SMReadWorkerPool;
-static SMReadWorkerPool tsMReadPool;
+static SMReadWorkerPool tsMReadWP;
static taos_qset tsMReadQset;
static taos_queue tsMReadQueue;
-static void *dnodeProcessMnodeReadQueue(void *param);
+static void *dnodeProcessMReadQueue(void *param);
-int32_t dnodeInitMnodeRead() {
+int32_t dnodeInitMRead() {
tsMReadQset = taosOpenQset();
- tsMReadPool.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2;
- tsMReadPool.maxNum = MAX(2, tsMReadPool.maxNum);
- tsMReadPool.maxNum = MIN(4, tsMReadPool.maxNum);
- tsMReadPool.curNum = 0;
- tsMReadPool.readWorker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadPool.maxNum);
+ tsMReadWP.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2;
+ tsMReadWP.maxNum = MAX(2, tsMReadWP.maxNum);
+ tsMReadWP.maxNum = MIN(4, tsMReadWP.maxNum);
+ tsMReadWP.curNum = 0;
+ tsMReadWP.worker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadWP.maxNum);
- if (tsMReadPool.readWorker == NULL) return -1;
- for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
- SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+ if (tsMReadWP.worker == NULL) return -1;
+ for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
+ SMReadWorker *pWorker = tsMReadWP.worker + i;
pWorker->workerId = i;
dDebug("dnode mread worker:%d is created", i);
}
- dDebug("dnode mread is opened, workers:%d qset:%p", tsMReadPool.maxNum, tsMReadQset);
+ dDebug("dnode mread is initialized, workers:%d qset:%p", tsMReadWP.maxNum, tsMReadQset);
return 0;
}
-void dnodeCleanupMnodeRead() {
- for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
- SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+void dnodeCleanupMRead() {
+ for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
+ SMReadWorker *pWorker = tsMReadWP.worker + i;
if (pWorker->thread) {
taosQsetThreadResume(tsMReadQset);
}
dDebug("dnode mread worker:%d is closed", i);
}
- for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
- SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+ for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
+ SMReadWorker *pWorker = tsMReadWP.worker + i;
dDebug("dnode mread worker:%d start to join", i);
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
@@ -86,64 +86,63 @@ void dnodeCleanupMnodeRead() {
taosCloseQset(tsMReadQset);
tsMReadQset = NULL;
- free(tsMReadPool.readWorker);
+ free(tsMReadWP.worker);
}
-int32_t dnodeAllocateMnodeRqueue() {
+int32_t dnodeAllocMReadQueue() {
tsMReadQueue = taosOpenQueue();
if (tsMReadQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
taosAddIntoQset(tsMReadQset, tsMReadQueue, NULL);
- for (int32_t i = tsMReadPool.curNum; i < tsMReadPool.maxNum; ++i) {
- SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+ for (int32_t i = tsMReadWP.curNum; i < tsMReadWP.maxNum; ++i) {
+ SMReadWorker *pWorker = tsMReadWP.worker + i;
pWorker->workerId = i;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
- if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeReadQueue, pWorker) != 0) {
+ if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMReadQueue, pWorker) != 0) {
dError("failed to create thread to process mread queue, reason:%s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
- tsMReadPool.curNum = i + 1;
- dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadPool.maxNum);
+ tsMReadWP.curNum = i + 1;
+ dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadWP.maxNum);
}
dDebug("dnode mread queue:%p is allocated", tsMReadQueue);
return TSDB_CODE_SUCCESS;
}
-void dnodeFreeMnodeRqueue() {
+void dnodeFreeMReadQueue() {
dDebug("dnode mread queue:%p is freed", tsMReadQueue);
taosCloseQueue(tsMReadQueue);
tsMReadQueue = NULL;
}
-void dnodeDispatchToMnodeReadQueue(SRpcMsg *pMsg) {
+void dnodeDispatchToMReadQueue(SRpcMsg *pMsg) {
if (!mnodeIsRunning() || tsMReadQueue == NULL) {
dnodeSendRedirectMsg(pMsg, true);
- rpcFreeCont(pMsg->pCont);
- return;
+ } else {
+ SMnodeMsg *pRead = mnodeCreateMsg(pMsg);
+ taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead);
}
- SMnodeMsg *pRead = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
- mnodeCreateMsg(pRead, pMsg);
- taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead);
+ rpcFreeCont(pMsg->pCont);
}
-static void dnodeFreeMnodeReadMsg(SMnodeMsg *pRead) {
+static void dnodeFreeMReadMsg(SMnodeMsg *pRead) {
mnodeCleanupMsg(pRead);
taosFreeQitem(pRead);
}
-static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) {
+static void dnodeSendRpcMReadRsp(SMnodeMsg *pRead, int32_t code) {
if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) {
// may be a auto create req, should put into write queue
- dnodeReprocessMnodeWriteMsg(pRead);
+ dnodeReprocessMWriteMsg(pRead);
return;
}
@@ -155,23 +154,23 @@ static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) {
};
rpcSendResponse(&rpcRsp);
- dnodeFreeMnodeReadMsg(pRead);
+ dnodeFreeMReadMsg(pRead);
}
-static void *dnodeProcessMnodeReadQueue(void *param) {
- SMnodeMsg *pReadMsg;
+static void *dnodeProcessMReadQueue(void *param) {
+ SMnodeMsg *pRead;
int32_t type;
void * unUsed;
-
+
while (1) {
- if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pReadMsg, &unUsed) == 0) {
+ if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pRead, &unUsed) == 0) {
dDebug("qset:%p, mnode read got no message from qset, exiting", tsMReadQset);
break;
}
- dDebug("%p, msg:%s will be processed in mread queue", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]);
- int32_t code = mnodeProcessRead(pReadMsg);
- dnodeSendRpcMnodeReadRsp(pReadMsg, code);
+ dDebug("%p, msg:%s will be processed in mread queue", pRead->rpcMsg.ahandle, taosMsg[pRead->rpcMsg.msgType]);
+ int32_t code = mnodeProcessRead(pRead);
+ dnodeSendRpcMReadRsp(pRead, code);
}
return NULL;
diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c
index 384a0fae75088197d7fb01dd67c6e1b9d38739cd..bde4b95bc6d0019178312cfa76a34bb381608097 100644
--- a/src/dnode/src/dnodeMWrite.c
+++ b/src/dnode/src/dnodeMWrite.c
@@ -36,45 +36,45 @@ typedef struct {
typedef struct {
int32_t curNum;
int32_t maxNum;
- SMWriteWorker *writeWorker;
+ SMWriteWorker *worker;
} SMWriteWorkerPool;
-static SMWriteWorkerPool tsMWritePool;
+static SMWriteWorkerPool tsMWriteWP;
static taos_qset tsMWriteQset;
static taos_queue tsMWriteQueue;
extern void * tsDnodeTmr;
-static void *dnodeProcessMnodeWriteQueue(void *param);
+static void *dnodeProcessMWriteQueue(void *param);
-int32_t dnodeInitMnodeWrite() {
+int32_t dnodeInitMWrite() {
tsMWriteQset = taosOpenQset();
- tsMWritePool.maxNum = 1;
- tsMWritePool.curNum = 0;
- tsMWritePool.writeWorker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWritePool.maxNum);
+ tsMWriteWP.maxNum = 1;
+ tsMWriteWP.curNum = 0;
+ tsMWriteWP.worker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWriteWP.maxNum);
- if (tsMWritePool.writeWorker == NULL) return -1;
- for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
- SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+ if (tsMWriteWP.worker == NULL) return -1;
+ for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
+ SMWriteWorker *pWorker = tsMWriteWP.worker + i;
pWorker->workerId = i;
dDebug("dnode mwrite worker:%d is created", i);
}
- dDebug("dnode mwrite is opened, workers:%d qset:%p", tsMWritePool.maxNum, tsMWriteQset);
+ dDebug("dnode mwrite is initialized, workers:%d qset:%p", tsMWriteWP.maxNum, tsMWriteQset);
return 0;
}
-void dnodeCleanupMnodeWrite() {
- for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
- SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+void dnodeCleanupMWrite() {
+ for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
+ SMWriteWorker *pWorker = tsMWriteWP.worker + i;
if (pWorker->thread) {
taosQsetThreadResume(tsMWriteQset);
}
dDebug("dnode mwrite worker:%d is closed", i);
}
- for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
- SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+ for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
+ SMWriteWorker *pWorker = tsMWriteWP.worker + i;
dDebug("dnode mwrite worker:%d start to join", i);
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
@@ -86,58 +86,56 @@ void dnodeCleanupMnodeWrite() {
taosCloseQset(tsMWriteQset);
tsMWriteQset = NULL;
- taosTFree(tsMWritePool.writeWorker);
+ tfree(tsMWriteWP.worker);
}
-int32_t dnodeAllocateMnodeWqueue() {
+int32_t dnodeAllocMWritequeue() {
tsMWriteQueue = taosOpenQueue();
if (tsMWriteQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
taosAddIntoQset(tsMWriteQset, tsMWriteQueue, NULL);
- for (int32_t i = tsMWritePool.curNum; i < tsMWritePool.maxNum; ++i) {
- SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+ for (int32_t i = tsMWriteWP.curNum; i < tsMWriteWP.maxNum; ++i) {
+ SMWriteWorker *pWorker = tsMWriteWP.worker + i;
pWorker->workerId = i;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
- if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeWriteQueue, pWorker) != 0) {
+ if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMWriteQueue, pWorker) != 0) {
dError("failed to create thread to process mwrite queue, reason:%s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
- tsMWritePool.curNum = i + 1;
- dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWritePool.maxNum);
+ tsMWriteWP.curNum = i + 1;
+ dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWriteWP.maxNum);
}
dDebug("dnode mwrite queue:%p is allocated", tsMWriteQueue);
return TSDB_CODE_SUCCESS;
}
-void dnodeFreeMnodeWqueue() {
+void dnodeFreeMWritequeue() {
dDebug("dnode mwrite queue:%p is freed", tsMWriteQueue);
taosCloseQueue(tsMWriteQueue);
tsMWriteQueue = NULL;
}
-void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) {
+void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) {
if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
dnodeSendRedirectMsg(pMsg, true);
- rpcFreeCont(pMsg->pCont);
- return;
+ } else {
+ SMnodeMsg *pWrite = mnodeCreateMsg(pMsg);
+ dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
+ taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
+ taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
}
- SMnodeMsg *pWrite = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
- mnodeCreateMsg(pWrite, pMsg);
-
- dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
- taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
- taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
+ rpcFreeCont(pMsg->pCont);
}
-static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
+static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) {
dDebug("app:%p:%p, msg:%s is freed from mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
@@ -145,12 +143,12 @@ static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
taosFreeQitem(pWrite);
}
-void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) {
+void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code) {
SMnodeMsg *pWrite = pMsg;
if (pWrite == NULL) return;
if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) {
- dnodeReprocessMnodeWriteMsg(pWrite);
+ dnodeReprocessMWriteMsg(pWrite);
return;
}
@@ -162,10 +160,10 @@ void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) {
};
rpcSendResponse(&rpcRsp);
- dnodeFreeMnodeWriteMsg(pWrite);
+ dnodeFreeMWriteMsg(pWrite);
}
-static void *dnodeProcessMnodeWriteQueue(void *param) {
+static void *dnodeProcessMWriteQueue(void *param) {
SMnodeMsg *pWrite;
int32_t type;
void * unUsed;
@@ -180,13 +178,13 @@ static void *dnodeProcessMnodeWriteQueue(void *param) {
taosMsg[pWrite->rpcMsg.msgType]);
int32_t code = mnodeProcessWrite(pWrite);
- dnodeSendRpcMnodeWriteRsp(pWrite, code);
+ dnodeSendRpcMWriteRsp(pWrite, code);
}
return NULL;
}
-void dnodeReprocessMnodeWriteMsg(void *pMsg) {
+void dnodeReprocessMWriteMsg(void *pMsg) {
SMnodeMsg *pWrite = pMsg;
if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
@@ -194,7 +192,7 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) {
taosMsg[pWrite->rpcMsg.msgType], pWrite->retry);
dnodeSendRedirectMsg(pMsg, true);
- dnodeFreeMnodeWriteMsg(pWrite);
+ dnodeFreeMWriteMsg(pWrite);
} else {
dDebug("app:%p:%p, msg:%s is reput into mwrite queue:%p, retry times:%d", pWrite->rpcMsg.ahandle, pWrite,
taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue, pWrite->retry);
@@ -203,12 +201,12 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) {
}
}
-static void dnodeDoDelayReprocessMnodeWriteMsg(void *param, void *tmrId) {
- dnodeReprocessMnodeWriteMsg(param);
+static void dnodeDoDelayReprocessMWriteMsg(void *param, void *tmrId) {
+ dnodeReprocessMWriteMsg(param);
}
-void dnodeDelayReprocessMnodeWriteMsg(void *pMsg) {
+void dnodeDelayReprocessMWriteMsg(void *pMsg) {
SMnodeMsg *mnodeMsg = pMsg;
void *unUsed = NULL;
- taosTmrReset(dnodeDoDelayReprocessMnodeWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed);
+ taosTmrReset(dnodeDoDelayReprocessMWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed);
}
diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c
index 7d65fd673a6f37b12ca9cc170b20114bdb8b307c..c4c5dcc9119c02936f27e48657dc5f017c641a4c 100644
--- a/src/dnode/src/dnodeMain.c
+++ b/src/dnode/src/dnodeMain.c
@@ -19,11 +19,16 @@
#include "tutil.h"
#include "tconfig.h"
#include "tglobal.h"
+#include "twal.h"
+#include "trpc.h"
#include "dnode.h"
#include "dnodeInt.h"
#include "dnodeMgmt.h"
#include "dnodePeer.h"
#include "dnodeModule.h"
+#include "dnodeEps.h"
+#include "dnodeMInfos.h"
+#include "dnodeCfg.h"
#include "dnodeCheck.h"
#include "dnodeVRead.h"
#include "dnodeVWrite.h"
@@ -35,29 +40,36 @@
#include "tpath.h"
#include "tdisk.h"
+static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED;
+
static int32_t dnodeInitStorage();
-static void dnodeCleanupStorage();
-static void dnodeSetRunStatus(SDnodeRunStatus status);
-static void dnodeCheckDataDirOpenned(char *dir);
-static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED;
+static void dnodeCleanupStorage();
+static void dnodeSetRunStatus(SRunStatus status);
+static void dnodeCheckDataDirOpenned(char *dir);
static int32_t dnodeInitComponents();
-static void dnodeCleanupComponents(int32_t stepId);
-static int dnodeCreateDir(const char *dir);
+static void dnodeCleanupComponents(int32_t stepId);
+static int dnodeCreateDir(const char *dir);
typedef struct {
const char *const name;
- int (*init)();
- void (*cleanup)();
+ int32_t (*init)();
+ void (*cleanup)();
} SDnodeComponent;
static const SDnodeComponent tsDnodeComponents[] = {
+ {"rpc", rpcInit, rpcCleanup},
{"storage", dnodeInitStorage, dnodeCleanupStorage},
+ {"dnodecfg", dnodeInitCfg, dnodeCleanupCfg},
+ {"dnodeeps", dnodeInitEps, dnodeCleanupEps},
+ {"globalcfg" ,taosCheckGlobalCfg, NULL},
+ {"mnodeinfos",dnodeInitMInfos, dnodeCleanupMInfos},
+ {"wal", walInit, walCleanUp},
{"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
- {"vread", dnodeInitVnodeRead, dnodeCleanupVnodeRead},
- {"vwrite", dnodeInitVnodeWrite, dnodeCleanupVnodeWrite},
- {"mread", dnodeInitMnodeRead, dnodeCleanupMnodeRead},
- {"mwrite", dnodeInitMnodeWrite, dnodeCleanupMnodeWrite},
- {"mpeer", dnodeInitMnodePeer, dnodeCleanupMnodePeer},
+ {"vread", dnodeInitVRead, dnodeCleanupVRead},
+ {"vwrite", dnodeInitVWrite, dnodeCleanupVWrite},
+ {"mread", dnodeInitMRead, dnodeCleanupMRead},
+ {"mwrite", dnodeInitMWrite, dnodeCleanupMWrite},
+ {"mpeer", dnodeInitMPeer, dnodeCleanupMPeer},
{"client", dnodeInitClient, dnodeCleanupClient},
{"server", dnodeInitServer, dnodeCleanupServer},
{"mgmt", dnodeInitMgmt, dnodeCleanupMgmt},
@@ -77,7 +89,9 @@ static int dnodeCreateDir(const char *dir) {
static void dnodeCleanupComponents(int32_t stepId) {
for (int32_t i = stepId; i >= 0; i--) {
- tsDnodeComponents[i].cleanup();
+ if (tsDnodeComponents[i].cleanup) {
+ (*tsDnodeComponents[i].cleanup)();
+ }
}
}
@@ -94,7 +108,7 @@ static int32_t dnodeInitComponents() {
}
int32_t dnodeInitSystem() {
- dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_INITIALIZE);
+ dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE);
tscEmbedded = 1;
taosBlockSIGPIPE();
taosResolveCRC();
@@ -114,21 +128,20 @@ int32_t dnodeInitSystem() {
printf("failed to init log file\n");
}
- if (!taosReadGlobalCfg() || !taosCheckGlobalCfg()) {
+ if (!taosReadGlobalCfg()) {
taosPrintGlobalCfg();
dError("TDengine read global config failed");
return -1;
}
- taosPrintGlobalCfg();
- dInfo("start to initialize TDengine on %s", tsLocalEp);
+ dInfo("start to initialize TDengine");
if (dnodeInitComponents() != 0) {
return -1;
}
dnodeStartModules();
- dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_RUNING);
+ dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING);
dInfo("TDengine is initialized successfully");
@@ -136,20 +149,20 @@ int32_t dnodeInitSystem() {
}
void dnodeCleanUpSystem() {
- if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) {
- dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED);
+ if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) {
+ dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED);
dnodeCleanupComponents(sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]) - 1);
taos_cleanup();
taosCloseLog();
}
}
-SDnodeRunStatus dnodeGetRunStatus() {
- return tsDnodeRunStatus;
+SRunStatus dnodeGetRunStatus() {
+ return tsRunStatus;
}
-static void dnodeSetRunStatus(SDnodeRunStatus status) {
- tsDnodeRunStatus = status;
+static void dnodeSetRunStatus(SRunStatus status) {
+ tsRunStatus = status;
}
static void dnodeCheckDataDirOpenned(char *dir) {
@@ -219,7 +232,7 @@ static int32_t dnodeInitStorage() {
dnodeCheckDataDirOpenned(tsDnodeDir);
- dInfo("storage directory is initialized");
+ dInfo("dnode storage is initialized at %s", tsDnodeDir);
return 0;
}
diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c
index 968a8d9759e5618753996476b40efc3be77f7925..dcb48f7833abf921d1085e1cbb14b9cfbf2aa8b7 100644
--- a/src/dnode/src/dnodeMgmt.c
+++ b/src/dnode/src/dnodeMgmt.c
@@ -31,12 +31,13 @@
#include "mnode.h"
#include "dnodeInt.h"
#include "dnodeMgmt.h"
+#include "dnodeEps.h"
+#include "dnodeCfg.h"
+#include "dnodeMInfos.h"
#include "dnodeVRead.h"
#include "dnodeVWrite.h"
#include "dnodeModule.h"
-#define MPEER_CONTENT_LEN 2000
-
typedef struct {
pthread_t thread;
int32_t threadIndex;
@@ -46,23 +47,18 @@ typedef struct {
int32_t * vnodeList;
} SOpenVnodeThread;
-void * tsDnodeTmr = NULL;
-static void * tsStatusTimer = NULL;
-static uint32_t tsRebootTime;
-
-static SRpcEpSet tsDMnodeEpSet = {0};
-static SDMMnodeInfos tsDMnodeInfos = {0};
-static SDMDnodeCfg tsDnodeCfg = {0};
-static taos_qset tsMgmtQset = NULL;
-static taos_queue tsMgmtQueue = NULL;
-static pthread_t tsQthread;
-
-static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes);
-static bool dnodeReadMnodeInfos();
-static void dnodeSaveMnodeInfos();
-static void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg);
-static bool dnodeReadDnodeCfg();
-static void dnodeSaveDnodeCfg();
+typedef struct {
+ SRpcMsg rpcMsg;
+ char pCont[];
+} SMgmtMsg;
+
+void * tsDnodeTmr = NULL;
+static void * tsStatusTimer = NULL;
+static uint32_t tsRebootTime;
+static taos_qset tsMgmtQset = NULL;
+static taos_queue tsMgmtQueue = NULL;
+static pthread_t tsQthread;
+
static void dnodeProcessStatusRsp(SRpcMsg *pMsg);
static void dnodeSendStatusMsg(void *handle, void *tmrId);
static void *dnodeProcessMgmtQueue(void *param);
@@ -74,7 +70,7 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
-static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
+static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);
int32_t dnodeInitMgmt() {
@@ -86,28 +82,8 @@ int32_t dnodeInitMgmt() {
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg;
dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP, dnodeProcessStatusRsp);
- dnodeReadDnodeCfg();
tsRebootTime = taosGetTimestampSec();
- if (!dnodeReadMnodeInfos()) {
- memset(&tsDMnodeEpSet, 0, sizeof(SRpcEpSet));
- memset(&tsDMnodeInfos, 0, sizeof(SDMMnodeInfos));
-
- tsDMnodeEpSet.numOfEps = 1;
- taosGetFqdnPortFromEp(tsFirst, tsDMnodeEpSet.fqdn[0], &tsDMnodeEpSet.port[0]);
-
- if (strcmp(tsSecond, tsFirst) != 0) {
- tsDMnodeEpSet.numOfEps = 2;
- taosGetFqdnPortFromEp(tsSecond, tsDMnodeEpSet.fqdn[1], &tsDMnodeEpSet.port[1]);
- }
- } else {
- tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse;
- tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum;
- for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
- taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]);
- }
- }
-
int32_t code = vnodeInitResources();
if (code != TSDB_CODE_SUCCESS) {
dnodeCleanupMgmt();
@@ -201,38 +177,46 @@ void dnodeCleanupMgmt() {
vnodeCleanupResources();
}
-void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
- void *item;
+static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
+ int32_t size = sizeof(SMgmtMsg) + pMsg->contLen;
+ SMgmtMsg *pMgmt = taosAllocateQitem(size);
+ if (pMgmt == NULL) {
+ return TSDB_CODE_DND_OUT_OF_MEMORY;
+ }
- item = taosAllocateQitem(sizeof(SRpcMsg));
- if (item) {
- memcpy(item, pMsg, sizeof(SRpcMsg));
- taosWriteQitem(tsMgmtQueue, 1, item);
- } else {
- SRpcMsg rsp = {
- .handle = pMsg->handle,
- .pCont = NULL,
- .code = TSDB_CODE_DND_OUT_OF_MEMORY
- };
-
+ pMgmt->rpcMsg = *pMsg;
+ pMgmt->rpcMsg.pCont = pMgmt->pCont;
+ memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen);
+ taosWriteQitem(tsMgmtQueue, TAOS_QTYPE_RPC, pMgmt);
+
+ return TSDB_CODE_SUCCESS;
+}
+
+void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
+ int32_t code = dnodeWriteToMgmtQueue(pMsg);
+ if (code != TSDB_CODE_SUCCESS) {
+ SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
rpcSendResponse(&rsp);
- rpcFreeCont(pMsg->pCont);
}
+
+ rpcFreeCont(pMsg->pCont);
}
static void *dnodeProcessMgmtQueue(void *param) {
- SRpcMsg *pMsg;
- SRpcMsg rsp = {0};
- int type;
- void * handle;
+ SMgmtMsg *pMgmt;
+ SRpcMsg * pMsg;
+ SRpcMsg rsp = {0};
+ int32_t qtype;
+ void * handle;
while (1) {
- if (taosReadQitemFromQset(tsMgmtQset, &type, (void **) &pMsg, &handle) == 0) {
+ if (taosReadQitemFromQset(tsMgmtQset, &qtype, (void **)&pMgmt, &handle) == 0) {
dDebug("qset:%p, dnode mgmt got no message from qset, exit", tsMgmtQset);
break;
}
- dDebug("%p, msg:%s will be processed", pMsg->ahandle, taosMsg[pMsg->msgType]);
+ pMsg = &pMgmt->rpcMsg;
+ dDebug("%p, msg:%p:%s will be processed", pMsg->ahandle, pMgmt, taosMsg[pMsg->msgType]);
if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
} else {
@@ -240,10 +224,9 @@ static void *dnodeProcessMgmtQueue(void *param) {
}
rsp.handle = pMsg->handle;
- rsp.pCont = NULL;
+ rsp.pCont = NULL;
rpcSendResponse(&rsp);
- rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
@@ -381,7 +364,7 @@ static void dnodeCloseVnodes() {
}
static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
- SMDCreateVnodeMsg *pCreate = rpcMsg->pCont;
+ SCreateVnodeMsg *pCreate = rpcMsg->pCont;
pCreate->cfg.vgId = htonl(pCreate->cfg.vgId);
pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion);
pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables);
@@ -404,7 +387,7 @@ static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
}
static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
- SMDCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);
+ SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);
void *pVnode = vnodeAcquire(pCreate->cfg.vgId);
if (pVnode != NULL) {
@@ -418,7 +401,7 @@ static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
}
static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
- SMDAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);
+ SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);
void *pVnode = vnodeAcquire(pAlter->cfg.vgId);
if (pVnode != NULL) {
@@ -433,14 +416,14 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
}
static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
- SMDDropVnodeMsg *pDrop = rpcMsg->pCont;
+ SDropVnodeMsg *pDrop = rpcMsg->pCont;
pDrop->vgId = htonl(pDrop->vgId);
return vnodeDrop(pDrop->vgId);
}
static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
-// SMDAlterStreamMsg *pStream = pCont;
+// SAlterStreamMsg *pStream = pCont;
// pStream->uid = htobe64(pStream->uid);
// pStream->stime = htobe64(pStream->stime);
// pStream->vnode = htonl(pStream->vnode);
@@ -453,12 +436,12 @@ static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
}
static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
- SMDCfgDnodeMsg *pCfg = pMsg->pCont;
+ SCfgDnodeMsg *pCfg = pMsg->pCont;
return taosCfgDynamicOptions(pCfg->config);
}
static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
- SMDCreateMnodeMsg *pCfg = pMsg->pCont;
+ SCreateMnodeMsg *pCfg = pMsg->pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId);
if (pCfg->dnodeId != dnodeGetDnodeId()) {
dError("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
@@ -470,10 +453,10 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED;
}
- dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.nodeNum);
- for (int i = 0; i < pCfg->mnodes.nodeNum; ++i) {
- pCfg->mnodes.nodeInfos[i].nodeId = htonl(pCfg->mnodes.nodeInfos[i].nodeId);
- dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.nodeInfos[i].nodeId, pCfg->mnodes.nodeInfos[i].nodeEp);
+ dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum);
+ for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) {
+ pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId);
+ dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp);
}
dnodeStartMnode(&pCfg->mnodes);
@@ -481,34 +464,6 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
return TSDB_CODE_SUCCESS;
}
-void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet) {
- if (pEpSet->numOfEps <= 0) {
- dError("mnode EP list for peer is changed, but content is invalid, discard it");
- return;
- }
-
- dInfo("mnode EP list for peer is changed, numOfEps:%d inUse:%d", pEpSet->numOfEps, pEpSet->inUse);
- for (int i = 0; i < pEpSet->numOfEps; ++i) {
- pEpSet->port[i] -= TSDB_PORT_DNODEDNODE;
- dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]);
- }
-
- tsDMnodeEpSet = *pEpSet;
-}
-
-void dnodeGetMnodeEpSetForPeer(void *epSetRaw) {
- SRpcEpSet *epSet = epSetRaw;
- *epSet = tsDMnodeEpSet;
-
- for (int i=0; inumOfEps; ++i)
- epSet->port[i] += TSDB_PORT_DNODEDNODE;
-}
-
-void dnodeGetMnodeEpSetForShell(void *epSetRaw) {
- SRpcEpSet *epSet = epSetRaw;
- *epSet = tsDMnodeEpSet;
-}
-
static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
if (pMsg->code != TSDB_CODE_SUCCESS) {
dError("status rsp is received, error:%s", tstrerror(pMsg->code));
@@ -516,202 +471,24 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
return;
}
- SDMStatusRsp *pStatusRsp = pMsg->pCont;
- SDMMnodeInfos *pMnodes = &pStatusRsp->mnodes;
- if (pMnodes->nodeNum <= 0) {
- dError("status msg is invalid, num of ips is %d", pMnodes->nodeNum);
- taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
- return;
- }
+ SStatusRsp *pStatusRsp = pMsg->pCont;
+ SMnodeInfos *minfos = &pStatusRsp->mnodes;
+ dnodeUpdateMInfos(minfos);
- SDMDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
- pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
+ SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
+ pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
pCfg->moduleStatus = htonl(pCfg->moduleStatus);
pCfg->dnodeId = htonl(pCfg->dnodeId);
-
- for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
- SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
- pMnodeInfo->nodeId = htonl(pMnodeInfo->nodeId);
- }
+ dnodeUpdateCfg(pCfg);
vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
- // will not set mnode in status msg
- // dnodeProcessModuleStatus(pCfg->moduleStatus);
- dnodeUpdateDnodeCfg(pCfg);
+ SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess));
+ dnodeUpdateEps(pEps);
- dnodeUpdateMnodeInfos(pMnodes);
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
}
-static bool dnodeCheckMnodeInfos(SDMMnodeInfos *pMnodes) {
- if (pMnodes->nodeNum <= 0 || pMnodes->nodeNum > 3) {
- dError("invalid mnode infos, num:%d", pMnodes->nodeNum);
- return false;
- }
-
- for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
- SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
- if (pMnodeInfo->nodeId <= 0 || strlen(pMnodeInfo->nodeEp) <= 5) {
- dError("invalid mnode info:%d, nodeId:%d nodeEp:%s", i, pMnodeInfo->nodeId, pMnodeInfo->nodeEp);
- return false;
- }
- }
-
- return true;
-}
-
-static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
- bool mnodesChanged = (memcmp(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos)) != 0);
- bool mnodesNotInit = (tsDMnodeInfos.nodeNum == 0);
- if (!(mnodesChanged || mnodesNotInit)) return;
-
- if (!dnodeCheckMnodeInfos(pMnodes)) return;
-
- memcpy(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos));
- dInfo("mnode infos is changed, nodeNum:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse);
- for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
- dInfo("mnode index:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp);
- }
-
- tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse;
- tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum;
- for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
- taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]);
- }
-
- dnodeSaveMnodeInfos();
- sdbUpdateAsync();
-}
-
-static bool dnodeReadMnodeInfos() {
- char ipFile[TSDB_FILENAME_LEN*2] = {0};
-
- sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir);
- FILE *fp = fopen(ipFile, "r");
- if (!fp) {
- dDebug("failed to read mnodeEpSet.json, file not exist");
- return false;
- }
-
- bool ret = false;
- int maxLen = 2000;
- char *content = calloc(1, maxLen + 1);
- int len = fread(content, 1, maxLen, fp);
- if (len <= 0) {
- free(content);
- fclose(fp);
- dError("failed to read mnodeEpSet.json, content is null");
- return false;
- }
-
- content[len] = 0;
- cJSON* root = cJSON_Parse(content);
- if (root == NULL) {
- dError("failed to read mnodeEpSet.json, invalid json format");
- goto PARSE_OVER;
- }
-
- cJSON* inUse = cJSON_GetObjectItem(root, "inUse");
- if (!inUse || inUse->type != cJSON_Number) {
- dError("failed to read mnodeEpSet.json, inUse not found");
- goto PARSE_OVER;
- }
- tsDMnodeInfos.inUse = inUse->valueint;
-
- cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum");
- if (!nodeNum || nodeNum->type != cJSON_Number) {
- dError("failed to read mnodeEpSet.json, nodeNum not found");
- goto PARSE_OVER;
- }
- tsDMnodeInfos.nodeNum = nodeNum->valueint;
-
- cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
- if (!nodeInfos || nodeInfos->type != cJSON_Array) {
- dError("failed to read mnodeEpSet.json, nodeInfos not found");
- goto PARSE_OVER;
- }
-
- int size = cJSON_GetArraySize(nodeInfos);
- if (size != tsDMnodeInfos.nodeNum) {
- dError("failed to read mnodeEpSet.json, nodeInfos size not matched");
- goto PARSE_OVER;
- }
-
- for (int i = 0; i < size; ++i) {
- cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
- if (nodeInfo == NULL) continue;
-
- cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
- if (!nodeId || nodeId->type != cJSON_Number) {
- dError("failed to read mnodeEpSet.json, nodeId not found");
- goto PARSE_OVER;
- }
- tsDMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint;
-
- cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
- if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
- dError("failed to read mnodeEpSet.json, nodeName not found");
- goto PARSE_OVER;
- }
- strncpy(tsDMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
- }
-
- ret = true;
-
- dInfo("read mnode epSet successed, numOfEps:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse);
- for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
- dInfo("mnode:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp);
- }
-
-PARSE_OVER:
- free(content);
- cJSON_Delete(root);
- fclose(fp);
- return ret;
-}
-
-static void dnodeSaveMnodeInfos() {
- char ipFile[TSDB_FILENAME_LEN] = {0};
- sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir);
- FILE *fp = fopen(ipFile, "w");
- if (!fp) return;
-
- int32_t len = 0;
- int32_t maxLen = 2000;
- char * content = calloc(1, maxLen + 1);
-
- len += snprintf(content + len, maxLen - len, "{\n");
- len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsDMnodeInfos.inUse);
- len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsDMnodeInfos.nodeNum);
- len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
- for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
- len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsDMnodeInfos.nodeInfos[i].nodeId);
- len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", tsDMnodeInfos.nodeInfos[i].nodeEp);
- if (i < tsDMnodeInfos.nodeNum -1) {
- len += snprintf(content + len, maxLen - len, " },{\n");
- } else {
- len += snprintf(content + len, maxLen - len, " }]\n");
- }
- }
- len += snprintf(content + len, maxLen - len, "}\n");
-
- fwrite(content, 1, len, fp);
- fflush(fp);
- fclose(fp);
- free(content);
-
- dInfo("save mnode epSet successed");
-}
-
-char *dnodeGetMnodeMasterEp() {
- return tsDMnodeInfos.nodeInfos[tsDMnodeEpSet.inUse].nodeEp;
-}
-
-void* dnodeGetMnodeInfos() {
- return &tsDMnodeInfos;
-}
-
static void dnodeSendStatusMsg(void *handle, void *tmrId) {
if (tsDnodeTmr == NULL) {
dError("dnode timer is already released");
@@ -724,22 +501,21 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
return;
}
- int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
- SDMStatusMsg *pStatus = rpcMallocCont(contLen);
+ int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
+ SStatusMsg *pStatus = rpcMallocCont(contLen);
if (pStatus == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to malloc status message");
return;
}
- //strcpy(pStatus->dnodeName, tsDnodeName);
+ dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId);
+ pStatus->dnodeId = htonl(dnodeGetDnodeId());
pStatus->version = htonl(tsVersion);
- pStatus->dnodeId = htonl(tsDnodeCfg.dnodeId);
pStatus->lastReboot = htonl(tsRebootTime);
pStatus->numOfCores = htons((uint16_t) tsNumOfCores);
pStatus->diskAvailable = tsAvailDataDirGB;
pStatus->alternativeRole = (uint8_t) tsAlternativeRole;
- tstrncpy(pStatus->clusterId, tsDnodeCfg.clusterId, TSDB_CLUSTER_ID_LEN);
tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);
// fill cluster cfg parameters
@@ -759,7 +535,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);
vnodeBuildStatusMsg(pStatus);
- contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
+ contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
pStatus->openVnodes = htons(pStatus->openVnodes);
SRpcMsg rpcMsg = {
@@ -769,110 +545,19 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
};
SRpcEpSet epSet;
- dnodeGetMnodeEpSetForPeer(&epSet);
+ dnodeGetEpSetForPeer(&epSet);
dnodeSendMsgToDnode(&epSet, &rpcMsg);
}
-static bool dnodeReadDnodeCfg() {
- char dnodeCfgFile[TSDB_FILENAME_LEN*2] = {0};
-
- sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir);
-
- FILE *fp = fopen(dnodeCfgFile, "r");
- if (!fp) {
- dDebug("failed to read dnodeCfg.json, file not exist");
- return false;
- }
-
- bool ret = false;
- int maxLen = 100;
- char *content = calloc(1, maxLen + 1);
- int len = fread(content, 1, maxLen, fp);
- if (len <= 0) {
- free(content);
- fclose(fp);
- dError("failed to read dnodeCfg.json, content is null");
- return false;
- }
-
- content[len] = 0;
- cJSON* root = cJSON_Parse(content);
- if (root == NULL) {
- dError("failed to read dnodeCfg.json, invalid json format");
- goto PARSE_CFG_OVER;
- }
-
- cJSON* dnodeId = cJSON_GetObjectItem(root, "dnodeId");
- if (!dnodeId || dnodeId->type != cJSON_Number) {
- dError("failed to read dnodeCfg.json, dnodeId not found");
- goto PARSE_CFG_OVER;
- }
- tsDnodeCfg.dnodeId = dnodeId->valueint;
-
- cJSON* clusterId = cJSON_GetObjectItem(root, "clusterId");
- if (!clusterId || clusterId->type != cJSON_String) {
- dError("failed to read dnodeCfg.json, clusterId not found");
- goto PARSE_CFG_OVER;
- }
- tstrncpy(tsDnodeCfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN);
-
- ret = true;
-
- dInfo("read numOfVnodes successed, dnodeId:%d", tsDnodeCfg.dnodeId);
-
-PARSE_CFG_OVER:
- free(content);
- cJSON_Delete(root);
- fclose(fp);
- return ret;
-}
-
-static void dnodeSaveDnodeCfg() {
- char dnodeCfgFile[TSDB_FILENAME_LEN] = {0};
- sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir);
-
- FILE *fp = fopen(dnodeCfgFile, "w");
- if (!fp) return;
-
- int32_t len = 0;
- int32_t maxLen = 200;
- char * content = calloc(1, maxLen + 1);
-
- len += snprintf(content + len, maxLen - len, "{\n");
- len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", tsDnodeCfg.dnodeId);
- len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%s\"\n", tsDnodeCfg.clusterId);
- len += snprintf(content + len, maxLen - len, "}\n");
-
- fwrite(content, 1, len, fp);
- fflush(fp);
- fclose(fp);
- free(content);
-
- dInfo("save dnodeId successed");
-}
-
-void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) {
- if (tsDnodeCfg.dnodeId == 0) {
- dInfo("dnodeId is set to %d, clusterId is set to %s", pCfg->dnodeId, pCfg->clusterId);
- tsDnodeCfg.dnodeId = pCfg->dnodeId;
- tstrncpy(tsDnodeCfg.clusterId, pCfg->clusterId, TSDB_CLUSTER_ID_LEN);
- dnodeSaveDnodeCfg();
- }
-}
-
-int32_t dnodeGetDnodeId() {
- return tsDnodeCfg.dnodeId;
-}
-
void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
SRpcConnInfo connInfo = {0};
rpcGetConnInfo(rpcMsg->handle, &connInfo);
SRpcEpSet epSet = {0};
if (forShell) {
- dnodeGetMnodeEpSetForShell(&epSet);
+ dnodeGetEpSetForShell(&epSet);
} else {
- dnodeGetMnodeEpSetForPeer(&epSet);
+ dnodeGetEpSetForPeer(&epSet);
}
dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c
index 46376159c6782efde7adbd19c75af83aa4cde397..bd9500ba51226d138fd3fe52f027d144d289681b 100644
--- a/src/dnode/src/dnodeModule.c
+++ b/src/dnode/src/dnodeModule.c
@@ -114,6 +114,7 @@ int32_t dnodeInitModules() {
}
}
+ dInfo("dnode modules is initialized");
return 0;
}
@@ -146,8 +147,8 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) {
}
}
-bool dnodeStartMnode(void *pMnodes) {
- SDMMnodeInfos *mnodes = pMnodes;
+bool dnodeStartMnode(SMnodeInfos *minfos) {
+ SMnodeInfos *mnodes = minfos;
if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) {
dDebug("mnode module is already started, module status:%d", tsModuleStatus);
diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c
index 3bc2f7b48b319f3c9e6215a05463ebedc74035fa..4c44924cd0b79bae5b0c83f9ddad8ae30afcf292 100644
--- a/src/dnode/src/dnodePeer.c
+++ b/src/dnode/src/dnodePeer.c
@@ -19,6 +19,7 @@
* to dnode. All theses messages are handled from here
*/
+#define _DEFAULT_SOURCE
#include "os.h"
#include "taosmsg.h"
#include "tglobal.h"
@@ -28,20 +29,20 @@
#include "dnodeMgmt.h"
#include "dnodeVWrite.h"
#include "dnodeMPeer.h"
+#include "dnodeMInfos.h"
-extern void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet);
static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *);
static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg);
static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet);
-static void *tsDnodeServerRpc = NULL;
-static void *tsDnodeClientRpc = NULL;
+static void *tsServerRpc = NULL;
+static void *tsClientRpc = NULL;
int32_t dnodeInitServer() {
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVnodeWriteQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = dnodeDispatchToVnodeWriteQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = dnodeDispatchToVnodeWriteQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVnodeWriteQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVWriteQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = dnodeDispatchToVWriteQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = dnodeDispatchToVWriteQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVWriteQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeDispatchToMgmtQueue;
@@ -50,11 +51,11 @@ int32_t dnodeInitServer() {
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToMgmtQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMnodePeerQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMnodePeerQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = dnodeDispatchToMnodePeerQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = dnodeDispatchToMnodePeerQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = dnodeDispatchToMnodePeerQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = dnodeDispatchToMPeerQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = dnodeDispatchToMPeerQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = dnodeDispatchToMPeerQueue;
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
@@ -66,20 +67,20 @@ int32_t dnodeInitServer() {
rpcInit.connType = TAOS_CONN_SERVER;
rpcInit.idleTime = tsShellActivityTimer * 1000;
- tsDnodeServerRpc = rpcOpen(&rpcInit);
- if (tsDnodeServerRpc == NULL) {
+ tsServerRpc = rpcOpen(&rpcInit);
+ if (tsServerRpc == NULL) {
dError("failed to init inter-dnodes RPC server");
return -1;
}
- dInfo("inter-dnodes RPC server is opened");
+ dInfo("dnode inter-dnodes RPC server is initialized");
return 0;
}
void dnodeCleanupServer() {
- if (tsDnodeServerRpc) {
- rpcClose(tsDnodeServerRpc);
- tsDnodeServerRpc = NULL;
+ if (tsServerRpc) {
+ rpcClose(tsServerRpc);
+ tsServerRpc = NULL;
dInfo("inter-dnodes RPC server is closed");
}
}
@@ -93,7 +94,7 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
if (pMsg->pCont == NULL) return;
- if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
+ if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
rspMsg.code = TSDB_CODE_APP_NOT_READY;
rpcSendResponse(&rspMsg);
rpcFreeCont(pMsg->pCont);
@@ -131,27 +132,34 @@ int32_t dnodeInitClient() {
rpcInit.ckey = "key";
rpcInit.secret = secret;
- tsDnodeClientRpc = rpcOpen(&rpcInit);
- if (tsDnodeClientRpc == NULL) {
+ tsClientRpc = rpcOpen(&rpcInit);
+ if (tsClientRpc == NULL) {
dError("failed to init mnode rpc client");
return -1;
}
- dInfo("inter-dnodes rpc client is opened");
+ dInfo("dnode inter-dnodes rpc client is initialized");
return 0;
}
void dnodeCleanupClient() {
- if (tsDnodeClientRpc) {
- rpcClose(tsDnodeClientRpc);
- tsDnodeClientRpc = NULL;
- dInfo("inter-dnodes rpc client is closed");
+ if (tsClientRpc) {
+ rpcClose(tsClientRpc);
+ tsClientRpc = NULL;
+ dInfo("dnode inter-dnodes rpc client is closed");
}
}
static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
+ if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
+ if (pMsg == NULL || pMsg->pCont == NULL) return;
+ dDebug("msg:%p is ignored since dnode not running", pMsg);
+ rpcFreeCont(pMsg->pCont);
+ return;
+ }
+
if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) {
- dnodeUpdateMnodeEpSetForPeer(pEpSet);
+ dnodeUpdateEpSetForPeer(pEpSet);
}
if (dnodeProcessRspMsgFp[pMsg->msgType]) {
@@ -168,15 +176,15 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
}
void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
- rpcSendRequest(tsDnodeClientRpc, epSet, rpcMsg);
+ rpcSendRequest(tsClientRpc, epSet, rpcMsg, NULL);
}
void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
SRpcEpSet epSet = {0};
- dnodeGetMnodeEpSetForPeer(&epSet);
- rpcSendRecv(tsDnodeClientRpc, &epSet, rpcMsg, rpcRsp);
+ dnodeGetEpSetForPeer(&epSet);
+ rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp);
}
void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) {
- rpcSendRecv(tsDnodeClientRpc, epSet, rpcMsg, rpcRsp);
-}
\ No newline at end of file
+ rpcSendRecv(tsClientRpc, epSet, rpcMsg, rpcRsp);
+}
diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c
index 4c6c2100e048e9aaf23f4155d3c54604104a8f9b..89f657f78986b8e57c0b5e1dedb841c451db00ba 100644
--- a/src/dnode/src/dnodeShell.c
+++ b/src/dnode/src/dnodeShell.c
@@ -33,46 +33,46 @@
static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *);
static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey);
-static void * tsDnodeShellRpc = NULL;
-static int32_t tsDnodeQueryReqNum = 0;
-static int32_t tsDnodeSubmitReqNum = 0;
+static void * tsShellRpc = NULL;
+static int32_t tsQueryReqNum = 0;
+static int32_t tsSubmitReqNum = 0;
int32_t dnodeInitShell() {
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVnodeWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue;
// the following message shall be treated as mnode write
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMnodeWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMWriteQueue;
// the following message shall be treated as mnode query
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMnodeReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue;
int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore;
numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0);
@@ -91,24 +91,24 @@ int32_t dnodeInitShell() {
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.afp = dnodeRetrieveUserAuthInfo;
- tsDnodeShellRpc = rpcOpen(&rpcInit);
- if (tsDnodeShellRpc == NULL) {
+ tsShellRpc = rpcOpen(&rpcInit);
+ if (tsShellRpc == NULL) {
dError("failed to init shell rpc server");
return -1;
}
- dInfo("shell rpc server is opened");
+ dInfo("dnode shell rpc server is initialized");
return 0;
}
void dnodeCleanupShell() {
- if (tsDnodeShellRpc) {
- rpcClose(tsDnodeShellRpc);
- tsDnodeShellRpc = NULL;
+ if (tsShellRpc) {
+ rpcClose(tsShellRpc);
+ tsShellRpc = NULL;
}
}
-void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
+static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
SRpcMsg rpcMsg = {
.handle = pMsg->handle,
.pCont = NULL,
@@ -117,7 +117,7 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
if (pMsg->pCont == NULL) return;
- if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
+ if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
rpcMsg.code = TSDB_CODE_APP_NOT_READY;
rpcSendResponse(&rpcMsg);
@@ -126,9 +126,9 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
}
if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) {
- atomic_fetch_add_32(&tsDnodeQueryReqNum, 1);
+ atomic_fetch_add_32(&tsQueryReqNum, 1);
} else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
- atomic_fetch_add_32(&tsDnodeSubmitReqNum, 1);
+ atomic_fetch_add_32(&tsSubmitReqNum, 1);
} else {}
if ( dnodeProcessShellMsgFp[pMsg->msgType] ) {
@@ -146,12 +146,12 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey);
if (code != TSDB_CODE_APP_NOT_READY) return code;
- SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg));
+ SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg));
tstrncpy(pMsg->user, user, sizeof(pMsg->user));
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pMsg;
- rpcMsg.contLen = sizeof(SDMAuthMsg);
+ rpcMsg.contLen = sizeof(SAuthMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH;
dDebug("user:%s, send auth msg to mnodes", user);
@@ -161,7 +161,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
if (rpcRsp.code != 0) {
dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code));
} else {
- SDMAuthRsp *pRsp = rpcRsp.pCont;
+ SAuthRsp *pRsp = rpcRsp.pCont;
dDebug("user:%s, auth msg received from mnodes", user);
memcpy(secret, pRsp->secret, TSDB_KEY_LEN);
memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN);
@@ -176,8 +176,8 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) {
dDebug("vgId:%d, tid:%d send config table msg to mnode", vgId, tid);
- int32_t contLen = sizeof(SDMConfigTableMsg);
- SDMConfigTableMsg *pMsg = rpcMallocCont(contLen);
+ int32_t contLen = sizeof(SConfigTableMsg);
+ SConfigTableMsg *pMsg = rpcMallocCont(contLen);
pMsg->dnodeId = htonl(dnodeGetDnodeId());
pMsg->vgId = htonl(vgId);
@@ -211,12 +211,12 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) {
}
}
-SDnodeStatisInfo dnodeGetStatisInfo() {
- SDnodeStatisInfo info = {0};
- if (dnodeGetRunStatus() == TSDB_DNODE_RUN_STATUS_RUNING) {
+SStatisInfo dnodeGetStatisInfo() {
+ SStatisInfo info = {0};
+ if (dnodeGetRunStatus() == TSDB_RUN_STATUS_RUNING) {
info.httpReqNum = httpGetReqCount();
- info.queryReqNum = atomic_exchange_32(&tsDnodeQueryReqNum, 0);
- info.submitReqNum = atomic_exchange_32(&tsDnodeSubmitReqNum, 0);
+ info.queryReqNum = atomic_exchange_32(&tsQueryReqNum, 0);
+ info.submitReqNum = atomic_exchange_32(&tsSubmitReqNum, 0);
}
return info;
diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c
index 4fdc0b8a73b58efa801f33ea99ffb7d123a72dcf..e973f9901f19b7aa6f4d4958d88e01ba258a82bb 100644
--- a/src/dnode/src/dnodeTelemetry.c
+++ b/src/dnode/src/dnodeTelemetry.c
@@ -268,7 +268,7 @@ static void dnodeGetEmail(char* filepath) {
return;
}
- if (taosTRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) {
+ if (taosRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) {
dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
}
@@ -299,6 +299,7 @@ int32_t dnodeInitTelemetry() {
dTrace("failed to create telemetry thread, reason:%s", strerror(errno));
}
+ dInfo("dnode telemetry is initialized");
return 0;
}
diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c
index e61158ef30dddd6037af36c61be1fd522f8af10b..34df11adcce72a6a21718b81c1895c30c66bea1c 100644
--- a/src/dnode/src/dnodeVRead.c
+++ b/src/dnode/src/dnodeVRead.c
@@ -17,225 +17,184 @@
#include "os.h"
#include "taoserror.h"
#include "taosmsg.h"
-#include "tutil.h"
-#include "tqueue.h"
-#include "twal.h"
#include "tglobal.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
-#include "dnodeVRead.h"
+#include "tqueue.h"
#include "vnode.h"
+#include "dnodeInt.h"
typedef struct {
- pthread_t thread; // thread
- int32_t workerId; // worker ID
-} SReadWorker;
+ pthread_t thread; // thread
+ int32_t workerId; // worker ID
+} SVReadWorker;
typedef struct {
- int32_t max; // max number of workers
- int32_t min; // min number of workers
- int32_t num; // current number of workers
- SReadWorker *readWorker;
+ int32_t max; // max number of workers
+ int32_t min; // min number of workers
+ int32_t num; // current number of workers
+ SVReadWorker * worker;
pthread_mutex_t mutex;
-} SReadWorkerPool;
+} SVReadWorkerPool;
static void *dnodeProcessReadQueue(void *param);
-static void dnodeHandleIdleReadWorker(SReadWorker *);
// module global variable
-static SReadWorkerPool readPool;
-static taos_qset readQset;
+static SVReadWorkerPool tsVReadWP;
+static taos_qset tsVReadQset;
-int32_t dnodeInitVnodeRead() {
- readQset = taosOpenQset();
+int32_t dnodeInitVRead() {
+ tsVReadQset = taosOpenQset();
- readPool.min = tsNumOfCores;
- readPool.max = tsNumOfCores * tsNumOfThreadsPerCore;
- if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min;
- readPool.readWorker = (SReadWorker *)calloc(sizeof(SReadWorker), readPool.max);
- pthread_mutex_init(&readPool.mutex, NULL);
+ tsVReadWP.min = tsNumOfCores;
+ tsVReadWP.max = tsNumOfCores * tsNumOfThreadsPerCore;
+ if (tsVReadWP.max <= tsVReadWP.min * 2) tsVReadWP.max = 2 * tsVReadWP.min;
+ tsVReadWP.worker = (SVReadWorker *)calloc(sizeof(SVReadWorker), tsVReadWP.max);
+ pthread_mutex_init(&tsVReadWP.mutex, NULL);
- if (readPool.readWorker == NULL) return -1;
- for (int i = 0; i < readPool.max; ++i) {
- SReadWorker *pWorker = readPool.readWorker + i;
+ if (tsVReadWP.worker == NULL) return -1;
+ for (int i = 0; i < tsVReadWP.max; ++i) {
+ SVReadWorker *pWorker = tsVReadWP.worker + i;
pWorker->workerId = i;
}
- dInfo("dnode read is opened, min worker:%d max worker:%d", readPool.min, readPool.max);
+ dInfo("dnode vread is initialized, min worker:%d max worker:%d", tsVReadWP.min, tsVReadWP.max);
return 0;
}
-void dnodeCleanupVnodeRead() {
- for (int i = 0; i < readPool.max; ++i) {
- SReadWorker *pWorker = readPool.readWorker + i;
+void dnodeCleanupVRead() {
+ for (int i = 0; i < tsVReadWP.max; ++i) {
+ SVReadWorker *pWorker = tsVReadWP.worker + i;
if (pWorker->thread) {
- taosQsetThreadResume(readQset);
+ taosQsetThreadResume(tsVReadQset);
}
}
- for (int i = 0; i < readPool.max; ++i) {
- SReadWorker *pWorker = readPool.readWorker + i;
+ for (int i = 0; i < tsVReadWP.max; ++i) {
+ SVReadWorker *pWorker = tsVReadWP.worker + i;
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
}
}
- free(readPool.readWorker);
- taosCloseQset(readQset);
- pthread_mutex_destroy(&readPool.mutex);
+ free(tsVReadWP.worker);
+ taosCloseQset(tsVReadQset);
+ pthread_mutex_destroy(&tsVReadWP.mutex);
- dInfo("dnode read is closed");
+ dInfo("dnode vread is closed");
}
-void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) {
- int32_t queuedMsgNum = 0;
- int32_t leftLen = pMsg->contLen;
- char *pCont = (char *) pMsg->pCont;
+void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
+ int32_t queuedMsgNum = 0;
+ int32_t leftLen = pMsg->contLen;
+ char * pCont = (char *)pMsg->pCont;
while (leftLen > 0) {
- SMsgHead *pHead = (SMsgHead *) pCont;
- pHead->vgId = htonl(pHead->vgId);
+ SMsgHead *pHead = (SMsgHead *)pCont;
+ pHead->vgId = htonl(pHead->vgId);
pHead->contLen = htonl(pHead->contLen);
- taos_queue queue = vnodeAcquireRqueue(pHead->vgId);
-
- if (queue == NULL) {
- leftLen -= pHead->contLen;
- pCont -= pHead->contLen;
- continue;
+ void *pVnode = vnodeAcquire(pHead->vgId);
+ if (pVnode != NULL) {
+ int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
+ if (code == TSDB_CODE_SUCCESS) queuedMsgNum++;
+ vnodeRelease(pVnode);
}
- // put message into queue
- SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg));
- pRead->rpcMsg = *pMsg;
- pRead->pCont = pCont;
- pRead->contLen = pHead->contLen;
-
- // next vnode
leftLen -= pHead->contLen;
pCont -= pHead->contLen;
- queuedMsgNum++;
-
- taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead);
}
if (queuedMsgNum == 0) {
- SRpcMsg rpcRsp = {
- .handle = pMsg->handle,
- .pCont = NULL,
- .contLen = 0,
- .code = TSDB_CODE_VND_INVALID_VGROUP_ID,
- .msgType = 0
- };
+ SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID};
rpcSendResponse(&rpcRsp);
- rpcFreeCont(pMsg->pCont);
}
+
+ rpcFreeCont(pMsg->pCont);
}
-void *dnodeAllocateVnodeRqueue(void *pVnode) {
- pthread_mutex_lock(&readPool.mutex);
+void *dnodeAllocVReadQueue(void *pVnode) {
+ pthread_mutex_lock(&tsVReadWP.mutex);
taos_queue queue = taosOpenQueue();
if (queue == NULL) {
- pthread_mutex_unlock(&readPool.mutex);
+ pthread_mutex_unlock(&tsVReadWP.mutex);
return NULL;
}
- taosAddIntoQset(readQset, queue, pVnode);
+ taosAddIntoQset(tsVReadQset, queue, pVnode);
// spawn a thread to process queue
- if (readPool.num < readPool.max) {
+ if (tsVReadWP.num < tsVReadWP.max) {
do {
- SReadWorker *pWorker = readPool.readWorker + readPool.num;
+ SVReadWorker *pWorker = tsVReadWP.worker + tsVReadWP.num;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessReadQueue, pWorker) != 0) {
- dError("failed to create thread to process read queue, reason:%s", strerror(errno));
+ dError("failed to create thread to process vread vqueue since %s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
- readPool.num++;
- dDebug("read worker:%d is launched, total:%d", pWorker->workerId, readPool.num);
- } while (readPool.num < readPool.min);
+ tsVReadWP.num++;
+ dDebug("dnode vread worker:%d is launched, total:%d", pWorker->workerId, tsVReadWP.num);
+ } while (tsVReadWP.num < tsVReadWP.min);
}
- pthread_mutex_unlock(&readPool.mutex);
- dDebug("pVnode:%p, read queue:%p is allocated", pVnode, queue);
+ pthread_mutex_unlock(&tsVReadWP.mutex);
+ dDebug("pVnode:%p, dnode vread queue:%p is allocated", pVnode, queue);
return queue;
}
-void dnodeFreeVnodeRqueue(void *rqueue) {
+void dnodeFreeVReadQueue(void *rqueue) {
taosCloseQueue(rqueue);
-
- // dynamically adjust the number of threads
}
-void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) {
+void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) {
SRpcMsg rpcRsp = {
- .handle = pRead->rpcMsg.handle,
+ .handle = pRead->rpcHandle,
.pCont = pRead->rspRet.rsp,
.contLen = pRead->rspRet.len,
.code = code,
};
rpcSendResponse(&rpcRsp);
- rpcFreeCont(pRead->rpcMsg.pCont);
vnodeRelease(pVnode);
}
-void dnodeDispatchNonRspMsg(void *pVnode, SReadMsg *pRead, int32_t code) {
- rpcFreeCont(pRead->rpcMsg.pCont);
+void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) {
vnodeRelease(pVnode);
}
static void *dnodeProcessReadQueue(void *param) {
- SReadMsg *pReadMsg;
- int type;
- void *pVnode;
+ SVReadMsg *pRead;
+ int32_t qtype;
+ void * pVnode;
while (1) {
- if (taosReadQitemFromQset(readQset, &type, (void **)&pReadMsg, &pVnode) == 0) {
- dDebug("qset:%p dnode read got no message from qset, exiting", readQset);
+ if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pRead, &pVnode) == 0) {
+ dDebug("qset:%p dnode vread got no message from qset, exiting", tsVReadQset);
break;
}
- dDebug("%p, msg:%s will be processed in vread queue, qtype:%d, msg:%p", pReadMsg->rpcMsg.ahandle,
- taosMsg[pReadMsg->rpcMsg.msgType], type, pReadMsg);
+ dDebug("%p, msg:%p:%s will be processed in vread queue, qtype:%d", pRead->rpcAhandle, pRead,
+ taosMsg[pRead->msgType], qtype);
- int32_t code = vnodeProcessRead(pVnode, pReadMsg);
+ int32_t code = vnodeProcessRead(pVnode, pRead);
- if (type == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) {
- dnodeSendRpcReadRsp(pVnode, pReadMsg, code);
+ if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) {
+ dnodeSendRpcVReadRsp(pVnode, pRead, code);
} else {
if (code == TSDB_CODE_QRY_HAS_RSP) {
- dnodeSendRpcReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code);
- } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client
- assert(pReadMsg->rpcMsg.handle == NULL || (pReadMsg->rpcMsg.handle != NULL && pReadMsg->rpcMsg.msgType == 5));
- dnodeDispatchNonRspMsg(pVnode, pReadMsg, code);
+ dnodeSendRpcVReadRsp(pVnode, pRead, pRead->code);
+ } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client
+ assert(pRead->rpcHandle == NULL || (pRead->rpcHandle != NULL && pRead->msgType == 5));
+ dnodeDispatchNonRspMsg(pVnode, pRead, code);
}
}
- taosFreeQitem(pReadMsg);
+ taosFreeQitem(pRead);
}
return NULL;
}
-
-
-UNUSED_FUNC
-static void dnodeHandleIdleReadWorker(SReadWorker *pWorker) {
- int32_t num = taosGetQueueNumber(readQset);
-
- if (num == 0 || (num <= readPool.min && readPool.num > readPool.min)) {
- readPool.num--;
- dDebug("read worker:%d is released, total:%d", pWorker->workerId, readPool.num);
- pthread_exit(NULL);
- } else {
- usleep(30000);
- sched_yield();
- }
-}
-
diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c
index f2740bf6b810197283926a602db4fc423067e231..da1a902fb3f6f3d84298d15cec840a7cc352f028 100644
--- a/src/dnode/src/dnodeVWrite.c
+++ b/src/dnode/src/dnodeVWrite.c
@@ -15,74 +15,55 @@
#define _DEFAULT_SOURCE
#include "os.h"
-#include "taosmsg.h"
#include "taoserror.h"
-#include "tutil.h"
+#include "taosmsg.h"
+#include "tglobal.h"
#include "tqueue.h"
-#include "trpc.h"
-#include "tsdb.h"
#include "twal.h"
-#include "tdataformat.h"
-#include "tglobal.h"
-#include "tsync.h"
#include "vnode.h"
#include "dnodeInt.h"
-#include "syncInt.h"
-#include "dnodeVWrite.h"
-#include "dnodeMgmt.h"
-
-typedef struct {
- taos_qall qall;
- taos_qset qset; // queue set
- pthread_t thread; // thread
- int32_t workerId; // worker ID
-} SWriteWorker;
typedef struct {
- SRspRet rspRet;
- int32_t processedCount;
- int32_t code;
- void *pCont;
- int32_t contLen;
- SRpcMsg rpcMsg;
-} SWriteMsg;
+ taos_qall qall;
+ taos_qset qset; // queue set
+ int32_t workerId; // worker ID
+ pthread_t thread; // thread
+} SVWriteWorker;
typedef struct {
- int32_t max; // max number of workers
- int32_t nextId; // from 0 to max-1, cyclic
- SWriteWorker *writeWorker;
+ int32_t max; // max number of workers
+ int32_t nextId; // from 0 to max-1, cyclic
+ SVWriteWorker * worker;
pthread_mutex_t mutex;
-} SWriteWorkerPool;
+} SVWriteWorkerPool;
-static void *dnodeProcessWriteQueue(void *param);
-static void dnodeHandleIdleWorker(SWriteWorker *pWorker);
+static SVWriteWorkerPool tsVWriteWP;
+static void *dnodeProcessVWriteQueue(void *pWorker);
-SWriteWorkerPool wWorkerPool;
+int32_t dnodeInitVWrite() {
+ tsVWriteWP.max = tsNumOfCores;
+ tsVWriteWP.worker = tcalloc(sizeof(SVWriteWorker), tsVWriteWP.max);
+ if (tsVWriteWP.worker == NULL) return -1;
+ pthread_mutex_init(&tsVWriteWP.mutex, NULL);
-int32_t dnodeInitVnodeWrite() {
- wWorkerPool.max = tsNumOfCores;
- wWorkerPool.writeWorker = (SWriteWorker *)calloc(sizeof(SWriteWorker), wWorkerPool.max);
- if (wWorkerPool.writeWorker == NULL) return -1;
- pthread_mutex_init(&wWorkerPool.mutex, NULL);
-
- for (int32_t i = 0; i < wWorkerPool.max; ++i) {
- wWorkerPool.writeWorker[i].workerId = i;
+ for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
+ tsVWriteWP.worker[i].workerId = i;
}
- dInfo("dnode write is opened, max worker %d", wWorkerPool.max);
+ dInfo("dnode vwrite is initialized, max worker %d", tsVWriteWP.max);
return 0;
}
-void dnodeCleanupVnodeWrite() {
- for (int32_t i = 0; i < wWorkerPool.max; ++i) {
- SWriteWorker *pWorker = wWorkerPool.writeWorker + i;
+void dnodeCleanupVWrite() {
+ for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
+ SVWriteWorker *pWorker = tsVWriteWP.worker + i;
if (pWorker->thread) {
taosQsetThreadResume(pWorker->qset);
}
}
- for (int32_t i = 0; i < wWorkerPool.max; ++i) {
- SWriteWorker *pWorker = wWorkerPool.writeWorker + i;
+ for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
+ SVWriteWorker *pWorker = tsVWriteWP.worker + i;
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
taosFreeQall(pWorker->qall);
@@ -90,52 +71,51 @@ void dnodeCleanupVnodeWrite() {
}
}
- pthread_mutex_destroy(&wWorkerPool.mutex);
- free(wWorkerPool.writeWorker);
- dInfo("dnode write is closed");
+ pthread_mutex_destroy(&tsVWriteWP.mutex);
+ tfree(tsVWriteWP.worker);
+ dInfo("dnode vwrite is closed");
}
-void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) {
- char *pCont = (char *)pMsg->pCont;
+void dnodeDispatchToVWriteQueue(SRpcMsg *pRpcMsg) {
+ int32_t code;
+ char *pCont = pRpcMsg->pCont;
- if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
+ if (pRpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
SMsgDesc *pDesc = (SMsgDesc *)pCont;
pDesc->numOfVnodes = htonl(pDesc->numOfVnodes);
pCont += sizeof(SMsgDesc);
}
- SMsgHead *pHead = (SMsgHead *) pCont;
- pHead->vgId = htonl(pHead->vgId);
- pHead->contLen = htonl(pHead->contLen);
-
- taos_queue queue = vnodeAcquireWqueue(pHead->vgId);
- if (queue) {
- // put message into queue
- SWriteMsg *pWrite = (SWriteMsg *)taosAllocateQitem(sizeof(SWriteMsg));
- pWrite->rpcMsg = *pMsg;
- pWrite->pCont = pCont;
- pWrite->contLen = pHead->contLen;
+ SMsgHead *pMsg = (SMsgHead *)pCont;
+ pMsg->vgId = htonl(pMsg->vgId);
+ pMsg->contLen = htonl(pMsg->contLen);
- taosWriteQitem(queue, TAOS_QTYPE_RPC, pWrite);
+ void *pVnode = vnodeAcquire(pMsg->vgId);
+ if (pVnode == NULL) {
+ code = TSDB_CODE_VND_INVALID_VGROUP_ID;
} else {
- SRpcMsg rpcRsp = {
- .handle = pMsg->handle,
- .pCont = NULL,
- .contLen = 0,
- .code = TSDB_CODE_VND_INVALID_VGROUP_ID,
- .msgType = 0
- };
+ SWalHead *pHead = (SWalHead *)(pCont - sizeof(SWalHead));
+ pHead->msgType = pRpcMsg->msgType;
+ pHead->version = 0;
+ pHead->len = pMsg->contLen;
+ code = vnodeWriteToWQueue(pVnode, pHead, TAOS_QTYPE_RPC, pRpcMsg);
+ }
+
+ if (code != TSDB_CODE_SUCCESS) {
+ SRpcMsg rpcRsp = {.handle = pRpcMsg->handle, .code = code};
rpcSendResponse(&rpcRsp);
- rpcFreeCont(pMsg->pCont);
}
+
+ vnodeRelease(pVnode);
+ rpcFreeCont(pRpcMsg->pCont);
}
-void *dnodeAllocateVnodeWqueue(void *pVnode) {
- pthread_mutex_lock(&wWorkerPool.mutex);
- SWriteWorker *pWorker = wWorkerPool.writeWorker + wWorkerPool.nextId;
- void *queue = taosOpenQueue();
+void *dnodeAllocVWriteQueue(void *pVnode) {
+ pthread_mutex_lock(&tsVWriteWP.mutex);
+ SVWriteWorker *pWorker = tsVWriteWP.worker + tsVWriteWP.nextId;
+ taos_queue *queue = taosOpenQueue();
if (queue == NULL) {
- pthread_mutex_unlock(&wWorkerPool.mutex);
+ pthread_mutex_unlock(&tsVWriteWP.mutex);
return NULL;
}
@@ -143,7 +123,7 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) {
pWorker->qset = taosOpenQset();
if (pWorker->qset == NULL) {
taosCloseQueue(queue);
- pthread_mutex_unlock(&wWorkerPool.mutex);
+ pthread_mutex_unlock(&tsVWriteWP.mutex);
return NULL;
}
@@ -152,45 +132,43 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) {
if (pWorker->qall == NULL) {
taosCloseQset(pWorker->qset);
taosCloseQueue(queue);
- pthread_mutex_unlock(&wWorkerPool.mutex);
+ pthread_mutex_unlock(&tsVWriteWP.mutex);
return NULL;
}
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
- if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessWriteQueue, pWorker) != 0) {
- dError("failed to create thread to process read queue, reason:%s", strerror(errno));
+ if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessVWriteQueue, pWorker) != 0) {
+ dError("failed to create thread to process vwrite queue since %s", strerror(errno));
taosFreeQall(pWorker->qall);
taosCloseQset(pWorker->qset);
taosCloseQueue(queue);
queue = NULL;
} else {
- dDebug("write worker:%d is launched", pWorker->workerId);
- wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
+ dDebug("dnode vwrite worker:%d is launched", pWorker->workerId);
+ tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max;
}
pthread_attr_destroy(&thAttr);
} else {
taosAddIntoQset(pWorker->qset, queue, pVnode);
- wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
+ tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max;
}
- pthread_mutex_unlock(&wWorkerPool.mutex);
- dDebug("pVnode:%p, write queue:%p is allocated", pVnode, queue);
+ pthread_mutex_unlock(&tsVWriteWP.mutex);
+ dDebug("pVnode:%p, dnode vwrite queue:%p is allocated", pVnode, queue);
return queue;
}
-void dnodeFreeVnodeWqueue(void *wqueue) {
- taosCloseQueue(wqueue);
-
- // dynamically adjust the number of threads
+void dnodeFreeVWriteQueue(void *pWqueue) {
+ taosCloseQueue(pWqueue);
}
-void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {
- SWriteMsg *pWrite = (SWriteMsg *)param;
- if (pWrite == NULL) return;
+void dnodeSendRpcVWriteRsp(void *pVnode, void *wparam, int32_t code) {
+ if (wparam == NULL) return;
+ SVWriteMsg *pWrite = wparam;
if (code < 0) pWrite->code = code;
int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1);
@@ -198,106 +176,64 @@ void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {
if (count <= 1) return;
SRpcMsg rpcRsp = {
- .handle = pWrite->rpcMsg.handle,
+ .handle = pWrite->rpcHandle,
.pCont = pWrite->rspRet.rsp,
.contLen = pWrite->rspRet.len,
.code = pWrite->code,
};
rpcSendResponse(&rpcRsp);
- rpcFreeCont(pWrite->rpcMsg.pCont);
- taosFreeQitem(pWrite);
-
- vnodeRelease(pVnode);
+ vnodeFreeFromWQueue(pVnode, pWrite);
}
-static void *dnodeProcessWriteQueue(void *param) {
- SWriteWorker *pWorker = (SWriteWorker *)param;
- SWriteMsg * pWrite;
- SWalHead * pHead;
- int32_t numOfMsgs;
- int type;
- void * pVnode, *item;
- SRspRet * pRspRet;
+static void *dnodeProcessVWriteQueue(void *wparam) {
+ SVWriteWorker *pWorker = wparam;
+ SVWriteMsg * pWrite;
+ void * pVnode;
+ int32_t numOfMsgs;
+ int32_t qtype;
- dDebug("write worker:%d is running", pWorker->workerId);
+ dDebug("dnode vwrite worker:%d is running", pWorker->workerId);
while (1) {
numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode);
if (numOfMsgs == 0) {
- dDebug("qset:%p, dnode write got no message from qset, exiting", pWorker->qset);
+ dDebug("qset:%p, dnode vwrite got no message from qset, exiting", pWorker->qset);
break;
}
+ bool forceFsync = false;
for (int32_t i = 0; i < numOfMsgs; ++i) {
- pWrite = NULL;
- pRspRet = NULL;
- taosGetQitem(pWorker->qall, &type, &item);
- if (type == TAOS_QTYPE_RPC) {
- pWrite = (SWriteMsg *)item;
- pRspRet = &pWrite->rspRet;
- pHead = (SWalHead *)(pWrite->pCont - sizeof(SWalHead));
- pHead->msgType = pWrite->rpcMsg.msgType;
- pHead->version = 0;
- pHead->len = pWrite->contLen;
- dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle,
- taosMsg[pWrite->rpcMsg.msgType]);
- } else if (type == TAOS_QTYPE_CQ) {
- pHead = (SWalHead *)((char*)item + sizeof(SSyncHead));
- dTrace("%p, CQ wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType],
- pHead->version);
- } else {
- pHead = (SWalHead *)item;
- dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType],
- pHead->version);
- }
+ taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite);
+ dTrace("%p, msg:%p:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite->rpcAhandle, pWrite,
+ taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version);
- int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet);
- dTrace("%p, msg:%s is processed in vwrite queue, version:%" PRIu64 ", result:%s", pHead, taosMsg[pHead->msgType],
- pHead->version, tstrerror(code));
+ pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet);
+ if (pWrite->code <= 0) pWrite->processedCount = 1;
+ if (pWrite->code == 0 && pWrite->pHead->msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true;
- if (pWrite) {
- pWrite->rpcMsg.code = code;
- if (code <= 0) pWrite->processedCount = 1;
- }
+ dTrace("msg:%p is processed in vwrite queue, result:%s", pWrite, tstrerror(pWrite->code));
}
- walFsync(vnodeGetWal(pVnode));
+ walFsync(vnodeGetWal(pVnode), forceFsync);
// browse all items, and process them one by one
taosResetQitems(pWorker->qall);
for (int32_t i = 0; i < numOfMsgs; ++i) {
- taosGetQitem(pWorker->qall, &type, &item);
- if (type == TAOS_QTYPE_RPC) {
- pWrite = (SWriteMsg *)item;
- dnodeSendRpcVnodeWriteRsp(pVnode, item, pWrite->rpcMsg.code);
- } else if (type == TAOS_QTYPE_FWD) {
- pHead = (SWalHead *)item;
- vnodeConfirmForward(pVnode, pHead->version, 0);
- taosFreeQitem(item);
- vnodeRelease(pVnode);
+ taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite);
+ if (qtype == TAOS_QTYPE_RPC) {
+ dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code);
} else {
- taosFreeQitem(item);
- vnodeRelease(pVnode);
+ if (qtype == TAOS_QTYPE_FWD) {
+ vnodeConfirmForward(pVnode, pWrite->pHead->version, 0);
+ }
+ if (pWrite->rspRet.rsp) {
+ rpcFreeCont(pWrite->rspRet.rsp);
+ }
+ vnodeFreeFromWQueue(pVnode, pWrite);
}
}
}
return NULL;
}
-
-UNUSED_FUNC
-static void dnodeHandleIdleWorker(SWriteWorker *pWorker) {
- int32_t num = taosGetQueueNumber(pWorker->qset);
-
- if (num > 0) {
- usleep(30000);
- sched_yield();
- } else {
- taosFreeQall(pWorker->qall);
- taosCloseQset(pWorker->qset);
- pWorker->qset = NULL;
- dDebug("write worker:%d is released", pWorker->workerId);
- pthread_exit(NULL);
- }
-}
diff --git a/src/inc/dnode.h b/src/inc/dnode.h
index 3efd125a4dc321760afe9931002fce42669e4336..bd835a89884696a38dd1cd7530f67cfc15f845f2 100644
--- a/src/inc/dnode.h
+++ b/src/inc/dnode.h
@@ -25,29 +25,31 @@ extern "C" {
#include "hash.h"
#include "taoserror.h"
#include "trpc.h"
+#include "taosmsg.h"
typedef struct {
int32_t queryReqNum;
int32_t submitReqNum;
int32_t httpReqNum;
-} SDnodeStatisInfo;
+} SStatisInfo;
typedef enum {
- TSDB_DNODE_RUN_STATUS_INITIALIZE,
- TSDB_DNODE_RUN_STATUS_RUNING,
- TSDB_DNODE_RUN_STATUS_STOPPED
-} SDnodeRunStatus;
+ TSDB_RUN_STATUS_INITIALIZE,
+ TSDB_RUN_STATUS_RUNING,
+ TSDB_RUN_STATUS_STOPPED
+} SRunStatus;
-SDnodeRunStatus dnodeGetRunStatus();
-SDnodeStatisInfo dnodeGetStatisInfo();
+SRunStatus dnodeGetRunStatus();
+SStatisInfo dnodeGetStatisInfo();
bool dnodeIsFirstDeploy();
-char * dnodeGetMnodeMasterEp();
-void dnodeGetMnodeEpSetForPeer(void *epSet);
-void dnodeGetMnodeEpSetForShell(void *epSet);
-void * dnodeGetMnodeInfos();
+bool dnodeIsMasterEp(char *ep);
+void dnodeGetEpSetForPeer(SRpcEpSet *epSet);
+void dnodeGetEpSetForShell(SRpcEpSet *epSet);
int32_t dnodeGetDnodeId();
-bool dnodeStartMnode(void *pModes);
+void dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port);
+bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr);
+bool dnodeStartMnode(SMnodeInfos *minfos);
void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg);
@@ -55,21 +57,21 @@ void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp);
void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet);
void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid);
-void *dnodeAllocateVnodeWqueue(void *pVnode);
-void dnodeFreeVnodeWqueue(void *queue);
-void *dnodeAllocateVnodeRqueue(void *pVnode);
-void dnodeFreeVnodeRqueue(void *rqueue);
-void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code);
+void *dnodeAllocVWriteQueue(void *pVnode);
+void dnodeFreeVWriteQueue(void *pWqueue);
+void dnodeSendRpcVWriteRsp(void *pVnode, void *pWrite, int32_t code);
+void *dnodeAllocVReadQueue(void *pVnode);
+void dnodeFreeVReadQueue(void *rqueue);
-int32_t dnodeAllocateMnodePqueue();
-void dnodeFreeMnodePqueue();
-int32_t dnodeAllocateMnodeRqueue();
-void dnodeFreeMnodeRqueue();
-int32_t dnodeAllocateMnodeWqueue();
-void dnodeFreeMnodeWqueue();
-void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code);
-void dnodeReprocessMnodeWriteMsg(void *pMsg);
-void dnodeDelayReprocessMnodeWriteMsg(void *pMsg);
+int32_t dnodeAllocateMPeerQueue();
+void dnodeFreeMPeerQueue();
+int32_t dnodeAllocMReadQueue();
+void dnodeFreeMReadQueue();
+int32_t dnodeAllocMWritequeue();
+void dnodeFreeMWritequeue();
+void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code);
+void dnodeReprocessMWriteMsg(void *pMsg);
+void dnodeDelayReprocessMWriteMsg(void *pMsg);
void dnodeSendStatusMsgToMnode();
diff --git a/src/inc/mnode.h b/src/inc/mnode.h
index 5bef7402e30a0cc9f1964cc2430e0b1f5141590c..bdc30b0c46ced0961715bd48623fdb9e52fb440e 100644
--- a/src/inc/mnode.h
+++ b/src/inc/mnode.h
@@ -35,24 +35,26 @@ typedef struct {
} SMnodeRsp;
typedef struct SMnodeMsg {
- SRpcMsg rpcMsg;
+ struct SAcctObj * pAcct;
+ struct SDnodeObj *pDnode;
+ struct SUserObj * pUser;
+ struct SDbObj * pDb;
+ struct SVgObj * pVgroup;
+ struct STableObj *pTable;
+ struct SSTableObj*pSTable;
SMnodeRsp rpcRsp;
int8_t received;
int8_t successed;
int8_t expected;
int8_t retry;
+ int32_t incomingTs;
int32_t code;
void * pObj;
- struct SAcctObj * pAcct;
- struct SDnodeObj *pDnode;
- struct SUserObj * pUser;
- struct SDbObj * pDb;
- struct SVgObj * pVgroup;
- struct STableObj *pTable;
- struct SSuperTableObj *pSTable;
+ SRpcMsg rpcMsg;
+ char pCont[];
} SMnodeMsg;
-void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg);
+void * mnodeCreateMsg(SRpcMsg *pRpcMsg);
int32_t mnodeInitMsg(SMnodeMsg *pMsg);
void mnodeCleanupMsg(SMnodeMsg *pMsg);
diff --git a/src/inc/taos.h b/src/inc/taos.h
index 315313734753de73bf477b1f67783a45c38c87c9..2c6454ced1911de603f120c79af5237c2f064c48 100644
--- a/src/inc/taos.h
+++ b/src/inc/taos.h
@@ -64,7 +64,7 @@ typedef struct taosField {
#endif
DLL_EXPORT void taos_init();
-DLL_EXPORT void taos_cleanup();
+DLL_EXPORT void taos_cleanup(void);
DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...);
DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
DLL_EXPORT void taos_close(TAOS *taos);
diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index 162be1583fe7ad30a427e22765ded3492839c275..bea6375aa49bb8a2c7acfb12b18d02ebc5bbf698 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -61,13 +61,23 @@ typedef struct tstr {
// Bytes for each type.
extern const int32_t TYPE_BYTES[11];
+
// TODO: replace and remove code below
-#define CHAR_BYTES sizeof(char)
-#define SHORT_BYTES sizeof(int16_t)
-#define INT_BYTES sizeof(int32_t)
-#define LONG_BYTES sizeof(int64_t)
-#define FLOAT_BYTES sizeof(float)
-#define DOUBLE_BYTES sizeof(double)
+#define CHAR_BYTES sizeof(char)
+#define SHORT_BYTES sizeof(int16_t)
+#define INT_BYTES sizeof(int32_t)
+#define LONG_BYTES sizeof(int64_t)
+#define FLOAT_BYTES sizeof(float)
+#define DOUBLE_BYTES sizeof(double)
+#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*)
+
+#define TSDB_KEYSIZE sizeof(TSKEY)
+
+#if LINUX
+#define TSDB_NCHAR_SIZE sizeof(wchar_t)
+#else
+#define TSDB_NCHAR_SIZE sizeof(int32_t)
+#endif
// NULL definition
#define TSDB_DATA_BOOL_NULL 0x02
@@ -75,6 +85,7 @@ extern const int32_t TYPE_BYTES[11];
#define TSDB_DATA_SMALLINT_NULL 0x8000
#define TSDB_DATA_INT_NULL 0x80000000L
#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L
+#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL
#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN
#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN
@@ -101,10 +112,12 @@ extern const int32_t TYPE_BYTES[11];
#define TSDB_TIME_PRECISION_MILLI 0
#define TSDB_TIME_PRECISION_MICRO 1
#define TSDB_TIME_PRECISION_NANO 2
-#define TSDB_TICK_PER_SECOND(precision) ((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))
#define TSDB_TIME_PRECISION_MILLI_STR "ms"
#define TSDB_TIME_PRECISION_MICRO_STR "us"
+#define TSDB_TIME_PRECISION_NANO_STR "ns"
+
+#define TSDB_TICK_PER_SECOND(precision) ((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))
#define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member)
#define T_APPEND_MEMBER(dst, ptr, type, member) \
@@ -118,15 +131,6 @@ do { \
(src) = (void *)((char *)src + sizeof(type));\
} while(0)
-#define TSDB_KEYSIZE sizeof(TSKEY)
-
-#if LINUX
- #define TSDB_NCHAR_SIZE sizeof(wchar_t)
-#else
- #define TSDB_NCHAR_SIZE 4
-#endif
-//#define TSDB_CHAR_TERMINATED_SPACE 1
-
#define GET_INT8_VAL(x) (*(int8_t *)(x))
#define GET_INT16_VAL(x) (*(int16_t *)(x))
#define GET_INT32_VAL(x) (*(int32_t *)(x))
@@ -172,7 +176,6 @@ typedef struct tDataTypeDescriptor {
} tDataTypeDescriptor;
extern tDataTypeDescriptor tDataTypeDesc[11];
-#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*)
bool isValidDataType(int32_t type);
//bool isNull(const char *val, int32_t type);
@@ -266,10 +269,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_AUTH_LEN 16
#define TSDB_KEY_LEN 16
#define TSDB_VERSION_LEN 12
-#define TSDB_STREET_LEN 64
-#define TSDB_CITY_LEN 20
-#define TSDB_STATE_LEN 20
-#define TSDB_COUNTRY_LEN 20
#define TSDB_LOCALE_LEN 64
#define TSDB_TIMEZONE_LEN 96
#define TSDB_LABEL_LEN 8
@@ -333,7 +332,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_MIN_DAYS_PER_FILE 1
#define TSDB_MAX_DAYS_PER_FILE 3650
-#define TSDB_DEFAULT_DAYS_PER_FILE 2
+#define TSDB_DEFAULT_DAYS_PER_FILE 10
#define TSDB_MIN_KEEP 1 // data in db to be reserved.
#define TSDB_MAX_KEEP 365000 // data in db to be reserved.
@@ -363,6 +362,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_MAX_WAL_LEVEL 2
#define TSDB_DEFAULT_WAL_LEVEL 1
+#define TSDB_MIN_DB_UPDATE 0
+#define TSDB_MAX_DB_UPDATE 1
+#define TSDB_DEFAULT_DB_UPDATE_OPTION 0
+
#define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond
#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second
@@ -388,27 +391,27 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
* 1. ordinary sub query for select * from super_table
* 2. all sqlobj generated by createSubqueryObj with this flag
*/
-#define TSDB_QUERY_TYPE_SUBQUERY 0x02u
-#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table
+#define TSDB_QUERY_TYPE_SUBQUERY 0x02u
+#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table
-#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side
-#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table
-#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query
-#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query
-#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage
+#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side
+#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table
+#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query
+#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query
+#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage
-#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u
-#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type
-#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u
-#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type
+#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u
+#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type
+#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u
+#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type
#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0)
#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type))
#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type))
#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE)
-#define TSDB_ORDER_ASC 1
-#define TSDB_ORDER_DESC 2
+#define TSDB_ORDER_ASC 1
+#define TSDB_ORDER_DESC 2
#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1
#define TSDB_DEFAULT_MNODES_HASH_SIZE 5
@@ -420,50 +423,60 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_DEFAULT_STABLES_HASH_SIZE 100
#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000
-#define TSDB_PORT_DNODESHELL 0
-#define TSDB_PORT_DNODEDNODE 5
-#define TSDB_PORT_SYNC 10
-#define TSDB_PORT_HTTP 11
-#define TSDB_PORT_ARBITRATOR 12
+#define TSDB_PORT_DNODESHELL 0
+#define TSDB_PORT_DNODEDNODE 5
+#define TSDB_PORT_SYNC 10
+#define TSDB_PORT_HTTP 11
+#define TSDB_PORT_ARBITRATOR 12
+#define TSDB_PORT_DNODESHELL 0
+#define TSDB_PORT_DNODEDNODE 5
+#define TSDB_PORT_SYNC 10
+#define TSDB_PORT_HTTP 11
+#define TSDB_PORT_ARBITRATOR 12
+
+#define TSDB_MAX_WAL_SIZE (1024*1024)
-#define TAOS_QTYPE_RPC 0
-#define TAOS_QTYPE_FWD 1
-#define TAOS_QTYPE_WAL 2
-#define TAOS_QTYPE_CQ 3
-#define TAOS_QTYPE_QUERY 4
+typedef enum {
+ TAOS_QTYPE_RPC = 0,
+ TAOS_QTYPE_FWD = 1,
+ TAOS_QTYPE_WAL = 2,
+ TAOS_QTYPE_CQ = 3,
+ TAOS_QTYPE_QUERY = 4
+} EQType;
#define TSDB_MAX_TIERS 3
#define TSDB_MAX_DISKS_PER_TIER 16
#define TSDB_MAX_DISKS (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER)
typedef enum {
- TSDB_SUPER_TABLE = 0, // super table
- TSDB_CHILD_TABLE = 1, // table created from super table
- TSDB_NORMAL_TABLE = 2, // ordinary table
- TSDB_STREAM_TABLE = 3, // table created from stream computing
- TSDB_TABLE_MAX = 4
+ TSDB_SUPER_TABLE = 0, // super table
+ TSDB_CHILD_TABLE = 1, // table created from super table
+ TSDB_NORMAL_TABLE = 2, // ordinary table
+ TSDB_STREAM_TABLE = 3, // table created from stream computing
+ TSDB_TABLE_MAX = 4
} ETableType;
typedef enum {
- TSDB_MOD_MNODE,
- TSDB_MOD_HTTP,
- TSDB_MOD_MONITOR,
- TSDB_MOD_MQTT,
- TSDB_MOD_MAX
+ TSDB_MOD_MNODE = 0,
+ TSDB_MOD_HTTP = 1,
+ TSDB_MOD_MONITOR = 2,
+ TSDB_MOD_MQTT = 3,
+ TSDB_MOD_MAX = 4
} EModuleType;
- typedef enum {
- TSDB_CHECK_ITEM_NETWORK,
- TSDB_CHECK_ITEM_MEM,
- TSDB_CHECK_ITEM_CPU,
- TSDB_CHECK_ITEM_DISK,
- TSDB_CHECK_ITEM_OS,
- TSDB_CHECK_ITEM_ACCESS,
- TSDB_CHECK_ITEM_VERSION,
- TSDB_CHECK_ITEM_DATAFILE,
- TSDB_CHECK_ITEM_MAX
- } ECheckItemType;
-
+typedef enum {
+ TSDB_CHECK_ITEM_NETWORK,
+ TSDB_CHECK_ITEM_MEM,
+ TSDB_CHECK_ITEM_CPU,
+ TSDB_CHECK_ITEM_DISK,
+ TSDB_CHECK_ITEM_OS,
+ TSDB_CHECK_ITEM_ACCESS,
+ TSDB_CHECK_ITEM_VERSION,
+ TSDB_CHECK_ITEM_DATAFILE,
+ TSDB_CHECK_ITEM_MAX
+} ECheckItemType;
+
+extern char *qtypeStr[];
#ifdef __cplusplus
}
diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h
index b78a091951d47d793db1a65e210c163ebf079213..01bf0808051204a8e03d1c6d0529cbd975ebb530 100644
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@@ -28,7 +28,7 @@ extern "C" {
#else
#define TAOS_DEFINE_ERROR(name, mod, code, msg) static const int32_t name = (0x80000000 | ((mod)<<16) | (code));
#endif
-
+
#define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code))
#define TAOS_SUCCEEDED(err) ((err) >= 0)
#define TAOS_FAILED(err) ((err) < 0)
@@ -37,7 +37,7 @@ const char* tstrerror(int32_t err);
int32_t* taosGetErrno();
#define terrno (*taosGetErrno())
-
+
#define TSDB_CODE_SUCCESS 0
#ifdef TAOS_ERROR_C
@@ -74,6 +74,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_COM_MEMORY_CORRUPTED, 0, 0x0101, "Memory cor
TAOS_DEFINE_ERROR(TSDB_CODE_COM_OUT_OF_MEMORY, 0, 0x0102, "Out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_COM_INVALID_CFG_MSG, 0, 0x0103, "Invalid config message")
TAOS_DEFINE_ERROR(TSDB_CODE_COM_FILE_CORRUPTED, 0, 0x0104, "Data file corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, 0, 0x0105, "Ref out of memory")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, 0, 0x0106, "too many Ref Objs")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, 0, 0x0107, "Ref ID is removed")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID, 0, 0x0108, "Invalid Ref ID")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_ALREADY_EXIST, 0, 0x0109, "Ref is already there")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, 0, 0x010A, "Ref is not there")
//client
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_SQL, 0, 0x0200, "Invalid SQL statement")
@@ -175,6 +181,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, 0, 0x0383, "Invalid da
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN, 0, 0x0384, "Cannot delete monitor database")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, 0, 0x0385, "Too many databases for account")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, 0, 0x0386, "Database not available")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY, 0, 0x0387, "Database unsynced")
// dnode
TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, 0, 0x0400, "Message not processed")
@@ -190,7 +197,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DND_DUPLICATE_PRIMARY_DISK, 0, 0x0409, "Duplicate
TAOS_DEFINE_ERROR(TSDB_CODE_DND_LACK_PRIMARY_DISK, 0, 0x040A, "Lack primary disk")
TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_DISK_AT_TIER, 0, 0x040B, "No disk at tier")
-// vnode
+// vnode
TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, 0, 0x0500, "Action in progress")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_MSG_NOT_PROCESSED, 0, 0x0501, "Message not processed")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_NEED_REPROCESSED, 0, 0x0502, "Action need to be reprocessed")
@@ -202,6 +209,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR, 0, 0x0507, "Missing da
TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY, 0, 0x0508, "Out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, 0, 0x0509, "Unexpected generic error in vnode")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE, 0, 0x050A, "Invalid version file")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL, 0, 0x050B, "Vnode memory is full because commit failed")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, 0, 0x0511, "Database suspended")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, 0, 0x0512, "Write operation denied")
@@ -238,7 +246,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query")
-TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, 0, 0x070B, "Query buffer limit has reached")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, 0, 0x070B, "Query buffer limit has reached")
// grant
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired")
@@ -262,6 +270,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, 0, 0x0902, "Invalid Sy
// wal
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, 0, 0x1001, "WAL file is corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, 0, 0x1002, "WAL size exceeds limit")
// http
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE, 0, 0x1100, "http server is not onlin")
diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h
index 600347c44fdb2a89e03bf24413f22240bb2815f2..681fa4492959347d5af5591615c0b39dbacbef7e 100644
--- a/src/inc/taosmsg.h
+++ b/src/inc/taosmsg.h
@@ -106,6 +106,10 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" )
+
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "network-test" )
+
+
#ifndef TAOS_MESSAGE_C
TSDB_MSG_TYPE_MAX // 105
#endif
@@ -291,7 +295,7 @@ typedef struct {
SSchema schema[];
// tagVal is padded after schema
// char tagVal[];
-} SCMAlterTableMsg;
+} SAlterTableMsg;
typedef struct {
SMsgHead head;
@@ -308,12 +312,12 @@ typedef struct {
} SUpdateTableTagValMsg;
typedef struct {
- char clientVersion[TSDB_VERSION_LEN];
- char msgVersion[TSDB_VERSION_LEN];
- char db[TSDB_TABLE_FNAME_LEN];
- char appName[TSDB_APPNAME_LEN];
+ char clientVersion[TSDB_VERSION_LEN];
+ char msgVersion[TSDB_VERSION_LEN];
+ char db[TSDB_TABLE_FNAME_LEN];
+ char appName[TSDB_APPNAME_LEN];
int32_t pid;
-} SCMConnectMsg;
+} SConnectMsg;
typedef struct {
char acctId[TSDB_ACCT_LEN];
@@ -324,7 +328,7 @@ typedef struct {
int8_t reserved2;
int32_t connId;
SRpcEpSet epSet;
-} SCMConnectRsp;
+} SConnectRsp;
typedef struct {
int32_t maxUsers;
@@ -344,18 +348,18 @@ typedef struct {
char user[TSDB_USER_LEN];
char pass[TSDB_KEY_LEN];
SAcctCfg cfg;
-} SCMCreateAcctMsg, SCMAlterAcctMsg;
+} SCreateAcctMsg, SAlterAcctMsg;
typedef struct {
char user[TSDB_USER_LEN];
-} SCMDropUserMsg, SCMDropAcctMsg;
+} SDropUserMsg, SDropAcctMsg;
typedef struct {
char user[TSDB_USER_LEN];
char pass[TSDB_KEY_LEN];
int8_t privilege;
int8_t flag;
-} SCMCreateUserMsg, SCMAlterUserMsg;
+} SCreateUserMsg, SAlterUserMsg;
typedef struct {
int32_t contLen;
@@ -370,11 +374,11 @@ typedef struct {
int32_t vgId;
uint64_t uid;
char tableId[TSDB_TABLE_FNAME_LEN];
-} SMDDropSTableMsg;
+} SDropSTableMsg;
typedef struct {
int32_t vgId;
-} SMDDropVnodeMsg;
+} SDropVnodeMsg;
typedef struct SColIndex {
int16_t colId; // column id
@@ -388,6 +392,7 @@ typedef struct SColIndex {
typedef struct SSqlFuncMsg {
int16_t functionId;
int16_t numOfParams;
+ int16_t resColId; // result column id, id of the current output column
SColIndex colInfo;
struct ArgElem {
@@ -457,11 +462,6 @@ typedef struct STimeWindow {
TSKEY ekey;
} STimeWindow;
-/*
- * the outputCols is equalled to or larger than numOfCols
- * e.g., select min(colName), max(colName), avg(colName) from table
- * the outputCols will be 3 while the numOfCols is 1.
- */
typedef struct {
SMsgHead head;
STimeWindow window;
@@ -477,10 +477,11 @@ typedef struct {
int64_t limit;
int64_t offset;
uint32_t queryType; // denote another query process
- int16_t numOfOutput; // final output columns numbers
+ int16_t numOfOutput; // final output columns numbers
int16_t tagNameRelType; // relation of tag criteria and tbname criteria
int16_t fillType; // interpolate type
uint64_t fillVal; // default value array list
+ int32_t secondStageOutput;
int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed
int32_t tsLen; // total length of ts comp block
int32_t tsNumOfBlocks; // ts comp block numbers
@@ -540,12 +541,14 @@ typedef struct {
int8_t replications;
int8_t quorum;
int8_t ignoreExist;
-} SCMCreateDbMsg, SCMAlterDbMsg;
+ int8_t update;
+ int8_t reserve[9];
+} SCreateDbMsg, SAlterDbMsg;
typedef struct {
char db[TSDB_TABLE_FNAME_LEN];
uint8_t ignoreNotExists;
-} SCMDropDbMsg, SCMUseDbMsg;
+} SDropDbMsg, SUseDbMsg;
// IMPORTANT: sizeof(SVnodeStatisticInfo) should not exceed
// TSDB_FILE_HEADER_LEN/4 - TSDB_FILE_HEADER_VERSION_SIZE
@@ -560,7 +563,7 @@ typedef struct {
typedef struct {
int32_t vgId;
int8_t accessState;
-} SDMVgroupAccess;
+} SVgroupAccess;
typedef struct {
int32_t dnodeId;
@@ -568,18 +571,29 @@ typedef struct {
uint32_t numOfVnodes;
char clusterId[TSDB_CLUSTER_ID_LEN];
char reserved[16];
-} SDMDnodeCfg;
+} SDnodeCfg;
typedef struct {
- int32_t nodeId;
- char nodeEp[TSDB_EP_LEN];
-} SDMMnodeInfo;
+ int32_t dnodeId;
+ uint16_t dnodePort;
+ char dnodeFqdn[TSDB_FQDN_LEN];
+} SDnodeEp;
typedef struct {
- int8_t inUse;
- int8_t nodeNum;
- SDMMnodeInfo nodeInfos[TSDB_MAX_REPLICA];
-} SDMMnodeInfos;
+ int32_t dnodeNum;
+ SDnodeEp dnodeEps[];
+} SDnodeEps;
+
+typedef struct {
+ int32_t mnodeId;
+ char mnodeEp[TSDB_EP_LEN];
+} SMnodeInfo;
+
+typedef struct {
+ int8_t inUse;
+ int8_t mnodeNum;
+ SMnodeInfo mnodeInfos[TSDB_MAX_REPLICA];
+} SMnodeInfos;
typedef struct {
int32_t numOfMnodes; // tsNumOfMnodes
@@ -611,13 +625,13 @@ typedef struct {
uint8_t reserve2[15];
SClusterCfg clusterCfg;
SVnodeLoad load[];
-} SDMStatusMsg;
+} SStatusMsg;
typedef struct {
- SDMMnodeInfos mnodes;
- SDMDnodeCfg dnodeCfg;
- SDMVgroupAccess vgAccess[];
-} SDMStatusRsp;
+ SMnodeInfos mnodes;
+ SDnodeCfg dnodeCfg;
+ SVgroupAccess vgAccess[];
+} SStatusRsp;
typedef struct {
uint32_t vgId;
@@ -639,55 +653,56 @@ typedef struct {
int8_t replications;
int8_t wals;
int8_t quorum;
- int8_t reserved[16];
-} SMDVnodeCfg;
+ int8_t update;
+ int8_t reserved[15];
+} SVnodeCfg;
typedef struct {
int32_t nodeId;
char nodeEp[TSDB_EP_LEN];
-} SMDVnodeDesc;
+} SVnodeDesc;
typedef struct {
- char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
- SMDVnodeCfg cfg;
- SMDVnodeDesc nodes[TSDB_MAX_REPLICA];
-} SMDCreateVnodeMsg, SMDAlterVnodeMsg;
+ char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+ SVnodeCfg cfg;
+ SVnodeDesc nodes[TSDB_MAX_REPLICA];
+} SCreateVnodeMsg, SAlterVnodeMsg;
typedef struct {
char tableId[TSDB_TABLE_FNAME_LEN];
int16_t createFlag;
char tags[];
-} SCMTableInfoMsg;
+} STableInfoMsg;
typedef struct {
int32_t numOfTables;
char tableIds[];
-} SCMMultiTableInfoMsg;
+} SMultiTableInfoMsg;
-typedef struct SCMSTableVgroupMsg {
+typedef struct SSTableVgroupMsg {
int32_t numOfTables;
-} SCMSTableVgroupMsg, SCMSTableVgroupRspMsg;
+} SSTableVgroupMsg, SSTableVgroupRspMsg;
typedef struct {
int32_t vgId;
int8_t numOfEps;
SEpAddr1 epAddr[TSDB_MAX_REPLICA];
-} SCMVgroupInfo;
+} SVgroupInfo;
typedef struct {
int32_t vgId;
int8_t numOfEps;
SEpAddrMsg epAddr[TSDB_MAX_REPLICA];
-} SCMVgroupMsg;
+} SVgroupMsg;
typedef struct {
int32_t numOfVgroups;
- SCMVgroupInfo vgroups[];
+ SVgroupInfo vgroups[];
} SVgroupsInfo;
typedef struct {
int32_t numOfVgroups;
- SCMVgroupMsg vgroups[];
+ SVgroupMsg vgroups[];
} SVgroupsMsg;
typedef struct STableMetaMsg {
@@ -702,7 +717,7 @@ typedef struct STableMetaMsg {
int16_t tversion;
int32_t tid;
uint64_t uid;
- SCMVgroupMsg vgroup;
+ SVgroupMsg vgroup;
SSchema schema[];
} STableMetaMsg;
@@ -728,38 +743,38 @@ typedef struct {
char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
uint16_t payloadLen;
char payload[];
-} SCMShowMsg;
+} SShowMsg;
-typedef struct SCMShowRsp {
+typedef struct SShowRsp {
uint64_t qhandle;
STableMetaMsg tableMeta;
-} SCMShowRsp;
+} SShowRsp;
typedef struct {
- char ep[TSDB_EP_LEN]; // end point, hostname:port
-} SCMCreateDnodeMsg, SCMDropDnodeMsg;
+ char ep[TSDB_EP_LEN]; // end point, hostname:port
+} SCreateDnodeMsg, SDropDnodeMsg;
typedef struct {
int32_t dnodeId;
char dnodeEp[TSDB_EP_LEN]; // end point, hostname:port
- SDMMnodeInfos mnodes;
-} SMDCreateMnodeMsg;
+ SMnodeInfos mnodes;
+} SCreateMnodeMsg;
typedef struct {
int32_t dnodeId;
int32_t vgId;
int32_t tid;
-} SDMConfigTableMsg;
+} SConfigTableMsg;
typedef struct {
uint32_t dnodeId;
int32_t vgId;
-} SDMConfigVnodeMsg;
+} SConfigVnodeMsg;
typedef struct {
char ep[TSDB_EP_LEN]; // end point, hostname:port
char config[64];
-} SMDCfgDnodeMsg, SCMCfgDnodeMsg;
+} SCfgDnodeMsg;
typedef struct {
char sql[TSDB_SHOW_SQL_LEN];
@@ -781,13 +796,14 @@ typedef struct {
} SStreamDesc;
typedef struct {
+ char clientVer[TSDB_VERSION_LEN];
uint32_t connId;
int32_t pid;
int32_t numOfQueries;
int32_t numOfStreams;
char appName[TSDB_APPNAME_LEN];
char pData[];
-} SCMHeartBeatMsg;
+} SHeartBeatMsg;
typedef struct {
uint32_t queryId;
@@ -797,11 +813,11 @@ typedef struct {
uint32_t connId;
int8_t killConnection;
SRpcEpSet epSet;
-} SCMHeartBeatRsp;
+} SHeartBeatRsp;
typedef struct {
char queryId[TSDB_KILL_MSG_LEN + 1];
-} SCMKillQueryMsg, SCMKillStreamMsg, SCMKillConnMsg;
+} SKillQueryMsg, SKillStreamMsg, SKillConnMsg;
typedef struct {
int32_t vnode;
@@ -810,7 +826,7 @@ typedef struct {
uint64_t stime; // stream starting time
int32_t status;
char tableId[TSDB_TABLE_FNAME_LEN];
-} SMDAlterStreamMsg;
+} SAlterStreamMsg;
typedef struct {
char user[TSDB_USER_LEN];
@@ -818,7 +834,7 @@ typedef struct {
char encrypt;
char secret[TSDB_KEY_LEN];
char ckey[TSDB_KEY_LEN];
-} SDMAuthMsg, SDMAuthRsp;
+} SAuthMsg, SAuthRsp;
#pragma pack(pop)
diff --git a/src/inc/tcq.h b/src/inc/tcq.h
index 32b75674c3278b3273fd4b98dd645f4168543155..7a0727f1b8dd7cdea1a815c174b9f85004eedf87 100644
--- a/src/inc/tcq.h
+++ b/src/inc/tcq.h
@@ -21,10 +21,10 @@ extern "C" {
#include "tdataformat.h"
-typedef int (*FCqWrite)(void *ahandle, void *pHead, int type);
+typedef int32_t (*FCqWrite)(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
typedef struct {
- int vgId;
+ int32_t vgId;
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char db[TSDB_DB_NAME_LEN];
@@ -42,12 +42,12 @@ void cqStart(void *handle);
void cqStop(void *handle);
// cqCreate is called by TSDB to start an instance of CQ
-void *cqCreate(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema);
+void *cqCreate(void *handle, uint64_t uid, int32_t sid, char *sqlStr, STSchema *pSchema);
// cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate
void cqDrop(void *handle);
-extern int cqDebugFlag;
+extern int32_t cqDebugFlag;
#ifdef __cplusplus
diff --git a/src/inc/trpc.h b/src/inc/trpc.h
index bdee917b5e8203743a79ca27d8fbc987569c35ab..0ce2e3da14d1cec204fc755db13da53f08295bff 100644
--- a/src/inc/trpc.h
+++ b/src/inc/trpc.h
@@ -78,18 +78,20 @@ typedef struct SRpcInit {
int (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
} SRpcInit;
+int32_t rpcInit();
+void rpcCleanup();
void *rpcOpen(const SRpcInit *pRpc);
void rpcClose(void *);
void *rpcMallocCont(int contLen);
void rpcFreeCont(void *pCont);
void *rpcReallocCont(void *ptr, int contLen);
-void rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg);
+void rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
void rpcSendResponse(const SRpcMsg *pMsg);
void rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet);
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
int rpcReportProgress(void *pConn, char *pCont, int contLen);
-void rpcCancelRequest(void *pContext);
+void rpcCancelRequest(int64_t rid);
#ifdef __cplusplus
}
diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h
index 85f9b3bdc7f1cf806309f597ffad6955151b4c36..58859f42bc80daa3317d789950c1625c1533cf5f 100644
--- a/src/inc/tsdb.h
+++ b/src/inc/tsdb.h
@@ -46,7 +46,7 @@ extern "C" {
typedef struct {
void *appH;
void *cqH;
- int (*notifyStatus)(void *, int status);
+ int (*notifyStatus)(void *, int status, int eno);
int (*eventCallBack)(void *);
void *(*cqCreateFunc)(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema);
void (*cqDropFunc)(void *handle);
@@ -65,6 +65,7 @@ typedef struct {
int32_t maxRowsPerFileBlock; // maximum rows per file block
int8_t precision;
int8_t compression;
+ int8_t update;
} STsdbCfg;
// --------- TSDB REPOSITORY USAGE STATISTICS
@@ -82,7 +83,7 @@ STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo);
int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg);
int32_t tsdbDropRepo(char *rootDir);
TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH);
-void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit);
+int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit);
int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg);
int tsdbGetState(TSDB_REPO_T *repo);
@@ -125,7 +126,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
// the TSDB repository info
typedef struct STsdbRepoInfo {
STsdbCfg tsdbCfg;
- int64_t version; // version of the repository
+ uint64_t version; // version of the repository
int64_t tsdbTotalDataSize; // the original inserted data size
int64_t tsdbTotalDiskSize; // the total disk size taken by this TSDB repository
// TODO: Other informations to add
@@ -135,7 +136,7 @@ STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo);
// the meter information report structure
typedef struct {
STableCfg tableCfg;
- int64_t version;
+ uint64_t version;
int64_t tableTotalDataSize; // In bytes
int64_t tableTotalDiskSize; // In bytes
} STableInfo;
@@ -163,6 +164,12 @@ typedef struct STsdbQueryCond {
SColumnInfo *colList;
} STsdbQueryCond;
+typedef struct SMemRef {
+ int32_t ref;
+ void *mem;
+ void *imem;
+} SMemRef;
+
typedef struct SDataBlockInfo {
STimeWindow window;
int32_t rows;
@@ -192,7 +199,7 @@ typedef struct {
* @param qinfo query info handle from query processor
* @return
*/
-TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo);
+TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo, SMemRef* pRef);
/**
* Get the last row of the given query time window for all the tables in STableGroupInfo object.
@@ -204,7 +211,7 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
* @param tableInfo table list.
* @return
*/
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo);
+TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo, SMemRef* pRef);
/**
* get the queried table object list
@@ -222,7 +229,7 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
* @return
*/
TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
- void *qinfo);
+ void *qinfo, SMemRef* pRef);
/**
* move to next block if exists
@@ -314,6 +321,12 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
*/
void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage);
+int tsdbInitCommitQueue(int nthreads);
+void tsdbDestroyCommitQueue();
+int tsdbSyncCommit(TSDB_REPO_T *repo);
+void tsdbIncCommitRef(int vgId);
+void tsdbDecCommitRef(int vgId);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/inc/tsync.h b/src/inc/tsync.h
index ca0f70d104d603d176d89dd5b92979433f390466..d57433eba9bacb622081032f4ed6a2a72406248d 100644
--- a/src/inc/tsync.h
+++ b/src/inc/tsync.h
@@ -51,9 +51,9 @@ typedef struct {
} SSyncCfg;
typedef struct {
- int selfIndex;
- uint32_t nodeId[TAOS_SYNC_MAX_REPLICA];
- int role[TAOS_SYNC_MAX_REPLICA];
+ int32_t selfIndex;
+ uint32_t nodeId[TAOS_SYNC_MAX_REPLICA];
+ int32_t role[TAOS_SYNC_MAX_REPLICA];
} SNodesRole;
/*
@@ -68,10 +68,10 @@ typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uin
// get the wal file from index or after
// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
-typedef int (*FGetWalInfo)(void *ahandle, char *name, uint32_t *index);
+typedef int32_t (*FGetWalInfo)(void *ahandle, char *fileName, int64_t *fileId);
-// when a forward pkt is received, call this to handle data
-typedef int (*FWriteToCache)(void *ahandle, void *pHead, int type);
+// when a forward pkt is received, call this to handle data
+typedef int32_t (*FWriteToCache)(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
// when forward is confirmed by peer, master call this API to notify app
typedef void (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code);
@@ -83,48 +83,47 @@ typedef void (*FNotifyRole)(void *ahandle, int8_t role);
typedef void (*FNotifyFlowCtrl)(void *ahandle, int32_t mseconds);
// when data file is synced successfully, notity app
-typedef int (*FNotifyFileSynced)(void *ahandle, uint64_t fversion);
+typedef int32_t (*FNotifyFileSynced)(void *ahandle, uint64_t fversion);
typedef struct {
- int32_t vgId; // vgroup ID
- uint64_t version; // initial version
- SSyncCfg syncCfg; // configuration from mgmt
- char path[128]; // path to the file
-
- void *ahandle; // handle provided by APP
- FGetFileInfo getFileInfo;
- FGetWalInfo getWalInfo;
- FWriteToCache writeToCache;
- FConfirmForward confirmForward;
- FNotifyRole notifyRole;
- FNotifyFlowCtrl notifyFlowCtrl;
+ int32_t vgId; // vgroup ID
+ uint64_t version; // initial version
+ SSyncCfg syncCfg; // configuration from mgmt
+ char path[128]; // path to the file
+ void * ahandle; // handle provided by APP
+ FGetFileInfo getFileInfo;
+ FGetWalInfo getWalInfo;
+ FWriteToCache writeToCache;
+ FConfirmForward confirmForward;
+ FNotifyRole notifyRole;
+ FNotifyFlowCtrl notifyFlowCtrl;
FNotifyFileSynced notifyFileSynced;
} SSyncInfo;
-typedef void* tsync_h;
+typedef void *tsync_h;
int32_t syncInit();
void syncCleanUp();
-tsync_h syncStart(const SSyncInfo *);
-void syncStop(tsync_h shandle);
-int32_t syncReconfig(tsync_h shandle, const SSyncCfg *);
-int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype);
-void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code);
-void syncRecover(tsync_h shandle); // recover from other nodes:
-int syncGetNodesRole(tsync_h shandle, SNodesRole *);
+int64_t syncStart(const SSyncInfo *);
+void syncStop(int64_t rid);
+int32_t syncReconfig(int64_t rid, const SSyncCfg *);
+int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype);
+void syncConfirmForward(int64_t rid, uint64_t version, int32_t code);
+void syncRecover(int64_t rid); // recover from other nodes:
+int32_t syncGetNodesRole(int64_t rid, SNodesRole *);
-extern char *syncRole[];
+extern char *syncRole[];
//global configurable parameters
-extern int tsMaxSyncNum;
-extern int tsSyncTcpThreads;
-extern int tsMaxWatchFiles;
-extern int tsSyncTimer;
-extern int tsMaxFwdInfo;
-extern int sDebugFlag;
-extern char tsArbitrator[];
-extern uint16_t tsSyncPort;
+extern int32_t tsMaxSyncNum;
+extern int32_t tsSyncTcpThreads;
+extern int32_t tsMaxWatchFiles;
+extern int32_t tsSyncTimer;
+extern int32_t tsMaxFwdInfo;
+extern int32_t sDebugFlag;
+extern char tsArbitrator[];
+extern uint16_t tsSyncPort;
#ifdef __cplusplus
}
diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h
index a94cdaad15a5c2abe35b7a02502ff8ae95102cd9..0a5a3d2fa402c3dd03f1feef8cf05e446922972b 100644
--- a/src/inc/ttokendef.h
+++ b/src/inc/ttokendef.h
@@ -16,7 +16,6 @@
#ifndef TDENGINE_TTOKENDEF_H
#define TDENGINE_TTOKENDEF_H
-
#define TK_ID 1
#define TK_BOOL 2
#define TK_TINYINT 3
@@ -114,114 +113,116 @@
#define TK_FSYNC 95
#define TK_COMP 96
#define TK_PRECISION 97
-#define TK_LP 98
-#define TK_RP 99
-#define TK_TAGS 100
-#define TK_USING 101
-#define TK_AS 102
-#define TK_COMMA 103
-#define TK_NULL 104
-#define TK_SELECT 105
-#define TK_UNION 106
-#define TK_ALL 107
-#define TK_FROM 108
-#define TK_VARIABLE 109
-#define TK_INTERVAL 110
-#define TK_FILL 111
-#define TK_SLIDING 112
-#define TK_ORDER 113
-#define TK_BY 114
-#define TK_ASC 115
-#define TK_DESC 116
-#define TK_GROUP 117
-#define TK_HAVING 118
-#define TK_LIMIT 119
-#define TK_OFFSET 120
-#define TK_SLIMIT 121
-#define TK_SOFFSET 122
-#define TK_WHERE 123
-#define TK_NOW 124
-#define TK_RESET 125
-#define TK_QUERY 126
-#define TK_ADD 127
-#define TK_COLUMN 128
-#define TK_TAG 129
-#define TK_CHANGE 130
-#define TK_SET 131
-#define TK_KILL 132
-#define TK_CONNECTION 133
-#define TK_STREAM 134
-#define TK_COLON 135
-#define TK_ABORT 136
-#define TK_AFTER 137
-#define TK_ATTACH 138
-#define TK_BEFORE 139
-#define TK_BEGIN 140
-#define TK_CASCADE 141
-#define TK_CLUSTER 142
-#define TK_CONFLICT 143
-#define TK_COPY 144
-#define TK_DEFERRED 145
-#define TK_DELIMITERS 146
-#define TK_DETACH 147
-#define TK_EACH 148
-#define TK_END 149
-#define TK_EXPLAIN 150
-#define TK_FAIL 151
-#define TK_FOR 152
-#define TK_IGNORE 153
-#define TK_IMMEDIATE 154
-#define TK_INITIALLY 155
-#define TK_INSTEAD 156
-#define TK_MATCH 157
-#define TK_KEY 158
-#define TK_OF 159
-#define TK_RAISE 160
-#define TK_REPLACE 161
-#define TK_RESTRICT 162
-#define TK_ROW 163
-#define TK_STATEMENT 164
-#define TK_TRIGGER 165
-#define TK_VIEW 166
-#define TK_COUNT 167
-#define TK_SUM 168
-#define TK_AVG 169
-#define TK_MIN 170
-#define TK_MAX 171
-#define TK_FIRST 172
-#define TK_LAST 173
-#define TK_TOP 174
-#define TK_BOTTOM 175
-#define TK_STDDEV 176
-#define TK_PERCENTILE 177
-#define TK_APERCENTILE 178
-#define TK_LEASTSQUARES 179
-#define TK_HISTOGRAM 180
-#define TK_DIFF 181
-#define TK_SPREAD 182
-#define TK_TWA 183
-#define TK_INTERP 184
-#define TK_LAST_ROW 185
-#define TK_RATE 186
-#define TK_IRATE 187
-#define TK_SUM_RATE 188
-#define TK_SUM_IRATE 189
-#define TK_AVG_RATE 190
-#define TK_AVG_IRATE 191
-#define TK_TBID 192
-#define TK_SEMI 193
-#define TK_NONE 194
-#define TK_PREV 195
-#define TK_LINEAR 196
-#define TK_IMPORT 197
-#define TK_METRIC 198
-#define TK_TBNAME 199
-#define TK_JOIN 200
-#define TK_METRICS 201
-#define TK_STABLE 202
-#define TK_INSERT 203
-#define TK_INTO 204
-#define TK_VALUES 205
+#define TK_UPDATE 98
+#define TK_LP 99
+#define TK_RP 100
+#define TK_TAGS 101
+#define TK_USING 102
+#define TK_AS 103
+#define TK_COMMA 104
+#define TK_NULL 105
+#define TK_SELECT 106
+#define TK_UNION 107
+#define TK_ALL 108
+#define TK_FROM 109
+#define TK_VARIABLE 110
+#define TK_INTERVAL 111
+#define TK_FILL 112
+#define TK_SLIDING 113
+#define TK_ORDER 114
+#define TK_BY 115
+#define TK_ASC 116
+#define TK_DESC 117
+#define TK_GROUP 118
+#define TK_HAVING 119
+#define TK_LIMIT 120
+#define TK_OFFSET 121
+#define TK_SLIMIT 122
+#define TK_SOFFSET 123
+#define TK_WHERE 124
+#define TK_NOW 125
+#define TK_RESET 126
+#define TK_QUERY 127
+#define TK_ADD 128
+#define TK_COLUMN 129
+#define TK_TAG 130
+#define TK_CHANGE 131
+#define TK_SET 132
+#define TK_KILL 133
+#define TK_CONNECTION 134
+#define TK_STREAM 135
+#define TK_COLON 136
+#define TK_ABORT 137
+#define TK_AFTER 138
+#define TK_ATTACH 139
+#define TK_BEFORE 140
+#define TK_BEGIN 141
+#define TK_CASCADE 142
+#define TK_CLUSTER 143
+#define TK_CONFLICT 144
+#define TK_COPY 145
+#define TK_DEFERRED 146
+#define TK_DELIMITERS 147
+#define TK_DETACH 148
+#define TK_EACH 149
+#define TK_END 150
+#define TK_EXPLAIN 151
+#define TK_FAIL 152
+#define TK_FOR 153
+#define TK_IGNORE 154
+#define TK_IMMEDIATE 155
+#define TK_INITIALLY 156
+#define TK_INSTEAD 157
+#define TK_MATCH 158
+#define TK_KEY 159
+#define TK_OF 160
+#define TK_RAISE 161
+#define TK_REPLACE 162
+#define TK_RESTRICT 163
+#define TK_ROW 164
+#define TK_STATEMENT 165
+#define TK_TRIGGER 166
+#define TK_VIEW 167
+#define TK_COUNT 168
+#define TK_SUM 169
+#define TK_AVG 170
+#define TK_MIN 171
+#define TK_MAX 172
+#define TK_FIRST 173
+#define TK_LAST 174
+#define TK_TOP 175
+#define TK_BOTTOM 176
+#define TK_STDDEV 177
+#define TK_PERCENTILE 178
+#define TK_APERCENTILE 179
+#define TK_LEASTSQUARES 180
+#define TK_HISTOGRAM 181
+#define TK_DIFF 182
+#define TK_SPREAD 183
+#define TK_TWA 184
+#define TK_INTERP 185
+#define TK_LAST_ROW 186
+#define TK_RATE 187
+#define TK_IRATE 188
+#define TK_SUM_RATE 189
+#define TK_SUM_IRATE 190
+#define TK_AVG_RATE 191
+#define TK_AVG_IRATE 192
+#define TK_TBID 193
+#define TK_SEMI 194
+#define TK_NONE 195
+#define TK_PREV 196
+#define TK_LINEAR 197
+#define TK_IMPORT 198
+#define TK_METRIC 199
+#define TK_TBNAME 200
+#define TK_JOIN 201
+#define TK_METRICS 202
+#define TK_STABLE 203
+#define TK_INSERT 204
+#define TK_INTO 205
+#define TK_VALUES 206
+
#define TK_SPACE 300
#define TK_COMMENT 301
diff --git a/src/inc/ttype.h b/src/inc/ttype.h
new file mode 100644
index 0000000000000000000000000000000000000000..7d5779c43f7c05ed7675cb59ef3036c14f852938
--- /dev/null
+++ b/src/inc/ttype.h
@@ -0,0 +1,36 @@
+#ifndef TDENGINE_TTYPE_H
+#define TDENGINE_TTYPE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taosdef.h"
+
+#define GET_TYPED_DATA(_v, _finalType, _type, _data) \
+ switch (_type) { \
+ case TSDB_DATA_TYPE_TINYINT: \
+ (_v) = (_finalType)GET_INT8_VAL(_data); \
+ break; \
+ case TSDB_DATA_TYPE_SMALLINT: \
+ (_v) = (_finalType)GET_INT16_VAL(_data); \
+ break; \
+ case TSDB_DATA_TYPE_BIGINT: \
+ (_v) = (_finalType)(GET_INT64_VAL(_data)); \
+ break; \
+ case TSDB_DATA_TYPE_FLOAT: \
+ (_v) = (_finalType)GET_FLOAT_VAL(_data); \
+ break; \
+ case TSDB_DATA_TYPE_DOUBLE: \
+ (_v) = (_finalType)GET_DOUBLE_VAL(_data); \
+ break; \
+ default: \
+ (_v) = (_finalType)GET_INT32_VAL(_data); \
+ break; \
+ };
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TDENGINE_TTYPE_H
diff --git a/src/inc/twal.h b/src/inc/twal.h
index 92204abd7d34a9ee2eebf1b74c2e8d58b9599f17..8dd3a8a91209e840abeb9560f94a52ce362492a9 100644
--- a/src/inc/twal.h
+++ b/src/inc/twal.h
@@ -19,42 +19,53 @@
extern "C" {
#endif
-#define TAOS_WAL_NOLOG 0
-#define TAOS_WAL_WRITE 1
-#define TAOS_WAL_FSYNC 2
-
+typedef enum {
+ TAOS_WAL_NOLOG = 0,
+ TAOS_WAL_WRITE = 1,
+ TAOS_WAL_FSYNC = 2
+} EWalType;
+
+typedef enum {
+ TAOS_WAL_NOT_KEEP = 0,
+ TAOS_WAL_KEEP = 1
+} EWalKeep;
+
typedef struct {
- int8_t msgType;
- int8_t reserved[3];
- int32_t len;
- uint64_t version;
- uint32_t signature;
- uint32_t cksum;
- char cont[];
+ int8_t msgType;
+ int8_t sver;
+ int8_t reserved[2];
+ int32_t len;
+ uint64_t version;
+ uint32_t signature;
+ uint32_t cksum;
+ char cont[];
} SWalHead;
typedef struct {
- int8_t walLevel; // wal level
- int32_t fsyncPeriod; // millisecond
- int8_t wals; // number of WAL files;
- int8_t keep; // keep the wal file when closed
+ int32_t vgId;
+ int32_t fsyncPeriod; // millisecond
+ EWalType walLevel; // wal level
+ EWalKeep keep; // keep the wal file when closed
} SWalCfg;
-typedef void* twalh; // WAL HANDLE
-typedef int (*FWalWrite)(void *ahandle, void *pHead, int type);
-
-twalh walOpen(const char *path, const SWalCfg *pCfg);
-int walAlter(twalh pWal, const SWalCfg *pCfg);
-void walClose(twalh);
-int walRenew(twalh);
-int walWrite(twalh, SWalHead *);
-void walFsync(twalh);
-int walRestore(twalh, void *pVnode, FWalWrite writeFp);
-int walGetWalFile(twalh, char *name, uint32_t *index);
-int64_t walGetVersion(twalh);
+typedef void * twalh; // WAL HANDLE
+typedef int32_t FWalWrite(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
-extern int wDebugFlag;
+int32_t walInit();
+void walCleanUp();
+twalh walOpen(char *path, SWalCfg *pCfg);
+int32_t walAlter(twalh pWal, SWalCfg *pCfg);
+void walStop(twalh);
+void walClose(twalh);
+int32_t walRenew(twalh);
+void walRemoveOneOldFile(twalh);
+void walRemoveAllOldFiles(twalh);
+int32_t walWrite(twalh, SWalHead *);
+void walFsync(twalh, bool forceFsync);
+int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp);
+int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId);
+uint64_t walGetVersion(twalh);
#ifdef __cplusplus
}
diff --git a/src/inc/vnode.h b/src/inc/vnode.h
index fdce4d62794075bd2e7027b125780fbd7a2deaed..563d035898758a69fb57d6e1e5d8dc7e99886af4 100644
--- a/src/inc/vnode.h
+++ b/src/inc/vnode.h
@@ -20,6 +20,8 @@
extern "C" {
#endif
+#include "twal.h"
+
typedef enum _VN_STATUS {
TAOS_VN_STATUS_INIT,
TAOS_VN_STATUS_READY,
@@ -29,44 +31,58 @@ typedef enum _VN_STATUS {
} EVnStatus;
typedef struct {
- int len;
- void *rsp;
- void *qhandle; //used by query and retrieve msg
+ int32_t len;
+ void * rsp;
+ void * qhandle; // used by query and retrieve msg
} SRspRet;
typedef struct {
+ int32_t code;
+ int32_t contLen;
+ void * rpcHandle;
+ void * rpcAhandle;
+ void * qhandle;
+ int8_t qtype;
+ int8_t msgType;
+ SRspRet rspRet;
+ char pCont[];
+} SVReadMsg;
+
+typedef struct {
+ int32_t code;
+ int32_t processedCount;
+ void * rpcHandle;
+ void * rpcAhandle;
SRspRet rspRet;
- void *pCont;
- int32_t contLen;
- SRpcMsg rpcMsg;
-} SReadMsg;
+ char reserveForSync[16];
+ SWalHead pHead[];
+} SVWriteMsg;
extern char *vnodeStatus[];
-int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg);
+int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg);
int32_t vnodeDrop(int32_t vgId);
int32_t vnodeOpen(int32_t vgId, char *rootDir);
-int32_t vnodeAlter(void *pVnode, SMDCreateVnodeMsg *pVnodeCfg);
+int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg);
int32_t vnodeClose(int32_t vgId);
void* vnodeAcquire(int32_t vgId); // add refcount
-void* vnodeAcquireRqueue(int32_t vgId); // add refCount, get read queue
-void* vnodeAcquireWqueue(int32_t vgId); // add recCount, get write queue
void vnodeRelease(void *pVnode); // dec refCount
void* vnodeGetWal(void *pVnode);
-int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item);
-int32_t vnodeCheckWrite(void *pVnode);
+int32_t vnodeWriteToWQueue(void *pVnode, void *pHead, int32_t qtype, void *pRpcMsg);
+void vnodeFreeFromWQueue(void *pVnode, SVWriteMsg *pWrite);
+int32_t vnodeProcessWrite(void *pVnode, void *pHead, int32_t qtype, void *pRspRet);
int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes);
-void vnodeBuildStatusMsg(void *param);
-void vnodeConfirmForward(void *param, uint64_t version, int32_t code);
-void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes);
+void vnodeBuildStatusMsg(void *pStatus);
+void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code);
+void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes);
int32_t vnodeInitResources();
void vnodeCleanupResources();
-int32_t vnodeProcessRead(void *pVnode, SReadMsg *pReadMsg);
-int32_t vnodeCheckRead(void *pVnode);
+int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam);
+int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead);
#ifdef __cplusplus
}
diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt
index 77db79e22003d04701bf7417cc9ebc06b202533e..66e8cf73988ab25db7544b9a52215d2279630c63 100644
--- a/src/kit/CMakeLists.txt
+++ b/src/kit/CMakeLists.txt
@@ -3,3 +3,4 @@ PROJECT(TDengine)
ADD_SUBDIRECTORY(shell)
ADD_SUBDIRECTORY(taosdemo)
+ADD_SUBDIRECTORY(taosdump)
diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h
index f508d186083c84482080f2f8fe251173733f1366..d65c943e28d7d5a63aba4fc1839a4ba9cf744746 100644
--- a/src/kit/shell/inc/shell.h
+++ b/src/kit/shell/inc/shell.h
@@ -60,7 +60,7 @@ typedef struct SShellArguments {
extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments);
extern TAOS* shellInit(SShellArguments* args);
extern void* shellLoopQuery(void* arg);
-extern void taos_error(TAOS_RES* tres);
+extern void taos_error(TAOS_RES* tres, int64_t st);
extern int regex_match(const char* s, const char* reg, int cflags);
void shellReadCommand(TAOS* con, char command[]);
int32_t shellRunCommand(TAOS* con, char* command);
diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c
index ffe537dd91834247460f04c47efff5ebe8687d7e..995b56f341b36b84e413aec154ae1abcc832741d 100644
--- a/src/kit/shell/src/shellDarwin.c
+++ b/src/kit/shell/src/shellDarwin.c
@@ -229,8 +229,8 @@ void shellReadCommand(TAOS *con, char *command) {
printf("\n");
if (isReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
- taosTFree(cmd.buffer);
- taosTFree(cmd.command);
+ tfree(cmd.buffer);
+ tfree(cmd.command);
return;
} else {
updateBuffer(&cmd);
diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c
index 748b7e792982352de611227d82a844448d251ee2..22f01ac142c72603b9fa0595ce8c1fc89c626d7e 100644
--- a/src/kit/shell/src/shellEngine.c
+++ b/src/kit/shell/src/shellEngine.c
@@ -193,7 +193,7 @@ int32_t shellRunCommand(TAOS* con, char* command) {
history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE] == NULL ||
strcmp(command, history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE]) != 0) {
if (history.hist[history.hend] != NULL) {
- taosTFree(history.hist[history.hend]);
+ tfree(history.hist[history.hend]);
}
history.hist[history.hend] = strdup(command);
@@ -244,7 +244,7 @@ int32_t shellRunCommand(TAOS* con, char* command) {
}
*p++ = c;
- if (c == ';') {
+ if (c == ';' && quote == 0) {
c = *p;
*p = 0;
if (shellRunSingleCommand(con, cmd) < 0) {
@@ -296,7 +296,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
TAOS_RES* pSql = taos_query_h(con, command, &result);
if (taos_errno(pSql)) {
- taos_error(pSql);
+ taos_error(pSql, st);
return;
}
@@ -770,7 +770,7 @@ void read_history() {
return;
}
- while ((read_size = taosGetline(&line, &line_size, f)) != -1) {
+ while ((read_size = tgetline(&line, &line_size, f)) != -1) {
line[read_size - 1] = '\0';
history.hist[history.hend] = strdup(line);
@@ -800,16 +800,17 @@ void write_history() {
for (int i = history.hstart; i != history.hend;) {
if (history.hist[i] != NULL) {
fprintf(f, "%s\n", history.hist[i]);
- taosTFree(history.hist[i]);
+ tfree(history.hist[i]);
}
i = (i + 1) % MAX_HISTORY_SIZE;
}
fclose(f);
}
-void taos_error(TAOS_RES *tres) {
+void taos_error(TAOS_RES *tres, int64_t st) {
+ int64_t et = taosGetTimestampUs();
atomic_store_ptr(&result, 0);
- fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres));
+ fprintf(stderr, "\nDB error: %s (%.6fs)\n", taos_errstr(tres), (et - st) / 1E6);
taos_free_result(tres);
}
@@ -853,7 +854,7 @@ void source_file(TAOS *con, char *fptr) {
return;
}
- while ((read_len = taosGetline(&line, &line_len, f)) != -1) {
+ while ((read_len = tgetline(&line, &line_len, f)) != -1) {
if (read_len >= tsMaxSQLStringLen) continue;
line[--read_len] = '\0';
diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c
index 6f5ea33d79f09219f49ee4094de2c377648b2290..04f5824d8da547557a4dc7fce599d721c1885611 100644
--- a/src/kit/shell/src/shellLinux.c
+++ b/src/kit/shell/src/shellLinux.c
@@ -46,7 +46,7 @@ static struct argp_option options[] = {
{"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."},
{"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."},
{"timezone", 't', "TIMEZONE", 0, "Time zone of the shell, default is local."},
- {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is NULL, valid option: client | server."},
+ {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is NULL, options: client|clients|server."},
{"endport", 'e', "ENDPORT", 0, "Net test end port, default is 6042."},
{"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."},
{0}};
@@ -232,8 +232,8 @@ void shellReadCommand(TAOS *con, char *command) {
printf("\n");
if (isReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
- taosTFree(cmd.buffer);
- taosTFree(cmd.command);
+ tfree(cmd.buffer);
+ tfree(cmd.command);
return;
} else {
updateBuffer(&cmd);
@@ -351,7 +351,7 @@ void *shellLoopQuery(void *arg) {
reset_terminal_mode();
} while (shellRunCommand(con, command) == 0);
- taosTFree(command);
+ tfree(command);
exitShell();
pthread_cleanup_pop(1);
diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c
index 6cb7c669cc7a08434b2558588067d007b51b3595..2083ad3e9b7d5a101f35af4fe20dcd38957d5be2 100644
--- a/src/kit/shell/src/shellMain.c
+++ b/src/kit/shell/src/shellMain.c
@@ -80,7 +80,10 @@ int main(int argc, char* argv[]) {
shellParseArgument(argc, argv, &args);
if (args.netTestRole && args.netTestRole[0] != 0) {
- taosNetTest(args.host, (uint16_t)args.port, (uint16_t)args.endPort, args.pktLen, args.netTestRole);
+ taos_init();
+ CmdArguments cmdArgs;
+ memcpy(&cmdArgs, &args, sizeof(SShellArguments));
+ taosNetTest(&cmdArgs);
exit(0);
}
diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c
index 699e96428e201e998c1a4bb9cb8cd3c835e61364..53e7d2398450fe22a11da1e7254e0c2ab5f02ea4 100644
--- a/src/kit/taosdemo/taosdemo.c
+++ b/src/kit/taosdemo/taosdemo.c
@@ -475,6 +475,7 @@ typedef struct {
tsem_t mutex_sem;
int notFinished;
tsem_t lock_sem;
+ int counter;
} info;
typedef struct {
@@ -766,6 +767,7 @@ int main(int argc, char *argv[]) {
t_info->data_of_rate = rate;
t_info->end_table_id = i < b ? last + a : last + a - 1;
last = t_info->end_table_id + 1;
+ t_info->counter = 0;
tsem_init(&(t_info->mutex_sem), 0, 1);
t_info->notFinished = t_info->end_table_id - t_info->start_table_id + 1;
@@ -788,14 +790,14 @@ int main(int argc, char *argv[]) {
printf("ASYNC Insert with %d connections:\n", threads);
}
- fprintf(fp, "|%10.d | %10.2f | %10.2f | %10.4f |\n\n",
- ntables * nrecords_per_table, ntables * nrecords_per_table / t,
- (ntables * nrecords_per_table) / (t * nrecords_per_request),
+ fprintf(fp, "|%"PRIu64" | %10.2f | %10.2f | %10.4f |\n\n",
+ (int64_t)ntables * nrecords_per_table, ntables * nrecords_per_table / t,
+ ((int64_t)ntables * nrecords_per_table) / (t * nrecords_per_request),
t * 1000);
- printf("Spent %.4f seconds to insert %d records with %d record(s) per request: %.2f records/second\n",
- t, ntables * nrecords_per_table, nrecords_per_request,
- ntables * nrecords_per_table / t);
+ printf("Spent %.4f seconds to insert %"PRIu64" records with %d record(s) per request: %.2f records/second\n",
+ t, (int64_t)ntables * nrecords_per_table, nrecords_per_request,
+ (int64_t)ntables * nrecords_per_table / t);
for (int i = 0; i < threads; i++) {
info *t_info = infos + i;
@@ -879,6 +881,7 @@ int main(int argc, char *argv[]) {
taos_close(rInfo->taos);
}
+ taos_cleanup();
return 0;
}
@@ -955,7 +958,7 @@ void querySqlFile(TAOS* taos, char* sqlFile)
double t = getCurrentTime();
- while ((read_len = taosGetline(&line, &line_len, fp)) != -1) {
+ while ((read_len = tgetline(&line, &line_len, fp)) != -1) {
if (read_len >= MAX_SQL_SIZE) continue;
line[--read_len] = '\0';
@@ -1283,68 +1286,39 @@ void *syncWrite(void *sarg) {
void *asyncWrite(void *sarg) {
info *winfo = (info *)sarg;
-
- sTable *tb_infos = (sTable *)malloc(sizeof(sTable) * (winfo->end_table_id - winfo->start_table_id + 1));
-
- for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) {
- sTable *tb_info = tb_infos + tID - winfo->start_table_id;
- tb_info->data_type = winfo->datatype;
- tb_info->ncols_per_record = winfo->ncols_per_record;
- tb_info->taos = winfo->taos;
- sprintf(tb_info->tb_name, "%s.%s%d", winfo->db_name, winfo->tb_prefix, tID);
- tb_info->timestamp = winfo->start_time;
- tb_info->counter = 0;
- tb_info->target = winfo->nrecords_per_table;
- tb_info->len_of_binary = winfo->len_of_binary;
- tb_info->nrecords_per_request = winfo->nrecords_per_request;
- tb_info->mutex_sem = &(winfo->mutex_sem);
- tb_info->notFinished = &(winfo->notFinished);
- tb_info->lock_sem = &(winfo->lock_sem);
- tb_info->data_of_order = winfo->data_of_order;
- tb_info->data_of_rate = winfo->data_of_rate;
-
- /* char buff[BUFFER_SIZE] = "\0"; */
- /* sprintf(buff, "insert into %s values (0, 0)", tb_info->tb_name); */
- /* queryDB(tb_info->taos,buff); */
-
- taos_query_a(winfo->taos, "show databases", callBack, tb_info);
- }
+ taos_query_a(winfo->taos, "show databases", callBack, winfo);
tsem_wait(&(winfo->lock_sem));
- free(tb_infos);
return NULL;
}
void callBack(void *param, TAOS_RES *res, int code) {
- sTable *tb_info = (sTable *)param;
- char **datatype = tb_info->data_type;
- int ncols_per_record = tb_info->ncols_per_record;
- int len_of_binary = tb_info->len_of_binary;
- int64_t tmp_time = tb_info->timestamp;
-
- if (code < 0) {
- fprintf(stderr, "failed to insert data %d:reason; %s\n", code, taos_errstr(res));
- exit(EXIT_FAILURE);
- }
+ info* winfo = (info*)param;
+ char **datatype = winfo->datatype;
+ int ncols_per_record = winfo->ncols_per_record;
+ int len_of_binary = winfo->len_of_binary;
- // If finished;
- if (tb_info->counter >= tb_info->target) {
- tsem_wait(tb_info->mutex_sem);
- (*(tb_info->notFinished))--;
- if (*(tb_info->notFinished) == 0) tsem_post(tb_info->lock_sem);
- tsem_post(tb_info->mutex_sem);
+ int64_t tmp_time = winfo->start_time;
+ char *buffer = calloc(1, BUFFER_SIZE);
+ char *data = calloc(1, MAX_DATA_SIZE);
+ char *pstr = buffer;
+ pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, winfo->start_table_id);
+ if (winfo->counter >= winfo->nrecords_per_table) {
+ winfo->start_table_id++;
+ winfo->counter = 0;
+ }
+ if (winfo->start_table_id > winfo->end_table_id) {
+ tsem_post(&winfo->lock_sem);
+ free(buffer);
+ free(data);
+ taos_free_result(res);
return;
}
-
- char buffer[BUFFER_SIZE] = "\0";
- char data[MAX_DATA_SIZE];
- char *pstr = buffer;
- pstr += sprintf(pstr, "insert into %s values", tb_info->tb_name);
-
- for (int i = 0; i < tb_info->nrecords_per_request; i++) {
+
+ for (int i = 0; i < winfo->nrecords_per_request; i++) {
int rand_num = rand() % 100;
- if (tb_info->data_of_order ==1 && rand_num < tb_info->data_of_rate)
+ if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate)
{
int64_t d = tmp_time - rand() % 1000000 + rand_num;
generateData(data, datatype, ncols_per_record, d, len_of_binary);
@@ -1353,15 +1327,15 @@ void callBack(void *param, TAOS_RES *res, int code) {
generateData(data, datatype, ncols_per_record, tmp_time += 1000, len_of_binary);
}
pstr += sprintf(pstr, "%s", data);
- tb_info->counter++;
+ winfo->counter++;
- if (tb_info->counter >= tb_info->target) {
+ if (winfo->counter >= winfo->nrecords_per_table) {
break;
}
}
- tb_info->timestamp = tmp_time;
-
- taos_query_a(tb_info->taos, buffer, callBack, tb_info);
+ taos_query_a(winfo->taos, buffer, callBack, winfo);
+ free(buffer);
+ free(data);
taos_free_result(res);
}
diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..dcdbd486154b2673459e25754dc75457851a4df4
--- /dev/null
+++ b/src/kit/taosdump/CMakeLists.txt
@@ -0,0 +1,16 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
+INCLUDE_DIRECTORIES(inc)
+AUX_SOURCE_DIRECTORY(. SRC)
+
+IF (TD_LINUX)
+ ADD_EXECUTABLE(taosdump ${SRC})
+ IF (TD_SOMODE_STATIC)
+ TARGET_LINK_LIBRARIES(taosdump taos_static)
+ ELSE ()
+ TARGET_LINK_LIBRARIES(taosdump taos)
+ ENDIF ()
+ENDIF ()
diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c
new file mode 100644
index 0000000000000000000000000000000000000000..88f07ee60238609f4ae0d100bcae368f1980d8d2
--- /dev/null
+++ b/src/kit/taosdump/taosdump.c
@@ -0,0 +1,2282 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include
+#include "os.h"
+#include "taos.h"
+#include "taosdef.h"
+#include "taosmsg.h"
+#include "tglobal.h"
+#include "tsclient.h"
+#include "tsdb.h"
+#include "tutil.h"
+
+#define COMMAND_SIZE 65536
+//#define DEFAULT_DUMP_FILE "taosdump.sql"
+
+int converStringToReadable(char *str, int size, char *buf, int bufsize);
+int convertNCharToReadable(char *str, int size, char *buf, int bufsize);
+void taosDumpCharset(FILE *fp);
+void taosLoadFileCharset(FILE *fp, char *fcharset);
+
+typedef struct {
+ short bytes;
+ int8_t type;
+} SOColInfo;
+
+// -------------------------- SHOW DATABASE INTERFACE-----------------------
+enum _show_db_index {
+ TSDB_SHOW_DB_NAME_INDEX,
+ TSDB_SHOW_DB_CREATED_TIME_INDEX,
+ TSDB_SHOW_DB_NTABLES_INDEX,
+ TSDB_SHOW_DB_VGROUPS_INDEX,
+ TSDB_SHOW_DB_REPLICA_INDEX,
+ TSDB_SHOW_DB_QUORUM_INDEX,
+ TSDB_SHOW_DB_DAYS_INDEX,
+ TSDB_SHOW_DB_KEEP_INDEX,
+ TSDB_SHOW_DB_CACHE_INDEX,
+ TSDB_SHOW_DB_BLOCKS_INDEX,
+ TSDB_SHOW_DB_MINROWS_INDEX,
+ TSDB_SHOW_DB_MAXROWS_INDEX,
+ TSDB_SHOW_DB_WALLEVEL_INDEX,
+ TSDB_SHOW_DB_FSYNC_INDEX,
+ TSDB_SHOW_DB_COMP_INDEX,
+ TSDB_SHOW_DB_PRECISION_INDEX,
+ TSDB_SHOW_DB_UPDATE_INDEX,
+ TSDB_SHOW_DB_STATUS_INDEX,
+ TSDB_MAX_SHOW_DB
+};
+
+// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
+enum _show_tables_index {
+ TSDB_SHOW_TABLES_NAME_INDEX,
+ TSDB_SHOW_TABLES_CREATED_TIME_INDEX,
+ TSDB_SHOW_TABLES_COLUMNS_INDEX,
+ TSDB_SHOW_TABLES_METRIC_INDEX,
+ TSDB_MAX_SHOW_TABLES
+};
+
+// ---------------------------------- DESCRIBE METRIC CONFIGURE ------------------------------
+enum _describe_table_index {
+ TSDB_DESCRIBE_METRIC_FIELD_INDEX,
+ TSDB_DESCRIBE_METRIC_TYPE_INDEX,
+ TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
+ TSDB_DESCRIBE_METRIC_NOTE_INDEX,
+ TSDB_MAX_DESCRIBE_METRIC
+};
+
+typedef struct {
+ char field[TSDB_COL_NAME_LEN + 1];
+ char type[16];
+ int length;
+ char note[128];
+} SColDes;
+
+typedef struct {
+ char name[TSDB_COL_NAME_LEN + 1];
+ SColDes cols[];
+} STableDef;
+
+extern char version[];
+
+typedef struct {
+ char name[TSDB_DB_NAME_LEN + 1];
+ int32_t tables;
+ int32_t vgroups;
+ int16_t replications;
+ int16_t quorum;
+ int16_t daysPerFile;
+ int16_t daysToKeep;
+ int16_t daysToKeep1;
+ int16_t daysToKeep2;
+ int32_t cacheBlockSize; //MB
+ int32_t totalBlocks;
+ int32_t minRowsPerFileBlock;
+ int32_t maxRowsPerFileBlock;
+ int8_t walLevel;
+ int32_t fsyncPeriod;
+ int8_t compression;
+ int8_t precision; // time resolution
+ int8_t update;
+} SDbInfo;
+
+typedef struct {
+ char name[TSDB_TABLE_NAME_LEN + 1];
+ char metric[TSDB_TABLE_NAME_LEN + 1];
+} STableRecord;
+
+typedef struct {
+ bool isMetric;
+ STableRecord tableRecord;
+} STableRecordInfo;
+
+typedef struct {
+ pthread_t threadID;
+ int32_t threadIndex;
+ int32_t totalThreads;
+ char dbName[TSDB_TABLE_NAME_LEN + 1];
+ void *taosCon;
+} SThreadParaObj;
+
+static int64_t totalDumpOutRows = 0;
+
+SDbInfo **dbInfos = NULL;
+
+const char *argp_program_version = version;
+const char *argp_program_bug_address = "";
+
+/* Program documentation. */
+static char doc[] = "";
+/* "Argp example #4 -- a program with somewhat more complicated\ */
+/* options\ */
+/* \vThis part of the documentation comes *after* the options;\ */
+/* note that the text is automatically filled, but it's possible\ */
+/* to force a line-break, e.g.\n<-- here."; */
+
+/* A description of the arguments we accept. */
+static char args_doc[] = "dbname [tbname ...]\n--databases dbname ...\n--all-databases\n-i inpath\n-o outpath";
+
+/* Keys for options without short-options. */
+#define OPT_ABORT 1 /* –abort */
+
+/* The options we understand. */
+static struct argp_option options[] = {
+ // connection option
+ {"host", 'h', "HOST", 0, "Server host dumping data from. Default is localhost.", 0},
+ {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0},
+ #ifdef _TD_POWER_
+ {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is powerdb.", 0},
+ #else
+ {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0},
+ #endif
+ {"port", 'P', "PORT", 0, "Port to connect", 0},
+ {"cversion", 'v', "CVERION", 0, "client version", 0},
+ {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0},
+ // input/output file
+ {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1},
+ {"inpath", 'i', "INPATH", 0, "Input file path.", 1},
+ #ifdef _TD_POWER_
+ {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1},
+ #else
+ {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1},
+ #endif
+ {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1},
+ // dump unit options
+ {"all-databases", 'A', 0, 0, "Dump all databases.", 2},
+ {"databases", 'B', 0, 0, "Dump assigned databases", 2},
+ // dump format options
+ {"schemaonly", 's', 0, 0, "Only dump schema.", 3},
+ {"with-property", 'M', 0, 0, "Dump schema with properties.", 3},
+ {"start-time", 'S', "START_TIME", 0, "Start time to dump.", 3},
+ {"end-time", 'E', "END_TIME", 0, "End time to dump.", 3},
+ {"data-batch", 'N', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3},
+ {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3},
+ {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3},
+ {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3},
+ {"allow-sys", 'a', 0, 0, "Allow to dump sys database", 3},
+ {0}};
+
+/* Used by main to communicate with parse_opt. */
+struct arguments {
+ // connection option
+ char *host;
+ char *user;
+ char *password;
+ uint16_t port;
+ char cversion[12];
+ uint16_t mysqlFlag;
+ // output file
+ char outpath[TSDB_FILENAME_LEN+1];
+ char inpath[TSDB_FILENAME_LEN+1];
+ char *encode;
+ // dump unit option
+ bool all_databases;
+ bool databases;
+ // dump format option
+ bool schemaonly;
+ bool with_property;
+ int64_t start_time;
+ int64_t end_time;
+ int32_t data_batch;
+ int32_t max_sql_len;
+ int32_t table_batch; // num of table which will be dump into one output file.
+ bool allow_sys;
+ // other options
+ int32_t thread_num;
+ int abort;
+ char **arg_list;
+ int arg_list_len;
+ bool isDumpIn;
+};
+
+/* Parse a single option. */
+static error_t parse_opt(int key, char *arg, struct argp_state *state) {
+ /* Get the input argument from argp_parse, which we
+ know is a pointer to our arguments structure. */
+ struct arguments *arguments = state->input;
+ wordexp_t full_path;
+
+ switch (key) {
+ // connection option
+ case 'a':
+ arguments->allow_sys = true;
+ break;
+ case 'h':
+ arguments->host = arg;
+ break;
+ case 'u':
+ arguments->user = arg;
+ break;
+ case 'p':
+ arguments->password = arg;
+ break;
+ case 'P':
+ arguments->port = atoi(arg);
+ break;
+ case 'q':
+ arguments->mysqlFlag = atoi(arg);
+ break;
+ case 'v':
+ if (wordexp(arg, &full_path, 0) != 0) {
+ fprintf(stderr, "Invalid client vesion %s\n", arg);
+ return -1;
+ }
+ tstrncpy(arguments->cversion, full_path.we_wordv[0], 11);
+ wordfree(&full_path);
+ break;
+ // output file path
+ case 'o':
+ if (wordexp(arg, &full_path, 0) != 0) {
+ fprintf(stderr, "Invalid path %s\n", arg);
+ return -1;
+ }
+ tstrncpy(arguments->outpath, full_path.we_wordv[0], TSDB_FILENAME_LEN);
+ wordfree(&full_path);
+ break;
+ case 'i':
+ arguments->isDumpIn = true;
+ if (wordexp(arg, &full_path, 0) != 0) {
+ fprintf(stderr, "Invalid path %s\n", arg);
+ return -1;
+ }
+ tstrncpy(arguments->inpath, full_path.we_wordv[0], TSDB_FILENAME_LEN);
+ wordfree(&full_path);
+ break;
+ case 'c':
+ if (wordexp(arg, &full_path, 0) != 0) {
+ fprintf(stderr, "Invalid path %s\n", arg);
+ return -1;
+ }
+ tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
+ wordfree(&full_path);
+ break;
+ case 'e':
+ arguments->encode = arg;
+ break;
+ // dump unit option
+ case 'A':
+ arguments->all_databases = true;
+ break;
+ case 'B':
+ arguments->databases = true;
+ break;
+ // dump format option
+ case 's':
+ arguments->schemaonly = true;
+ break;
+ case 'M':
+ arguments->with_property = true;
+ break;
+ case 'S':
+ // parse time here.
+ arguments->start_time = atol(arg);
+ break;
+ case 'E':
+ arguments->end_time = atol(arg);
+ break;
+ case 'N':
+ arguments->data_batch = atoi(arg);
+ break;
+ case 'L':
+ {
+ int32_t len = atoi(arg);
+ if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
+ len = TSDB_MAX_ALLOWED_SQL_LEN;
+ } else if (len < TSDB_MAX_SQL_LEN) {
+ len = TSDB_MAX_SQL_LEN;
+ }
+ arguments->max_sql_len = len;
+ break;
+ }
+ case 't':
+ arguments->table_batch = atoi(arg);
+ break;
+ case 'T':
+ arguments->thread_num = atoi(arg);
+ break;
+ case OPT_ABORT:
+ arguments->abort = 1;
+ break;
+ case ARGP_KEY_ARG:
+ arguments->arg_list = &state->argv[state->next - 1];
+ arguments->arg_list_len = state->argc - state->next + 1;
+ state->next = state->argc;
+ break;
+
+ default:
+ return ARGP_ERR_UNKNOWN;
+ }
+ return 0;
+}
+
+/* Our argp parser. */
+static struct argp argp = {options, parse_opt, args_doc, doc};
+
+int taosDumpOut(struct arguments *arguments);
+int taosDumpIn(struct arguments *arguments);
+void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp);
+int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon);
+int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon);
+void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp);
+void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp);
+int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon);
+int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon);
+int taosCheckParam(struct arguments *arguments);
+void taosFreeDbInfos();
+static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfThread, char *dbName);
+
+struct arguments tsArguments = {
+ // connection option
+ NULL,
+ "root",
+ #ifdef _TD_POWER_
+ "powerdb",
+ #else
+ "taosdata",
+ #endif
+ 0,
+ "",
+ 0,
+ // outpath and inpath
+ "",
+ "",
+ NULL,
+ // dump unit option
+ false,
+ false,
+ // dump format option
+ false,
+ false,
+ 0,
+ INT64_MAX,
+ 1,
+ TSDB_MAX_SQL_LEN,
+ 1,
+ false,
+ // other options
+ 5,
+ 0,
+ NULL,
+ 0,
+ false
+};
+
+int queryDB(TAOS *taos, char *command) {
+ TAOS_RES *pSql = NULL;
+ int32_t code = -1;
+
+ pSql = taos_query(taos, command);
+ code = taos_errno(pSql);
+ if (code) {
+ fprintf(stderr, "sql error: %s, reason:%s\n", command, taos_errstr(pSql));
+ }
+ taos_free_result(pSql);
+ return code;
+}
+
+int main(int argc, char *argv[]) {
+
+ /* Parse our arguments; every option seen by parse_opt will be
+ reflected in arguments. */
+ argp_parse(&argp, argc, argv, 0, 0, &tsArguments);
+
+ if (tsArguments.abort) {
+ #ifndef _ALPINE
+ error(10, 0, "ABORTED");
+ #else
+ abort();
+ #endif
+ }
+
+ printf("====== arguments config ======\n");
+ {
+ printf("host: %s\n", tsArguments.host);
+ printf("user: %s\n", tsArguments.user);
+ printf("password: %s\n", tsArguments.password);
+ printf("port: %u\n", tsArguments.port);
+ printf("cversion: %s\n", tsArguments.cversion);
+ printf("mysqlFlag: %d\n", tsArguments.mysqlFlag);
+ printf("outpath: %s\n", tsArguments.outpath);
+ printf("inpath: %s\n", tsArguments.inpath);
+ printf("encode: %s\n", tsArguments.encode);
+ printf("all_databases: %d\n", tsArguments.all_databases);
+ printf("databases: %d\n", tsArguments.databases);
+ printf("schemaonly: %d\n", tsArguments.schemaonly);
+ printf("with_property: %d\n", tsArguments.with_property);
+ printf("start_time: %" PRId64 "\n", tsArguments.start_time);
+ printf("end_time: %" PRId64 "\n", tsArguments.end_time);
+ printf("data_batch: %d\n", tsArguments.data_batch);
+ printf("max_sql_len: %d\n", tsArguments.max_sql_len);
+ printf("table_batch: %d\n", tsArguments.table_batch);
+ printf("thread_num: %d\n", tsArguments.thread_num);
+ printf("allow_sys: %d\n", tsArguments.allow_sys);
+ printf("abort: %d\n", tsArguments.abort);
+ printf("isDumpIn: %d\n", tsArguments.isDumpIn);
+ printf("arg_list_len: %d\n", tsArguments.arg_list_len);
+
+ for (int32_t i = 0; i < tsArguments.arg_list_len; i++) {
+ printf("arg_list[%d]: %s\n", i, tsArguments.arg_list[i]);
+ }
+ }
+ printf("==============================\n");
+
+ if (tsArguments.cversion[0] != 0){
+ tstrncpy(version, tsArguments.cversion, 11);
+ }
+
+ if (taosCheckParam(&tsArguments) < 0) {
+ exit(EXIT_FAILURE);
+ }
+
+ if (tsArguments.isDumpIn) {
+ if (taosDumpIn(&tsArguments) < 0) return -1;
+ } else {
+ if (taosDumpOut(&tsArguments) < 0) return -1;
+ }
+
+ return 0;
+}
+
+void taosFreeDbInfos() {
+ if (dbInfos == NULL) return;
+ for (int i = 0; i < 128; i++) tfree(dbInfos[i]);
+ tfree(dbInfos);
+}
+
+// check table is normal table or super table
+int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo, TAOS *taosCon) {
+ TAOS_ROW row = NULL;
+ bool isSet = false;
+ TAOS_RES *result = NULL;
+
+ memset(pTableRecordInfo, 0, sizeof(STableRecordInfo));
+
+ char* tempCommand = (char *)malloc(COMMAND_SIZE);
+ if (tempCommand == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ return -1;
+ }
+
+ sprintf(tempCommand, "show tables like %s", table);
+
+ result = taos_query(taosCon, tempCommand);
+ int32_t code = taos_errno(result);
+
+ if (code != 0) {
+ fprintf(stderr, "failed to run command %s\n", tempCommand);
+ free(tempCommand);
+ taos_free_result(result);
+ return -1;
+ }
+
+ TAOS_FIELD *fields = taos_fetch_fields(result);
+
+ while ((row = taos_fetch_row(result)) != NULL) {
+ isSet = true;
+ pTableRecordInfo->isMetric = false;
+ strncpy(pTableRecordInfo->tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX],
+ fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
+ strncpy(pTableRecordInfo->tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX],
+ fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes);
+ break;
+ }
+
+ taos_free_result(result);
+ result = NULL;
+
+ if (isSet) {
+ free(tempCommand);
+ return 0;
+ }
+
+ sprintf(tempCommand, "show stables like %s", table);
+
+ result = taos_query(taosCon, tempCommand);
+ code = taos_errno(result);
+
+ if (code != 0) {
+ fprintf(stderr, "failed to run command %s\n", tempCommand);
+ free(tempCommand);
+ taos_free_result(result);
+ return -1;
+ }
+
+ while ((row = taos_fetch_row(result)) != NULL) {
+ isSet = true;
+ pTableRecordInfo->isMetric = true;
+ tstrncpy(pTableRecordInfo->tableRecord.metric, table, TSDB_TABLE_NAME_LEN);
+ break;
+ }
+
+ taos_free_result(result);
+ result = NULL;
+
+ if (isSet) {
+ free(tempCommand);
+ return 0;
+ }
+ fprintf(stderr, "invalid table/metric %s\n", table);
+ free(tempCommand);
+ return -1;
+}
+
+
+int32_t taosSaveAllNormalTableToTempFile(TAOS *taosCon, char*meter, char* metric, int* fd) {
+ STableRecord tableRecord;
+
+ if (-1 == *fd) {
+ *fd = open(".tables.tmp.0", O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
+ if (*fd == -1) {
+ fprintf(stderr, "failed to open temp file: .tables.tmp.0\n");
+ return -1;
+ }
+ }
+
+ memset(&tableRecord, 0, sizeof(STableRecord));
+ tstrncpy(tableRecord.name, meter, TSDB_TABLE_NAME_LEN);
+ tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
+
+ taosWrite(*fd, &tableRecord, sizeof(STableRecord));
+ return 0;
+}
+
+
+int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, struct arguments *arguments, int32_t* totalNumOfThread) {
+ TAOS_ROW row;
+ int fd = -1;
+ STableRecord tableRecord;
+
+ char* tmpCommand = (char *)malloc(COMMAND_SIZE);
+ if (tmpCommand == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ return -1;
+ }
+
+ sprintf(tmpCommand, "select tbname from %s", metric);
+
+ TAOS_RES *result = taos_query(taosCon, tmpCommand);
+ int32_t code = taos_errno(result);
+ if (code != 0) {
+ fprintf(stderr, "failed to run command %s\n", tmpCommand);
+ free(tmpCommand);
+ taos_free_result(result);
+ return -1;
+ }
+
+ TAOS_FIELD *fields = taos_fetch_fields(result);
+
+ int32_t numOfTable = 0;
+ int32_t numOfThread = *totalNumOfThread;
+ char tmpFileName[TSDB_FILENAME_LEN + 1];
+ while ((row = taos_fetch_row(result)) != NULL) {
+ if (0 == numOfTable) {
+ memset(tmpFileName, 0, TSDB_FILENAME_LEN);
+ sprintf(tmpFileName, ".tables.tmp.%d", numOfThread);
+ fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
+ if (fd == -1) {
+ fprintf(stderr, "failed to open temp file: %s\n", tmpFileName);
+ taos_free_result(result);
+ for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) {
+ sprintf(tmpFileName, ".tables.tmp.%d", loopCnt);
+ (void)remove(tmpFileName);
+ }
+ free(tmpCommand);
+ return -1;
+ }
+
+ numOfThread++;
+ }
+
+ memset(&tableRecord, 0, sizeof(STableRecord));
+ tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes);
+ tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
+
+ taosWrite(fd, &tableRecord, sizeof(STableRecord));
+
+ numOfTable++;
+
+ if (numOfTable >= arguments->table_batch) {
+ numOfTable = 0;
+ close(fd);
+ fd = -1;
+ }
+ }
+
+ if (fd >= 0) {
+ close(fd);
+ fd = -1;
+ }
+
+ taos_free_result(result);
+
+ *totalNumOfThread = numOfThread;
+
+ free(tmpCommand);
+
+ return 0;
+}
+
+int taosDumpOut(struct arguments *arguments) {
+ TAOS *taos = NULL;
+ TAOS_RES *result = NULL;
+ char *command = NULL;
+
+ TAOS_ROW row;
+ FILE *fp = NULL;
+ int32_t count = 0;
+ STableRecordInfo tableRecordInfo;
+
+ char tmpBuf[TSDB_FILENAME_LEN+9] = {0};
+ if (arguments->outpath[0] != 0) {
+ sprintf(tmpBuf, "%s/dbs.sql", arguments->outpath);
+ } else {
+ sprintf(tmpBuf, "dbs.sql");
+ }
+
+ fp = fopen(tmpBuf, "w");
+ if (fp == NULL) {
+ fprintf(stderr, "failed to open file %s\n", tmpBuf);
+ return -1;
+ }
+
+ dbInfos = (SDbInfo **)calloc(128, sizeof(SDbInfo *));
+ if (dbInfos == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ goto _exit_failure;
+ }
+
+ command = (char *)malloc(COMMAND_SIZE);
+ if (command == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ goto _exit_failure;
+ }
+
+ /* Connect to server */
+ taos = taos_connect(arguments->host, arguments->user, arguments->password, NULL, arguments->port);
+ if (taos == NULL) {
+ fprintf(stderr, "failed to connect to TDengine server\n");
+ goto _exit_failure;
+ }
+
+ /* --------------------------------- Main Code -------------------------------- */
+ /* if (arguments->databases || arguments->all_databases) { // dump part of databases or all databases */
+ /* */
+ taosDumpCharset(fp);
+
+ sprintf(command, "show databases");
+ result = taos_query(taos, command);
+ int32_t code = taos_errno(result);
+
+ if (code != 0) {
+ fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(taos));
+ goto _exit_failure;
+ }
+
+ TAOS_FIELD *fields = taos_fetch_fields(result);
+
+ while ((row = taos_fetch_row(result)) != NULL) {
+ // sys database name : 'log', but subsequent version changed to 'log'
+ if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0 &&
+ (!arguments->allow_sys))
+ continue;
+
+ if (arguments->databases) { // input multi dbs
+ for (int i = 0; arguments->arg_list[i]; i++) {
+ if (strncasecmp(arguments->arg_list[i], (char *)row[TSDB_SHOW_DB_NAME_INDEX],
+ fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0)
+ goto _dump_db_point;
+ }
+ continue;
+ } else if (!arguments->all_databases) { // only input one db
+ if (strncasecmp(arguments->arg_list[0], (char *)row[TSDB_SHOW_DB_NAME_INDEX],
+ fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0)
+ goto _dump_db_point;
+ else
+ continue;
+ }
+
+ _dump_db_point:
+
+ dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
+ if (dbInfos[count] == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ goto _exit_failure;
+ }
+
+ strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
+#if 0
+ if (arguments->with_property) {
+ dbInfos[count]->tables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
+ dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
+ dbInfos[count]->replications = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
+ dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
+ dbInfos[count]->daysPerFile = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
+ dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]);
+ dbInfos[count]->daysToKeep1;
+ dbInfos[count]->daysToKeep2;
+ dbInfos[count]->cacheBlockSize = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
+ dbInfos[count]->totalBlocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
+ dbInfos[count]->minRowsPerFileBlock = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
+ dbInfos[count]->maxRowsPerFileBlock = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
+ dbInfos[count]->walLevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
+ dbInfos[count]->fsyncPeriod = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
+ dbInfos[count]->compression = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
+ dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]);
+ dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
+ }
+#endif
+ count++;
+
+ if (arguments->databases) {
+ if (count > arguments->arg_list_len) break;
+
+ } else if (!arguments->all_databases) {
+ if (count >= 1) break;
+ }
+ }
+
+ if (count == 0) {
+ fprintf(stderr, "No databases valid to dump\n");
+ goto _exit_failure;
+ }
+
+ if (arguments->databases || arguments->all_databases) { // case: taosdump --databases dbx dby ... OR taosdump --all-databases
+ for (int i = 0; i < count; i++) {
+ taosDumpDb(dbInfos[i], arguments, fp, taos);
+ }
+ } else {
+ if (arguments->arg_list_len == 1) { // case: taosdump
+ taosDumpDb(dbInfos[0], arguments, fp, taos);
+ } else { // case: taosdump tablex tabley ...
+ taosDumpCreateDbClause(dbInfos[0], arguments->with_property, fp);
+
+ sprintf(command, "use %s", dbInfos[0]->name);
+
+ result = taos_query(taos, command);
+ int32_t code = taos_errno(result);
+ if (code != 0) {
+ fprintf(stderr, "invalid database %s\n", dbInfos[0]->name);
+ goto _exit_failure;
+ }
+
+ fprintf(fp, "USE %s;\n\n", dbInfos[0]->name);
+
+ int32_t totalNumOfThread = 1; // 0: all normal talbe into .tables.tmp.0
+ int normalTblFd = -1;
+ int32_t retCode;
+ for (int i = 1; arguments->arg_list[i]; i++) {
+ if (taosGetTableRecordInfo(arguments->arg_list[i], &tableRecordInfo, taos) < 0) {
+ fprintf(stderr, "input the invalide table %s\n", arguments->arg_list[i]);
+ continue;
+ }
+
+ if (tableRecordInfo.isMetric) { // dump all table of this metric
+ (void)taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos);
+ retCode = taosSaveTableOfMetricToTempFile(taos, tableRecordInfo.tableRecord.metric, arguments, &totalNumOfThread);
+ } else {
+ if (tableRecordInfo.tableRecord.metric[0] != '\0') { // dump this sub table and it's metric
+ (void)taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos);
+ }
+ retCode = taosSaveAllNormalTableToTempFile(taos, tableRecordInfo.tableRecord.name, tableRecordInfo.tableRecord.metric, &normalTblFd);
+ }
+
+ if (retCode < 0) {
+ if (-1 != normalTblFd){
+ taosClose(normalTblFd);
+ }
+ goto _clean_tmp_file;
+ }
+ }
+
+ if (-1 != normalTblFd){
+ taosClose(normalTblFd);
+ }
+
+ // start multi threads to dumpout
+ taosStartDumpOutWorkThreads(arguments, totalNumOfThread, dbInfos[0]->name);
+
+ char tmpFileName[TSDB_FILENAME_LEN + 1];
+ _clean_tmp_file:
+ for (int loopCnt = 0; loopCnt < totalNumOfThread; loopCnt++) {
+ sprintf(tmpFileName, ".tables.tmp.%d", loopCnt);
+ remove(tmpFileName);
+ }
+ }
+ }
+
+ /* Close the handle and return */
+ fclose(fp);
+ taos_close(taos);
+ taos_free_result(result);
+ tfree(command);
+ taosFreeDbInfos();
+ fprintf(stderr, "dump out rows: %" PRId64 "\n", totalDumpOutRows);
+ return 0;
+
+_exit_failure:
+ fclose(fp);
+ taos_close(taos);
+ taos_free_result(result);
+ tfree(command);
+ taosFreeDbInfos();
+ fprintf(stderr, "dump out rows: %" PRId64 "\n", totalDumpOutRows);
+ return -1;
+}
+
+int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) {
+ TAOS_ROW row = NULL;
+ TAOS_RES *tmpResult = NULL;
+ int count = 0;
+
+ char* tempCommand = (char *)malloc(COMMAND_SIZE);
+ if (tempCommand == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ return -1;
+ }
+
+ char* tbuf = (char *)malloc(COMMAND_SIZE);
+ if (tbuf == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ free(tempCommand);
+ return -1;
+ }
+
+ sprintf(tempCommand, "describe %s", table);
+
+ tmpResult = taos_query(taosCon, tempCommand);
+ int32_t code = taos_errno(tmpResult);
+ if (code != 0) {
+ fprintf(stderr, "failed to run command %s\n", tempCommand);
+ free(tempCommand);
+ free(tbuf);
+ taos_free_result(tmpResult);
+ return -1;
+ }
+
+ TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
+
+ tstrncpy(tableDes->name, table, TSDB_COL_NAME_LEN);
+
+ while ((row = taos_fetch_row(tmpResult)) != NULL) {
+ strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
+ fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
+ strncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
+ fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
+ tableDes->cols[count].length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
+ strncpy(tableDes->cols[count].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
+ fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
+
+ count++;
+ }
+
+ taos_free_result(tmpResult);
+ tmpResult = NULL;
+
+ if (isSuperTable) {
+ free(tempCommand);
+ free(tbuf);
+ return count;
+ }
+
+ // if chidl-table have tag, using select tagName from table to get tagValue
+ for (int i = 0 ; i < count; i++) {
+ if (strcmp(tableDes->cols[i].note, "TAG") != 0) continue;
+
+
+ sprintf(tempCommand, "select %s from %s", tableDes->cols[i].field, table);
+
+ tmpResult = taos_query(taosCon, tempCommand);
+ code = taos_errno(tmpResult);
+ if (code != 0) {
+ fprintf(stderr, "failed to run command %s\n", tempCommand);
+ free(tempCommand);
+ free(tbuf);
+ taos_free_result(tmpResult);
+ return -1;
+ }
+
+ fields = taos_fetch_fields(tmpResult);
+
+ row = taos_fetch_row(tmpResult);
+ if (NULL == row) {
+ fprintf(stderr, " fetch failed to run command %s\n", tempCommand);
+ free(tempCommand);
+ free(tbuf);
+ taos_free_result(tmpResult);
+ return -1;
+ }
+
+ if (row[0] == NULL) {
+ sprintf(tableDes->cols[i].note, "%s", "NULL");
+ taos_free_result(tmpResult);
+ tmpResult = NULL;
+ continue;
+ }
+
+ int32_t* length = taos_fetch_lengths(tmpResult);
+
+ //int32_t* length = taos_fetch_lengths(tmpResult);
+ switch (fields[0].type) {
+ case TSDB_DATA_TYPE_BOOL:
+ sprintf(tableDes->cols[i].note, "%d", ((((int32_t)(*((char *)row[0]))) == 1) ? 1 : 0));
+ break;
+ case TSDB_DATA_TYPE_TINYINT:
+ sprintf(tableDes->cols[i].note, "%d", *((int8_t *)row[0]));
+ break;
+ case TSDB_DATA_TYPE_SMALLINT:
+ sprintf(tableDes->cols[i].note, "%d", *((int16_t *)row[0]));
+ break;
+ case TSDB_DATA_TYPE_INT:
+ sprintf(tableDes->cols[i].note, "%d", *((int32_t *)row[0]));
+ break;
+ case TSDB_DATA_TYPE_BIGINT:
+ sprintf(tableDes->cols[i].note, "%" PRId64 "", *((int64_t *)row[0]));
+ break;
+ case TSDB_DATA_TYPE_FLOAT:
+ sprintf(tableDes->cols[i].note, "%f", GET_FLOAT_VAL(row[0]));
+ break;
+ case TSDB_DATA_TYPE_DOUBLE:
+ sprintf(tableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[0]));
+ break;
+ case TSDB_DATA_TYPE_BINARY:
+ memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note));
+ tableDes->cols[i].note[0] = '\'';
+ converStringToReadable((char *)row[0], length[0], tbuf, COMMAND_SIZE);
+ char* pstr = stpcpy(&(tableDes->cols[i].note[1]), tbuf);
+ *(pstr++) = '\'';
+ break;
+ case TSDB_DATA_TYPE_NCHAR:
+ memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note));
+ convertNCharToReadable((char *)row[0], length[0], tbuf, COMMAND_SIZE);
+ sprintf(tableDes->cols[i].note, "\'%s\'", tbuf);
+ break;
+ case TSDB_DATA_TYPE_TIMESTAMP:
+ sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]);
+ #if 0
+ if (!arguments->mysqlFlag) {
+ sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]);
+ } else {
+ char buf[64] = "\0";
+ int64_t ts = *((int64_t *)row[0]);
+ time_t tt = (time_t)(ts / 1000);
+ struct tm *ptm = localtime(&tt);
+ strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm);
+ sprintf(tableDes->cols[i].note, "\'%s.%03d\'", buf, (int)(ts % 1000));
+ }
+ #endif
+ break;
+ default:
+ break;
+ }
+
+ taos_free_result(tmpResult);
+ tmpResult = NULL;
+ }
+
+ free(tempCommand);
+ free(tbuf);
+
+ return count;
+}
+
+int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon) {
+ int count = 0;
+
+ STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
+
+ if (metric != NULL && metric[0] != '\0') { // dump table schema which is created by using super table
+ /*
+ count = taosGetTableDes(metric, tableDes, taosCon);
+
+ if (count < 0) {
+ free(tableDes);
+ return -1;
+ }
+
+ taosDumpCreateTableClause(tableDes, count, fp);
+
+ memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
+ */
+
+ count = taosGetTableDes(table, tableDes, taosCon, false);
+
+ if (count < 0) {
+ free(tableDes);
+ return -1;
+ }
+
+ // create child-table using super-table
+ taosDumpCreateMTableClause(tableDes, metric, count, fp);
+
+ } else { // dump table definition
+ count = taosGetTableDes(table, tableDes, taosCon, false);
+
+ if (count < 0) {
+ free(tableDes);
+ return -1;
+ }
+
+ // create normal-table or super-table
+ taosDumpCreateTableClause(tableDes, count, fp);
+ }
+
+ free(tableDes);
+
+ return taosDumpTableData(fp, table, arguments, taosCon);
+}
+
+void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) {
+
+ char* tmpCommand = (char *)malloc(COMMAND_SIZE);
+ if (tmpCommand == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ return;
+ }
+
+ char *pstr = tmpCommand;
+
+ pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s", dbInfo->name);
+ if (isDumpProperty) {
+ #if 0
+ pstr += sprintf(pstr,
+ "TABLES %d vgroups %d REPLICA %d quorum %d DAYS %d KEEP %d CACHE %d BLOCKS %d MINROWS %d MAXROWS %d WALLEVEL %d FYNC %d COMP %d PRECISION %s UPDATE %d",
+ dbInfo->tables, dbInfo->vgroups, dbInfo->replications, dbInfo->quorum, dbInfo->daysPerFile, dbInfo->daysToKeep, dbInfo->cacheBlockSize,
+ dbInfo->totalBlocks, dbInfo->minRowsPerFileBlock, dbInfo->maxRowsPerFileBlock, dbInfo->walLevel, dbInfo->fsyncPeriod, dbInfo->compression,
+ dbInfo->precision, dbInfo->update);
+ #endif
+ }
+
+ pstr += sprintf(pstr, ";");
+
+ fprintf(fp, "%s\n\n", tmpCommand);
+ free(tmpCommand);
+}
+
+void* taosDumpOutWorkThreadFp(void *arg)
+{
+ SThreadParaObj *pThread = (SThreadParaObj*)arg;
+ STableRecord tableRecord;
+ int fd;
+
+ char tmpFileName[TSDB_FILENAME_LEN*4] = {0};
+ sprintf(tmpFileName, ".tables.tmp.%d", pThread->threadIndex);
+ fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
+ if (fd == -1) {
+ fprintf(stderr, "taosDumpTableFp() failed to open temp file: %s\n", tmpFileName);
+ return NULL;
+ }
+
+ FILE *fp = NULL;
+ memset(tmpFileName, 0, TSDB_FILENAME_LEN + 128);
+
+ if (tsArguments.outpath[0] != 0) {
+ sprintf(tmpFileName, "%s/%s.tables.%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex);
+ } else {
+ sprintf(tmpFileName, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex);
+ }
+
+ fp = fopen(tmpFileName, "w");
+ if (fp == NULL) {
+ fprintf(stderr, "failed to open file %s\n", tmpFileName);
+ close(fd);
+ return NULL;
+ }
+
+ memset(tmpFileName, 0, TSDB_FILENAME_LEN);
+ sprintf(tmpFileName, "use %s", pThread->dbName);
+
+ TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpFileName);
+ int32_t code = taos_errno(tmpResult);
+ if (code != 0) {
+ fprintf(stderr, "invalid database %s\n", pThread->dbName);
+ taos_free_result(tmpResult);
+ fclose(fp);
+ close(fd);
+ return NULL;
+ }
+
+ fprintf(fp, "USE %s;\n\n", pThread->dbName);
+ while (1) {
+ ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord));
+ if (readLen <= 0) break;
+ taosDumpTable(tableRecord.name, tableRecord.metric, &tsArguments, fp, pThread->taosCon);
+ }
+
+ taos_free_result(tmpResult);
+ close(fd);
+ fclose(fp);
+
+ return NULL;
+}
+
+static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfThread, char *dbName)
+{
+ pthread_attr_t thattr;
+ SThreadParaObj *threadObj = (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj));
+ for (int t = 0; t < numOfThread; ++t) {
+ SThreadParaObj *pThread = threadObj + t;
+ pThread->threadIndex = t;
+ pThread->totalThreads = numOfThread;
+ tstrncpy(pThread->dbName, dbName, TSDB_TABLE_NAME_LEN);
+ pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port);
+
+ if (pThread->taosCon == NULL) {
+ fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taosCon));
+ exit(0);
+ }
+
+ pthread_attr_init(&thattr);
+ pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+
+ if (pthread_create(&(pThread->threadID), &thattr, taosDumpOutWorkThreadFp, (void*)pThread) != 0) {
+ fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex);
+ exit(0);
+ }
+ }
+
+ for (int32_t t = 0; t < numOfThread; ++t) {
+ pthread_join(threadObj[t].threadID, NULL);
+ }
+
+ for (int32_t t = 0; t < numOfThread; ++t) {
+ taos_close(threadObj[t].taosCon);
+ }
+ free(threadObj);
+}
+
+
+
+int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon) {
+ int count = 0;
+
+ STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
+ if (NULL == tableDes) {
+ fprintf(stderr, "failed to allocate memory\n");
+ exit(-1);
+ }
+
+ count = taosGetTableDes(table, tableDes, taosCon, true);
+
+ if (count < 0) {
+ free(tableDes);
+ fprintf(stderr, "failed to get stable schema\n");
+ exit(-1);
+ }
+
+ taosDumpCreateTableClause(tableDes, count, fp);
+
+ free(tableDes);
+ return 0;
+}
+
+
+int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp)
+{
+ TAOS_ROW row;
+ int fd = -1;
+ STableRecord tableRecord;
+
+ char* tmpCommand = (char *)malloc(COMMAND_SIZE);
+ if (tmpCommand == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ exit(-1);
+ }
+
+ sprintf(tmpCommand, "use %s", dbName);
+
+ TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand);
+ int32_t code = taos_errno(tmpResult);
+ if (code != 0) {
+ fprintf(stderr, "invalid database %s, error: %s\n", dbName, taos_errstr(taosCon));
+ free(tmpCommand);
+ taos_free_result(tmpResult);
+ exit(-1);
+ }
+
+ taos_free_result(tmpResult);
+
+ sprintf(tmpCommand, "show stables");
+
+ tmpResult = taos_query(taosCon, tmpCommand);
+ code = taos_errno(tmpResult);
+ if (code != 0) {
+ fprintf(stderr, "failed to run command %s, error: %s\n", tmpCommand, taos_errstr(taosCon));
+ free(tmpCommand);
+ taos_free_result(tmpResult);
+ exit(-1);
+ }
+
+ TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
+
+ char tmpFileName[TSDB_FILENAME_LEN + 1];
+ memset(tmpFileName, 0, TSDB_FILENAME_LEN);
+ sprintf(tmpFileName, ".stables.tmp");
+ fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
+ if (fd == -1) {
+ fprintf(stderr, "failed to open temp file: %s\n", tmpFileName);
+ taos_free_result(tmpResult);
+ free(tmpCommand);
+ (void)remove(".stables.tmp");
+ exit(-1);
+ }
+
+ while ((row = taos_fetch_row(tmpResult)) != NULL) {
+ memset(&tableRecord, 0, sizeof(STableRecord));
+ strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
+ taosWrite(fd, &tableRecord, sizeof(STableRecord));
+ }
+
+ taos_free_result(tmpResult);
+ (void)lseek(fd, 0, SEEK_SET);
+
+ while (1) {
+ ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord));
+ if (readLen <= 0) break;
+
+ (void)taosDumpStable(tableRecord.name, fp, taosCon);
+ }
+
+ close(fd);
+ (void)remove(".stables.tmp");
+
+ free(tmpCommand);
+ return 0;
+}
+
+
+int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon) {
+ TAOS_ROW row;
+ int fd = -1;
+ STableRecord tableRecord;
+
+ taosDumpCreateDbClause(dbInfo, arguments->with_property, fp);
+
+ char* tmpCommand = (char *)malloc(COMMAND_SIZE);
+ if (tmpCommand == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ return -1;
+ }
+
+ sprintf(tmpCommand, "use %s", dbInfo->name);
+
+ TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand);
+ int32_t code = taos_errno(tmpResult);
+ if (code != 0) {
+ fprintf(stderr, "invalid database %s\n", dbInfo->name);
+ free(tmpCommand);
+ taos_free_result(tmpResult);
+ return -1;
+ }
+ taos_free_result(tmpResult);
+
+ fprintf(fp, "USE %s;\n\n", dbInfo->name);
+
+ (void)taosDumpCreateSuperTableClause(taosCon, dbInfo->name, fp);
+
+ sprintf(tmpCommand, "show tables");
+
+ tmpResult = taos_query(taosCon, tmpCommand);
+ code = taos_errno(tmpResult);
+ if (code != 0) {
+ fprintf(stderr, "failed to run command %s\n", tmpCommand);
+ free(tmpCommand);
+ taos_free_result(tmpResult);
+ return -1;
+ }
+
+ TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
+
+ int32_t numOfTable = 0;
+ int32_t numOfThread = 0;
+ char tmpFileName[TSDB_FILENAME_LEN + 1];
+ while ((row = taos_fetch_row(tmpResult)) != NULL) {
+ if (0 == numOfTable) {
+ memset(tmpFileName, 0, TSDB_FILENAME_LEN);
+ sprintf(tmpFileName, ".tables.tmp.%d", numOfThread);
+ fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
+ if (fd == -1) {
+ fprintf(stderr, "failed to open temp file: %s\n", tmpFileName);
+ taos_free_result(tmpResult);
+ for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) {
+ sprintf(tmpFileName, ".tables.tmp.%d", loopCnt);
+ (void)remove(tmpFileName);
+ }
+ free(tmpCommand);
+ return -1;
+ }
+
+ numOfThread++;
+ }
+
+ memset(&tableRecord, 0, sizeof(STableRecord));
+ tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
+ tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes);
+
+ taosWrite(fd, &tableRecord, sizeof(STableRecord));
+
+ numOfTable++;
+
+ if (numOfTable >= arguments->table_batch) {
+ numOfTable = 0;
+ close(fd);
+ fd = -1;
+ }
+ }
+
+ if (fd >= 0) {
+ close(fd);
+ fd = -1;
+ }
+
+ taos_free_result(tmpResult);
+
+ // start multi threads to dumpout
+ taosStartDumpOutWorkThreads(arguments, numOfThread, dbInfo->name);
+ for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) {
+ sprintf(tmpFileName, ".tables.tmp.%d", loopCnt);
+ (void)remove(tmpFileName);
+ }
+
+ free(tmpCommand);
+
+ return 0;
+}
+
+void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp) {
+ int counter = 0;
+ int count_temp = 0;
+
+ char* tmpBuf = (char *)malloc(COMMAND_SIZE);
+ if (tmpBuf == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ return;
+ }
+
+ char* pstr = tmpBuf;
+
+ pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s", tableDes->name);
+
+ for (; counter < numOfCols; counter++) {
+ if (tableDes->cols[counter].note[0] != '\0') break;
+
+ if (counter == 0) {
+ pstr += sprintf(pstr, " (%s %s", tableDes->cols[counter].field, tableDes->cols[counter].type);
+ } else {
+ pstr += sprintf(pstr, ", %s %s", tableDes->cols[counter].field, tableDes->cols[counter].type);
+ }
+
+ if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
+ strcasecmp(tableDes->cols[counter].type, "nchar") == 0) {
+ pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length);
+ }
+ }
+
+ count_temp = counter;
+
+ for (; counter < numOfCols; counter++) {
+ if (counter == count_temp) {
+ pstr += sprintf(pstr, ") TAGS (%s %s", tableDes->cols[counter].field, tableDes->cols[counter].type);
+ } else {
+ pstr += sprintf(pstr, ", %s %s", tableDes->cols[counter].field, tableDes->cols[counter].type);
+ }
+
+ if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
+ strcasecmp(tableDes->cols[counter].type, "nchar") == 0) {
+ pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length);
+ }
+ }
+
+ pstr += sprintf(pstr, ");");
+
+ fprintf(fp, "%s\n", tmpBuf);
+
+ free(tmpBuf);
+}
+
+void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp) {
+ int counter = 0;
+ int count_temp = 0;
+
+ char* tmpBuf = (char *)malloc(COMMAND_SIZE);
+ if (tmpBuf == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ return;
+ }
+
+ char *pstr = NULL;
+ pstr = tmpBuf;
+
+ pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s USING %s TAGS (", tableDes->name, metric);
+
+ for (; counter < numOfCols; counter++) {
+ if (tableDes->cols[counter].note[0] != '\0') break;
+ }
+
+ assert(counter < numOfCols);
+ count_temp = counter;
+
+ for (; counter < numOfCols; counter++) {
+ if (counter != count_temp) {
+ if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
+ strcasecmp(tableDes->cols[counter].type, "nchar") == 0) {
+ //pstr += sprintf(pstr, ", \'%s\'", tableDes->cols[counter].note);
+ pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note);
+ } else {
+ pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note);
+ }
+ } else {
+ if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
+ strcasecmp(tableDes->cols[counter].type, "nchar") == 0) {
+ //pstr += sprintf(pstr, "\'%s\'", tableDes->cols[counter].note);
+ pstr += sprintf(pstr, "%s", tableDes->cols[counter].note);
+ } else {
+ pstr += sprintf(pstr, "%s", tableDes->cols[counter].note);
+ }
+ /* pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); */
+ }
+
+ /* if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || strcasecmp(tableDes->cols[counter].type, "nchar")
+ * == 0) { */
+ /* pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length); */
+ /* } */
+ }
+
+ pstr += sprintf(pstr, ");");
+
+ fprintf(fp, "%s\n", tmpBuf);
+ free(tmpBuf);
+}
+
+int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon) {
+ /* char temp[MAX_COMMAND_SIZE] = "\0"; */
+ int64_t totalRows = 0;
+ int count = 0;
+ char *pstr = NULL;
+ TAOS_ROW row = NULL;
+ int numFields = 0;
+ char *tbuf = NULL;
+
+ char* tmpCommand = (char *)calloc(1, COMMAND_SIZE);
+ if (tmpCommand == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ return -1;
+ }
+
+ int32_t sql_buf_len = arguments->max_sql_len;
+ char* tmpBuffer = (char *)calloc(1, sql_buf_len + 128);
+ if (tmpBuffer == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ free(tmpCommand);
+ return -1;
+ }
+
+ pstr = tmpBuffer;
+
+ if (arguments->schemaonly) {
+ free(tmpCommand);
+ free(tmpBuffer);
+ return 0;
+ }
+
+ sprintf(tmpCommand,
+ "select * from %s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc",
+ tbname,
+ arguments->start_time,
+ arguments->end_time);
+
+ TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand);
+ int32_t code = taos_errno(tmpResult);
+ if (code != 0) {
+ fprintf(stderr, "failed to run command %s, reason: %s\n", tmpCommand, taos_errstr(taosCon));
+ free(tmpCommand);
+ free(tmpBuffer);
+ taos_free_result(tmpResult);
+ return -1;
+ }
+
+ numFields = taos_field_count(tmpResult);
+ assert(numFields > 0);
+ TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
+ tbuf = (char *)malloc(COMMAND_SIZE);
+ if (tbuf == NULL) {
+ fprintf(stderr, "No enough memory\n");
+ free(tmpCommand);
+ free(tmpBuffer);
+ taos_free_result(tmpResult);
+ return -1;
+ }
+
+ int rowFlag = 0;
+ int32_t curr_sqlstr_len = 0;
+ int32_t total_sqlstr_len = 0;
+ count = 0;
+ while ((row = taos_fetch_row(tmpResult)) != NULL) {
+ pstr = tmpBuffer;
+ curr_sqlstr_len = 0;
+
+ int32_t* length = taos_fetch_lengths(tmpResult); // act len
+
+ if (count == 0) {
+ total_sqlstr_len = 0;
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "INSERT INTO %s VALUES (", tbname);
+ } else {
+ if (arguments->mysqlFlag) {
+ if (0 == rowFlag) {
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "(");
+ rowFlag++;
+ } else {
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ", (");
+ }
+ } else {
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "(");
+ }
+ }
+
+ for (int col = 0; col < numFields; col++) {
+ if (col != 0) curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ", ");
+
+ if (row[col] == NULL) {
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "NULL");
+ continue;
+ }
+
+ switch (fields[col].type) {
+ case TSDB_DATA_TYPE_BOOL:
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", ((((int32_t)(*((char *)row[col]))) == 1) ? 1 : 0));
+ break;
+ case TSDB_DATA_TYPE_TINYINT:
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int8_t *)row[col]));
+ break;
+ case TSDB_DATA_TYPE_SMALLINT:
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int16_t *)row[col]));
+ break;
+ case TSDB_DATA_TYPE_INT:
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int32_t *)row[col]));
+ break;
+ case TSDB_DATA_TYPE_BIGINT:
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", *((int64_t *)row[col]));
+ break;
+ case TSDB_DATA_TYPE_FLOAT:
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_FLOAT_VAL(row[col]));
+ break;
+ case TSDB_DATA_TYPE_DOUBLE:
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_DOUBLE_VAL(row[col]));
+ break;
+ case TSDB_DATA_TYPE_BINARY:
+ //*(pstr++) = '\'';
+ converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE);
+ //pstr = stpcpy(pstr, tbuf);
+ //*(pstr++) = '\'';
+ pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf);
+ break;
+ case TSDB_DATA_TYPE_NCHAR:
+ convertNCharToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE);
+ pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf);
+ break;
+ case TSDB_DATA_TYPE_TIMESTAMP:
+ if (!arguments->mysqlFlag) {
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", *(int64_t *)row[col]);
+ } else {
+ char buf[64] = "\0";
+ int64_t ts = *((int64_t *)row[col]);
+ time_t tt = (time_t)(ts / 1000);
+ struct tm *ptm = localtime(&tt);
+ strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm);
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s.%03d\'", buf, (int)(ts % 1000));
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ") ");
+
+ totalRows++;
+ count++;
+ fprintf(fp, "%s", tmpBuffer);
+
+ total_sqlstr_len += curr_sqlstr_len;
+
+ if ((count >= arguments->data_batch) || (sql_buf_len - total_sqlstr_len < TSDB_MAX_BYTES_PER_ROW)) {
+ fprintf(fp, ";\n");
+ count = 0;
+ } //else {
+ //fprintf(fp, "\\\n");
+ //}
+ }
+
+ atomic_add_fetch_64(&totalDumpOutRows, totalRows);
+
+ fprintf(fp, "\n");
+
+ if (tbuf) {
+ free(tbuf);
+ }
+
+ taos_free_result(tmpResult);
+ tmpResult = NULL;
+ free(tmpCommand);
+ free(tmpBuffer);
+ return 0;
+}
+
+int taosCheckParam(struct arguments *arguments) {
+ if (arguments->all_databases && arguments->databases) {
+ fprintf(stderr, "conflict option --all-databases and --databases\n");
+ return -1;
+ }
+
+ if (arguments->start_time > arguments->end_time) {
+ fprintf(stderr, "start time is larger than end time\n");
+ return -1;
+ }
+
+ if (arguments->arg_list_len == 0) {
+ if ((!arguments->all_databases) && (!arguments->isDumpIn)) {
+ fprintf(stderr, "taosdump requires parameters\n");
+ return -1;
+ }
+ }
+/*
+ if (arguments->isDumpIn && (strcmp(arguments->outpath, DEFAULT_DUMP_FILE) != 0)) {
+ fprintf(stderr, "duplicate parameter input and output file path\n");
+ return -1;
+ }
+*/
+ if (!arguments->isDumpIn && arguments->encode != NULL) {
+ fprintf(stderr, "invalid option in dump out\n");
+ return -1;
+ }
+
+ if (arguments->table_batch <= 0) {
+ fprintf(stderr, "invalid option in dump out\n");
+ return -1;
+ }
+
+ return 0;
+}
+
+bool isEmptyCommand(char *cmd) {
+ char *pchar = cmd;
+
+ while (*pchar != '\0') {
+ if (*pchar != ' ') return false;
+ pchar++;
+ }
+
+ return true;
+}
+
+void taosReplaceCtrlChar(char *str) {
+ _Bool ctrlOn = false;
+ char *pstr = NULL;
+
+ for (pstr = str; *str != '\0'; ++str) {
+ if (ctrlOn) {
+ switch (*str) {
+ case 'n':
+ *pstr = '\n';
+ pstr++;
+ break;
+ case 'r':
+ *pstr = '\r';
+ pstr++;
+ break;
+ case 't':
+ *pstr = '\t';
+ pstr++;
+ break;
+ case '\\':
+ *pstr = '\\';
+ pstr++;
+ break;
+ case '\'':
+ *pstr = '\'';
+ pstr++;
+ break;
+ default:
+ break;
+ }
+ ctrlOn = false;
+ } else {
+ if (*str == '\\') {
+ ctrlOn = true;
+ } else {
+ *pstr = *str;
+ pstr++;
+ }
+ }
+ }
+
+ *pstr = '\0';
+}
+
+char *ascii_literal_list[] = {
+ "\\x00", "\\x01", "\\x02", "\\x03", "\\x04", "\\x05", "\\x06", "\\x07", "\\x08", "\\t", "\\n", "\\x0b", "\\x0c",
+ "\\r", "\\x0e", "\\x0f", "\\x10", "\\x11", "\\x12", "\\x13", "\\x14", "\\x15", "\\x16", "\\x17", "\\x18", "\\x19",
+ "\\x1a", "\\x1b", "\\x1c", "\\x1d", "\\x1e", "\\x1f", " ", "!", "\\\"", "#", "$", "%", "&",
+ "\\'", "(", ")", "*", "+", ",", "-", ".", "/", "0", "1", "2", "3",
+ "4", "5", "6", "7", "8", "9", ":", ";", "<", "=", ">", "?", "@",
+ "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M",
+ "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z",
+ "[", "\\\\", "]", "^", "_", "`", "a", "b", "c", "d", "e", "f", "g",
+ "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t",
+ "u", "v", "w", "x", "y", "z", "{", "|", "}", "~", "\\x7f", "\\x80", "\\x81",
+ "\\x82", "\\x83", "\\x84", "\\x85", "\\x86", "\\x87", "\\x88", "\\x89", "\\x8a", "\\x8b", "\\x8c", "\\x8d", "\\x8e",
+ "\\x8f", "\\x90", "\\x91", "\\x92", "\\x93", "\\x94", "\\x95", "\\x96", "\\x97", "\\x98", "\\x99", "\\x9a", "\\x9b",
+ "\\x9c", "\\x9d", "\\x9e", "\\x9f", "\\xa0", "\\xa1", "\\xa2", "\\xa3", "\\xa4", "\\xa5", "\\xa6", "\\xa7", "\\xa8",
+ "\\xa9", "\\xaa", "\\xab", "\\xac", "\\xad", "\\xae", "\\xaf", "\\xb0", "\\xb1", "\\xb2", "\\xb3", "\\xb4", "\\xb5",
+ "\\xb6", "\\xb7", "\\xb8", "\\xb9", "\\xba", "\\xbb", "\\xbc", "\\xbd", "\\xbe", "\\xbf", "\\xc0", "\\xc1", "\\xc2",
+ "\\xc3", "\\xc4", "\\xc5", "\\xc6", "\\xc7", "\\xc8", "\\xc9", "\\xca", "\\xcb", "\\xcc", "\\xcd", "\\xce", "\\xcf",
+ "\\xd0", "\\xd1", "\\xd2", "\\xd3", "\\xd4", "\\xd5", "\\xd6", "\\xd7", "\\xd8", "\\xd9", "\\xda", "\\xdb", "\\xdc",
+ "\\xdd", "\\xde", "\\xdf", "\\xe0", "\\xe1", "\\xe2", "\\xe3", "\\xe4", "\\xe5", "\\xe6", "\\xe7", "\\xe8", "\\xe9",
+ "\\xea", "\\xeb", "\\xec", "\\xed", "\\xee", "\\xef", "\\xf0", "\\xf1", "\\xf2", "\\xf3", "\\xf4", "\\xf5", "\\xf6",
+ "\\xf7", "\\xf8", "\\xf9", "\\xfa", "\\xfb", "\\xfc", "\\xfd", "\\xfe", "\\xff"};
+
+int converStringToReadable(char *str, int size, char *buf, int bufsize) {
+ char *pstr = str;
+ char *pbuf = buf;
+ while (size > 0) {
+ if (*pstr == '\0') break;
+ pbuf = stpcpy(pbuf, ascii_literal_list[((uint8_t)(*pstr))]);
+ pstr++;
+ size--;
+ }
+ *pbuf = '\0';
+ return 0;
+}
+
+int convertNCharToReadable(char *str, int size, char *buf, int bufsize) {
+ char *pstr = str;
+ char *pbuf = buf;
+ // TODO
+ wchar_t wc;
+ while (size > 0) {
+ if (*pstr == '\0') break;
+ int byte_width = mbtowc(&wc, pstr, MB_CUR_MAX);
+ if (byte_width < 0) {
+ fprintf(stderr, "mbtowc() return fail.\n");
+ exit(-1);
+ }
+
+ if ((int)wc < 256) {
+ pbuf = stpcpy(pbuf, ascii_literal_list[(int)wc]);
+ } else {
+ memcpy(pbuf, pstr, byte_width);
+ pbuf += byte_width;
+ }
+ pstr += byte_width;
+ }
+
+ *pbuf = '\0';
+
+ return 0;
+}
+
+void taosDumpCharset(FILE *fp) {
+ char charsetline[256];
+
+ (void)fseek(fp, 0, SEEK_SET);
+ sprintf(charsetline, "#!%s\n", tsCharset);
+ (void)fwrite(charsetline, strlen(charsetline), 1, fp);
+}
+
+void taosLoadFileCharset(FILE *fp, char *fcharset) {
+ char * line = NULL;
+ size_t line_size = 0;
+
+ (void)fseek(fp, 0, SEEK_SET);
+ ssize_t size = getline(&line, &line_size, fp);
+ if (size <= 2) {
+ goto _exit_no_charset;
+ }
+
+ if (strncmp(line, "#!", 2) != 0) {
+ goto _exit_no_charset;
+ }
+ if (line[size - 1] == '\n') {
+ line[size - 1] = '\0';
+ size--;
+ }
+ strcpy(fcharset, line + 2);
+
+ tfree(line);
+ return;
+
+_exit_no_charset:
+ (void)fseek(fp, 0, SEEK_SET);
+ *fcharset = '\0';
+ tfree(line);
+ return;
+}
+
+// ======== dumpIn support multi threads functions ================================//
+
+static char **tsDumpInSqlFiles = NULL;
+static int32_t tsSqlFileNum = 0;
+static char tsDbSqlFile[TSDB_FILENAME_LEN] = {0};
+static char tsfCharset[64] = {0};
+static int taosGetFilesNum(const char *directoryName, const char *prefix)
+{
+ char cmd[1024] = { 0 };
+ sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix);
+
+ FILE *fp = popen(cmd, "r");
+ if (fp == NULL) {
+ fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
+ exit(0);
+ }
+
+ int fileNum = 0;
+ if (fscanf(fp, "%d", &fileNum) != 1) {
+ fprintf(stderr, "ERROR: failed to execute:%s, parse result error\n", cmd);
+ exit(0);
+ }
+
+ if (fileNum <= 0) {
+ fprintf(stderr, "ERROR: directory:%s is empry\n", directoryName);
+ exit(0);
+ }
+
+ pclose(fp);
+ return fileNum;
+}
+
+static void taosParseDirectory(const char *directoryName, const char *prefix, char **fileArray, int totalFiles)
+{
+ char cmd[1024] = { 0 };
+ sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix);
+
+ FILE *fp = popen(cmd, "r");
+ if (fp == NULL) {
+ fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
+ exit(0);
+ }
+
+ int fileNum = 0;
+ while (fscanf(fp, "%128s", fileArray[fileNum++])) {
+ if (strcmp(fileArray[fileNum-1], tsDbSqlFile) == 0) {
+ fileNum--;
+ }
+ if (fileNum >= totalFiles) {
+ break;
+ }
+ }
+
+ if (fileNum != totalFiles) {
+ fprintf(stderr, "ERROR: directory:%s changed while read\n", directoryName);
+ pclose(fp);
+ exit(0);
+ }
+
+ pclose(fp);
+}
+
+static void taosCheckTablesSQLFile(const char *directoryName)
+{
+ char cmd[1024] = { 0 };
+ sprintf(cmd, "ls %s/dbs.sql", directoryName);
+
+ FILE *fp = popen(cmd, "r");
+ if (fp == NULL) {
+ fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
+ exit(0);
+ }
+
+ while (fscanf(fp, "%128s", tsDbSqlFile)) {
+ break;
+ }
+
+ pclose(fp);
+}
+
+static void taosMallocSQLFiles()
+{
+ tsDumpInSqlFiles = (char**)calloc(tsSqlFileNum, sizeof(char*));
+ for (int i = 0; i < tsSqlFileNum; i++) {
+ tsDumpInSqlFiles[i] = calloc(1, TSDB_FILENAME_LEN);
+ }
+}
+
+static void taosFreeSQLFiles()
+{
+ for (int i = 0; i < tsSqlFileNum; i++) {
+ tfree(tsDumpInSqlFiles[i]);
+ }
+ tfree(tsDumpInSqlFiles);
+}
+
+static void taosGetDirectoryFileList(char *inputDir)
+{
+ struct stat fileStat;
+ if (stat(inputDir, &fileStat) < 0) {
+ fprintf(stderr, "ERROR: %s not exist\n", inputDir);
+ exit(0);
+ }
+
+ if (fileStat.st_mode & S_IFDIR) {
+ taosCheckTablesSQLFile(inputDir);
+ tsSqlFileNum = taosGetFilesNum(inputDir, "sql");
+ int totalSQLFileNum = tsSqlFileNum;
+ if (tsDbSqlFile[0] != 0) {
+ tsSqlFileNum--;
+ }
+ taosMallocSQLFiles();
+ taosParseDirectory(inputDir, "sql", tsDumpInSqlFiles, tsSqlFileNum);
+ fprintf(stdout, "\nstart to dispose %d files in %s\n", totalSQLFileNum, inputDir);
+ }
+ else {
+ fprintf(stderr, "ERROR: %s is not a directory\n", inputDir);
+ exit(0);
+ }
+}
+
+static FILE* taosOpenDumpInFile(char *fptr) {
+ wordexp_t full_path;
+
+ if (wordexp(fptr, &full_path, 0) != 0) {
+ fprintf(stderr, "ERROR: illegal file name: %s\n", fptr);
+ return NULL;
+ }
+
+ char *fname = full_path.we_wordv[0];
+
+ FILE *f = fopen(fname, "r");
+ if (f == NULL) {
+ fprintf(stderr, "ERROR: failed to open file %s\n", fname);
+ wordfree(&full_path);
+ return NULL;
+ }
+
+ wordfree(&full_path);
+
+ return f;
+}
+
+int taosDumpInOneFile_old(TAOS * taos, FILE* fp, char* fcharset, char* encode) {
+ char *command = NULL;
+ char *lcommand = NULL;
+ int tsize = 0;
+ char *line = NULL;
+ _Bool isRun = true;
+ size_t line_size = 0;
+ char *pstr = NULL;
+ char *lstr = NULL;
+ size_t inbytesleft = 0;
+ size_t outbytesleft = COMMAND_SIZE;
+ char *tcommand = NULL;
+ char *charsetOfFile = NULL;
+ iconv_t cd = (iconv_t)(-1);
+
+ command = (char *)malloc(COMMAND_SIZE);
+ lcommand = (char *)malloc(COMMAND_SIZE);
+ if (command == NULL || lcommand == NULL) {
+ fprintf(stderr, "failed to connect to allocate memory\n");
+ goto _dumpin_exit_failure;
+ }
+
+ // Resolve locale
+ if (*fcharset != '\0') {
+ charsetOfFile = fcharset;
+ } else {
+ charsetOfFile = encode;
+ }
+
+ if (charsetOfFile != NULL && strcasecmp(tsCharset, charsetOfFile) != 0) {
+ cd = iconv_open(tsCharset, charsetOfFile);
+ if (cd == ((iconv_t)(-1))) {
+ fprintf(stderr, "Failed to open iconv handle\n");
+ goto _dumpin_exit_failure;
+ }
+ }
+
+ pstr = command;
+ int64_t linenu = 0;
+ while (1) {
+ ssize_t size = getline(&line, &line_size, fp);
+ linenu++;
+ if (size <= 0) break;
+ if (size == 1) {
+ if (pstr != command) {
+ inbytesleft = pstr - command;
+ memset(lcommand, 0, COMMAND_SIZE);
+ pstr = command;
+ lstr = lcommand;
+ outbytesleft = COMMAND_SIZE;
+ if (cd != ((iconv_t)(-1))) {
+ iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft);
+ tcommand = lcommand;
+ } else {
+ tcommand = command;
+ }
+
+ taosReplaceCtrlChar(tcommand);
+
+ if (queryDB(taos, tcommand) != 0) {
+ fprintf(stderr, "error sql: linenu: %" PRId64 " failed\n", linenu);
+ exit(0);
+ }
+
+ pstr = command;
+ pstr[0] = '\0';
+ tsize = 0;
+ isRun = true;
+ }
+
+ continue;
+ }
+
+ /* if (line[0] == '-' && line[1] == '-') continue; */
+
+ line[size - 1] = 0;
+
+ if (tsize + size - 1 > COMMAND_SIZE) {
+ fprintf(stderr, "command is too long\n");
+ goto _dumpin_exit_failure;
+ }
+
+ if (line[size - 2] == '\\') {
+ line[size - 2] = ' ';
+ isRun = false;
+ } else {
+ isRun = true;
+ }
+
+ memcpy(pstr, line, size - 1);
+ pstr += (size - 1);
+ *pstr = '\0';
+
+ if (!isRun) continue;
+
+ if (command != pstr && !isEmptyCommand(command)) {
+ inbytesleft = pstr - command;
+ memset(lcommand, 0, COMMAND_SIZE);
+ pstr = command;
+ lstr = lcommand;
+ outbytesleft = COMMAND_SIZE;
+ if (cd != ((iconv_t)(-1))) {
+ iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft);
+ tcommand = lcommand;
+ } else {
+ tcommand = command;
+ }
+ taosReplaceCtrlChar(tcommand);
+ if (queryDB(taos, tcommand) != 0) {
+ fprintf(stderr, "error sql: linenu:%" PRId64 " failed\n", linenu);
+ exit(0);
+ }
+ }
+
+ pstr = command;
+ *pstr = '\0';
+ tsize = 0;
+ }
+
+ if (pstr != command) {
+ inbytesleft = pstr - command;
+ memset(lcommand, 0, COMMAND_SIZE);
+ pstr = command;
+ lstr = lcommand;
+ outbytesleft = COMMAND_SIZE;
+ if (cd != ((iconv_t)(-1))) {
+ iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft);
+ tcommand = lcommand;
+ } else {
+ tcommand = command;
+ }
+ taosReplaceCtrlChar(lcommand);
+ if (queryDB(taos, tcommand) != 0)
+ fprintf(stderr, "error sql: linenu:%" PRId64 " failed \n", linenu);
+ }
+
+ if (cd != ((iconv_t)(-1))) iconv_close(cd);
+ tfree(line);
+ tfree(command);
+ tfree(lcommand);
+ taos_close(taos);
+ fclose(fp);
+ return 0;
+
+_dumpin_exit_failure:
+ if (cd != ((iconv_t)(-1))) iconv_close(cd);
+ tfree(command);
+ tfree(lcommand);
+ taos_close(taos);
+ fclose(fp);
+ return -1;
+}
+
+int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, char* fileName) {
+ int read_len = 0;
+ char * cmd = NULL;
+ size_t cmd_len = 0;
+ char * line = NULL;
+ size_t line_len = 0;
+
+ cmd = (char *)malloc(COMMAND_SIZE);
+ if (cmd == NULL) {
+ fprintf(stderr, "failed to allocate memory\n");
+ return -1;
+ }
+
+ int lineNo = 0;
+ while ((read_len = getline(&line, &line_len, fp)) != -1) {
+ ++lineNo;
+ if (read_len >= COMMAND_SIZE) continue;
+ line[--read_len] = '\0';
+
+ //if (read_len == 0 || isCommentLine(line)) { // line starts with #
+ if (read_len == 0 ) {
+ continue;
+ }
+
+ if (line[read_len - 1] == '\\') {
+ line[read_len - 1] = ' ';
+ memcpy(cmd + cmd_len, line, read_len);
+ cmd_len += read_len;
+ continue;
+ }
+
+ memcpy(cmd + cmd_len, line, read_len);
+ cmd[read_len + cmd_len]= '\0';
+ if (queryDB(taos, cmd)) {
+ fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName);
+ }
+
+ memset(cmd, 0, COMMAND_SIZE);
+ cmd_len = 0;
+ }
+
+ tfree(cmd);
+ tfree(line);
+ fclose(fp);
+ return 0;
+}
+
+void* taosDumpInWorkThreadFp(void *arg)
+{
+ SThreadParaObj *pThread = (SThreadParaObj*)arg;
+ for (int32_t f = 0; f < tsSqlFileNum; ++f) {
+ if (f % pThread->totalThreads == pThread->threadIndex) {
+ char *SQLFileName = tsDumpInSqlFiles[f];
+ FILE* fp = taosOpenDumpInFile(SQLFileName);
+ if (NULL == fp) {
+ continue;
+ }
+ fprintf(stderr, "Success Open input file: %s\n", SQLFileName);
+ taosDumpInOneFile(pThread->taosCon, fp, tsfCharset, tsArguments.encode, SQLFileName);
+ }
+ }
+
+ return NULL;
+}
+
+static void taosStartDumpInWorkThreads(struct arguments *args)
+{
+ pthread_attr_t thattr;
+ SThreadParaObj *pThread;
+ int32_t totalThreads = args->thread_num;
+
+ if (totalThreads > tsSqlFileNum) {
+ totalThreads = tsSqlFileNum;
+ }
+
+ SThreadParaObj *threadObj = (SThreadParaObj *)calloc(totalThreads, sizeof(SThreadParaObj));
+ for (int32_t t = 0; t < totalThreads; ++t) {
+ pThread = threadObj + t;
+ pThread->threadIndex = t;
+ pThread->totalThreads = totalThreads;
+ pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port);
+ if (pThread->taosCon == NULL) {
+ fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taosCon));
+ exit(0);
+ }
+
+ pthread_attr_init(&thattr);
+ pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+
+ if (pthread_create(&(pThread->threadID), &thattr, taosDumpInWorkThreadFp, (void*)pThread) != 0) {
+ fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex);
+ exit(0);
+ }
+ }
+
+ for (int t = 0; t < totalThreads; ++t) {
+ pthread_join(threadObj[t].threadID, NULL);
+ }
+
+ for (int t = 0; t < totalThreads; ++t) {
+ taos_close(threadObj[t].taosCon);
+ }
+ free(threadObj);
+}
+
+
+int taosDumpIn(struct arguments *arguments) {
+ assert(arguments->isDumpIn);
+
+ TAOS *taos = NULL;
+ FILE *fp = NULL;
+
+ taos = taos_connect(arguments->host, arguments->user, arguments->password, NULL, arguments->port);
+ if (taos == NULL) {
+ fprintf(stderr, "failed to connect to TDengine server\n");
+ return -1;
+ }
+
+ taosGetDirectoryFileList(arguments->inpath);
+
+ if (tsDbSqlFile[0] != 0) {
+ fp = taosOpenDumpInFile(tsDbSqlFile);
+ if (NULL == fp) {
+ fprintf(stderr, "failed to open input file %s\n", tsDbSqlFile);
+ return -1;
+ }
+ fprintf(stderr, "Success Open input file: %s\n", tsDbSqlFile);
+
+ taosLoadFileCharset(fp, tsfCharset);
+
+ taosDumpInOneFile(taos, fp, tsfCharset, arguments->encode, tsDbSqlFile);
+ }
+
+ taosStartDumpInWorkThreads(arguments);
+
+ taos_close(taos);
+ taosFreeSQLFiles();
+ return 0;
+}
+
+
diff --git a/src/kit/taosdump/taosdump.sh b/src/kit/taosdump/taosdump.sh
new file mode 100755
index 0000000000000000000000000000000000000000..6d32c090dbb0f538b0fc0abb4a9588ee08037a95
--- /dev/null
+++ b/src/kit/taosdump/taosdump.sh
@@ -0,0 +1,48 @@
+taos1_6="/root/mnt/work/test/td1.6/build/bin/taos"
+taosdump1_6="/root/mnt/work/test/td1.6/build/bin/taosdump"
+taoscfg1_6="/root/mnt/work/test/td1.6/test/cfg"
+
+taos2_0="/root/mnt/work/test/td2.0/build/bin/taos"
+taosdump2_0="/root/mnt/work/test/td2.0/build/bin/taosdump"
+taoscfg2_0="/root/mnt/work/test/td2.0/test/cfg"
+
+data_dir="/root/mnt/work/test/td1.6/output"
+table_list="/root/mnt/work/test/td1.6/tables"
+
+DBNAME="test"
+NTABLES=$(wc -l ${table_list} | awk '{print $1;}')
+NTABLES_PER_DUMP=101
+
+mkdir -p ${data_dir}
+i=0
+round=0
+command="${taosdump1_6} -c ${taoscfg1_6} -o ${data_dir} -N 100 -T 20 ${DBNAME}"
+while IFS= read -r line
+do
+ i=$((i+1))
+
+ command="${command} ${line}"
+
+ if [[ "$i" -eq ${NTABLES_PER_DUMP} ]]; then
+ round=$((round+1))
+ echo "Starting round ${round} dump out..."
+ rm -f ${data_dir}/*
+ ${command}
+ echo "Starting round ${round} dump in..."
+ ${taosdump2_0} -c ${taoscfg2_0} -i ${data_dir}
+
+ # Reset variables
+ # command="${taosdump1_6} -c ${taoscfg1_6} -o ${data_dir} -N 100 ${DBNAME}"
+ command="${taosdump1_6} -c ${taoscfg1_6} -o ${data_dir} -N 100 -T 20 ${DBNAME}"
+ i=0
+ fi
+done < "${table_list}"
+
+if [[ ${i} -ne "0" ]]; then
+ round=$((round+1))
+ echo "Starting round ${round} dump out..."
+ rm -f ${data_dir}/*
+ ${command}
+ echo "Starting round ${round} dump in..."
+ ${taosdump2_0} -c ${taoscfg2_0} -i ${data_dir}
+fi
diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h
index 4bc840f026d41dc4fe4e8d61c84d53140501bff6..6d3061c4269f54f0075a07ee75115f43e856479b 100644
--- a/src/mnode/inc/mnodeDef.h
+++ b/src/mnode/inc/mnodeDef.h
@@ -89,7 +89,7 @@ typedef struct STableObj {
int8_t type;
} STableObj;
-typedef struct SSuperTableObj {
+typedef struct SSTableObj {
STableObj info;
int8_t reserved0[9]; // for fill struct STableObj to 4byte align
int16_t nextColId;
@@ -104,7 +104,7 @@ typedef struct SSuperTableObj {
int32_t numOfTables;
SSchema * schema;
void * vgHash;
-} SSuperTableObj;
+} SSTableObj;
typedef struct {
STableObj info;
@@ -122,8 +122,8 @@ typedef struct {
int32_t refCount;
char* sql; //used by normal table
SSchema* schema; //used by normal table
- SSuperTableObj *superTable;
-} SChildTableObj;
+ SSTableObj*superTable;
+} SCTableObj;
typedef struct {
int32_t dnodeId;
@@ -172,7 +172,8 @@ typedef struct {
int8_t walLevel;
int8_t replications;
int8_t quorum;
- int8_t reserved[12];
+ int8_t update;
+ int8_t reserved[11];
} SDbCfg;
typedef struct SDbObj {
diff --git a/src/mnode/inc/mnodeProfile.h b/src/mnode/inc/mnodeProfile.h
index 1e5b1c0f9c7c1c220a88339f00d46538c78cf938..ee57c5da1f66b620e8624a74ba774643c589e818 100644
--- a/src/mnode/inc/mnodeProfile.h
+++ b/src/mnode/inc/mnodeProfile.h
@@ -45,7 +45,7 @@ void mnodeCleanupProfile();
SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port, int32_t pid, const char* app);
SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t port);
void mnodeReleaseConn(SConnObj *pConn);
-int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg);
+int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg);
#ifdef __cplusplus
}
diff --git a/src/mnode/inc/mnodeTable.h b/src/mnode/inc/mnodeTable.h
index ed0dbe4ecfc4b344052c70487c3b740d81eaa0f3..7c0077aa609843aa99e9f5114573aea8f7446fc1 100644
--- a/src/mnode/inc/mnodeTable.h
+++ b/src/mnode/inc/mnodeTable.h
@@ -29,8 +29,8 @@ int64_t mnodeGetChildTableNum();
void * mnodeGetTable(char *tableId);
void mnodeIncTableRef(void *pTable);
void mnodeDecTableRef(void *pTable);
-void * mnodeGetNextChildTable(void *pIter, SChildTableObj **pTable);
-void * mnodeGetNextSuperTable(void *pIter, SSuperTableObj **pTable);
+void * mnodeGetNextChildTable(void *pIter, SCTableObj **pTable);
+void * mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable);
void mnodeDropAllChildTables(SDbObj *pDropDb);
void mnodeDropAllSuperTables(SDbObj *pDropDb);
void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup);
diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h
index 7aa662b81cc08c90d6453ff5c73191ad81416073..0e6d9dfde463dc50810654921b2006fda212e496 100644
--- a/src/mnode/inc/mnodeVgroup.h
+++ b/src/mnode/inc/mnodeVgroup.h
@@ -43,8 +43,8 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle);
void mnodeAlterVgroup(SVgObj *pVgroup, void *ahandle);
int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_t *sid);
-void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
-void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
+void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable);
+void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable);
void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle);
void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
void mnodeSendAlterVgroupMsg(SVgObj *pVgroup);
diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c
index e161940a2b1a07572d9beff5222b8eb8d3829075..365cf656de283a53899ccd866682e8d0a06caaf8 100644
--- a/src/mnode/src/mnodeAcct.c
+++ b/src/mnode/src/mnodeAcct.c
@@ -34,7 +34,7 @@ static int32_t mnodeCreateRootAcct();
static int32_t mnodeAcctActionDestroy(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
pthread_mutex_destroy(&pAcct->mutex);
- taosTFree(pOper->pObj);
+ tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c
index 35b6a67ab24d5ee50a4f4da7bb902ab0d7232b7e..98587cf53d673b847a321ded9c369e2085195081 100644
--- a/src/mnode/src/mnodeCluster.c
+++ b/src/mnode/src/mnodeCluster.c
@@ -33,7 +33,7 @@ static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *
static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mnodeClusterActionDestroy(SSdbOper *pOper) {
- taosTFree(pOper->pObj);
+ tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c
index 8d7c267ab7cb328bb187518a88be3d344b0d53e2..3b3a21aeb18f08d9af20a17ee78dcba124bd00d7 100644
--- a/src/mnode/src/mnodeDb.c
+++ b/src/mnode/src/mnodeDb.c
@@ -41,7 +41,7 @@
static void * tsDbSdb = NULL;
static int32_t tsDbUpdateSize;
-static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg);
+static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *pMsg);
static int32_t mnodeDropDb(SMnodeMsg *newMsg);
static int32_t mnodeSetDbDropping(SDbObj *pDb);
static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
@@ -52,8 +52,8 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg);
static void mnodeDestroyDb(SDbObj *pDb) {
pthread_mutex_destroy(&pDb->mutex);
- taosTFree(pDb->vgList);
- taosTFree(pDb);
+ tfree(pDb->vgList);
+ tfree(pDb);
}
static int32_t mnodeDbActionDestroy(SSdbOper *pOper) {
@@ -319,6 +319,11 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
}
#endif
+ if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) {
+ mError("invalid db option update:%d valid range: [%d, %d]", pCfg->update, TSDB_MIN_DB_UPDATE, TSDB_MAX_DB_UPDATE);
+ return TSDB_CODE_MND_INVALID_DB_OPTION;
+ }
+
return TSDB_CODE_SUCCESS;
}
@@ -339,6 +344,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL;
if (pCfg->replications < 0) pCfg->replications = tsReplications;
if (pCfg->quorum < 0) pCfg->quorum = tsQuorum;
+ if (pCfg->update < 0) pCfg->update = tsUpdate;
}
static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
@@ -352,7 +358,7 @@ static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
return code;
}
-static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg) {
+static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *pMsg) {
int32_t code = acctCheck(pAcct, ACCT_GRANT_DB);
if (code != 0) return code;
@@ -391,14 +397,15 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg
.compression = pCreate->compression,
.walLevel = pCreate->walLevel,
.replications = pCreate->replications,
- .quorum = pCreate->quorum
+ .quorum = pCreate->quorum,
+ .update = pCreate->update
};
mnodeSetDefaultDbCfg(&pDb->cfg);
code = mnodeCheckDbCfg(&pDb->cfg);
if (code != TSDB_CODE_SUCCESS) {
- taosTFree(pDb);
+ tfree(pDb);
return code;
}
@@ -610,6 +617,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
+ pShow->bytes[cols] = 1;
+ pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
+ strcpy(pSchema[cols].name, "update");
+ pSchema[cols].bytes = htons(pShow->bytes[cols]);
+ cols++;
+
pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "status");
@@ -749,6 +762,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2);
cols++;
+ pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+ *(int8_t *)pWrite = pDb->cfg.update;
+ cols++;
+
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
if (pDb->status == TSDB_DB_STATUS_READY) {
const char *src = "ready";
@@ -805,7 +822,7 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb) {
}
static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) {
- SCMCreateDbMsg *pCreate = pMsg->rpcMsg.pCont;
+ SCreateDbMsg *pCreate = pMsg->rpcMsg.pCont;
pCreate->maxTables = htonl(pCreate->maxTables);
pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize);
pCreate->totalBlocks = htonl(pCreate->totalBlocks);
@@ -830,7 +847,7 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) {
return code;
}
-static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
+static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
SDbCfg newCfg = pDb->cfg;
int32_t maxTables = htonl(pAlter->maxTables);
int32_t cacheBlockSize = htonl(pAlter->cacheBlockSize);
@@ -848,6 +865,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
int8_t replications = pAlter->replications;
int8_t quorum = pAlter->quorum;
int8_t precision = pAlter->precision;
+ int8_t update = pAlter->update;
terrno = TSDB_CODE_SUCCESS;
@@ -950,6 +968,16 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
newCfg.quorum = quorum;
}
+ if (update >= 0 && update != pDb->cfg.update) {
+#if 0
+ mDebug("db:%s, update:%d change to %d", pDb->name, pDb->cfg.update, update);
+ newCfg.update = update;
+#else
+ mError("db:%s, can't alter update option", pDb->name);
+ terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
+#endif
+ }
+
return newCfg;
}
@@ -977,7 +1005,7 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) {
+static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) {
SDbCfg newCfg = mnodeGetAlterDbOption(pDb, pAlter);
if (terrno != TSDB_CODE_SUCCESS) {
return terrno;
@@ -1009,7 +1037,7 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) {
}
static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) {
- SCMAlterDbMsg *pAlter = pMsg->rpcMsg.pCont;
+ SAlterDbMsg *pAlter = pMsg->rpcMsg.pCont;
mDebug("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->rpcMsg.handle);
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pAlter->db);
@@ -1060,7 +1088,7 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) {
}
static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) {
- SCMDropDbMsg *pDrop = pMsg->rpcMsg.pCont;
+ SDropDbMsg *pDrop = pMsg->rpcMsg.pCont;
mDebug("db:%s, drop db msg is received from thandle:%p", pDrop->db, pMsg->rpcMsg.handle);
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pDrop->db);
diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c
index 1cd861e22303d5f6f0dcb75c7ef83cf66696578f..7e34e0837389e8a288ba70a2b18d4627f91219ae 100644
--- a/src/mnode/src/mnodeDnode.c
+++ b/src/mnode/src/mnodeDnode.c
@@ -39,11 +39,15 @@
#include "mnodeCluster.h"
int32_t tsAccessSquence = 0;
-static void *tsDnodeSdb = NULL;
+static void * tsDnodeSdb = NULL;
static int32_t tsDnodeUpdateSize = 0;
extern void * tsMnodeSdb;
extern void * tsVgroupSdb;
+static SDnodeEps*tsDnodeEps;
+static int32_t tsDnodeEpsSize;
+static pthread_mutex_t tsDnodeEpsMutex;
+
static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg);
static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg);
static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg);
@@ -59,6 +63,7 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo
static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole);
+static void mnodeUpdateDnodeEps();
static char* offlineReason[] = {
"",
@@ -83,7 +88,7 @@ static char* offlineReason[] = {
};
static int32_t mnodeDnodeActionDestroy(SSdbOper *pOper) {
- taosTFree(pOper->pObj);
+ tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
@@ -95,6 +100,9 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) {
pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED;
}
+ dnodeUpdateEp(pDnode->dnodeId, pDnode->dnodeEp, pDnode->dnodeFqdn, &pDnode->dnodePort);
+ mnodeUpdateDnodeEps();
+
mInfo("dnode:%d, fqdn:%s ep:%s port:%d, do insert action", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort);
return TSDB_CODE_SUCCESS;
}
@@ -107,6 +115,7 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
#endif
mnodeDropMnodeLocal(pDnode->dnodeId);
balanceAsyncNotify();
+ mnodeUpdateDnodeEps();
mDebug("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId);
return TSDB_CODE_SUCCESS;
@@ -121,6 +130,7 @@ static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) {
}
mnodeDecDnodeRef(pDnode);
+ mnodeUpdateDnodeEps();
return TSDB_CODE_SUCCESS;
}
@@ -152,12 +162,14 @@ static int32_t mnodeDnodeActionRestored() {
}
}
+ mnodeUpdateDnodeEps();
return TSDB_CODE_SUCCESS;
}
int32_t mnodeInitDnodes() {
SDnodeObj tObj;
tsDnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
+ pthread_mutex_init(&tsDnodeEpsMutex, NULL);
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_DNODE,
@@ -201,6 +213,9 @@ int32_t mnodeInitDnodes() {
void mnodeCleanupDnodes() {
sdbCloseTable(tsDnodeSdb);
+ pthread_mutex_destroy(&tsDnodeEpsMutex);
+ free(tsDnodeEps);
+ tsDnodeEps = NULL;
tsDnodeSdb = NULL;
}
@@ -299,7 +314,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_NO_RIGHTS;
}
- SCMCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont;
+ SCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont;
if (pCmCfgDnode->ep[0] == 0) {
tstrncpy(pCmCfgDnode->ep, tsLocalEp, TSDB_EP_LEN);
}
@@ -334,7 +349,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
mnodeDecDnodeRef(pDnode);
return code;
} else {
- SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg));
+ SCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SCfgDnodeMsg));
strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep);
strcpy(pMdCfgDnode->config, pCmCfgDnode->config);
@@ -343,7 +358,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
.code = 0,
.msgType = TSDB_MSG_TYPE_MD_CONFIG_DNODE,
.pCont = pMdCfgDnode,
- .contLen = sizeof(SMDCfgDnodeMsg)
+ .contLen = sizeof(SCfgDnodeMsg)
};
mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user);
@@ -418,9 +433,54 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) {
return 0;
}
+static int32_t mnodeGetDnodeEpsSize() {
+ pthread_mutex_lock(&tsDnodeEpsMutex);
+ int32_t size = tsDnodeEpsSize;
+ pthread_mutex_unlock(&tsDnodeEpsMutex);
+ return size;
+}
+
+static void mnodeGetDnodeEpsData(SDnodeEps *pEps, int32_t epsSize) {
+ pthread_mutex_lock(&tsDnodeEpsMutex);
+ if (epsSize == tsDnodeEpsSize) {
+ memcpy(pEps, tsDnodeEps, tsDnodeEpsSize);
+ }
+ pthread_mutex_unlock(&tsDnodeEpsMutex);
+}
+
+static void mnodeUpdateDnodeEps() {
+ pthread_mutex_lock(&tsDnodeEpsMutex);
+
+ int32_t totalDnodes = mnodeGetDnodesNum();
+ tsDnodeEpsSize = sizeof(SDnodeEps) + totalDnodes * sizeof(SDnodeEp);
+ free(tsDnodeEps);
+ tsDnodeEps = calloc(1, tsDnodeEpsSize);
+ tsDnodeEps->dnodeNum = htonl(totalDnodes);
+
+ SDnodeObj *pDnode = NULL;
+ void * pIter = NULL;
+ int32_t dnodesNum = 0;
+
+ while (1) {
+ pIter = mnodeGetNextDnode(pIter, &pDnode);
+ if (pDnode == NULL) break;
+ if (dnodesNum >= totalDnodes) break;
+
+ SDnodeEp *pEp = &tsDnodeEps->dnodeEps[dnodesNum];
+ dnodesNum++;
+ pEp->dnodeId = htonl(pDnode->dnodeId);
+ pEp->dnodePort = htons(pDnode->dnodePort);
+ tstrncpy(pEp->dnodeFqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN);
+ mnodeDecDnodeRef(pDnode);
+ }
+
+ sdbFreeIter(pIter);
+ pthread_mutex_unlock(&tsDnodeEpsMutex);
+}
+
static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
SDnodeObj *pDnode = NULL;
- SDMStatusMsg *pStatus = pMsg->rpcMsg.pCont;
+ SStatusMsg *pStatus = pMsg->rpcMsg.pCont;
pStatus->dnodeId = htonl(pStatus->dnodeId);
pStatus->moduleStatus = htonl(pStatus->moduleStatus);
pStatus->lastReboot = htonl(pStatus->lastReboot);
@@ -477,8 +537,11 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
}
int32_t openVnodes = htons(pStatus->openVnodes);
- int32_t contLen = sizeof(SDMStatusRsp) + openVnodes * sizeof(SDMVgroupAccess);
- SDMStatusRsp *pRsp = rpcMallocCont(contLen);
+ int32_t epsSize = mnodeGetDnodeEpsSize();
+ int32_t vgAccessSize = openVnodes * sizeof(SVgroupAccess);
+ int32_t contLen = sizeof(SStatusRsp) + vgAccessSize + epsSize;
+
+ SStatusRsp *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) {
mnodeDecDnodeRef(pDnode);
return TSDB_CODE_MND_OUT_OF_MEMORY;
@@ -488,8 +551,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt);
pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes);
tstrncpy(pRsp->dnodeCfg.clusterId, mnodeGetClusterId(), TSDB_CLUSTER_ID_LEN);
- SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp));
-
+ SVgroupAccess *pAccess = (SVgroupAccess *)((char *)pRsp + sizeof(SStatusRsp));
+
for (int32_t j = 0; j < openVnodes; ++j) {
SVnodeLoad *pVload = &pStatus->load[j];
pVload->vgId = htonl(pVload->vgId);
@@ -521,7 +584,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT;
}
- mDebug("dnode:%d, from offline to online", pDnode->dnodeId);
+ mInfo("dnode:%d, from offline to online", pDnode->dnodeId);
pDnode->status = TAOS_DN_STATUS_READY;
pDnode->offlineReason = TAOS_DN_OFF_ONLINE;
balanceSyncNotify();
@@ -539,6 +602,9 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
mnodeDecDnodeRef(pDnode);
+ SDnodeEps *pEps = (SDnodeEps *)((char *)pRsp + sizeof(SStatusRsp) + vgAccessSize);
+ mnodeGetDnodeEpsData(pEps, epsSize);
+
pMsg->rpcRsp.len = contLen;
pMsg->rpcRsp.rsp = pRsp;
@@ -589,7 +655,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) {
int32_t code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
int dnodeId = pDnode->dnodeId;
- taosTFree(pDnode);
+ tfree(pDnode);
mError("failed to create dnode:%d, reason:%s", dnodeId, tstrerror(code));
} else {
mLInfo("dnode:%d is created", pDnode->dnodeId);
@@ -645,7 +711,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
}
static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) {
- SCMCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont;
+ SCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont;
if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
return TSDB_CODE_MND_NO_RIGHTS;
@@ -655,7 +721,7 @@ static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) {
}
static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg) {
- SCMDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont;
+ SDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont;
if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
return TSDB_CODE_MND_NO_RIGHTS;
diff --git a/src/mnode/src/mnodeInt.c b/src/mnode/src/mnodeInt.c
index fb1b8741a9f8895c579ad0e549dc8f3c15f7618c..8431baf4f455a2c577321ec18ff3bfcd7fbe5d46 100644
--- a/src/mnode/src/mnodeInt.c
+++ b/src/mnode/src/mnodeInt.c
@@ -18,7 +18,7 @@
#include "taosmsg.h"
#include "taoserror.h"
#include "trpc.h"
-#include "tcache.h"
+#include "tqueue.h"
#include "mnode.h"
#include "dnode.h"
#include "mnodeDef.h"
@@ -34,8 +34,16 @@
#include "mnodeUser.h"
#include "mnodeVgroup.h"
-void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg) {
- pMsg->rpcMsg = *rpcMsg;
+void *mnodeCreateMsg(SRpcMsg *pRpcMsg) {
+ int32_t size = sizeof(SMnodeMsg) + pRpcMsg->contLen;
+ SMnodeMsg *pMsg = taosAllocateQitem(size);
+
+ pMsg->rpcMsg = *pRpcMsg;
+ pMsg->rpcMsg.pCont = pMsg->pCont;
+ pMsg->incomingTs = taosGetTimestampSec();
+ memcpy(pMsg->pCont, pRpcMsg->pCont, pRpcMsg->contLen);
+
+ return pMsg;
}
int32_t mnodeInitMsg(SMnodeMsg *pMsg) {
@@ -54,7 +62,9 @@ int32_t mnodeInitMsg(SMnodeMsg *pMsg) {
void mnodeCleanupMsg(SMnodeMsg *pMsg) {
if (pMsg != NULL) {
- if (pMsg->rpcMsg.pCont) rpcFreeCont(pMsg->rpcMsg.pCont);
+ if (pMsg->rpcMsg.pCont != pMsg->pCont) {
+ tfree(pMsg->rpcMsg.pCont);
+ }
if (pMsg->pUser) mnodeDecUserRef(pMsg->pUser);
if (pMsg->pDb) mnodeDecDbRef(pMsg->pDb);
if (pMsg->pVgroup) mnodeDecVgroupRef(pMsg->pVgroup);
diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c
index 2bb8a810566a676a4ae991bc79e5eb78234747c7..1f5ad42bdead75064c60f8626ce4916e7b08fd9f 100644
--- a/src/mnode/src/mnodeMain.c
+++ b/src/mnode/src/mnodeMain.c
@@ -96,9 +96,9 @@ int32_t mnodeStartSystem() {
return -1;
}
- dnodeAllocateMnodeWqueue();
- dnodeAllocateMnodeRqueue();
- dnodeAllocateMnodePqueue();
+ dnodeAllocMWritequeue();
+ dnodeAllocMReadQueue();
+ dnodeAllocateMPeerQueue();
if (mnodeInitComponents() != 0) {
return -1;
@@ -127,9 +127,9 @@ void mnodeCleanupSystem() {
mInfo("starting to clean up mnode");
tsMgmtIsRunning = false;
- dnodeFreeMnodeWqueue();
- dnodeFreeMnodeRqueue();
- dnodeFreeMnodePqueue();
+ dnodeFreeMWritequeue();
+ dnodeFreeMReadQueue();
+ dnodeFreeMPeerQueue();
mnodeCleanupTimer();
mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1);
diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c
index 89b2f50b731c90c1261957da93623ba16cebf2ab..092f246c139533f806ce1d311162e0516088d879 100644
--- a/src/mnode/src/mnodeMnode.c
+++ b/src/mnode/src/mnodeMnode.c
@@ -38,7 +38,7 @@ static void * tsMnodeSdb = NULL;
static int32_t tsMnodeUpdateSize = 0;
static SRpcEpSet tsMnodeEpSetForShell;
static SRpcEpSet tsMnodeEpSetForPeer;
-static SDMMnodeInfos tsMnodeInfos;
+static SMnodeInfos tsMnodeInfos;
static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
@@ -59,7 +59,7 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo
#endif
static int32_t mnodeMnodeActionDestroy(SSdbOper *pOper) {
- taosTFree(pOper->pObj);
+ tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
@@ -70,8 +70,9 @@ static int32_t mnodeMnodeActionInsert(SSdbOper *pOper) {
pDnode->isMgmt = true;
mnodeDecDnodeRef(pDnode);
-
- mInfo("mnode:%d, fqdn:%s ep:%s port:%d, do insert action", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort);
+
+ mInfo("mnode:%d, fqdn:%s ep:%s port:%u, do insert action", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp,
+ pDnode->dnodePort);
return TSDB_CODE_SUCCESS;
}
@@ -202,7 +203,7 @@ void mnodeUpdateMnodeEpSet() {
memset(&tsMnodeEpSetForShell, 0, sizeof(SRpcEpSet));
memset(&tsMnodeEpSetForPeer, 0, sizeof(SRpcEpSet));
- memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos));
+ memset(&tsMnodeInfos, 0, sizeof(SMnodeInfos));
int32_t index = 0;
void * pIter = NULL;
@@ -221,8 +222,8 @@ void mnodeUpdateMnodeEpSet() {
tsMnodeEpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE);
mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForPeer.fqdn[index], htons(tsMnodeEpSetForPeer.port[index]));
- tsMnodeInfos.nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
- strcpy(tsMnodeInfos.nodeInfos[index].nodeEp, pDnode->dnodeEp);
+ tsMnodeInfos.mnodeInfos[index].mnodeId = htonl(pMnode->mnodeId);
+ strcpy(tsMnodeInfos.mnodeInfos[index].mnodeEp, pDnode->dnodeEp);
if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
tsMnodeEpSetForShell.inUse = index;
@@ -238,7 +239,7 @@ void mnodeUpdateMnodeEpSet() {
mnodeDecMnodeRef(pMnode);
}
- tsMnodeInfos.nodeNum = index;
+ tsMnodeInfos.mnodeNum = index;
tsMnodeEpSetForShell.numOfEps = index;
tsMnodeEpSetForPeer.numOfEps = index;
@@ -260,19 +261,19 @@ void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) {
}
char* mnodeGetMnodeMasterEp() {
- return tsMnodeInfos.nodeInfos[tsMnodeInfos.inUse].nodeEp;
+ return tsMnodeInfos.mnodeInfos[tsMnodeInfos.inUse].mnodeEp;
}
void mnodeGetMnodeInfos(void *mnodeInfos) {
mnodeMnodeRdLock();
- *(SDMMnodeInfos *)mnodeInfos = tsMnodeInfos;
+ *(SMnodeInfos *)mnodeInfos = tsMnodeInfos;
mnodeMnodeUnLock();
}
static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) {
mDebug("dnode:%d, send create mnode msg to dnode %s", dnodeId, dnodeEp);
- SMDCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SMDCreateMnodeMsg));
+ SCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SCreateMnodeMsg));
if (pCreate == NULL) {
return TSDB_CODE_MND_OUT_OF_MEMORY;
} else {
@@ -280,21 +281,21 @@ static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) {
tstrncpy(pCreate->dnodeEp, dnodeEp, sizeof(pCreate->dnodeEp));
pCreate->mnodes = tsMnodeInfos;
bool found = false;
- for (int i = 0; i < pCreate->mnodes.nodeNum; ++i) {
- if (pCreate->mnodes.nodeInfos[i].nodeId == htonl(dnodeId)) {
+ for (int i = 0; i < pCreate->mnodes.mnodeNum; ++i) {
+ if (pCreate->mnodes.mnodeInfos[i].mnodeId == htonl(dnodeId)) {
found = true;
}
}
if (!found) {
- pCreate->mnodes.nodeInfos[pCreate->mnodes.nodeNum].nodeId = htonl(dnodeId);
- tstrncpy(pCreate->mnodes.nodeInfos[pCreate->mnodes.nodeNum].nodeEp, dnodeEp, sizeof(pCreate->dnodeEp));
- pCreate->mnodes.nodeNum++;
+ pCreate->mnodes.mnodeInfos[pCreate->mnodes.mnodeNum].mnodeId = htonl(dnodeId);
+ tstrncpy(pCreate->mnodes.mnodeInfos[pCreate->mnodes.mnodeNum].mnodeEp, dnodeEp, sizeof(pCreate->dnodeEp));
+ pCreate->mnodes.mnodeNum++;
}
}
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pCreate;
- rpcMsg.contLen = sizeof(SMDCreateMnodeMsg);
+ rpcMsg.contLen = sizeof(SCreateMnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_MD_CREATE_MNODE;
SRpcMsg rpcRsp = {0};
@@ -341,14 +342,14 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) {
}
if (code != TSDB_CODE_SUCCESS) {
- taosTFree(pMnode);
+ tfree(pMnode);
return;
}
code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("dnode:%d, failed to create mnode, ep:%s reason:%s", dnodeId, dnodeEp, tstrerror(code));
- taosTFree(pMnode);
+ tfree(pMnode);
}
}
diff --git a/src/mnode/src/mnodePeer.c b/src/mnode/src/mnodePeer.c
index e95018ee0b43179c2129271a57ba3e6b7de74dde..c7957db7a798e3ec974844c291488a5a2f06c3da 100644
--- a/src/mnode/src/mnodePeer.c
+++ b/src/mnode/src/mnodePeer.c
@@ -62,9 +62,9 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) {
for (int32_t i = 0; i < epSet->numOfEps; ++i) {
if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) {
epSet->inUse = (i + 1) % epSet->numOfEps;
- mDebug("mnode index:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
+ mDebug("mpeer:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
} else {
- mDebug("mnode index:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
+ mDebug("mpeer:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
}
}
diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c
index c29d1ec0b7c863ec86d78f28997d40ba86b4dd34..12ac64854c6834ca0cd1509f9bfedd7b65a0900c 100644
--- a/src/mnode/src/mnodeProfile.c
+++ b/src/mnode/src/mnodeProfile.c
@@ -131,8 +131,8 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po
static void mnodeFreeConn(void *data) {
SConnObj *pConn = data;
- taosTFree(pConn->pQueries);
- taosTFree(pConn->pStreams);
+ tfree(pConn->pQueries);
+ tfree(pConn->pStreams);
mDebug("connId:%d, is destroyed", pConn->connId);
}
@@ -280,7 +280,7 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi
}
// not thread safe, need optimized
-int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg) {
+int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg) {
pConn->numOfQueries = htonl(pHBMsg->numOfQueries);
if (pConn->numOfQueries > 0) {
if (pConn->pQueries == NULL) {
@@ -561,7 +561,7 @@ static int32_t mnodeProcessKillQueryMsg(SMnodeMsg *pMsg) {
SUserObj *pUser = pMsg->pUser;
if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
- SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
+ SKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
mInfo("kill query msg is received, queryId:%s", pKill->queryId);
const char delim = ':';
@@ -592,7 +592,7 @@ static int32_t mnodeProcessKillStreamMsg(SMnodeMsg *pMsg) {
SUserObj *pUser = pMsg->pUser;
if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
- SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
+ SKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
mInfo("kill stream msg is received, streamId:%s", pKill->queryId);
const char delim = ':';
@@ -623,7 +623,7 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg) {
SUserObj *pUser = pMsg->pUser;
if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
- SCMKillConnMsg *pKill = pMsg->rpcMsg.pCont;
+ SKillConnMsg *pKill = pMsg->rpcMsg.pCont;
int32_t connId = atoi(pKill->queryId);
SConnObj *pConn = taosCacheAcquireByKey(tsMnodeConnCache, &connId, sizeof(int32_t));
if (pConn == NULL) {
diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c
index 14558485aa743a42bee016226dfc2e16a4f81d0c..003ecd0d24498d51d5e82d041740aa6f0a180280 100644
--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@@ -71,8 +71,8 @@ typedef struct _SSdbTable {
typedef struct {
ESyncRole role;
ESdbStatus status;
- int64_t version;
- void * sync;
+ uint64_t version;
+ int64_t sync;
void * wal;
SSyncCfg cfg;
int32_t numOfTables;
@@ -98,8 +98,8 @@ static taos_qall tsSdbWriteQall;
static taos_queue tsSdbWriteQueue;
static SSdbWriteWorkerPool tsSdbPool;
-static int sdbWrite(void *param, void *data, int type);
-static int sdbWriteToQueue(void *param, void *data, int type);
+static int32_t sdbWrite(void *param, void *data, int32_t type, void *pMsg);
+static int32_t sdbWriteToQueue(void *param, void *data, int32_t type, void *pMsg);
static void * sdbWorkerFp(void *param);
static int32_t sdbInitWriteWorker();
static void sdbCleanupWriteWorker();
@@ -175,7 +175,7 @@ static void *sdbGetTableFromId(int32_t tableId) {
}
static int32_t sdbInitWal() {
- SWalCfg walCfg = {.walLevel = 2, .wals = 2, .keep = 1, .fsyncPeriod = 0};
+ SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0};
char temp[TSDB_FILENAME_LEN];
sprintf(temp, "%s/wal", tsMnodeDir);
tsSdbObj.wal = walOpen(temp, &walCfg);
@@ -212,7 +212,7 @@ static void sdbRestoreTables() {
}
void sdbUpdateMnodeRoles() {
- if (tsSdbObj.sync == NULL) return;
+ if (tsSdbObj.sync <= 0) return;
SNodesRole roles = {0};
syncGetNodesRole(tsSdbObj.sync, &roles);
@@ -237,8 +237,8 @@ static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint3
return 0;
}
-static int sdbGetWalInfo(void *ahandle, char *name, uint32_t *index) {
- return walGetWalFile(tsSdbObj.wal, name, index);
+static int32_t sdbGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) {
+ return walGetWalFile(tsSdbObj.wal, fileName, fileId);
}
static void sdbNotifyRole(void *ahandle, int8_t role) {
@@ -281,14 +281,21 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
((SSdbTable *)pOper->table)->tableName, pOper->pObj, sdbGetKeyStr(pOper->table, pHead->cont),
pHead->version, action, tstrerror(pOper->retCode));
if (action == SDB_ACTION_INSERT) {
- sdbDeleteHash(pOper->table, pOper);
+ // It's better to create a table in two stages, create it first and then set it success
+ //sdbDeleteHash(pOper->table, pOper);
+ SSdbOper oper = {
+ .type = SDB_OPER_GLOBAL,
+ .table = pOper->table,
+ .pObj = pOper->pObj
+ };
+ sdbDeleteRow(&oper);
}
}
if (pOper->writeCb != NULL) {
pOper->retCode = (*pOper->writeCb)(pMsg, pOper->retCode);
}
- dnodeSendRpcMnodeWriteRsp(pMsg, pOper->retCode);
+ dnodeSendRpcMWriteRsp(pMsg, pOper->retCode);
// if ahandle, means this func is called by sdb write
if (ahandle == NULL) {
@@ -305,7 +312,7 @@ void sdbUpdateAsync() {
}
void sdbUpdateSync(void *pMnodes) {
- SDMMnodeInfos *mnodes = pMnodes;
+ SMnodeInfos *mnodes = pMnodes;
if (!mnodeIsRunning()) {
mDebug("mnode not start yet, update sync config later");
return;
@@ -339,10 +346,10 @@ void sdbUpdateSync(void *pMnodes) {
syncCfg.replica = index;
mDebug("mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica);
} else {
- for (index = 0; index < mnodes->nodeNum; ++index) {
- SDMMnodeInfo *node = &mnodes->nodeInfos[index];
- syncCfg.nodeInfo[index].nodeId = node->nodeId;
- taosGetFqdnPortFromEp(node->nodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort);
+ for (index = 0; index < mnodes->mnodeNum; ++index) {
+ SMnodeInfo *node = &mnodes->mnodeInfos[index];
+ syncCfg.nodeInfo[index].nodeId = node->mnodeId;
+ taosGetFqdnPortFromEp(node->mnodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort);
syncCfg.nodeInfo[index].nodePort += TSDB_PORT_SYNC;
}
syncCfg.replica = index;
@@ -426,7 +433,7 @@ void sdbCleanUp() {
if (tsSdbObj.sync) {
syncStop(tsSdbObj.sync);
- tsSdbObj.sync = NULL;
+ tsSdbObj.sync = -1;
}
if (tsSdbObj.wal) {
@@ -568,7 +575,7 @@ static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int sdbWrite(void *param, void *data, int type) {
+static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) {
SSdbOper *pOper = param;
SWalHead *pHead = data;
int32_t tableId = pHead->msgType / 10;
@@ -977,7 +984,7 @@ void sdbCleanupWriteWorker() {
}
sdbFreeWritequeue();
- taosTFree(tsSdbPool.writeWorker);
+ tfree(tsSdbPool.writeWorker);
mInfo("sdb write is closed");
}
@@ -1033,13 +1040,13 @@ void sdbFreeWritequeue() {
tsSdbWriteQueue = NULL;
}
-int sdbWriteToQueue(void *param, void *data, int type) {
+int32_t sdbWriteToQueue(void *param, void *data, int32_t qtype, void *pMsg) {
SWalHead *pHead = data;
- int size = sizeof(SWalHead) + pHead->len;
- SWalHead *pWal = (SWalHead *)taosAllocateQitem(size);
+ int32_t size = sizeof(SWalHead) + pHead->len;
+ SWalHead *pWal = taosAllocateQitem(size);
memcpy(pWal, pHead, size);
- taosWriteQitem(tsSdbWriteQueue, type, pWal);
+ taosWriteQitem(tsSdbWriteQueue, qtype, pWal);
return 0;
}
@@ -1074,7 +1081,7 @@ static void *sdbWorkerFp(void *param) {
pOper = NULL;
}
- int32_t code = sdbWrite(pOper, pHead, type);
+ int32_t code = sdbWrite(pOper, pHead, type, NULL);
if (code > 0) code = 0;
if (pOper) {
pOper->retCode = code;
@@ -1083,7 +1090,7 @@ static void *sdbWorkerFp(void *param) {
}
}
- walFsync(tsSdbObj.wal);
+ walFsync(tsSdbObj.wal, true);
// browse all items, and process them one by one
taosResetQitems(tsSdbWriteQall);
diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c
index 80909e99aec6d752d35042ca2d761a6e8b923441..0d53fa9617b500d61457889e2228e5e756a244ae 100644
--- a/src/mnode/src/mnodeShow.c
+++ b/src/mnode/src/mnodeShow.c
@@ -110,7 +110,7 @@ static char *mnodeGetShowType(int32_t showType) {
}
static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
- SCMShowMsg *pShowMsg = pMsg->rpcMsg.pCont;
+ SShowMsg *pShowMsg = pMsg->rpcMsg.pCont;
if (pShowMsg->type >= TSDB_MGMT_TABLE_MAX) {
return TSDB_CODE_MND_INVALID_MSG_TYPE;
}
@@ -132,8 +132,8 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_OUT_OF_MEMORY;
}
- int32_t size = sizeof(SCMShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE;
- SCMShowRsp *pShowRsp = rpcMallocCont(size);
+ int32_t size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE;
+ SShowRsp *pShowRsp = rpcMallocCont(size);
if (pShowRsp == NULL) {
mnodeReleaseShowObj(pShow, true);
return TSDB_CODE_MND_OUT_OF_MEMORY;
@@ -146,7 +146,7 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
if (code == TSDB_CODE_SUCCESS) {
pMsg->rpcRsp.rsp = pShowRsp;
- pMsg->rpcRsp.len = sizeof(SCMShowRsp) + sizeof(SSchema) * pShow->numOfColumns;
+ pMsg->rpcRsp.len = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns;
mnodeReleaseShowObj(pShow, false);
return TSDB_CODE_SUCCESS;
} else {
@@ -232,12 +232,17 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
}
static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) {
- SCMHeartBeatRsp *pHBRsp = (SCMHeartBeatRsp *) rpcMallocCont(sizeof(SCMHeartBeatRsp));
- if (pHBRsp == NULL) {
+ SHeartBeatRsp *pRsp = (SHeartBeatRsp *)rpcMallocCont(sizeof(SHeartBeatRsp));
+ if (pRsp == NULL) {
return TSDB_CODE_MND_OUT_OF_MEMORY;
}
- SCMHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont;
+ SHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont;
+ if (taosCheckVersion(pHBMsg->clientVer, version, 3) != TSDB_CODE_SUCCESS) {
+ rpcFreeCont(pRsp);
+ return TSDB_CODE_TSC_INVALID_VERSION; // todo change the error code
+ }
+
SRpcConnInfo connInfo = {0};
rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo);
@@ -251,40 +256,40 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) {
if (pConn == NULL) {
// do not close existing links, otherwise
// mError("failed to create connId, close connect");
- // pHBRsp->killConnection = 1;
+ // pRsp->killConnection = 1;
} else {
- pHBRsp->connId = htonl(pConn->connId);
+ pRsp->connId = htonl(pConn->connId);
mnodeSaveQueryStreamList(pConn, pHBMsg);
if (pConn->killed != 0) {
- pHBRsp->killConnection = 1;
+ pRsp->killConnection = 1;
}
if (pConn->streamId != 0) {
- pHBRsp->streamId = htonl(pConn->streamId);
+ pRsp->streamId = htonl(pConn->streamId);
pConn->streamId = 0;
}
if (pConn->queryId != 0) {
- pHBRsp->queryId = htonl(pConn->queryId);
+ pRsp->queryId = htonl(pConn->queryId);
pConn->queryId = 0;
}
}
- pHBRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum());
- pHBRsp->totalDnodes = htonl(mnodeGetDnodesNum());
- mnodeGetMnodeEpSetForShell(&pHBRsp->epSet);
+ pRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum());
+ pRsp->totalDnodes = htonl(mnodeGetDnodesNum());
+ mnodeGetMnodeEpSetForShell(&pRsp->epSet);
+
+ pMsg->rpcRsp.rsp = pRsp;
+ pMsg->rpcRsp.len = sizeof(SHeartBeatRsp);
- pMsg->rpcRsp.rsp = pHBRsp;
- pMsg->rpcRsp.len = sizeof(SCMHeartBeatRsp);
-
mnodeReleaseConn(pConn);
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
- SCMConnectMsg *pConnectMsg = pMsg->rpcMsg.pCont;
- SCMConnectRsp *pConnectRsp = NULL;
+ SConnectMsg *pConnectMsg = pMsg->rpcMsg.pCont;
+ SConnectRsp *pConnectRsp = NULL;
int32_t code = TSDB_CODE_SUCCESS;
SRpcConnInfo connInfo = {0};
@@ -320,7 +325,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
mnodeDecDbRef(pDb);
}
- pConnectRsp = rpcMallocCont(sizeof(SCMConnectRsp));
+ pConnectRsp = rpcMallocCont(sizeof(SConnectRsp));
if (pConnectRsp == NULL) {
code = TSDB_CODE_MND_OUT_OF_MEMORY;
goto connect_over;
@@ -349,14 +354,14 @@ connect_over:
} else {
mLInfo("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code));
pMsg->rpcRsp.rsp = pConnectRsp;
- pMsg->rpcRsp.len = sizeof(SCMConnectRsp);
+ pMsg->rpcRsp.len = sizeof(SConnectRsp);
}
return code;
}
static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) {
- SCMUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont;
+ SUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont;
int32_t code = TSDB_CODE_SUCCESS;
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pUseDbMsg->db);
@@ -410,7 +415,7 @@ static void mnodeFreeShowObj(void *data) {
sdbFreeIter(pShow->pIter);
mDebug("%p, show is destroyed, data:%p index:%d", pShow, data, pShow->index);
- taosTFree(pShow);
+ tfree(pShow);
}
static void mnodeReleaseShowObj(SShowObj *pShow, bool forceRemove) {
diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index 82a062169a591d26d665437273183a4c5e1f27b4..2b5e6455c0ec731d0697f2b8f6b638a1ce139788 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -45,6 +45,10 @@
#include "mnodeRead.h"
#include "mnodePeer.h"
+#define ALTER_CTABLE_RETRY_TIMES 3
+#define CREATE_CTABLE_RETRY_TIMES 10
+#define CREATE_CTABLE_RETRY_SEC 14
+
static void * tsChildTableSdb;
static void * tsSuperTableSdb;
static int32_t tsChildTableUpdateSize;
@@ -52,9 +56,9 @@ static int32_t tsSuperTableUpdateSize;
static void * mnodeGetChildTable(char *tableId);
static void * mnodeGetSuperTable(char *tableId);
static void * mnodeGetSuperTableByUid(uint64_t uid);
-static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable);
-static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable);
-static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable);
+static void mnodeDropAllChildTablesInStable(SSTableObj *pStable);
+static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable);
+static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable);
static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn);
@@ -86,13 +90,13 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg);
static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg);
static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg);
-static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName);
+static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName);
-static void mnodeDestroyChildTable(SChildTableObj *pTable) {
- taosTFree(pTable->info.tableId);
- taosTFree(pTable->schema);
- taosTFree(pTable->sql);
- taosTFree(pTable);
+static void mnodeDestroyChildTable(SCTableObj *pTable) {
+ tfree(pTable->info.tableId);
+ tfree(pTable->schema);
+ tfree(pTable->sql);
+ tfree(pTable);
}
static int32_t mnodeChildTableActionDestroy(SSdbOper *pOper) {
@@ -101,7 +105,7 @@ static int32_t mnodeChildTableActionDestroy(SSdbOper *pOper) {
}
static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) {
- SChildTableObj *pTable = pOper->pObj;
+ SCTableObj *pTable = pOper->pObj;
SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId);
if (pVgroup == NULL) {
@@ -150,7 +154,7 @@ static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) {
}
static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) {
- SChildTableObj *pTable = pOper->pObj;
+ SCTableObj *pTable = pOper->pObj;
if (pTable->vgId == 0) {
return TSDB_CODE_MND_VGROUP_NOT_EXIST;
}
@@ -186,8 +190,8 @@ static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) {
}
static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) {
- SChildTableObj *pNew = pOper->pObj;
- SChildTableObj *pTable = mnodeGetChildTable(pNew->info.tableId);
+ SCTableObj *pNew = pOper->pObj;
+ SCTableObj *pTable = mnodeGetChildTable(pNew->info.tableId);
if (pTable != pNew) {
void *oldTableId = pTable->info.tableId;
void *oldSql = pTable->sql;
@@ -195,7 +199,7 @@ static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) {
void *oldSTable = pTable->superTable;
int32_t oldRefCount = pTable->refCount;
- memcpy(pTable, pNew, sizeof(SChildTableObj));
+ memcpy(pTable, pNew, sizeof(SCTableObj));
pTable->refCount = oldRefCount;
pTable->sql = pNew->sql;
@@ -213,7 +217,7 @@ static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) {
}
static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) {
- SChildTableObj *pTable = pOper->pObj;
+ SCTableObj *pTable = pOper->pObj;
assert(pTable != NULL && pOper->rowData != NULL);
int32_t len = strlen(pTable->info.tableId);
@@ -244,7 +248,7 @@ static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) {
static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) {
assert(pOper->rowData != NULL);
- SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj));
+ SCTableObj *pTable = calloc(1, sizeof(SCTableObj));
if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
int32_t len = strlen(pOper->rowData);
@@ -284,7 +288,7 @@ static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) {
static int32_t mnodeChildTableActionRestored() {
void *pIter = NULL;
- SChildTableObj *pTable = NULL;
+ SCTableObj *pTable = NULL;
while (1) {
pIter = mnodeGetNextChildTable(pIter, &pTable);
@@ -323,7 +327,7 @@ static int32_t mnodeChildTableActionRestored() {
}
if (pTable->info.type == TSDB_CHILD_TABLE) {
- SSuperTableObj *pSuperTable = mnodeGetSuperTableByUid(pTable->suid);
+ SSTableObj *pSuperTable = mnodeGetSuperTableByUid(pTable->suid);
if (pSuperTable == NULL) {
mError("ctable:%s, stable:%" PRIu64 " not exist", pTable->info.tableId, pTable->suid);
pTable->vgId = 0;
@@ -344,14 +348,14 @@ static int32_t mnodeChildTableActionRestored() {
}
static int32_t mnodeInitChildTables() {
- SChildTableObj tObj;
+ SCTableObj tObj;
tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_CTABLE,
.tableName = "ctables",
.hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE,
- .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE,
+ .maxRowSize = sizeof(SCTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_VAR_STRING,
.insertFp = mnodeChildTableActionInsert,
@@ -386,7 +390,7 @@ int64_t mnodeGetChildTableNum() {
return sdbGetNumOfRows(tsChildTableSdb);
}
-static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
+static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable) {
atomic_add_fetch_32(&pStable->numOfTables, 1);
if (pStable->vgHash == NULL) {
@@ -402,7 +406,7 @@ static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCt
}
}
-static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
+static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable) {
atomic_sub_fetch_32(&pStable->numOfTables, 1);
if (pStable->vgHash == NULL) return;
@@ -416,14 +420,14 @@ static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *
mnodeDecVgroupRef(pVgroup);
}
-static void mnodeDestroySuperTable(SSuperTableObj *pStable) {
+static void mnodeDestroySuperTable(SSTableObj *pStable) {
if (pStable->vgHash != NULL) {
taosHashCleanup(pStable->vgHash);
pStable->vgHash = NULL;
}
- taosTFree(pStable->info.tableId);
- taosTFree(pStable->schema);
- taosTFree(pStable);
+ tfree(pStable->info.tableId);
+ tfree(pStable->schema);
+ tfree(pStable);
}
static int32_t mnodeSuperTableActionDestroy(SSdbOper *pOper) {
@@ -432,7 +436,7 @@ static int32_t mnodeSuperTableActionDestroy(SSdbOper *pOper) {
}
static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) {
- SSuperTableObj *pStable = pOper->pObj;
+ SSTableObj *pStable = pOper->pObj;
SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId);
if (pDb != NULL && pDb->status == TSDB_DB_STATUS_READY) {
mnodeAddSuperTableIntoDb(pDb);
@@ -443,11 +447,11 @@ static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) {
}
static int32_t mnodeSuperTableActionDelete(SSdbOper *pOper) {
- SSuperTableObj *pStable = pOper->pObj;
+ SSTableObj *pStable = pOper->pObj;
SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId);
if (pDb != NULL) {
mnodeRemoveSuperTableFromDb(pDb);
- mnodeDropAllChildTablesInStable((SSuperTableObj *)pStable);
+ mnodeDropAllChildTablesInStable((SSTableObj *)pStable);
}
mnodeDecDbRef(pDb);
@@ -455,8 +459,8 @@ static int32_t mnodeSuperTableActionDelete(SSdbOper *pOper) {
}
static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) {
- SSuperTableObj *pNew = pOper->pObj;
- SSuperTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId);
+ SSTableObj *pNew = pOper->pObj;
+ SSTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId);
if (pTable != NULL && pTable != pNew) {
void *oldTableId = pTable->info.tableId;
void *oldSchema = pTable->schema;
@@ -464,7 +468,7 @@ static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) {
int32_t oldRefCount = pTable->refCount;
int32_t oldNumOfTables = pTable->numOfTables;
- memcpy(pTable, pNew, sizeof(SSuperTableObj));
+ memcpy(pTable, pNew, sizeof(SSTableObj));
pTable->vgHash = oldVgHash;
pTable->refCount = oldRefCount;
@@ -480,7 +484,7 @@ static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) {
}
static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) {
- SSuperTableObj *pStable = pOper->pObj;
+ SSTableObj *pStable = pOper->pObj;
assert(pOper->pObj != NULL && pOper->rowData != NULL);
int32_t len = strlen(pStable->info.tableId);
@@ -504,7 +508,7 @@ static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) {
static int32_t mnodeSuperTableActionDecode(SSdbOper *pOper) {
assert(pOper->rowData != NULL);
- SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj));
+ SSTableObj *pStable = (SSTableObj *) calloc(1, sizeof(SSTableObj));
if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
int32_t len = strlen(pOper->rowData);
@@ -537,14 +541,14 @@ static int32_t mnodeSuperTableActionRestored() {
}
static int32_t mnodeInitSuperTables() {
- SSuperTableObj tObj;
+ SSTableObj tObj;
tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_STABLE,
.tableName = "stables",
.hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE,
- .maxRowSize = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN,
+ .maxRowSize = sizeof(SSTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_VAR_STRING,
.insertFp = mnodeSuperTableActionInsert,
@@ -615,7 +619,7 @@ static void *mnodeGetSuperTable(char *tableId) {
}
static void *mnodeGetSuperTableByUid(uint64_t uid) {
- SSuperTableObj *pStable = NULL;
+ SSTableObj *pStable = NULL;
void *pIter = NULL;
while (1) {
@@ -647,11 +651,11 @@ void *mnodeGetTable(char *tableId) {
return NULL;
}
-void *mnodeGetNextChildTable(void *pIter, SChildTableObj **pTable) {
+void *mnodeGetNextChildTable(void *pIter, SCTableObj **pTable) {
return sdbFetchRow(tsChildTableSdb, pIter, (void **)pTable);
}
-void *mnodeGetNextSuperTable(void *pIter, SSuperTableObj **pTable) {
+void *mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable) {
return sdbFetchRow(tsSuperTableSdb, pIter, (void **)pTable);
}
@@ -765,12 +769,12 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
}
if (pMsg->pTable->type == TSDB_SUPER_TABLE) {
- SSuperTableObj *pSTable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pSTable = (SSTableObj *)pMsg->pTable;
mInfo("app:%p:%p, table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d",
pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pSTable->uid, pSTable->numOfTables, (int32_t)taosHashGetSize(pSTable->vgHash));
return mnodeProcessDropSuperTableMsg(pMsg);
} else {
- SChildTableObj *pCTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pCTable = (SCTableObj *)pMsg->pTable;
mInfo("app:%p:%p, table:%s, start to drop ctable, vgId:%d tid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
pDrop->tableId, pCTable->vgId, pCTable->tid, pCTable->uid);
return mnodeProcessDropChildTableMsg(pMsg);
@@ -778,7 +782,7 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
}
static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
- SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
+ STableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
pInfo->createFlag = htons(pInfo->createFlag);
mDebug("app:%p:%p, table:%s, table meta msg is received from thandle:%p, createFlag:%d", pMsg->rpcMsg.ahandle, pMsg,
pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag);
@@ -816,7 +820,7 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
}
static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pTable = (SSTableObj *)pMsg->pTable;
assert(pTable);
if (code == TSDB_CODE_SUCCESS) {
@@ -835,7 +839,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
- SSuperTableObj * pStable = calloc(1, sizeof(SSuperTableObj));
+ SSTableObj * pStable = calloc(1, sizeof(SSTableObj));
if (pStable == NULL) {
mError("app:%p:%p, table:%s, failed to create, no enough memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
return TSDB_CODE_MND_OUT_OF_MEMORY;
@@ -878,7 +882,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
- .rowSize = sizeof(SSuperTableObj) + schemaSize,
+ .rowSize = sizeof(SSTableObj) + schemaSize,
.pMsg = pMsg,
.writeCb = mnodeCreateSuperTableCb
};
@@ -894,7 +898,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
}
static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pTable = (SSTableObj *)pMsg->pTable;
if (code != TSDB_CODE_SUCCESS) {
mError("app:%p:%p, stable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
} else {
@@ -907,7 +911,7 @@ static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) {
SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash);
while (taosHashIterNext(pIter)) {
@@ -915,8 +919,8 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
SVgObj *pVgroup = mnodeGetVgroup(*pVgId);
if (pVgroup == NULL) break;
- SMDDropSTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropSTableMsg));
- pDrop->contLen = htonl(sizeof(SMDDropSTableMsg));
+ SDropSTableMsg *pDrop = rpcMallocCont(sizeof(SDropSTableMsg));
+ pDrop->contLen = htonl(sizeof(SDropSTableMsg));
pDrop->vgId = htonl(pVgroup->vgId);
pDrop->uid = htobe64(pStable->uid);
mnodeExtractTableName(pStable->info.tableId, pDrop->tableId);
@@ -924,7 +928,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
mInfo("app:%p:%p, stable:%s, send drop stable msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
pVgroup->vgId);
SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pVgroup);
- SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
+ SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
dnodeSendMsgToDnode(&epSet, &rpcMsg);
mnodeDecVgroupRef(pVgroup);
}
@@ -950,7 +954,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
return code;
}
-static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) {
+static int32_t mnodeFindSuperTableTagIndex(SSTableObj *pStable, const char *tagName) {
SSchema *schema = (SSchema *) pStable->schema;
for (int32_t tag = 0; tag < pStable->numOfTags; tag++) {
if (strcasecmp(schema[pStable->numOfColumns + tag].name, tagName) == 0) {
@@ -962,7 +966,7 @@ static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char *
}
static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, stable %s, add tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
tstrerror(code));
@@ -970,7 +974,7 @@ static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
}
static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t ntags) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) {
mError("app:%p:%p, stable:%s, add tag, too many tags", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId);
return TSDB_CODE_MND_TOO_MANY_TAGS;
@@ -1018,14 +1022,14 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t
}
static int32_t mnodeDropSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, stable %s, drop tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
tstrerror(code));
return code;
}
static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
int32_t col = mnodeFindSuperTableTagIndex(pStable, tagName);
if (col < 0) {
mError("app:%p:%p, stable:%s, drop tag, tag:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
@@ -1052,14 +1056,14 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) {
}
static int32_t mnodeModifySuperTableTagNameCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, stable %s, modify tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
tstrerror(code));
return code;
}
static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, char *newTagName) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
int32_t col = mnodeFindSuperTableTagIndex(pStable, oldTagName);
if (col < 0) {
mError("app:%p:%p, stable:%s, failed to modify table tag, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg,
@@ -1095,7 +1099,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c
return sdbUpdateRow(&oper);
}
-static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName) {
+static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName) {
SSchema *schema = (SSchema *) pStable->schema;
for (int32_t col = 0; col < pStable->numOfColumns; col++) {
if (strcasecmp(schema[col].name, colName) == 0) {
@@ -1107,7 +1111,7 @@ static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *col
}
static int32_t mnodeAddSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, stable %s, add column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
tstrerror(code));
return code;
@@ -1115,7 +1119,7 @@ static int32_t mnodeAddSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32_t ncols) {
SDbObj *pDb = pMsg->pDb;
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
if (ncols <= 0) {
mError("app:%p:%p, stable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, ncols);
return TSDB_CODE_MND_APP_ERROR;
@@ -1170,7 +1174,7 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32
}
static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, stable %s, delete column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
tstrerror(code));
return code;
@@ -1178,7 +1182,7 @@ static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) {
SDbObj *pDb = pMsg->pDb;
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
int32_t col = mnodeFindSuperTableColumnIndex(pStable, colName);
if (col <= 0) {
mError("app:%p:%p, stable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
@@ -1215,14 +1219,14 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) {
}
static int32_t mnodeChangeSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, stable %s, change column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
tstrerror(code));
return code;
}
static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
int32_t col = mnodeFindSuperTableColumnIndex(pStable, oldName);
if (col < 0) {
mError("app:%p:%p, stable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg,
@@ -1321,7 +1325,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
int32_t numOfRows = 0;
char * pWrite;
int32_t cols = 0;
- SSuperTableObj *pTable = NULL;
+ SSTableObj *pTable = NULL;
char prefix[64] = {0};
int32_t prefixLen;
@@ -1399,7 +1403,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
void mnodeDropAllSuperTables(SDbObj *pDropDb) {
void * pIter= NULL;
int32_t numOfTables = 0;
- SSuperTableObj *pTable = NULL;
+ SSTableObj *pTable = NULL;
char prefix[64] = {0};
tstrncpy(prefix, pDropDb->name, 64);
@@ -1430,7 +1434,7 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) {
mInfo("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables);
}
-static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) {
+static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSTableObj *pTable) {
int32_t numOfCols = pTable->numOfColumns + pTable->numOfTags;
assert(numOfCols <= TSDB_MAX_COLUMNS);
@@ -1446,7 +1450,7 @@ static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pT
}
static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) {
- SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pTable = (SSTableObj *)pMsg->pTable;
STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16));
if (pMeta == NULL) {
return TSDB_CODE_MND_OUT_OF_MEMORY;
@@ -1472,32 +1476,32 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) {
}
static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
- SCMSTableVgroupMsg *pInfo = pMsg->rpcMsg.pCont;
+ SSTableVgroupMsg *pInfo = pMsg->rpcMsg.pCont;
int32_t numOfTable = htonl(pInfo->numOfTables);
// reserve space
- int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupMsg) + sizeof(SVgroupsMsg);
+ int32_t contLen = sizeof(SSTableVgroupRspMsg) + 32 * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg);
for (int32_t i = 0; i < numOfTable; ++i) {
- char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN) * i;
- SSuperTableObj *pTable = mnodeGetSuperTable(stableName);
+ char *stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i;
+ SSTableObj *pTable = mnodeGetSuperTable(stableName);
if (pTable != NULL && pTable->vgHash != NULL) {
- contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupMsg) + sizeof(SVgroupsMsg));
- }
-
+ contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg));
+ }
+
mnodeDecTableRef(pTable);
}
- SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen);
+ SSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) {
return TSDB_CODE_MND_OUT_OF_MEMORY;
}
pRsp->numOfTables = 0;
- char *msg = (char *)pRsp + sizeof(SCMSTableVgroupRspMsg);
+ char *msg = (char *)pRsp + sizeof(SSTableVgroupRspMsg);
for (int32_t i = 0; i < numOfTable; ++i) {
- char * stableName = (char *)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i;
- SSuperTableObj *pTable = mnodeGetSuperTable(stableName);
+ char * stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i;
+ SSTableObj *pTable = mnodeGetSuperTable(stableName);
if (pTable == NULL) {
mError("app:%p:%p, stable:%s, not exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, stableName);
mnodeDecTableRef(pTable);
@@ -1548,7 +1552,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
pVgroupMsg->numOfVgroups = htonl(vgSize);
// one table is done, try the next table
- msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SCMVgroupMsg);
+ msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SVgroupMsg);
pRsp->numOfTables++;
}
}
@@ -1569,7 +1573,7 @@ static void mnodeProcessDropSuperTableRsp(SRpcMsg *rpcMsg) {
mInfo("drop stable rsp received, result:%s", tstrerror(rpcMsg->code));
}
-static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) {
+static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SCTableObj *pTable) {
STagData * pTagData = NULL;
int32_t tagDataLen = 0;
int32_t totalCols = 0;
@@ -1643,7 +1647,7 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableO
}
static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
assert(pTable);
mDebug("app:%p:%p, table:%s, created in mnode, vgId:%d sid:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
@@ -1669,7 +1673,7 @@ static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) {
}
static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
assert(pTable);
@@ -1679,12 +1683,12 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
pTable->info.tableId, pMsg->rpcMsg.handle);
pMsg->retry = 0;
- dnodeReprocessMnodeWriteMsg(pMsg);
+ dnodeReprocessMWriteMsg(pMsg);
} else {
mDebug("app:%p:%p, table:%s, created in dnode, thandle:%p", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
pMsg->rpcMsg.handle);
- dnodeSendRpcMnodeWriteRsp(pMsg, TSDB_CODE_SUCCESS);
+ dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS);
}
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
} else {
@@ -1699,7 +1703,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
SVgObj *pVgroup = pMsg->pVgroup;
SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
- SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj));
+ SCTableObj *pTable = calloc(1, sizeof(SCTableObj));
if (pTable == NULL) {
mError("app:%p:%p, table:%s, failed to alloc memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
return TSDB_CODE_MND_OUT_OF_MEMORY;
@@ -1842,7 +1846,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
}
static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, ctable:%s, is dropped from sdb", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg));
@@ -1880,7 +1884,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) {
static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) {
if (code != TSDB_CODE_SUCCESS) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
mError("app:%p:%p, ctable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
return code;
}
@@ -1889,7 +1893,7 @@ static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) {
}
static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
if (pMsg->pVgroup == NULL) {
mError("app:%p:%p, table:%s, failed to drop ctable, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg,
@@ -1914,7 +1918,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) {
return code;
}
-static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) {
+static int32_t mnodeFindNormalTableColumnIndex(SCTableObj *pTable, char *colName) {
SSchema *schema = (SSchema *) pTable->schema;
for (int32_t col = 0; col < pTable->numOfColumns; col++) {
if (strcasecmp(schema[col].name, colName) == 0) {
@@ -1926,7 +1930,7 @@ static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *col
}
static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
if (code != TSDB_CODE_SUCCESS) {
mError("app:%p:%p, ctable %s, failed to alter column, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
tstrerror(code));
@@ -1965,7 +1969,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
}
static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32_t ncols) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
SDbObj *pDb = pMsg->pDb;
if (ncols <= 0) {
mError("app:%p:%p, ctable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, ncols);
@@ -2014,7 +2018,7 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3
static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
SDbObj *pDb = pMsg->pDb;
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
int32_t col = mnodeFindNormalTableColumnIndex(pTable, colName);
if (col <= 0) {
mError("app:%p:%p, ctable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
@@ -2046,7 +2050,7 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
}
static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
int32_t col = mnodeFindNormalTableColumnIndex(pTable, oldName);
if (col < 0) {
mError("app:%p:%p, ctable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg,
@@ -2082,7 +2086,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char
return sdbUpdateRow(&oper);
}
-static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pTable) {
+static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SCTableObj *pTable) {
int32_t numOfCols = pTable->numOfColumns;
for (int32_t i = 0; i < numOfCols; ++i) {
strcpy(pSchema->name, pTable->schema[i].name);
@@ -2097,7 +2101,7 @@ static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *p
static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
SDbObj *pDb = pMsg->pDb;
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
pMeta->uid = htobe64(pTable->uid);
pMeta->tid = htonl(pTable->tid);
@@ -2146,7 +2150,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
}
static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
- SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
+ STableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
STagData *pTags = (STagData *)pInfo->tags;
int32_t tagLen = htonl(pTags->dataLen);
if (pTags->name[0] == 0) {
@@ -2156,7 +2160,7 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
}
int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + tagLen;
- SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen);
+ SCMCreateTableMsg *pCreateMsg = calloc(1, contLen);
if (pCreateMsg == NULL) {
mError("app:%p:%p, table:%s, failed to create table while get meta info, no enough memory", pMsg->rpcMsg.ahandle,
pMsg, pInfo->tableId);
@@ -2174,11 +2178,13 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
mDebug("app:%p:%p, table:%s, start to create on demand, tagLen:%d stable:%s",
pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId, tagLen, pTags->name);
- rpcFreeCont(pMsg->rpcMsg.pCont);
+ if (pMsg->rpcMsg.pCont != pMsg->pCont) {
+ tfree(pMsg->rpcMsg.pCont);
+ }
pMsg->rpcMsg.msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE;
pMsg->rpcMsg.pCont = pCreateMsg;
pMsg->rpcMsg.contLen = contLen;
-
+
return TSDB_CODE_MND_ACTION_NEED_REPROCESSED;
}
@@ -2203,7 +2209,7 @@ static int32_t mnodeGetChildTableMeta(SMnodeMsg *pMsg) {
void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) {
void * pIter = NULL;
int32_t numOfTables = 0;
- SChildTableObj *pTable = NULL;
+ SCTableObj *pTable = NULL;
mInfo("vgId:%d, all child tables will be dropped from sdb", pVgroup->vgId);
@@ -2231,7 +2237,7 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) {
void mnodeDropAllChildTables(SDbObj *pDropDb) {
void * pIter = NULL;
int32_t numOfTables = 0;
- SChildTableObj *pTable = NULL;
+ SCTableObj *pTable = NULL;
char prefix[64] = {0};
tstrncpy(prefix, pDropDb->name, 64);
@@ -2261,10 +2267,10 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) {
mInfo("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables);
}
-static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) {
+static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) {
void * pIter = NULL;
int32_t numOfTables = 0;
- SChildTableObj *pTable = NULL;
+ SCTableObj *pTable = NULL;
mInfo("stable:%s uid:%" PRIu64 ", all child tables:%d will be dropped from sdb", pStable->info.tableId, pStable->uid,
pStable->numOfTables);
@@ -2292,11 +2298,11 @@ static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) {
}
#if 0
-static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) {
+static SCTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) {
SVgObj *pVgroup = mnodeGetVgroup(vnode);
if (pVgroup == NULL) return NULL;
- SChildTableObj *pTable = pVgroup->tableList[tid - 1];
+ SCTableObj *pTable = pVgroup->tableList[tid - 1];
mnodeIncTableRef((STableObj *)pTable);
mnodeDecVgroupRef(pVgroup);
@@ -2307,14 +2313,14 @@ static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) {
static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_COM_OPS_NOT_SUPPORT;
#if 0
- SDMConfigTableMsg *pCfg = pMsg->rpcMsg.pCont;
+ SConfigTableMsg *pCfg = pMsg->rpcMsg.pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId);
pCfg->vgId = htonl(pCfg->vgId);
pCfg->sid = htonl(pCfg->sid);
mDebug("app:%p:%p, dnode:%d, vgId:%d sid:%d, receive table config msg", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId,
pCfg->vgId, pCfg->sid);
- SChildTableObj *pTable = mnodeGetTableByPos(pCfg->vgId, pCfg->sid);
+ SCTableObj *pTable = mnodeGetTableByPos(pCfg->vgId, pCfg->sid);
if (pTable == NULL) {
mError("app:%p:%p, dnode:%d, vgId:%d sid:%d, table not found", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId,
pCfg->vgId, pCfg->sid);
@@ -2322,7 +2328,7 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) {
}
SMDCreateTableMsg *pCreate = NULL;
- pCreate = mnodeBuildCreateChildTableMsg(NULL, (SChildTableObj *)pTable);
+ pCreate = mnodeBuildCreateChildTableMsg(NULL, (SCTableObj *)pTable);
mnodeDecTableRef(pTable);
if (pCreate == NULL) return terrno;
@@ -2340,7 +2346,7 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
SMnodeMsg *mnodeMsg = rpcMsg->ahandle;
mnodeMsg->received++;
- SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable;
assert(pTable);
mInfo("app:%p:%p, table:%s, drop table rsp received, vgId:%d sid:%d uid:%" PRIu64 ", thandle:%p result:%s",
@@ -2351,14 +2357,14 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
mError("app:%p:%p, table:%s, failed to drop in dnode, vgId:%d sid:%d uid:%" PRIu64 ", reason:%s",
mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid,
tstrerror(rpcMsg->code));
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
+ dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
return;
}
if (mnodeMsg->pVgroup == NULL) mnodeMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
if (mnodeMsg->pVgroup == NULL) {
mError("app:%p:%p, table:%s, failed to get vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId);
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST);
+ dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST);
return;
}
@@ -2368,7 +2374,7 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
mnodeDropVgroup(mnodeMsg->pVgroup, NULL);
}
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
+ dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
}
/*
@@ -2381,7 +2387,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
SMnodeMsg *mnodeMsg = rpcMsg->ahandle;
mnodeMsg->received++;
- SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable;
assert(pTable);
// If the table is deleted by another thread during creation, stop creating and send drop msg to vnode
@@ -2399,7 +2405,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
mnodeSendDropChildTableMsg(mnodeMsg, false);
rpcMsg->code = TSDB_CODE_SUCCESS;
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
+ dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
return;
}
@@ -2416,25 +2422,32 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mnodeMsg->pTable = NULL;
mnodeDestroyChildTable(pTable);
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, code);
+ dnodeSendRpcMWriteRsp(mnodeMsg, code);
}
} else {
- if (mnodeMsg->retry++ < 10) {
+ mnodeMsg->retry++;
+ int32_t sec = taosGetTimestampSec();
+ if (mnodeMsg->retry < CREATE_CTABLE_RETRY_TIMES && ABS(sec - mnodeMsg->incomingTs) < CREATE_CTABLE_RETRY_SEC) {
mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64
" result:%s thandle:%p",
mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->tid,
pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
- dnodeDelayReprocessMnodeWriteMsg(mnodeMsg);
+ dnodeDelayReprocessMWriteMsg(mnodeMsg);
} else {
- mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 ", result:%s thandle:%p",
+ mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64
+ ", result:%s thandle:%p incomingTs:%d curTs:%d retryTimes:%d",
mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid,
- tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
+ tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle, mnodeMsg->incomingTs, sec, mnodeMsg->retry);
SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable};
sdbDeleteRow(&oper);
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
+ if (rpcMsg->code == TSDB_CODE_APP_NOT_READY) {
+ //Avoid retry again in client
+ rpcMsg->code = TSDB_CODE_MND_VGROUP_NOT_READY;
+ }
+ dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
}
}
}
@@ -2445,31 +2458,31 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) {
SMnodeMsg *mnodeMsg = rpcMsg->ahandle;
mnodeMsg->received++;
- SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable;
assert(pTable);
if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) {
mDebug("app:%p:%p, ctable:%s, altered in dnode, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
+ dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
} else {
- if (mnodeMsg->retry++ < 3) {
+ if (mnodeMsg->retry++ < ALTER_CTABLE_RETRY_TIMES) {
mDebug("app:%p:%p, table:%s, alter table rsp received, need retry, times:%d result:%s thandle:%p",
mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code),
mnodeMsg->rpcMsg.handle);
- dnodeDelayReprocessMnodeWriteMsg(mnodeMsg);
+ dnodeDelayReprocessMWriteMsg(mnodeMsg);
} else {
mError("app:%p:%p, table:%s, failed to alter in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
+ dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
}
}
}
static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
- SCMMultiTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
+ SMultiTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
pInfo->numOfTables = htonl(pInfo->numOfTables);
int32_t totalMallocLen = 4 * 1024 * 1024; // first malloc 4 MB, subsequent reallocation as twice
@@ -2483,7 +2496,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
for (int32_t t = 0; t < pInfo->numOfTables; ++t) {
char * tableId = (char *)(pInfo->tableIds + t * TSDB_TABLE_FNAME_LEN);
- SChildTableObj *pTable = mnodeGetChildTable(tableId);
+ SCTableObj *pTable = mnodeGetChildTable(tableId);
if (pTable == NULL) continue;
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(tableId);
@@ -2607,7 +2620,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
int32_t cols = 0;
int32_t numOfRows = 0;
- SChildTableObj *pTable = NULL;
+ SCTableObj *pTable = NULL;
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
char prefix[64] = {0};
@@ -2705,7 +2718,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
}
static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
- SCMAlterTableMsg *pAlter = pMsg->rpcMsg.pCont;
+ SAlterTableMsg *pAlter = pMsg->rpcMsg.pCont;
mDebug("app:%p:%p, table:%s, alter table msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg,
pAlter->tableId, pMsg->rpcMsg.handle);
@@ -2843,7 +2856,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
}
int32_t numOfRows = 0;
- SChildTableObj *pTable = NULL;
+ SCTableObj *pTable = NULL;
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
char prefix[64] = {0};
diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c
index 779e25254897ff37f1ca884e83469233d7197b8c..cd02d5a935c3ef3cf822d518a9cc3bbd795db46b 100644
--- a/src/mnode/src/mnodeUser.c
+++ b/src/mnode/src/mnodeUser.c
@@ -43,7 +43,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg);
static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg);
static int32_t mnodeUserActionDestroy(SSdbOper *pOper) {
- taosTFree(pOper->pObj);
+ tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
@@ -270,7 +270,7 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) {
code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("user:%s, failed to create by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code));
- taosTFree(pUser);
+ tfree(pUser);
} else {
mLInfo("user:%s, is created by %s", pUser->user, mnodeGetUserFromMsg(pMsg));
}
@@ -414,7 +414,7 @@ static int32_t mnodeProcessCreateUserMsg(SMnodeMsg *pMsg) {
SUserObj *pOperUser = pMsg->pUser;
if (pOperUser->superAuth) {
- SCMCreateUserMsg *pCreate = pMsg->rpcMsg.pCont;
+ SCreateUserMsg *pCreate = pMsg->rpcMsg.pCont;
return mnodeCreateUser(pOperUser->pAcct, pCreate->user, pCreate->pass, pMsg);
} else {
mError("user:%s, no rights to create user", pOperUser->user);
@@ -426,7 +426,7 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) {
int32_t code;
SUserObj *pOperUser = pMsg->pUser;
- SCMAlterUserMsg *pAlter = pMsg->rpcMsg.pCont;
+ SAlterUserMsg *pAlter = pMsg->rpcMsg.pCont;
SUserObj *pUser = mnodeGetUser(pAlter->user);
if (pUser == NULL) {
return TSDB_CODE_MND_INVALID_USER;
@@ -514,7 +514,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) {
int32_t code;
SUserObj *pOperUser = pMsg->pUser;
- SCMDropUserMsg *pDrop = pMsg->rpcMsg.pCont;
+ SDropUserMsg *pDrop = pMsg->rpcMsg.pCont;
SUserObj *pUser = mnodeGetUser(pDrop->user);
if (pUser == NULL) {
return TSDB_CODE_MND_INVALID_USER;
@@ -604,11 +604,11 @@ int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, cha
}
static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg) {
- SDMAuthMsg *pAuthMsg = pMsg->rpcMsg.pCont;
- SDMAuthRsp *pAuthRsp = rpcMallocCont(sizeof(SDMAuthRsp));
+ SAuthMsg *pAuthMsg = pMsg->rpcMsg.pCont;
+ SAuthRsp *pAuthRsp = rpcMallocCont(sizeof(SAuthRsp));
pMsg->rpcRsp.rsp = pAuthRsp;
- pMsg->rpcRsp.len = sizeof(SDMAuthRsp);
+ pMsg->rpcRsp.len = sizeof(SAuthRsp);
return mnodeRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey);
}
diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c
index 5084f1276ae883289939bd74c50a0502f6b3c71d..9e398e94f136de7324f18ee6bef32fdd4a700b33 100644
--- a/src/mnode/src/mnodeVgroup.c
+++ b/src/mnode/src/mnodeVgroup.c
@@ -69,7 +69,7 @@ static void mnodeDestroyVgroup(SVgObj *pVgroup) {
pVgroup->idPool = NULL;
}
- taosTFree(pVgroup);
+ tfree(pVgroup);
}
static int32_t mnodeVgroupActionDestroy(SSdbOper *pOper) {
@@ -529,7 +529,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
(void)sdbUpdateRow(&desc);
- dnodeReprocessMnodeWriteMsg(pMsg);
+ dnodeReprocessMWriteMsg(pMsg);
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
// if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_READY) {
// mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
@@ -537,7 +537,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
// pVgroup->status = TAOS_VG_STATUS_READY;
// SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
// (void)sdbUpdateRow(&desc);
- // dnodeReprocessMnodeWriteMsg(pMsg);
+ // dnodeReprocessMWriteMsg(pMsg);
// return TSDB_CODE_MND_ACTION_IN_PROGRESS;
// } else {
// mError("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d, but vgroup is dropping", pMsg->rpcMsg.ahandle,
@@ -663,13 +663,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
for (int32_t i = 0; i < pShow->maxReplica; ++i) {
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
- snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "dnode%d", i + 1);
+ snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1);
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
- snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dstatus", i + 1);
+ snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1);
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
}
@@ -694,7 +694,7 @@ static bool mnodeFilterVgroups(SVgObj *pVgroup, STableObj *pTable) {
return true;
}
- SChildTableObj *pCTable = (SChildTableObj *)pTable;
+ SCTableObj *pCTable = (SCTableObj *)pTable;
if (pVgroup->vgId == pCTable->vgId) {
return true;
} else {
@@ -791,7 +791,7 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v
return numOfRows;
}
-void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
+void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable) {
int32_t idPoolSize = taosIdPoolMaxSize(pVgroup->idPool);
if (pTable->tid > idPoolSize) {
mnodeAllocVgroupIdPool(pVgroup);
@@ -807,7 +807,7 @@ void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
}
}
-void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
+void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable) {
if (pTable->tid >= 1) {
taosFreeId(pVgroup->idPool, pTable->tid);
pVgroup->numOfTables--;
@@ -818,11 +818,11 @@ void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
}
}
-static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
+static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
SDbObj *pDb = pVgroup->pDb;
if (pDb == NULL) return NULL;
- SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg));
+ SCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SCreateVnodeMsg));
if (pVnode == NULL) return NULL;
strcpy(pVnode->db, pVgroup->dbName);
@@ -830,7 +830,7 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
//TODO: dynamic alloc tables in tsdb
maxTables = MAX(10000, tsMaxTablePerVnode);
- SMDVnodeCfg *pCfg = &pVnode->cfg;
+ SVnodeCfg *pCfg = &pVnode->cfg;
pCfg->vgId = htonl(pVgroup->vgId);
pCfg->cfgVersion = htonl(pDb->cfgVersion);
pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize);
@@ -850,8 +850,9 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
pCfg->replications = (int8_t) pVgroup->numOfVnodes;
pCfg->wals = 3;
pCfg->quorum = pDb->cfg.quorum;
+ pCfg->update = pDb->cfg.update;
- SMDVnodeDesc *pNodes = pVnode->nodes;
+ SVnodeDesc *pNodes = pVnode->nodes;
for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) {
SDnodeObj *pDnode = pVgroup->vnodeGid[j].pDnode;
if (pDnode != NULL) {
@@ -886,11 +887,11 @@ SRpcEpSet mnodeGetEpSetFromIp(char *ep) {
}
static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet) {
- SMDAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup);
+ SAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup);
SRpcMsg rpcMsg = {
.ahandle = NULL,
.pCont = pAlter,
- .contLen = pAlter ? sizeof(SMDAlterVnodeMsg) : 0,
+ .contLen = pAlter ? sizeof(SAlterVnodeMsg) : 0,
.code = 0,
.msgType = TSDB_MSG_TYPE_MD_ALTER_VNODE
};
@@ -909,11 +910,11 @@ void mnodeSendAlterVgroupMsg(SVgObj *pVgroup) {
}
static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet, void *ahandle) {
- SMDCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup);
+ SCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup);
SRpcMsg rpcMsg = {
.ahandle = ahandle,
.pCont = pCreate,
- .contLen = pCreate ? sizeof(SMDCreateVnodeMsg) : 0,
+ .contLen = pCreate ? sizeof(SCreateVnodeMsg) : 0,
.code = 0,
.msgType = TSDB_MSG_TYPE_MD_CREATE_VNODE
};
@@ -969,7 +970,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mnodeMsg->pVgroup = NULL;
mnodeDestroyVgroup(pVgroup);
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, code);
+ dnodeSendRpcMWriteRsp(mnodeMsg, code);
}
} else {
SSdbOper oper = {
@@ -978,12 +979,12 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
.pObj = pVgroup
};
sdbDeleteRow(&oper);
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, mnodeMsg->code);
+ dnodeSendRpcMWriteRsp(mnodeMsg, mnodeMsg->code);
}
}
-static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) {
- SMDDropVnodeMsg *pDrop = rpcMallocCont(sizeof(SMDDropVnodeMsg));
+static SDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) {
+ SDropVnodeMsg *pDrop = rpcMallocCont(sizeof(SDropVnodeMsg));
if (pDrop == NULL) return NULL;
pDrop->vgId = htonl(vgId);
@@ -991,11 +992,11 @@ static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) {
}
void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle) {
- SMDDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId);
+ SDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId);
SRpcMsg rpcMsg = {
.ahandle = ahandle,
.pCont = pDrop,
- .contLen = pDrop ? sizeof(SMDDropVnodeMsg) : 0,
+ .contLen = pDrop ? sizeof(SDropVnodeMsg) : 0,
.code = 0,
.msgType = TSDB_MSG_TYPE_MD_DROP_VNODE
};
@@ -1040,11 +1041,11 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
code = TSDB_CODE_MND_SDB_ERROR;
}
- dnodeReprocessMnodeWriteMsg(mnodeMsg);
+ dnodeReprocessMWriteMsg(mnodeMsg);
}
static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) {
- SDMConfigVnodeMsg *pCfg = pMsg->rpcMsg.pCont;
+ SConfigVnodeMsg *pCfg = pMsg->rpcMsg.pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId);
pCfg->vgId = htonl(pCfg->vgId);
diff --git a/src/os/inc/os.h b/src/os/inc/os.h
index 86e16db8b1446308060945d3a7db2531287c62ec..9383ae48dc5c6151aea0d9e8a2641603b63da144 100644
--- a/src/os/inc/os.h
+++ b/src/os/inc/os.h
@@ -52,9 +52,9 @@ extern "C" {
#include "osWindows.h"
#endif
+#include "osDef.h"
#include "osAtomic.h"
#include "osCommon.h"
-#include "osDef.h"
#include "osDir.h"
#include "osFile.h"
#include "osLz4.h"
diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h
index c1a950fbe6375b14e3cf25277eb933b96e518f10..7bb844831e03dd1e21abd389a61976e14927ebbd 100644
--- a/src/os/inc/osDarwin.h
+++ b/src/os/inc/osDarwin.h
@@ -72,8 +72,6 @@ extern "C" {
#include
#define TAOS_OS_FUNC_FILE_SENDIFLE
- #define taosFSendFile(outfile, infile, offset, count) taosFSendFileImp(outfile, infile, offset, size)
- #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size)
#define TAOS_OS_FUNC_SEMPHONE
#define tsem_t dispatch_semaphore_t
diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h
index 75c2b749d175f8b074806132496b438aa24c4043..62e44d8eb0b70fb1526693895847637daa72247a 100644
--- a/src/os/inc/osFile.h
+++ b/src/os/inc/osFile.h
@@ -20,47 +20,52 @@
extern "C" {
#endif
-ssize_t taosTReadImp(int fd, void *buf, size_t count);
-ssize_t taosTWriteImp(int fd, void *buf, size_t count);
+#define tread(fd, buf, count) read(fd, buf, count)
+#define twrite(fd, buf, count) write(fd, buf, count)
+#define tlseek(fd, offset, whence) lseek(fd, offset, whence)
+#define tclose(fd) \
+ { \
+ if (FD_VALID(fd)) { \
+ close(fd); \
+ fd = FD_INITIALIZER; \
+ } \
+ }
-ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size);
-int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count);
+int64_t taosReadImp(int32_t fd, void *buf, int64_t count);
+int64_t taosWriteImp(int32_t fd, void *buf, int64_t count);
+int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence);
+int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath);
-#ifndef TAOS_OS_FUNC_FILE_SENDIFLE
- #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size)
- #define taosFSendFile(outfile, infile, offset, count) taosTSendFileImp(fileno(outfile), fileno(infile), offset, size)
-#endif
+#define taosRead(fd, buf, count) taosReadImp(fd, buf, count)
+#define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count)
+#define taosLSeek(fd, offset, whence) taosLSeekImp(fd, offset, whence)
+#define taosClose(x) tclose(x)
-#define taosTRead(fd, buf, count) taosTReadImp(fd, buf, count)
-#define taosTWrite(fd, buf, count) taosTWriteImp(fd, buf, count)
-#define taosLSeek(fd, offset, whence) lseek(fd, offset, whence)
-ssize_t taosTCopy(char *from, char *to);
+// TAOS_OS_FUNC_FILE_SENDIFLE
+int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size);
+int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size);
#ifdef TAOS_RANDOM_FILE_FAIL
- void taosSetRandomFileFailFactor(int factor);
+ void taosSetRandomFileFailFactor(int32_t factor);
void taosSetRandomFileFailOutput(const char *path);
#ifdef TAOS_RANDOM_FILE_FAIL_TEST
- ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
- ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
- off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line);
- #undef taosTRead
- #undef taosTWrite
+ int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line);
+ int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line);
+ int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line);
+ #undef taosRead
+ #undef taosWrite
#undef taosLSeek
- #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__)
- #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+ #define taosRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+ #define taosWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__)
#define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__)
#endif
#endif
-int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath);
-
// TAOS_OS_FUNC_FILE_GETTMPFILEPATH
void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath);
-#ifndef TAOS_OS_FUNC_FILE_FTRUNCATE
- #define taosFtruncate ftruncate
-#endif
-
+// TAOS_OS_FUNC_FILE_FTRUNCATE
+int32_t taosFtruncate(int32_t fd, int64_t length);
#ifdef __cplusplus
}
#endif
diff --git a/src/os/inc/osMemory.h b/src/os/inc/osMemory.h
index 37d9dc9828ec1f51f36542e6c297ec3df8709f7e..0616006650eb2d53cdf7bd70b68c9f60748deac5 100644
--- a/src/os/inc/osMemory.h
+++ b/src/os/inc/osMemory.h
@@ -31,6 +31,7 @@ typedef enum {
void taosSetAllocMode(int mode, const char *path, bool autoDump);
void taosDumpMemoryLeak();
+// used in tsdb module
void * taosTMalloc(size_t size);
void * taosTCalloc(size_t nmemb, size_t size);
void * taosTRealloc(void *ptr, size_t size);
@@ -38,7 +39,14 @@ void taosTZfree(void *ptr);
size_t taosTSizeof(void *ptr);
void taosTMemset(void *ptr, int c);
-#define taosTFree(x) \
+// used in other module
+#define tmalloc(size) malloc(size)
+#define tcalloc(num, size) calloc(num, size)
+#define trealloc(ptr, size) realloc(ptr, size)
+#define tstrdup(str) taosStrdupImp(str)
+#define tstrndup(str, size) taosStrndupImp(str, size)
+#define tgetline(lineptr, n, stream) taosGetlineImp(lineptr, n, stream)
+#define tfree(x) \
do { \
if (x) { \
free((void *)(x)); \
@@ -46,37 +54,30 @@ void taosTMemset(void *ptr, int c);
} \
} while (0);
-#define taosMalloc(size) malloc(size)
-#define taosCalloc(num, size) calloc(num, size)
-#define taosRealloc(ptr, size) realloc(ptr, size)
-#define taosFree(ptr) free(ptr)
-#define taosStrdup(str) taosStrdupImp(str)
-#define taosStrndup(str, size) taosStrndupImp(str, size)
-#define taosGetline(lineptr, n, stream) taosGetlineImp(lineptr, n, stream)
-
#ifdef TAOS_MEM_CHECK
#ifdef TAOS_MEM_CHECK_TEST
- void * taos_malloc(size_t size, const char *file, uint32_t line);
- void * taos_calloc(size_t num, size_t size, const char *file, uint32_t line);
- void * taos_realloc(void *ptr, size_t size, const char *file, uint32_t line);
- void taos_free(void *ptr, const char *file, uint32_t line);
- char * taos_strdup(const char *str, const char *file, uint32_t line);
- char * taos_strndup(const char *str, size_t size, const char *file, uint32_t line);
- ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line);
- #undef taosMalloc
- #undef taosCalloc
- #undef taosRealloc
- #undef taosFree
- #undef taosStrdup
- #undef taosStrndup
- #undef taosGetline
- #define taosMalloc(size) taos_malloc(size, __FILE__, __LINE__)
- #define taosCalloc(num, size) taos_calloc(num, size, __FILE__, __LINE__)
- #define taosRealloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__)
- #define taosFree(ptr) taos_free(ptr, __FILE__, __LINE__)
- //#define taosStrdup(str) taos_strdup(str, __FILE__, __LINE__)
- //#define taosStrndup(str, size) taos_strndup(str, size, __FILE__, __LINE__)
- //#define taosGetline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__)
+ void * taosMallocMem(size_t size, const char *file, uint32_t line);
+ void * taosCallocMem(size_t num, size_t size, const char *file, uint32_t line);
+ void * taosReallocMem(void *ptr, size_t size, const char *file, uint32_t line);
+ void taosFreeMem(void *ptr, const char *file, uint32_t line);
+ char * taosStrdupMem(const char *str, const char *file, uint32_t line);
+ char * taosStrndupMem(const char *str, size_t size, const char *file, uint32_t line);
+ ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line);
+ #undef tmalloc
+ #undef tcalloc
+ #undef trealloc
+ #undef tfree
+ #define tmalloc(size) taosMallocMem(size, __FILE__, __LINE__)
+ #define tcalloc(num, size) taosCallocMem(num, size, __FILE__, __LINE__)
+ #define trealloc(ptr, size) taosReallocMem(ptr, size, __FILE__, __LINE__)
+ #define tfree(ptr) taosFreeMem(ptr, __FILE__, __LINE__)
+
+ // #undef tstrdup
+ // #undef tstrndup
+ // #undef tgetline
+ // #define taosStrdup(str) taos_strdup(str, __FILE__, __LINE__)
+ // #define taosStrndup(str, size) taos_strndup(str, size, __FILE__, __LINE__)
+ // #define tgetline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__)
#endif
#endif
diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h
index 0ab3ff0fcafa43a404533f145a0bb1595bb29061..baf7687dd03e1ee4f6dd92e3204244b3d31b7a1f 100644
--- a/src/os/inc/osSocket.h
+++ b/src/os/inc/osSocket.h
@@ -33,21 +33,19 @@ extern "C" {
x = FD_INITIALIZER; \
} \
}
- typedef int SOCKET;
+ typedef int32_t SOCKET;
#endif
#ifndef TAOS_OS_DEF_EPOLL
- #define TAOS_EPOLL_WAIT_TIME -1
+ #define TAOS_EPOLL_WAIT_TIME 500
#endif
-#define taosClose(x) taosCloseSocket(x)
-
#ifdef TAOS_RANDOM_NETWORK_FAIL
#ifdef TAOS_RANDOM_NETWORK_FAIL_TEST
- ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags);
- ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen);
- ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count);
- ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count);
+ int64_t taosSendRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags);
+ int64_t taosSendToRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags, const struct sockaddr *dest_addr, socklen_t addrlen);
+ int64_t taosReadSocketRandomFail(int32_t fd, void *buf, size_t count);
+ int64_t taosWriteSocketRandomFail(int32_t fd, const void *buf, size_t count);
#undef taosSend
#undef taosSendto
#undef taosReadSocket
@@ -60,14 +58,14 @@ extern "C" {
#endif
// TAOS_OS_FUNC_SOCKET
-int taosSetNonblocking(SOCKET sock, int on);
-void taosBlockSIGPIPE();
+int32_t taosSetNonblocking(SOCKET sock, int32_t on);
+void taosBlockSIGPIPE();
// TAOS_OS_FUNC_SOCKET_SETSOCKETOPT
-int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen);
+int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen);
// TAOS_OS_FUNC_SOCKET_INET
-uint32_t taosInetAddr(char *ipAddr);
+uint32_t taosInetAddr(char *ipAddr);
const char *taosInetNtoa(struct in_addr ipInt);
#ifdef __cplusplus
diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h
index dc1da35037dc06c258856a1a143146492eeed0e5..36e30528bf9d84739fd391442145d7e65e1f7df2 100644
--- a/src/os/inc/osWindows.h
+++ b/src/os/inc/osWindows.h
@@ -62,11 +62,8 @@ extern "C" {
#define TAOS_OS_FUNC_FILE_ISDIR
#define TAOS_OS_FUNC_FILE_ISLNK
#define TAOS_OS_FUNC_FILE_SENDIFLE
- #define taosFSendFile(outfile, infile, offset, count) taosFSendFileImp(outfile, infile, offset, size)
- #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size)
#define TAOS_OS_FUNC_FILE_GETTMPFILEPATH
-#define TAOS_OS_FUNC_FILE_FTRUNCATE
- extern int taosFtruncate(int fd, int64_t length);
+#define TAOS_OS_FUNC_FILE_FTRUNCATE
#define TAOS_OS_FUNC_MATH
#define SWAP(a, b, c) \
@@ -139,7 +136,6 @@ typedef int (*__compar_fn_t)(const void *, const void *);
#define in_addr_t unsigned long
#define socklen_t int
#define htobe64 htonll
-#define twrite write
#define getpid _getpid
struct tm *localtime_r(const time_t *timep, struct tm *result);
diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c
index 66bdb5b939d77ab03b30f024466fe3e3ff4e9388..dacf4db74137b8cc58db2b390ba971cf9dfbc012 100644
--- a/src/os/src/darwin/darwinFile.c
+++ b/src/os/src/darwin/darwinFile.c
@@ -19,21 +19,19 @@
#define _SEND_FILE_STEP_ 1000
-int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) {
+int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
fseek(in_file, (int32_t)(*offset), 0);
- int writeLen = 0;
- uint8_t buffer[_SEND_FILE_STEP_] = { 0 };
+ int writeLen = 0;
+ uint8_t buffer[_SEND_FILE_STEP_] = {0};
for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file);
if (rlen <= 0) {
return writeLen;
- }
- else if (rlen < _SEND_FILE_STEP_) {
+ } else if (rlen < _SEND_FILE_STEP_) {
fwrite(buffer, 1, rlen, out_file);
return (int)(writeLen + rlen);
- }
- else {
+ } else {
fwrite(buffer, 1, _SEND_FILE_STEP_, in_file);
writeLen += _SEND_FILE_STEP_;
}
@@ -44,8 +42,7 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou
size_t rlen = fread(buffer, 1, remain, in_file);
if (rlen <= 0) {
return writeLen;
- }
- else {
+ } else {
fwrite(buffer, 1, remain, out_file);
writeLen += remain;
}
@@ -54,7 +51,7 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou
return writeLen;
}
-ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
- uError("not implemented yet");
+int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) {
+ uError("taosSendFile not implemented yet");
return -1;
}
\ No newline at end of file
diff --git a/src/os/src/detail/osFail.c b/src/os/src/detail/osFail.c
index e0eb200851f976e1c9dad47afbc152bfb021e4e5..a99bcd01dbccd0290de1fc38a1220b573ab74b22 100644
--- a/src/os/src/detail/osFail.c
+++ b/src/os/src/detail/osFail.c
@@ -20,7 +20,7 @@
#ifdef TAOS_RANDOM_NETWORK_FAIL
-ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags) {
+int64_t taosSendRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags) {
if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
errno = ECONNRESET;
return -1;
@@ -29,8 +29,8 @@ ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags) {
return send(sockfd, buf, len, flags);
}
-ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr,
- socklen_t addrlen) {
+int64_t taosSendToRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags,
+ const struct sockaddr *dest_addr, socklen_t addrlen) {
if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
errno = ECONNRESET;
return -1;
@@ -39,7 +39,7 @@ ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags,
return sendto(sockfd, buf, len, flags, dest_addr, addrlen);
}
-ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count) {
+int64_t taosReadSocketRandomFail(int32_t fd, void *buf, size_t count) {
if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
errno = ECONNRESET;
return -1;
@@ -48,7 +48,7 @@ ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count) {
return read(fd, buf, count);
}
-ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count) {
+int64_t taosWriteSocketRandomFail(int32_t fd, const void *buf, size_t count) {
if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
errno = EINTR;
return -1;
@@ -61,10 +61,10 @@ ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count) {
#ifdef TAOS_RANDOM_FILE_FAIL
-static int random_file_fail_factor = 20;
+static int32_t random_file_fail_factor = 20;
static FILE *fpRandomFileFailOutput = NULL;
-void taosSetRandomFileFailFactor(int factor) {
+void taosSetRandomFileFailFactor(int32_t factor) {
random_file_fail_factor = factor;
}
@@ -77,7 +77,7 @@ static void close_random_file_fail_output() {
}
}
-static void random_file_fail_output_sig(int sig) {
+static void random_file_fail_output_sig(int32_t sig) {
fprintf(fpRandomFileFailOutput, "signal %d received.\n", sig);
struct sigaction act = {0};
@@ -105,7 +105,7 @@ void taosSetRandomFileFailOutput(const char *path) {
sigaction(SIGILL, &act, NULL);
}
-ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) {
+int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line) {
if (random_file_fail_factor > 0) {
if (rand() % random_file_fail_factor == 0) {
errno = EIO;
@@ -113,10 +113,10 @@ ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file
}
}
- return taosTReadImp(fd, buf, count);
+ return taosReadImp(fd, buf, count);
}
-ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) {
+int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line) {
if (random_file_fail_factor > 0) {
if (rand() % random_file_fail_factor == 0) {
errno = EIO;
@@ -124,10 +124,10 @@ ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *fil
}
}
- return taosTWriteImp(fd, buf, count);
+ return taosWriteImp(fd, buf, count);
}
-off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line) {
+int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line) {
if (random_file_fail_factor > 0) {
if (rand() % random_file_fail_factor == 0) {
errno = EIO;
@@ -135,7 +135,7 @@ off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, ui
}
}
- return lseek(fd, offset, whence);
+ return taosLSeekImp(fd, offset, whence);
}
#endif //TAOS_RANDOM_FILE_FAIL
diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c
index 429aab77e4c8a094df8abdab204be0ffc69535fd..493796bdcec4dc800ee5d3f9a15fa5b3da150e10 100644
--- a/src/os/src/detail/osFile.c
+++ b/src/os/src/detail/osFile.c
@@ -15,16 +15,22 @@
#define _DEFAULT_SOURCE
#include "os.h"
+#include "tglobal.h"
#ifndef TAOS_OS_FUNC_FILE_GETTMPFILEPATH
+
void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
const char *tdengineTmpFileNamePrefix = "tdengine-";
char tmpPath[PATH_MAX];
- char *tmpDir = "/tmp/";
+ int32_t len = strlen(tsTempDir);
+ memcpy(tmpPath, tsTempDir, len);
+
+ if (tmpPath[len - 1] != '/') {
+ tmpPath[len++] = '/';
+ }
- strcpy(tmpPath, tmpDir);
- strcat(tmpPath, tdengineTmpFileNamePrefix);
+ strcpy(tmpPath + len, tdengineTmpFileNamePrefix);
if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
strcat(tmpPath, fileNamePrefix);
strcat(tmpPath, "-%d-%s");
@@ -34,10 +40,10 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
taosRandStr(rand, tListLen(rand) - 1);
snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
}
+
#endif
-// rename file name
-int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) {
+int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath) {
int32_t ts = taosGetTimestampSec();
char fname[PATH_MAX] = {0}; // max file name length must be less than 255
@@ -46,12 +52,13 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP
if (delimiterPos == NULL) return -1;
int32_t fileNameLen = 0;
- if (suffix)
+ if (suffix) {
fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix);
- else
+ } else {
fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts);
+ }
- size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1);
+ int32_t len = (int32_t)((delimiterPos - fullPath) + fileNameLen + 1);
if (*dstPath == NULL) {
*dstPath = calloc(1, len + 1);
if (*dstPath == NULL) return -1;
@@ -64,9 +71,9 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP
return rename(fullPath, *dstPath);
}
-ssize_t taosTReadImp(int fd, void *buf, size_t count) {
- size_t leftbytes = count;
- ssize_t readbytes;
+int64_t taosReadImp(int32_t fd, void *buf, int64_t count) {
+ int64_t leftbytes = count;
+ int64_t readbytes;
char * tbuf = (char *)buf;
while (leftbytes > 0) {
@@ -78,19 +85,19 @@ ssize_t taosTReadImp(int fd, void *buf, size_t count) {
return -1;
}
} else if (readbytes == 0) {
- return (ssize_t)(count - leftbytes);
+ return (int64_t)(count - leftbytes);
}
leftbytes -= readbytes;
tbuf += readbytes;
}
- return (ssize_t)count;
+ return count;
}
-ssize_t taosTWriteImp(int fd, void *buf, size_t n) {
- size_t nleft = n;
- ssize_t nwritten = 0;
+int64_t taosWriteImp(int32_t fd, void *buf, int64_t n) {
+ int64_t nleft = n;
+ int64_t nwritten = 0;
char * tbuf = (char *)buf;
while (nleft > 0) {
@@ -105,7 +112,11 @@ ssize_t taosTWriteImp(int fd, void *buf, size_t n) {
tbuf += nwritten;
}
- return (ssize_t)n;
+ return n;
+}
+
+int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) {
+ return (int64_t)tlseek(fd, (long)offset, whence);
}
ssize_t taosTCopy(char *from, char *to) {
@@ -142,9 +153,10 @@ _err:
}
#ifndef TAOS_OS_FUNC_FILE_SENDIFLE
-ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
- size_t leftbytes = size;
- ssize_t sentbytes;
+
+int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) {
+ int64_t leftbytes = size;
+ int64_t sentbytes;
while (leftbytes > 0) {
/*
@@ -153,13 +165,13 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
// if (leftbytes > 1000000000) leftbytes = 1000000000;
sentbytes = sendfile(dfd, sfd, offset, leftbytes);
if (sentbytes == -1) {
- if (errno == EINTR) {
+ if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
continue;
} else {
return -1;
}
} else if (sentbytes == 0) {
- return (ssize_t)(size - leftbytes);
+ return (int64_t)(size - leftbytes);
}
leftbytes -= sentbytes;
@@ -167,4 +179,17 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
return size;
}
+
+int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) {
+ return taosSendFile(fileno(outfile), fileno(infile), offset, size);
+}
+
+#endif
+
+#ifndef TAOS_OS_FUNC_FILE_FTRUNCATE
+
+int32_t taosFtruncate(int32_t fd, int64_t length) {
+ return ftruncate(fd, length);
+}
+
#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osMemory.c b/src/os/src/detail/osMemory.c
index dfd320be89a75a4700a4918f68d56ee66faa3135..53310d179c0090382e009de949e5158146dc282a 100644
--- a/src/os/src/detail/osMemory.c
+++ b/src/os/src/detail/osMemory.c
@@ -28,7 +28,7 @@ static FILE* fpAllocLog = NULL;
extern int32_t taosGetTimestampSec();
static int32_t startTime = INT32_MAX;
-static bool random_alloc_fail(size_t size, const char* file, uint32_t line) {
+static bool taosRandomAllocFail(size_t size, const char* file, uint32_t line) {
if (taosGetTimestampSec() < startTime) {
return false;
}
@@ -48,33 +48,33 @@ static bool random_alloc_fail(size_t size, const char* file, uint32_t line) {
return true;
}
-static void* malloc_random(size_t size, const char* file, uint32_t line) {
- return random_alloc_fail(size, file, line) ? NULL : malloc(size);
+static void* taosRandmoMalloc(size_t size, const char* file, uint32_t line) {
+ return taosRandomAllocFail(size, file, line) ? NULL : malloc(size);
}
-static void* calloc_random(size_t num, size_t size, const char* file, uint32_t line) {
- return random_alloc_fail(num * size, file, line) ? NULL : calloc(num, size);
+static void* taosRandomCalloc(size_t num, size_t size, const char* file, uint32_t line) {
+ return taosRandomAllocFail(num * size, file, line) ? NULL : calloc(num, size);
}
-static void* realloc_random(void* ptr, size_t size, const char* file, uint32_t line) {
- return random_alloc_fail(size, file, line) ? NULL : realloc(ptr, size);
+static void* taosRandomRealloc(void* ptr, size_t size, const char* file, uint32_t line) {
+ return taosRandomAllocFail(size, file, line) ? NULL : realloc(ptr, size);
}
-static char* strdup_random(const char* str, const char* file, uint32_t line) {
+static char* taosRandomStrdup(const char* str, const char* file, uint32_t line) {
size_t len = strlen(str);
- return random_alloc_fail(len + 1, file, line) ? NULL : taosStrdupImp(str);
+ return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrdupImp(str);
}
-static char* strndup_random(const char* str, size_t size, const char* file, uint32_t line) {
+static char* taosRandomStrndup(const char* str, size_t size, const char* file, uint32_t line) {
size_t len = strlen(str);
if (len > size) {
len = size;
}
- return random_alloc_fail(len + 1, file, line) ? NULL : taosStrndupImp(str, len);
+ return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrndupImp(str, len);
}
-static ssize_t getline_random(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
- return random_alloc_fail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream);
+static ssize_t taosRandomGetline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
+ return taosRandomAllocFail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream);
}
////////////////////////////////////////////////////////////////////////////////
@@ -96,7 +96,7 @@ typedef struct SMemBlock {
static SMemBlock *blocks = NULL;
static uintptr_t lock = 0;
-static void add_mem_block(SMemBlock* blk) {
+static void taosAddMemBlock(SMemBlock* blk) {
blk->prev = NULL;
while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0);
blk->next = blocks;
@@ -107,7 +107,7 @@ static void add_mem_block(SMemBlock* blk) {
atomic_store_ptr(&lock, 0);
}
-static void remove_mem_block(SMemBlock* blk) {
+static void taosRemoveMemBlock(SMemBlock* blk) {
while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0);
if (blocks == blk) {
@@ -126,7 +126,7 @@ static void remove_mem_block(SMemBlock* blk) {
blk->next = NULL;
}
-static void free_detect_leak(void* ptr, const char* file, uint32_t line) {
+static void taosFreeDetectLeak(void* ptr, const char* file, uint32_t line) {
if (ptr == NULL) {
return;
}
@@ -140,11 +140,11 @@ static void free_detect_leak(void* ptr, const char* file, uint32_t line) {
return;
}
- remove_mem_block(blk);
+ taosRemoveMemBlock(blk);
free(blk);
}
-static void* malloc_detect_leak(size_t size, const char* file, uint32_t line) {
+static void* taosMallocDetectLeak(size_t size, const char* file, uint32_t line) {
if (size == 0) {
return NULL;
}
@@ -166,28 +166,28 @@ static void* malloc_detect_leak(size_t size, const char* file, uint32_t line) {
blk->line = (uint16_t)line;
blk->magic = MEMBLK_MAGIC;
blk->size = size;
- add_mem_block(blk);
+ taosAddMemBlock(blk);
return blk->data;
}
-static void* calloc_detect_leak(size_t num, size_t size, const char* file, uint32_t line) {
+static void* taosCallocDetectLeak(size_t num, size_t size, const char* file, uint32_t line) {
size *= num;
- void* p = malloc_detect_leak(size, file, line);
+ void* p = taosMallocDetectLeak(size, file, line);
if (p != NULL) {
memset(p, 0, size);
}
return p;
}
-static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint32_t line) {
+static void* taosReallocDetectLeak(void* ptr, size_t size, const char* file, uint32_t line) {
if (size == 0) {
- free_detect_leak(ptr, file, line);
+ taosFreeDetectLeak(ptr, file, line);
return NULL;
}
if (ptr == NULL) {
- return malloc_detect_leak(size, file, line);
+ return taosMallocDetectLeak(size, file, line);
}
SMemBlock* blk = (SMemBlock *)((char*)ptr) - sizeof(SMemBlock);
@@ -198,11 +198,11 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3
return realloc(ptr, size);
}
- remove_mem_block(blk);
+ taosRemoveMemBlock(blk);
void* p = realloc(blk, size + sizeof(SMemBlock));
if (p == NULL) {
- add_mem_block(blk);
+ taosAddMemBlock(blk);
return NULL;
}
@@ -212,13 +212,13 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3
blk = (SMemBlock*)p;
blk->size = size;
- add_mem_block(blk);
+ taosAddMemBlock(blk);
return blk->data;
}
-static char* strdup_detect_leak(const char* str, const char* file, uint32_t line) {
+static char* taosStrdupDetectLeak(const char* str, const char* file, uint32_t line) {
size_t len = strlen(str);
- char *p = malloc_detect_leak(len + 1, file, line);
+ char *p = taosMallocDetectLeak(len + 1, file, line);
if (p != NULL) {
memcpy(p, str, len);
p[len] = 0;
@@ -226,12 +226,12 @@ static char* strdup_detect_leak(const char* str, const char* file, uint32_t line
return p;
}
-static char* strndup_detect_leak(const char* str, size_t size, const char* file, uint32_t line) {
+static char* taosStrndupDetectLeak(const char* str, size_t size, const char* file, uint32_t line) {
size_t len = strlen(str);
if (len > size) {
len = size;
}
- char *p = malloc_detect_leak(len + 1, file, line);
+ char *p = taosMallocDetectLeak(len + 1, file, line);
if (p != NULL) {
memcpy(p, str, len);
p[len] = 0;
@@ -239,13 +239,13 @@ static char* strndup_detect_leak(const char* str, size_t size, const char* file,
return p;
}
-static ssize_t getline_detect_leak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
+static ssize_t taosGetlineDetectLeak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
char* buf = NULL;
size_t bufSize = 0;
ssize_t size = taosGetlineImp(&buf, &bufSize, stream);
if (size != -1) {
if (*n < size + 1) {
- void* p = realloc_detect_leak(*lineptr, size + 1, file, line);
+ void* p = taosReallocDetectLeak(*lineptr, size + 1, file, line);
if (p == NULL) {
free(buf);
return -1;
@@ -260,7 +260,7 @@ static ssize_t getline_detect_leak(char **lineptr, size_t *n, FILE *stream, cons
return size;
}
-static void dump_memory_leak() {
+static void taosDumpMemoryLeakImp() {
const char* hex = "0123456789ABCDEF";
const char* fmt = ":%d: addr=%p, size=%d, content(first 16 bytes)=";
size_t numOfBlk = 0, totalSize = 0;
@@ -299,7 +299,7 @@ static void dump_memory_leak() {
fflush(fpAllocLog);
}
-static void dump_memory_leak_on_sig(int sig) {
+static void taosDumpMemoryLeakOnSig(int sig) {
fprintf(fpAllocLog, "signal %d received.\n", sig);
// restore default signal handler
@@ -307,55 +307,55 @@ static void dump_memory_leak_on_sig(int sig) {
act.sa_handler = SIG_DFL;
sigaction(sig, &act, NULL);
- dump_memory_leak();
+ taosDumpMemoryLeakImp();
}
////////////////////////////////////////////////////////////////////////////////
// interface functions
-void* taos_malloc(size_t size, const char* file, uint32_t line) {
+void* taosMallocMem(size_t size, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return malloc(size);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
- return malloc_random(size, file, line);
+ return taosRandmoMalloc(size, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return malloc_detect_leak(size, file, line);
+ return taosMallocDetectLeak(size, file, line);
}
return malloc(size);
}
-void* taos_calloc(size_t num, size_t size, const char* file, uint32_t line) {
+void* taosCallocMem(size_t num, size_t size, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return calloc(num, size);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
- return calloc_random(num, size, file, line);
+ return taosRandomCalloc(num, size, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return calloc_detect_leak(num, size, file, line);
+ return taosCallocDetectLeak(num, size, file, line);
}
return calloc(num, size);
}
-void* taos_realloc(void* ptr, size_t size, const char* file, uint32_t line) {
+void* taosReallocMem(void* ptr, size_t size, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return realloc(ptr, size);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
- return realloc_random(ptr, size, file, line);
+ return taosRandomRealloc(ptr, size, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return realloc_detect_leak(ptr, size, file, line);
+ return taosReallocDetectLeak(ptr, size, file, line);
}
return realloc(ptr, size);
}
-void taos_free(void* ptr, const char* file, uint32_t line) {
+void taosFreeMem(void* ptr, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return free(ptr);
@@ -364,54 +364,54 @@ void taos_free(void* ptr, const char* file, uint32_t line) {
return free(ptr);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return free_detect_leak(ptr, file, line);
+ return taosFreeDetectLeak(ptr, file, line);
}
return free(ptr);
}
-char* taos_strdup(const char* str, const char* file, uint32_t line) {
+char* taosStrdupMem(const char* str, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return taosStrdupImp(str);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
- return strdup_random(str, file, line);
+ return taosRandomStrdup(str, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return strdup_detect_leak(str, file, line);
+ return taosStrdupDetectLeak(str, file, line);
}
return taosStrdupImp(str);
}
-char* taos_strndup(const char* str, size_t size, const char* file, uint32_t line) {
+char* taosStrndupMem(const char* str, size_t size, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return taosStrndupImp(str, size);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
- return strndup_random(str, size, file, line);
+ return taosRandomStrndup(str, size, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return strndup_detect_leak(str, size, file, line);
+ return taosStrndupDetectLeak(str, size, file, line);
}
return taosStrndupImp(str, size);
}
-ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
+ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return taosGetlineImp(lineptr, n, stream);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
- return getline_random(lineptr, n, stream, file, line);
+ return taosRandomGetline(lineptr, n, stream, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return getline_detect_leak(lineptr, n, stream, file, line);
+ return taosGetlineDetectLeak(lineptr, n, stream, file, line);
}
return taosGetlineImp(lineptr, n, stream);
}
-static void close_alloc_log() {
+static void taosCloseAllocLog() {
if (fpAllocLog != NULL) {
if (fpAllocLog != stdout) {
fclose(fpAllocLog);
@@ -432,7 +432,7 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) {
if (path == NULL || path[0] == 0) {
fpAllocLog = stdout;
} else if ((fpAllocLog = fopen(path, "w")) != NULL) {
- atexit(close_alloc_log);
+ atexit(taosCloseAllocLog);
} else {
printf("failed to open memory allocation log file '%s', errno=%d\n", path, errno);
return;
@@ -446,10 +446,10 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) {
}
if (autoDump && mode == TAOS_ALLOC_MODE_DETECT_LEAK) {
- atexit(dump_memory_leak);
+ atexit(taosDumpMemoryLeakImp);
struct sigaction act = {0};
- act.sa_handler = dump_memory_leak_on_sig;
+ act.sa_handler = taosDumpMemoryLeakOnSig;
sigaction(SIGFPE, &act, NULL);
sigaction(SIGSEGV, &act, NULL);
sigaction(SIGILL, &act, NULL);
@@ -457,8 +457,8 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) {
}
void taosDumpMemoryLeak() {
- dump_memory_leak();
- close_alloc_log();
+ taosDumpMemoryLeakImp();
+ taosCloseAllocLog();
}
#else // 'TAOS_MEM_CHECK' not defined
diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c
index 8a51c389e9f2efdcf3eaad97d434269ff3c0061f..c7c9d774271555ae5989aa18b7e00d325a0eddde 100644
--- a/src/os/src/detail/osSocket.c
+++ b/src/os/src/detail/osSocket.c
@@ -19,8 +19,8 @@
#ifndef TAOS_OS_FUNC_SOCKET
-int taosSetNonblocking(SOCKET sock, int on) {
- int flags = 0;
+int32_t taosSetNonblocking(SOCKET sock, int32_t on) {
+ int32_t flags = 0;
if ((flags = fcntl(sock, F_GETFL, 0)) < 0) {
uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno));
return 1;
@@ -43,7 +43,7 @@ void taosBlockSIGPIPE() {
sigset_t signal_mask;
sigemptyset(&signal_mask);
sigaddset(&signal_mask, SIGPIPE);
- int rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
+ int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
if (rc != 0) {
uError("failed to block SIGPIPE");
}
@@ -53,7 +53,7 @@ void taosBlockSIGPIPE() {
#ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT
-int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) {
+int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) {
return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
}
diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c
index 8cac60bfd6c32c3f51edcc6d15e99132765b07d8..7baa58bfcd842a9bbbf40c7434db988d0510cdc0 100644
--- a/src/os/src/detail/osSysinfo.c
+++ b/src/os/src/detail/osSysinfo.c
@@ -62,7 +62,7 @@ bool taosGetProcMemory(float *memoryUsedMB) {
size_t len;
char * line = NULL;
while (!feof(fp)) {
- taosTFree(line);
+ tfree(line);
len = 0;
getline(&line, &len, fp);
if (line == NULL) {
@@ -84,7 +84,7 @@ bool taosGetProcMemory(float *memoryUsedMB) {
sscanf(line, "%s %" PRId64, tmp, &memKB);
*memoryUsedMB = (float)((double)memKB / 1024);
- taosTFree(line);
+ tfree(line);
fclose(fp);
return true;
}
@@ -108,7 +108,7 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
char cpu[10] = {0};
sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle);
- taosTFree(line);
+ tfree(line);
fclose(fp);
return true;
}
@@ -137,7 +137,7 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
}
}
- taosTFree(line);
+ tfree(line);
fclose(fp);
return true;
}
@@ -351,7 +351,7 @@ static bool taosGetCardInfo(int64_t *bytes) {
*bytes += (rbytes + tbytes);
}
- taosTFree(line);
+ tfree(line);
fclose(fp);
return true;
@@ -406,7 +406,7 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) {
int readIndex = 0;
while (!feof(fp)) {
- taosTFree(line);
+ tfree(line);
len = 0;
getline(&line, &len, fp);
if (line == NULL) {
@@ -424,7 +424,7 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) {
if (readIndex >= 2) break;
}
- taosTFree(line);
+ tfree(line);
fclose(fp);
if (readIndex < 2) {
diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c
index 22f7b94c3aea55432b3e59c4ad8664cb3ef6020e..9883a03a0933075616b69800fe6e34a36fc6c746 100644
--- a/src/os/src/detail/osTimer.c
+++ b/src/os/src/detail/osTimer.c
@@ -111,6 +111,9 @@ void taosUninitTimer() {
pthread_sigmask(SIG_BLOCK, &set, NULL);
*/
void taosMsleep(int mseconds) {
+#if 1
+ usleep(mseconds * 1000);
+#else
struct timeval timeout;
int seconds, useconds;
@@ -126,7 +129,8 @@ void taosMsleep(int mseconds) {
select(0, NULL, NULL, NULL, &timeout);
- /* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */
+/* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */
+#endif
}
#endif
\ No newline at end of file
diff --git a/src/os/src/windows/wAtomic.c b/src/os/src/windows/wAtomic.c
index a025cb8f0e67ff65ceb0cfadd420fd10cb99891b..b645893030b852428bff13744050b1f6f8ef1c42 100644
--- a/src/os/src/windows/wAtomic.c
+++ b/src/os/src/windows/wAtomic.c
@@ -44,7 +44,7 @@ long interlocked_add_fetch_32(long volatile* ptr, long val) {
__int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) {
//#ifdef _WIN64
- return _InterlockedExchangeAdd64(ptr, val) + val;
+ return InterlockedExchangeAdd64(ptr, val) + val;
//#else
// return _InterlockedExchangeAdd(ptr, val) + val;
//#endif
diff --git a/src/os/src/windows/wEnv.c b/src/os/src/windows/wEnv.c
index 8110a194904bbbc1166ac57de187de284db34a6b..19351eb7c964a4c2a8a4d1d5d4d1c8ec669908dc 100644
--- a/src/os/src/windows/wEnv.c
+++ b/src/os/src/windows/wEnv.c
@@ -46,5 +46,16 @@ void osInit() {
strcpy(tsDnodeDir, "");
strcpy(tsMnodeDir, "");
strcpy(tsOsName, "Windows");
+
+ const char *tmpDir = getenv("tmp");
+ if (tmpDir != NULL) {
+ tmpDir = getenv("temp");
+ }
+ if (tmpDir != NULL) {
+ strcpy(tsTempDir, tmpDir);
+ } else {
+ strcpy(tsTempDir, "C:\\Windows\\Temp");
+ }
+
taosWinSocketInit();
}
diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c
index 5549c078a5220d3e493ce21e2c19e38fb9611cbb..2204135ae657eb813144dc2bd1ae637d9510842e 100644
--- a/src/os/src/windows/wFile.c
+++ b/src/os/src/windows/wFile.c
@@ -16,17 +16,20 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "tulog.h"
+#include "tglobal.h"
void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
const char* tdengineTmpFileNamePrefix = "tdengine-";
- char tmpPath[PATH_MAX];
+ char tmpPath[PATH_MAX];
- char *tmpDir = getenv("tmp");
- if (tmpDir == NULL) {
- tmpDir = "";
+ int32_t len = (int32_t)strlen(tsTempDir);
+ memcpy(tmpPath, tsTempDir, len);
+
+ if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') {
+ tmpPath[len++] = '\\';
}
-
- strcpy(tmpPath, tmpDir);
+
+ strcpy(tmpPath + len, tdengineTmpFileNamePrefix);
strcat(tmpPath, tdengineTmpFileNamePrefix);
if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
strcat(tmpPath, fileNamePrefix);
@@ -40,19 +43,19 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
#define _SEND_FILE_STEP_ 1000
-int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) {
+int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
fseek(in_file, (int32_t)(*offset), 0);
- int writeLen = 0;
+ int64_t writeLen = 0;
uint8_t buffer[_SEND_FILE_STEP_] = { 0 };
- for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
+ for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file);
if (rlen <= 0) {
return writeLen;
}
else if (rlen < _SEND_FILE_STEP_) {
fwrite(buffer, 1, rlen, out_file);
- return (int)(writeLen + rlen);
+ return (int64_t)(writeLen + rlen);
}
else {
fwrite(buffer, 1, _SEND_FILE_STEP_, in_file);
@@ -60,14 +63,14 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou
}
}
- int remain = count - writeLen;
+ int64_t remain = count - writeLen;
if (remain > 0) {
- size_t rlen = fread(buffer, 1, remain, in_file);
+ size_t rlen = fread(buffer, 1, (size_t) remain, in_file);
if (rlen <= 0) {
return writeLen;
}
else {
- fwrite(buffer, 1, remain, out_file);
+ fwrite(buffer, 1, (size_t) remain, out_file);
writeLen += remain;
}
}
@@ -75,12 +78,12 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou
return writeLen;
}
-ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
- uError("taosTSendFileImp no implemented yet");
+int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) {
+ uError("taosSendFile no implemented yet");
return 0;
}
-int taosFtruncate(int fd, int64_t length) {
+int32_t taosFtruncate(int32_t fd, int64_t length) {
uError("taosFtruncate no implemented yet");
return 0;
}
\ No newline at end of file
diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c
index da9242d6a3bff471c4e749f0faf7c95b6c3a9d8c..3b091b269931e644d9f8c2c01ba3c9cb9ddc520c 100644
--- a/src/os/src/windows/wSocket.c
+++ b/src/os/src/windows/wSocket.c
@@ -34,7 +34,7 @@ void taosWinSocketInit() {
}
}
-int taosSetNonblocking(SOCKET sock, int on) {
+int32_t taosSetNonblocking(SOCKET sock, int32_t on) {
u_long mode;
if (on) {
mode = 1;
@@ -48,7 +48,7 @@ int taosSetNonblocking(SOCKET sock, int on) {
void taosBlockSIGPIPE() {}
-int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) {
+int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) {
if (level == SOL_SOCKET && optname == TCP_KEEPCNT) {
return 0;
}
@@ -72,7 +72,7 @@ int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int op
uint32_t taosInetAddr(char *ipAddr) {
uint32_t value;
- int ret = inet_pton(AF_INET, ipAddr, &value);
+ int32_t ret = inet_pton(AF_INET, ipAddr, &value);
if (ret <= 0) {
return INADDR_NONE;
} else {
diff --git a/src/plugins/http/inc/httpQueue.h b/src/plugins/http/inc/httpQueue.h
index a4590719ff24d48eee875b2f2c4ff2f28a0a31f6..1ffbd5148164ab4d1f74bdac781c8fcf731ad772 100644
--- a/src/plugins/http/inc/httpQueue.h
+++ b/src/plugins/http/inc/httpQueue.h
@@ -22,9 +22,11 @@ extern "C" {
#include
+typedef void (*FHttpResultFp)(void *param, void *result, int32_t code, int32_t rows);
+
bool httpInitResultQueue();
void httpCleanupResultQueue();
-void httpDispatchToResultQueue();
+void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp);
#ifdef __cplusplus
}
diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c
index ec60b984b2b0fcc72367295cef9d956165e678d5..22f464924e2c21cdeccb0c152457fad11a10294a 100644
--- a/src/plugins/http/src/httpContext.c
+++ b/src/plugins/http/src/httpContext.c
@@ -63,7 +63,7 @@ static void httpDestroyContext(void *data) {
pContext->parser = NULL;
}
- taosTFree(pContext);
+ tfree(pContext);
}
bool httpInitContexts() {
diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c
index 43a8ddbd1a0d207addf0576951b2af9b2be16555..1c039abb4d4cd546f6673648217a2410f57ae8e7 100644
--- a/src/plugins/http/src/httpQueue.c
+++ b/src/plugins/http/src/httpQueue.c
@@ -25,6 +25,7 @@
#include "httpResp.h"
#include "httpAuth.h"
#include "httpSession.h"
+#include "httpQueue.h"
typedef struct {
pthread_t thread;
@@ -37,42 +38,45 @@ typedef struct {
} SHttpWorkerPool;
typedef struct {
- void *param;
- void *result;
- int32_t numOfRows;
- void (*fp)(void *param, void *result, int32_t numOfRows);
+ void * param;
+ void * result;
+ int32_t code;
+ int32_t rows;
+ FHttpResultFp fp;
} SHttpResult;
static SHttpWorkerPool tsHttpPool;
static taos_qset tsHttpQset;
static taos_queue tsHttpQueue;
-void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t numOfRows, void (*fp)(void *param, void *result, int32_t numOfRows)) {
+void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp) {
if (tsHttpQueue != NULL) {
- SHttpResult *pMsg = (SHttpResult *)taosAllocateQitem(sizeof(SHttpResult));
+ SHttpResult *pMsg = taosAllocateQitem(sizeof(SHttpResult));
pMsg->param = param;
pMsg->result = result;
- pMsg->numOfRows = numOfRows;
+ pMsg->code = code;
+ pMsg->rows = rows;
pMsg->fp = fp;
taosWriteQitem(tsHttpQueue, TAOS_QTYPE_RPC, pMsg);
} else {
- (*fp)(param, result, numOfRows);
+ (*fp)(param, result, code, rows);
}
}
static void *httpProcessResultQueue(void *param) {
SHttpResult *pMsg;
- int32_t type;
- void *unUsed;
-
+ int32_t type;
+ void * unUsed;
+
while (1) {
if (taosReadQitemFromQset(tsHttpQset, &type, (void **)&pMsg, &unUsed) == 0) {
httpDebug("qset:%p, http queue got no message from qset, exiting", tsHttpQset);
break;
}
- httpTrace("context:%p, res:%p will be processed in result queue", pMsg->param, pMsg->result);
- (*pMsg->fp)(pMsg->param, pMsg->result, pMsg->numOfRows);
+ httpTrace("context:%p, res:%p will be processed in result queue, code:%d rows:%d", pMsg->param, pMsg->result,
+ pMsg->code, pMsg->rows);
+ (*pMsg->fp)(pMsg->param, pMsg->result, pMsg->code, pMsg->rows);
taosFreeQitem(pMsg);
}
diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c
index f0a7249b512d2c9161592abe91822aefba786c20..4896d50c6cabf6d924161c97f7fe020fe81e2ebf 100644
--- a/src/plugins/http/src/httpServer.c
+++ b/src/plugins/http/src/httpServer.c
@@ -85,7 +85,7 @@ static void httpProcessHttpData(void *param) {
while (1) {
struct epoll_event events[HTTP_MAX_EVENTS];
//-1 means uncertainty, 0-nowait, 1-wait 1 ms, set it from -1 to 1
- fdNum = epoll_wait(pThread->pollFd, events, HTTP_MAX_EVENTS, 1);
+ fdNum = epoll_wait(pThread->pollFd, events, HTTP_MAX_EVENTS, TAOS_EPOLL_WAIT_TIME);
if (pThread->stop) {
httpDebug("%p, http thread get stop event, exiting...", pThread);
break;
diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c
index 70d644146cf5ebe15a42c8f00a2e6e4bd603d081..564f555c403b6f5b3deb5832da6505722104eff8 100644
--- a/src/plugins/http/src/httpSql.c
+++ b/src/plugins/http/src/httpSql.c
@@ -29,9 +29,9 @@
void httpProcessMultiSql(HttpContext *pContext);
-void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows);
+void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows);
-void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int numOfRows) {
+void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
@@ -43,7 +43,7 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n
bool isContinue = false;
- if (numOfRows > 0) {
+ if (code == TSDB_CODE_SUCCESS && numOfRows > 0) {
if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->buildQueryJsonFp) {
isContinue = (encode->buildQueryJsonFp)(pContext, singleCmd, result, numOfRows);
}
@@ -58,9 +58,9 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, stop retrieve, numOfRows:%d, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, numOfRows, sql);
- if (numOfRows < 0) {
+ if (code < 0) {
httpError("context:%p, fd:%d, user:%s, process pos:%d, retrieve failed code:%s, sql:%s", pContext, pContext->fd,
- pContext->user, multiCmds->pos, tstrerror(numOfRows), sql);
+ pContext->user, multiCmds->pos, tstrerror(code), sql);
}
taos_free_result(result);
@@ -73,15 +73,15 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n
}
}
-void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) {
- httpDispatchToResultQueue(param, result, numOfRows, httpProcessMultiSqlRetrieveCallBackImp);
+void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) {
+ int32_t code = taos_errno(result);
+ httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessMultiSqlRetrieveCallBackImp);
}
-void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
+void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
- code = taos_errno(result);
HttpSqlCmds *multiCmds = pContext->multiCmds;
HttpEncodeMethod *encode = pContext->encodeMethod;
@@ -94,7 +94,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
return;
}
- if (code < 0) {
+ if (code != TSDB_CODE_SUCCESS) {
if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code)) {
singleCmd->code = code;
httpDebug("context:%p, fd:%d, user:%s, process pos jump to:%d, last code:%s, last sql:%s", pContext, pContext->fd,
@@ -119,7 +119,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
bool isUpdate = tscIsUpdateQuery(result);
if (isUpdate) {
// not select or show commands
- int affectRows = taos_affected_rows(result);
+ int32_t affectRows = taos_affected_rows(result);
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, affect rows:%d, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, affectRows, sql);
@@ -156,8 +156,10 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
}
}
-void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) {
- httpDispatchToResultQueue(param, result, unUsedCode, httpProcessMultiSqlCallBackImp);
+void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) {
+ int32_t code = taos_errno(result);
+ int32_t affectRows = taos_affected_rows(result);
+ httpDispatchToResultQueue(param, result, code, affectRows, httpProcessMultiSqlCallBackImp);
}
void httpProcessMultiSql(HttpContext *pContext) {
@@ -202,9 +204,9 @@ void httpProcessMultiSqlCmd(HttpContext *pContext) {
httpProcessMultiSql(pContext);
}
-void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows);
+void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows);
-void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int numOfRows) {
+void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
@@ -212,7 +214,7 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int
bool isContinue = false;
- if (numOfRows > 0) {
+ if (code == TSDB_CODE_SUCCESS && numOfRows > 0) {
if (encode->buildQueryJsonFp) {
isContinue = (encode->buildQueryJsonFp)(pContext, &pContext->singleCmd, result, numOfRows);
}
@@ -227,9 +229,9 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int
httpDebug("context:%p, fd:%d, user:%s, stop retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user,
numOfRows);
- if (numOfRows < 0) {
+ if (code < 0) {
httpError("context:%p, fd:%d, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->user,
- tstrerror(numOfRows));
+ tstrerror(code));
}
taos_free_result(result);
@@ -242,30 +244,30 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int
}
}
-void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) {
- httpDispatchToResultQueue(param, result, numOfRows, httpProcessSingleSqlRetrieveCallBackImp);
+void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) {
+ int32_t code = taos_errno(result);
+ httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessSingleSqlRetrieveCallBackImp);
}
-void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCode) {
+void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
- int32_t code = taos_errno(result);
-
HttpEncodeMethod *encode = pContext->encodeMethod;
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
httpError("context:%p, fd:%d, user:%s, query error, code:%s:inprogress, sqlObj:%p", pContext, pContext->fd,
- pContext->user, tstrerror(code), (SSqlObj *)result);
+ pContext->user, tstrerror(code), result);
return;
}
- if (code < 0) {
+ if (code != TSDB_CODE_SUCCESS) {
SSqlObj *pObj = (SSqlObj *)result;
if (code == TSDB_CODE_TSC_INVALID_SQL) {
- httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext,
- pContext->fd, pContext->user, tstrerror(code), pObj, pObj->cmd.payload);
- httpSendTaosdInvalidSqlErrorResp(pContext, pObj->cmd.payload);
+ terrno = code;
+ httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext, pContext->fd,
+ pContext->user, tstrerror(code), pObj, taos_errstr(pObj));
+ httpSendTaosdInvalidSqlErrorResp(pContext, taos_errstr(pObj));
} else {
httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd,
pContext->user, tstrerror(code), pObj);
@@ -278,7 +280,8 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo
bool isUpdate = tscIsUpdateQuery(result);
if (isUpdate) {
// not select or show commands
- int affectRows = taos_affected_rows(result);
+ int32_t affectRows = taos_affected_rows(result);
+ assert(affectRows == affectRowsInput);
httpDebug("context:%p, fd:%d, user:%s, affect rows:%d, stop query, sqlObj:%p", pContext, pContext->fd,
pContext->user, affectRows, result);
@@ -308,8 +311,10 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo
}
}
-void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) {
- httpDispatchToResultQueue(param, result, unUsedCode, httpProcessSingleSqlCallBackImp);
+void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) {
+ int32_t code = taos_errno(result);
+ int32_t affectRows = taos_affected_rows(result);
+ httpDispatchToResultQueue(param, result, code, affectRows, httpProcessSingleSqlCallBackImp);
}
void httpProcessSingleSqlCmd(HttpContext *pContext) {
@@ -373,7 +378,7 @@ void httpExecCmd(HttpContext *pContext) {
}
}
-void httpProcessRequestCb(void *param, TAOS_RES *result, int code) {
+void httpProcessRequestCb(void *param, TAOS_RES *result, int32_t code) {
HttpContext *pContext = param;
taos_free_result(result);
diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c
index 8993b233dd1dde9283679d337845d42f5d649715..3b8858b62e1297f0db5ece60c59a26e376b546f3 100644
--- a/src/plugins/http/src/httpSystem.c
+++ b/src/plugins/http/src/httpSystem.c
@@ -107,7 +107,7 @@ void httpCleanUpSystem() {
httpCleanupResultQueue();
pthread_mutex_destroy(&tsHttpServer.serverMutex);
- taosTFree(tsHttpServer.pThreads);
+ tfree(tsHttpServer.pThreads);
tsHttpServer.pThreads = NULL;
tsHttpServer.status = HTTP_SERVER_CLOSED;
diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c
index c182d85dac588954f5bd1610820c4fca1cdb7c22..4bea78e00e10850c1b60041db82820f2dbd03b36 100644
--- a/src/plugins/monitor/src/monitorMain.c
+++ b/src/plugins/monitor/src/monitorMain.c
@@ -27,12 +27,12 @@
#include "monitor.h"
#include "taoserror.h"
-#define monitorFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
-#define monitorError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
-#define monitorWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
-#define monitorInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }}
-#define monitorDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
-#define monitorTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
+#define mnFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
+#define mnError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
+#define mnWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
+#define mnInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }}
+#define mnDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
+#define mnTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
#define SQL_LENGTH 1030
#define LOG_LEN_STR 100
@@ -91,12 +91,12 @@ int32_t monitorInitSystem() {
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&tsMonitor.thread, &thAttr, monitorThreadFunc, NULL)) {
- monitorError("failed to create thread to for monitor module, reason:%s", strerror(errno));
+ mnError("failed to create thread to for monitor module, reason:%s", strerror(errno));
return -1;
}
pthread_attr_destroy(&thAttr);
- monitorDebug("monitor thread is launched");
+ mnDebug("monitor thread is launched");
monitorStartSystemFp = monitorStartSystem;
monitorStopSystemFp = monitorStopSystem;
@@ -107,12 +107,12 @@ int32_t monitorStartSystem() {
taos_init();
tsMonitor.start = 1;
monitorExecuteSQLFp = monitorExecuteSQL;
- monitorInfo("monitor module start");
+ mnInfo("monitor module start");
return 0;
}
static void *monitorThreadFunc(void *param) {
- monitorDebug("starting to initialize monitor module ...");
+ mnDebug("starting to initialize monitor module ...");
while (1) {
static int32_t accessTimes = 0;
@@ -121,7 +121,7 @@ static void *monitorThreadFunc(void *param) {
if (tsMonitor.quiting) {
tsMonitor.state = MON_STATE_NOT_INIT;
- monitorInfo("monitor thread will quit, for taosd is quiting");
+ mnInfo("monitor thread will quit, for taosd is quiting");
break;
} else {
taosGetDisk();
@@ -132,7 +132,7 @@ static void *monitorThreadFunc(void *param) {
}
if (dnodeGetDnodeId() <= 0) {
- monitorDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
+ mnDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
continue;
}
@@ -140,10 +140,10 @@ static void *monitorThreadFunc(void *param) {
tsMonitor.state = MON_STATE_NOT_INIT;
tsMonitor.conn = taos_connect(NULL, "monitor", tsInternalPass, "", 0);
if (tsMonitor.conn == NULL) {
- monitorError("failed to connect to database, reason:%s", tstrerror(terrno));
+ mnError("failed to connect to database, reason:%s", tstrerror(terrno));
continue;
} else {
- monitorDebug("connect to database success");
+ mnDebug("connect to database success");
}
}
@@ -155,10 +155,10 @@ static void *monitorThreadFunc(void *param) {
taos_free_result(res);
if (code != 0) {
- monitorError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
+ mnError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
break;
} else {
- monitorDebug("successfully to exec sql:%s", tsMonitor.sql);
+ mnDebug("successfully to exec sql:%s", tsMonitor.sql);
}
}
@@ -174,7 +174,7 @@ static void *monitorThreadFunc(void *param) {
}
}
- monitorInfo("monitor thread is stopped");
+ mnInfo("monitor thread is stopped");
return NULL;
}
@@ -238,7 +238,7 @@ void monitorStopSystem() {
tsMonitor.start = 0;
tsMonitor.state = MON_STATE_NOT_INIT;
monitorExecuteSQLFp = NULL;
- monitorInfo("monitor module stopped");
+ mnInfo("monitor module stopped");
}
void monitorCleanUpSystem() {
@@ -249,7 +249,7 @@ void monitorCleanUpSystem() {
taos_close(tsMonitor.conn);
tsMonitor.conn = NULL;
}
- monitorInfo("monitor module is cleaned up");
+ mnInfo("monitor module is cleaned up");
}
// unit is MB
@@ -257,13 +257,13 @@ static int32_t monitorBuildMemorySql(char *sql) {
float sysMemoryUsedMB = 0;
bool suc = taosGetSysMemory(&sysMemoryUsedMB);
if (!suc) {
- monitorDebug("failed to get sys memory info");
+ mnDebug("failed to get sys memory info");
}
float procMemoryUsedMB = 0;
suc = taosGetProcMemory(&procMemoryUsedMB);
if (!suc) {
- monitorDebug("failed to get proc memory info");
+ mnDebug("failed to get proc memory info");
}
return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB);
@@ -274,7 +274,7 @@ static int32_t monitorBuildCpuSql(char *sql) {
float sysCpuUsage = 0, procCpuUsage = 0;
bool suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage);
if (!suc) {
- monitorDebug("failed to get cpu usage");
+ mnDebug("failed to get cpu usage");
}
if (sysCpuUsage <= procCpuUsage) {
@@ -294,14 +294,14 @@ static int32_t monitorBuildBandSql(char *sql) {
float bandSpeedKb = 0;
bool suc = taosGetBandSpeed(&bandSpeedKb);
if (!suc) {
- monitorDebug("failed to get bandwidth speed");
+ mnDebug("failed to get bandwidth speed");
}
return sprintf(sql, ", %f", bandSpeedKb);
}
static int32_t monitorBuildReqSql(char *sql) {
- SDnodeStatisInfo info = dnodeGetStatisInfo();
+ SStatisInfo info = dnodeGetStatisInfo();
return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum);
}
@@ -309,7 +309,7 @@ static int32_t monitorBuildIoSql(char *sql) {
float readKB = 0, writeKB = 0;
bool suc = taosGetProcIO(&readKB, &writeKB);
if (!suc) {
- monitorDebug("failed to get io info");
+ mnDebug("failed to get io info");
}
return sprintf(sql, ", %f, %f", readKB, writeKB);
@@ -332,19 +332,19 @@ static void monitorSaveSystemInfo() {
taos_free_result(res);
if (code != 0) {
- monitorError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
+ mnError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
} else {
- monitorDebug("successfully to save system info, sql:%s", tsMonitor.sql);
+ mnDebug("successfully to save system info, sql:%s", tsMonitor.sql);
}
}
static void montiorExecSqlCb(void *param, TAOS_RES *result, int32_t code) {
int32_t c = taos_errno(result);
if (c != TSDB_CODE_SUCCESS) {
- monitorError("save %s failed, reason:%s", (char *)param, tstrerror(c));
+ mnError("save %s failed, reason:%s", (char *)param, tstrerror(c));
} else {
int32_t rows = taos_affected_rows(result);
- monitorDebug("save %s succ, rows:%d", (char *)param, rows);
+ mnDebug("save %s succ, rows:%d", (char *)param, rows);
}
taos_free_result(result);
@@ -380,7 +380,7 @@ void monitorSaveAcctLog(SAcctMonitorObj *pMon) {
pMon->totalConns, pMon->maxConns,
pMon->accessState);
- monitorDebug("save account info, sql:%s", sql);
+ mnDebug("save account info, sql:%s", sql);
taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "account info");
}
@@ -401,13 +401,13 @@ void monitorSaveLog(int32_t level, const char *const format, ...) {
len += sprintf(sql + len, "', '%s')", tsLocalEp);
sql[len++] = 0;
- monitorDebug("save log, sql: %s", sql);
+ mnDebug("save log, sql: %s", sql);
taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "log");
}
void monitorExecuteSQL(char *sql) {
if (tsMonitor.state != MON_STATE_INITED) return;
- monitorDebug("execute sql:%s", sql);
+ mnDebug("execute sql:%s", sql);
taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "sql");
}
diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h
index d3e60c21dc376e6a1e88f959fbf4b43bf47ab4b0..28c1c7b838236d10a3f67fa7bbfc15a9b36c4612 100644
--- a/src/query/inc/qAst.h
+++ b/src/query/inc/qAst.h
@@ -74,20 +74,18 @@ typedef struct tExprNode {
};
} tExprNode;
-void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*));
-
void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param);
void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
char *(*cb)(void *, const char*, int32_t));
-uint8_t getBinaryExprOptr(SStrToken *pToken);
+tExprNode* exprTreeFromBinary(const void* data, size_t size);
+tExprNode* exprTreeFromTableName(const char* tbnameCond);
-void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *));
void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
-tExprNode* exprTreeFromBinary(const void* data, size_t size);
-tExprNode* exprTreeFromTableName(const char* tbnameCond);
+void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *));
+void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*));
#ifdef __cplusplus
}
diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h
index f392644e6736f528fa75f21ee2857c570c0c22c9..6b4b188c5ede2fc10baa894a55ad5d2225283d0c 100644
--- a/src/query/inc/qExecutor.h
+++ b/src/query/inc/qExecutor.h
@@ -33,17 +33,26 @@ struct SColumnFilterElem;
typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2);
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
-typedef struct SPosInfo {
- int32_t pageId:20;
- int32_t rowId:12;
-} SPosInfo;
-
typedef struct SGroupResInfo {
int32_t groupId;
int32_t numOfDataPages;
- SPosInfo pos;
+ int32_t pageId;
+ int32_t rowId;
} SGroupResInfo;
+typedef struct SResultRowPool {
+ int32_t elemSize;
+ int32_t blockSize;
+ int32_t numOfElemPerBlock;
+
+ struct {
+ int32_t blockIndex;
+ int32_t pos;
+ } position;
+
+ SArray* pData; // SArray
+} SResultRowPool;
+
typedef struct SSqlGroupbyExpr {
int16_t tableIndex;
SArray* columnInfo; // SArray, group by columns information
@@ -52,13 +61,14 @@ typedef struct SSqlGroupbyExpr {
int16_t orderType; // order by type: asc/desc
} SSqlGroupbyExpr;
-typedef struct SWindowResult {
- SPosInfo pos; // Position of current result in disk-based output buffer
+typedef struct SResultRow {
+ int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer
+ int32_t rowId:15;
+ bool closed:1; // this result status: closed or opened
uint16_t numOfRows; // number of rows of current time window
- bool closed; // this result status: closed or opened
- SResultInfo* resultInfo; // For each result column, there is a resultInfo
+ SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo
union {STimeWindow win; char* key;}; // start key of current time window
-} SWindowResult;
+} SResultRow;
/**
* If the number of generated results is greater than this value,
@@ -72,16 +82,14 @@ typedef struct SResultRec {
} SResultRec;
typedef struct SWindowResInfo {
- SWindowResult* pResult; // result list
- SHashObj* hashList; // hash list for quick access
- int16_t type; // data type for hash key
+ SResultRow** pResult; // result list
+ int16_t type:8; // data type for hash key
+ int32_t size:24; // number of result set
+ int32_t threshold; // threshold to halt query and return the generated results.
int32_t capacity; // max capacity
int32_t curIndex; // current start active index
- int32_t size; // number of result set
int64_t startTime; // start time of the first time window for sliding query
int64_t prevSKey; // previous (not completed) sliding window start key
- int64_t threshold; // threshold to halt query and return the generated results.
- int64_t interval; // time window interval
} SWindowResInfo;
typedef struct SColumnFilterElem {
@@ -97,7 +105,7 @@ typedef struct SSingleColumnFilterInfo {
SColumnFilterElem* pFilters;
} SSingleColumnFilterInfo;
-typedef struct STableQueryInfo { // todo merge with the STableQueryInfo struct
+typedef struct STableQueryInfo {
TSKEY lastKey;
int32_t groupIndex; // group id in table list
int16_t queryRangeSet; // denote if the query range is set, only available for interval query
@@ -125,7 +133,9 @@ typedef struct SQueryCostInfo {
uint32_t discardBlocks;
uint64_t elapsedTime;
uint64_t firstStageMergeTime;
- uint64_t internalSupSize;
+ uint64_t winInfoSize;
+ uint64_t tableInfoSize;
+ uint64_t hashSize;
uint64_t numOfTimeWindows;
} SQueryCostInfo;
@@ -142,7 +152,10 @@ typedef struct SQuery {
SLimitVal limit;
int32_t rowSize;
SSqlGroupbyExpr* pGroupbyExpr;
- SExprInfo* pSelectExpr;
+ SExprInfo* pExpr1;
+ SExprInfo* pExpr2;
+ int32_t numOfExpr2;
+
SColumnInfo* colList;
SColumnInfo* tagColList;
int32_t numOfFilterCols;
@@ -158,11 +171,11 @@ typedef struct SQuery {
typedef struct SQueryRuntimeEnv {
jmp_buf env;
- SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo
+ SResultRow* pResultRow; // todo refactor to merge with SWindowResInfo
SQuery* pQuery;
SQLFunctionCtx* pCtx;
int32_t numOfRowsPerPage;
- int16_t offset[TSDB_MAX_COLUMNS];
+ uint16_t offset[TSDB_MAX_COLUMNS];
uint16_t scanFlag; // denotes reversed scan of data or not
SFillInfo* pFillInfo;
SWindowResInfo windowResInfo;
@@ -178,6 +191,11 @@ typedef struct SQueryRuntimeEnv {
int32_t interBufSize; // intermediate buffer sizse
int32_t prevGroupId; // previous executed group id
SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file
+ SHashObj* pResultRowHashTable; // quick locate the window object for each result
+ char* keyBuf; // window key buffer
+ SResultRowPool* pool; // window result object pool
+
+ int32_t* rowCellInfoOffset;// offset value for each row result cell info
} SQueryRuntimeEnv;
enum {
@@ -185,12 +203,6 @@ enum {
QUERY_RESULT_READY = 2,
};
-typedef struct SMemRef {
- int32_t ref;
- void *mem;
- void *imem;
-} SMemRef;
-
typedef struct SQInfo {
void* signature;
int32_t code; // error code to returned to client
diff --git a/src/query/inc/qFill.h b/src/query/inc/qFill.h
index 6d44fee09557fa8b8d73527677a2c7b238ecb42c..385ae8854327b036cbb6ee46de70e97ae56ce25b 100644
--- a/src/query/inc/qFill.h
+++ b/src/query/inc/qFill.h
@@ -28,6 +28,7 @@ typedef struct {
STColumn col; // column info
int16_t functionId; // sql function id
int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN
+ int16_t tagIndex; // index of current tag in SFillTagColInfo array list
union {int64_t i; double d;} fillVal;
} SFillColInfo;
@@ -37,27 +38,29 @@ typedef struct {
} SFillTagColInfo;
typedef struct SFillInfo {
- TSKEY start; // start timestamp
- TSKEY endKey; // endKey for fill
- int32_t order; // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC]
- int32_t fillType; // fill type
- int32_t numOfRows; // number of rows in the input data block
- int32_t rowIdx; // rowIdx
- int32_t numOfTotal; // number of filled rows in one round
- int32_t numOfCurrent; // number of filled rows in current results
-
- int32_t numOfTags; // number of tags
- int32_t numOfCols; // number of columns, including the tags columns
- int32_t rowSize; // size of each row
-// char ** pTags; // tags value for current interpolation
- SFillTagColInfo* pTags; // tags value for filling gap
+ TSKEY start; // start timestamp
+ TSKEY end; // endKey for fill
+ TSKEY currentKey; // current active timestamp, the value may be changed during the fill procedure.
+ int32_t order; // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC]
+ int32_t type; // fill type
+ int32_t numOfRows; // number of rows in the input data block
+ int32_t index; // active row index
+ int32_t numOfTotal; // number of filled rows in one round
+ int32_t numOfCurrent; // number of filled rows in current results
+
+ int32_t numOfTags; // number of tags
+ int32_t numOfCols; // number of columns, including the tags columns
+ int32_t rowSize; // size of each row
SInterval interval;
- char * prevValues; // previous row of data, to generate the interpolation results
- char * nextValues; // next row of data
- char** pData; // original result data block involved in filling data
- int32_t capacityInRows; // data buffer size in rows
- int8_t precision; // time resoluation
- SFillColInfo* pFillCol; // column info for fill operations
+ char * prevValues; // previous row of data, to generate the interpolation results
+ char * nextValues; // next row of data
+ char** pData; // original result data block involved in filling data
+ int32_t alloc; // data buffer size in rows
+ int8_t precision; // time resoluation
+
+ SFillColInfo* pFillCol; // column info for fill operations
+ SFillTagColInfo* pTags; // tags value for filling gap
+ void* handle; // for dubug purpose
} SFillInfo;
typedef struct SPoint {
@@ -67,25 +70,25 @@ typedef struct SPoint {
SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
- SFillColInfo* pFillCol);
+ SFillColInfo* pFillCol, void* handle);
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);
-void* taosDestoryFillInfo(SFillInfo *pFillInfo);
+void* taosDestroyFillInfo(SFillInfo *pFillInfo);
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
-void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput);
+void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput);
-void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput);
+void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput);
-int64_t getFilledNumOfRes(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
+int64_t getNumOfResWithFill(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
int32_t taosNumOfRemainRows(SFillInfo *pFillInfo);
-int taosDoLinearInterpolation(int32_t type, SPoint *point1, SPoint *point2, SPoint *point);
+int32_t taosGetLinearInterpolationVal(int32_t type, SPoint *point1, SPoint *point2, SPoint *point);
-int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity);
+int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity);
#ifdef __cplusplus
}
diff --git a/src/query/inc/qHistogram.h b/src/query/inc/qHistogram.h
index bb058449e806c8270dbf141ca3a81103f63c6e5c..442e61750b06d269ef48c5e74ede7646f7ac62e1 100644
--- a/src/query/inc/qHistogram.h
+++ b/src/query/inc/qHistogram.h
@@ -43,7 +43,8 @@ typedef struct SHistogramInfo {
int32_t numOfElems;
int32_t numOfEntries;
int32_t maxEntries;
-
+ double min;
+ double max;
#if defined(USE_ARRAYLIST)
SHistBin* elems;
#else
@@ -52,9 +53,6 @@ typedef struct SHistogramInfo {
int32_t maxIndex;
bool ordered;
#endif
-
- double min;
- double max;
} SHistogramInfo;
SHistogramInfo* tHistogramCreate(int32_t numOfBins);
diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h
index bc8f9a5e23df72f82bcb3bb5140bec42fe426268..513ab090f921b15ca18e384643ab79f51a119a8d 100644
--- a/src/query/inc/qSqlparser.h
+++ b/src/query/inc/qSqlparser.h
@@ -20,10 +20,10 @@
extern "C" {
#endif
-#include
#include "taos.h"
#include "taosmsg.h"
#include "tstoken.h"
+#include "tstrbuild.h"
#include "tvariant.h"
#define ParseTOKENTYPE SStrToken
@@ -37,12 +37,6 @@ extern char tTokenTypeSwitcher[13];
(x) = tTokenTypeSwitcher[(x)]; \
} \
} while (0)
-
-typedef struct tFieldList {
- int32_t nField;
- int32_t nAlloc;
- TAOS_FIELD *p;
-} tFieldList;
typedef struct SLimitVal {
int64_t limit;
@@ -59,12 +53,6 @@ typedef struct tVariantListItem {
uint8_t sortOrder;
} tVariantListItem;
-typedef struct tVariantList {
- int32_t nExpr; /* Number of expressions on the list */
- int32_t nAlloc; /* Number of entries allocated below */
- tVariantListItem *a; /* One entry for each expression */
-} tVariantList;
-
typedef struct SIntervalVal {
SStrToken interval;
SStrToken offset;
@@ -72,16 +60,16 @@ typedef struct SIntervalVal {
typedef struct SQuerySQL {
struct tSQLExprList *pSelection; // select clause
- tVariantList * from; // from clause
+ SArray * from; // from clause SArray
struct tSQLExpr * pWhere; // where clause [optional]
- tVariantList * pGroupby; // groupby clause, only for tags[optional]
- tVariantList * pSortOrder; // orderby [optional]
+ SArray * pGroupby; // groupby clause, only for tags[optional], SArray
+ SArray * pSortOrder; // orderby [optional], SArray
SStrToken interval; // interval [optional]
SStrToken offset; // offset window [optional]
SStrToken sliding; // sliding window [optional]
SLimitVal limit; // limit offset [optional]
SLimitVal slimit; // group limit offset [optional]
- tVariantList * fillType; // fill type[optional]
+ SArray * fillType; // fill type[optional], SArray
SStrToken selectToken; // sql string
} SQuerySQL;
@@ -91,26 +79,25 @@ typedef struct SCreateTableSQL {
int8_t type; // create normal table/from super table/ stream
struct {
- tFieldList *pTagColumns; // for normal table, pTagColumns = NULL;
- tFieldList *pColumns;
+ SArray *pTagColumns; // SArray
+ SArray *pColumns; // SArray
} colInfo;
struct {
- SStrToken stableName; // super table name, for using clause
- tVariantList *pTagVals; // create by using metric, tag value
- STagData tagdata;
+ SStrToken stableName; // super table name, for using clause
+ SArray *pTagVals; // create by using metric, tag value
+ STagData tagdata;
} usingInfo;
- SQuerySQL *pSelect;
+ SQuerySQL *pSelect;
} SCreateTableSQL;
typedef struct SAlterTableSQL {
SStrToken name;
int16_t type;
STagData tagData;
-
- tFieldList * pAddColumns;
- tVariantList *varList; // set t=val or: change src dst
+ SArray *pAddColumns; // SArray
+ SArray *varList; // set t=val or: change src dst, SArray
} SAlterTableSQL;
typedef struct SCreateDBInfo {
@@ -129,8 +116,9 @@ typedef struct SCreateDBInfo {
int32_t compressionLevel;
SStrToken precision;
bool ignoreExists;
+ int8_t update;
- tVariantList *keep;
+ SArray *keep;
} SCreateDBInfo;
typedef struct SCreateAcctSQL {
@@ -168,7 +156,7 @@ typedef struct tDCLSQL {
SCreateDBInfo dbOpt;
SCreateAcctSQL acctOpt;
SShowInfo showOpt;
- SStrToken ip;
+ SStrToken ip;
};
SUserInfo user;
@@ -181,33 +169,32 @@ typedef struct SSubclauseInfo { // "UNION" multiple select sub-clause
} SSubclauseInfo;
typedef struct SSqlInfo {
- int32_t type;
- bool valid;
+ int32_t type;
+ bool valid;
union {
SCreateTableSQL *pCreateTableInfo;
- SAlterTableSQL * pAlterInfo;
- tDCLSQL * pDCLInfo;
+ SAlterTableSQL *pAlterInfo;
+ tDCLSQL *pDCLInfo;
};
- SSubclauseInfo subclauseInfo;
- char pzErrMsg[256];
+ SSubclauseInfo subclauseInfo;
+ char pzErrMsg[256];
} SSqlInfo;
typedef struct tSQLExpr {
- // TK_FUNCTION: sql function, TK_LE: less than(binary expr)
- uint32_t nSQLOptr;
+ uint32_t nSQLOptr; // TK_FUNCTION: sql function, TK_LE: less than(binary expr)
// the full sql string of function(col, param), which is actually the raw
// field name, since the function name is kept in nSQLOptr already
- SStrToken operand;
- SStrToken colInfo; // field id
- tVariant val; // value only for string, float, int
-
+ SStrToken operand;
+ SStrToken colInfo; // field id
+ tVariant val; // value only for string, float, int
+ SStrToken token; // original sql expr string
+
struct tSQLExpr *pLeft; // left child
struct tSQLExpr *pRight; // right child
struct tSQLExprList *pParam; // function parameters
- SStrToken token; // original sql expr string
} tSQLExpr;
// used in select clause. select from xxx
@@ -223,13 +210,6 @@ typedef struct tSQLExprList {
tSQLExprItem *a; /* One entry for each expression */
} tSQLExprList;
-typedef struct tSQLExprListList {
- int32_t nList; /* Number of expressions on the list */
- int32_t nAlloc; /* Number of entries allocated below */
- tSQLExprList **a; /* one entry for each row */
-} tSQLExprListList;
-
-
/**
*
* @param yyp The parser
@@ -245,16 +225,9 @@ void Parse(void *yyp, int yymajor, ParseTOKENTYPE yyminor, SSqlInfo *);
*/
void ParseFree(void *p, void (*freeProc)(void *));
-tVariantList *tVariantListAppend(tVariantList *pList, tVariant *pVar, uint8_t sortOrder);
-
-tVariantList *tVariantListInsert(tVariantList *pList, tVariant *pVar, uint8_t sortOrder, int32_t index);
-
-tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pAliasToken, uint8_t sortOrder);
-void tVariantListDestroy(tVariantList *pList);
-
-tFieldList *tFieldListAppend(tFieldList *pList, TAOS_FIELD *pField);
-
-void tFieldListDestroy(tFieldList *pList);
+SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder);
+SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index);
+SArray *tVariantListAppendToken(SArray *pList, SStrToken *pAliasToken, uint8_t sortOrder);
tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optType);
@@ -264,17 +237,16 @@ tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken
void tSQLExprListDestroy(tSQLExprList *pList);
-SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere,
- tVariantList *pGroupby, tVariantList *pSortOrder, SIntervalVal *pInterval,
- SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit);
+SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere,
+ SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval,
+ SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit);
-SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pMetricName,
- tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type);
+SCreateTableSQL *tSetCreateSQLElems(SArray *pCols, SArray *pTags, SStrToken *pMetricName,
+ SArray *pTagVals, SQuerySQL *pSelect, int32_t type);
-void tSQLExprNodeDestroy(tSQLExpr *pExpr);
-tSQLExpr *tSQLExprNodeClone(tSQLExpr *pExpr);
+void tSQLExprNodeDestroy(tSQLExpr *pExpr);
-SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type);
+SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, SArray *pCols, SArray *pVals, int32_t type);
void destroyAllSelectClause(SSubclauseInfo *pSql);
void doDestroyQuerySql(SQuerySQL *pSql);
@@ -316,9 +288,6 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *pToken);
void *ParseAlloc(void *(*mallocProc)(size_t));
-// convert the sql filter expression into binary data
-int32_t tSQLExprToBinary(tSQLExpr* pExpr, SStringBuilder* sb);
-
enum {
TSQL_NODE_TYPE_EXPR = 0x1,
TSQL_NODE_TYPE_ID = 0x2,
diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h
index 6c2a955f47577a0da19f6520b2addc44e66b73ba..90bd64336fdeed91deb68b9b490224a7fb29bc80 100644
--- a/src/query/inc/qTsbuf.h
+++ b/src/query/inc/qTsbuf.h
@@ -26,7 +26,7 @@ extern "C" {
#define MEM_BUF_SIZE (1 << 20)
#define TS_COMP_FILE_MAGIC 0x87F5EC4C
-#define TS_COMP_FILE_VNODE_MAX 512
+#define TS_COMP_FILE_GROUP_MAX 512
typedef struct STSList {
char* rawBuf;
@@ -38,7 +38,7 @@ typedef struct STSList {
typedef struct STSElem {
TSKEY ts;
tVariant* tag;
- int32_t vnode;
+ int32_t id;
} STSElem;
typedef struct STSCursor {
@@ -60,17 +60,17 @@ typedef struct STSBlock {
* The size of buffer file should not be greater than 2G,
* and the offset of int32_t type is enough
*/
-typedef struct STSVnodeBlockInfo {
- int32_t vnode; // vnode id
+typedef struct STSGroupBlockInfo {
+ int32_t id; // group id
int32_t offset; // offset set value in file
int32_t numOfBlocks; // number of total blocks
int32_t compLen; // compressed size
-} STSVnodeBlockInfo;
+} STSGroupBlockInfo;
-typedef struct STSVnodeBlockInfoEx {
- STSVnodeBlockInfo info;
+typedef struct STSGroupBlockInfoEx {
+ STSGroupBlockInfo info;
int32_t len; // length before compress
-} STSVnodeBlockInfoEx;
+} STSGroupBlockInfoEx;
typedef struct STSBuf {
FILE* f;
@@ -78,9 +78,9 @@ typedef struct STSBuf {
uint32_t fileSize;
// todo use array
- STSVnodeBlockInfoEx* pData;
+ STSGroupBlockInfoEx* pData;
uint32_t numOfAlloc;
- uint32_t numOfVnodes;
+ uint32_t numOfGroups;
char* assistBuf;
int32_t bufSize;
@@ -94,22 +94,22 @@ typedef struct STSBuf {
typedef struct STSBufFileHeader {
uint32_t magic; // file magic number
- uint32_t numOfVnode; // number of vnode stored in current file
+ uint32_t numOfGroup; // number of group stored in current file
int32_t tsOrder; // timestamp order in current file
} STSBufFileHeader;
STSBuf* tsBufCreate(bool autoDelete, int32_t order);
STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete);
-STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t vnodeId);
+STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t id);
void* tsBufDestroy(STSBuf* pTSBuf);
-void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len);
+void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len);
int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf);
STSBuf* tsBufClone(STSBuf* pTSBuf);
-STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId);
+STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id);
void tsBufFlush(STSBuf* pTSBuf);
@@ -118,7 +118,7 @@ STSElem tsBufGetElem(STSBuf* pTSBuf);
bool tsBufNextPos(STSBuf* pTSBuf);
-STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag);
+STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag);
STSCursor tsBufGetCursor(STSBuf* pTSBuf);
void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order);
@@ -131,9 +131,15 @@ void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur);
*/
void tsBufDisplay(STSBuf* pTSBuf);
-int32_t tsBufGetNumOfVnodes(STSBuf* pTSBuf);
+int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf);
-void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId);
+void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id);
+
+int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t id, void* buf, int32_t* len, int32_t* numOfBlocks);
+
+STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag);
+
+bool tsBufIsValidElem(STSElem* pElem);
#ifdef __cplusplus
}
diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h
index 5320e5622e70a93c06edb4a1e5a3fe568498ef21..93dca42fdcfd85b8e11fde205392a15caaa59416 100644
--- a/src/query/inc/qUtil.h
+++ b/src/query/inc/qUtil.h
@@ -15,13 +15,22 @@
#ifndef TDENGINE_QUERYUTIL_H
#define TDENGINE_QUERYUTIL_H
+#define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \
+ do { \
+ assert(sizeof(_uid) == sizeof(uint64_t)); \
+ *(uint64_t *)(_k) = (_uid); \
+ memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \
+ } while (0)
+
+#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t))
+
int32_t getOutputInterResultBufSize(SQuery* pQuery);
-void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes);
-void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, const SWindowResult* src);
+void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow);
+void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src);
+SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index);
-int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size,
- int32_t threshold, int16_t type);
+int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type);
void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo);
void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo);
@@ -33,27 +42,27 @@ void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot);
void closeAllTimeWindow(SWindowResInfo* pWindowResInfo);
void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order);
-static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) {
+static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int32_t slot) {
assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size);
- return &pWindowResInfo->pResult[slot];
+ return pWindowResInfo->pResult[slot];
}
#define curTimeWindowIndex(_winres) ((_winres)->curIndex)
-#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pSelectExpr[1].base.arg->argValue.i64:1)
+#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pExpr1[1].base.arg->argValue.i64:1)
bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);
-int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize);
+int32_t initResultRow(SResultRow *pResultRow);
-static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult,
+static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SResultRow *pResult,
tFilePage* page) {
assert(pResult != NULL && pRuntimeEnv != NULL);
SQuery *pQuery = pRuntimeEnv->pQuery;
- int32_t realRowId = (int32_t)(pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery));
+ int32_t realRowId = (int32_t)(pResult->rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery));
return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage +
- pQuery->pSelectExpr[columnIndex].bytes * realRowId;
+ pQuery->pExpr1[columnIndex].bytes * realRowId;
}
bool isNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval);
@@ -62,4 +71,14 @@ bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval);
__filter_func_t *getRangeFilterFuncArray(int32_t type);
__filter_func_t *getValueFilterFuncArray(int32_t type);
+size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv);
+
+SResultRowPool* initResultRowPool(size_t size);
+SResultRow* getNewResultRow(SResultRowPool* p);
+int64_t getResultRowPoolMemSize(SResultRowPool* p);
+void* destroyResultRowPool(SResultRowPool* p);
+int32_t getNumOfAllocatedResultRows(SResultRowPool* p);
+int32_t getNumOfUsedResultRows(SResultRowPool* p);
+
+
#endif // TDENGINE_QUERYUTIL_H
diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y
index e5d1185330a47b79fec1becd74a01ed779db8c80..09b1e1592aa53dcec817472a068e0378b7f4adf1 100644
--- a/src/query/inc/sql.y
+++ b/src/query/inc/sql.y
@@ -223,8 +223,8 @@ acct_optr(Y) ::= pps(C) tseries(D) storage(P) streams(F) qtime(Q) dbs(E) users(K
Y.stat = M;
}
-%type keep {tVariantList*}
-%destructor keep {tVariantListDestroy($$);}
+%type keep {SArray*}
+%destructor keep {taosArrayDestroy($$);}
keep(Y) ::= KEEP tagitemlist(X). { Y = X; }
cache(Y) ::= CACHE INTEGER(X). { Y = X; }
@@ -239,6 +239,7 @@ wal(Y) ::= WAL INTEGER(X). { Y = X; }
fsync(Y) ::= FSYNC INTEGER(X). { Y = X; }
comp(Y) ::= COMP INTEGER(X). { Y = X; }
prec(Y) ::= PRECISION STRING(X). { Y = X; }
+update(Y) ::= UPDATE INTEGER(X). { Y = X; }
%type db_optr {SCreateDBInfo}
db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);}
@@ -256,6 +257,7 @@ db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z
db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; }
db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; }
+db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); }
%type alter_db_optr {SCreateDBInfo}
alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);}
@@ -267,6 +269,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = s
alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); }
+alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); }
%type typename {TAOS_FIELD}
typename(A) ::= ids(X). {
@@ -324,10 +327,10 @@ create_table_args(A) ::= AS select(S). {
}
%type column{TAOS_FIELD}
-%type columnlist{tFieldList*}
-%destructor columnlist {tFieldListDestroy($$);}
-columnlist(A) ::= columnlist(X) COMMA column(Y). {A = tFieldListAppend(X, &Y); }
-columnlist(A) ::= column(X). {A = tFieldListAppend(NULL, &X);}
+%type columnlist{SArray*}
+%destructor columnlist {taosArrayDestroy($$);}
+columnlist(A) ::= columnlist(X) COMMA column(Y). {taosArrayPush(X, &Y); A = X; }
+columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(A, &X);}
// The information used for a column is the name and type of column:
// tinyint smallint int bigint float double bool timestamp binary(x) nchar(x)
@@ -335,8 +338,8 @@ column(A) ::= ids(X) typename(Y). {
tSQLSetColumnInfo(&A, &X, &Y);
}
-%type tagitemlist {tVariantList*}
-%destructor tagitemlist {tVariantListDestroy($$);}
+%type tagitemlist {SArray*}
+%destructor tagitemlist {taosArrayDestroy($$);}
%type tagitem {tVariant}
tagitemlist(A) ::= tagitemlist(X) COMMA tagitem(Y). { A = tVariantListAppend(X, &Y, -1); }
@@ -429,11 +432,11 @@ as(X) ::= ids(Y). { X = Y; }
as(X) ::= . { X.n = 0; }
// A complete FROM clause.
-%type from {tVariantList*}
+%type from {SArray*}
// current not support query from no-table
from(A) ::= FROM tablelist(X). {A = X;}
-%type tablelist {tVariantList*}
+%type tablelist {SArray*}
tablelist(A) ::= ids(X) cpxName(Y). {
toTSDBType(X.type);
X.n += Y.n;
@@ -473,8 +476,8 @@ interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N.interval = E; N.offset.n = 0;
interval_opt(N) ::= INTERVAL LP tmvar(E) COMMA tmvar(O) RP. {N.interval = E; N.offset = O;}
interval_opt(N) ::= . {memset(&N, 0, sizeof(N));}
-%type fill_opt {tVariantList*}
-%destructor fill_opt {tVariantListDestroy($$);}
+%type fill_opt {SArray*}
+%destructor fill_opt {taosArrayDestroy($$);}
fill_opt(N) ::= . {N = 0; }
fill_opt(N) ::= FILL LP ID(Y) COMMA tagitemlist(X) RP. {
tVariant A = {0};
@@ -494,11 +497,11 @@ fill_opt(N) ::= FILL LP ID(Y) RP. {
sliding_opt(K) ::= SLIDING LP tmvar(E) RP. {K = E; }
sliding_opt(K) ::= . {K.n = 0; K.z = NULL; K.type = 0; }
-%type orderby_opt {tVariantList*}
-%destructor orderby_opt {tVariantListDestroy($$);}
+%type orderby_opt {SArray*}
+%destructor orderby_opt {taosArrayDestroy($$);}
-%type sortlist {tVariantList*}
-%destructor sortlist {tVariantListDestroy($$);}
+%type sortlist {SArray*}
+%destructor sortlist {taosArrayDestroy($$);}
%type sortitem {tVariant}
%destructor sortitem {tVariantDestroy(&$$);}
@@ -528,10 +531,10 @@ sortorder(A) ::= DESC. {A = TSDB_ORDER_DESC;}
sortorder(A) ::= . {A = TSDB_ORDER_ASC;} //default is descend order
//group by clause
-%type groupby_opt {tVariantList*}
-%destructor groupby_opt {tVariantListDestroy($$);}
-%type grouplist {tVariantList*}
-%destructor grouplist {tVariantListDestroy($$);}
+%type groupby_opt {SArray*}
+%destructor groupby_opt {taosArrayDestroy($$);}
+%type grouplist {SArray*}
+%destructor grouplist {taosArrayDestroy($$);}
groupby_opt(A) ::= . {A = 0;}
groupby_opt(A) ::= GROUP BY grouplist(X). {A = X;}
@@ -553,11 +556,11 @@ having_opt(A) ::= HAVING expr(X). {A = X;}
//limit-offset subclause
%type limit_opt {SLimitVal}
limit_opt(A) ::= . {A.limit = -1; A.offset = 0;}
-limit_opt(A) ::= LIMIT signed(X). {A.limit = X; A.offset = 0;}
+limit_opt(A) ::= LIMIT signed(X). {printf("aa1, %d\n", X); A.limit = X; A.offset = 0;}
limit_opt(A) ::= LIMIT signed(X) OFFSET signed(Y).
- {A.limit = X; A.offset = Y;}
+ {printf("aa2\n, %d\n", X); A.limit = X; A.offset = Y;}
limit_opt(A) ::= LIMIT signed(X) COMMA signed(Y).
- {A.limit = Y; A.offset = X;}
+ {printf("aa3\n, %d\n", X); A.limit = Y; A.offset = X;}
%type slimit_opt {SLimitVal}
slimit_opt(A) ::= . {A.limit = -1; A.offset = 0;}
@@ -654,7 +657,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). {
X.n += F.n;
toTSDBType(A.type);
- tVariantList* K = tVariantListAppendToken(NULL, &A, -1);
+ SArray* K = tVariantListAppendToken(NULL, &A, -1);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
@@ -670,7 +673,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). {
X.n += Z.n;
toTSDBType(Y.type);
- tVariantList* A = tVariantListAppendToken(NULL, &Y, -1);
+ SArray* A = tVariantListAppendToken(NULL, &Y, -1);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
@@ -680,7 +683,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). {
X.n += F.n;
toTSDBType(Y.type);
- tVariantList* A = tVariantListAppendToken(NULL, &Y, -1);
+ SArray* A = tVariantListAppendToken(NULL, &Y, -1);
toTSDBType(Z.type);
A = tVariantListAppendToken(A, &Z, -1);
@@ -693,7 +696,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). {
X.n += F.n;
toTSDBType(Y.type);
- tVariantList* A = tVariantListAppendToken(NULL, &Y, -1);
+ SArray* A = tVariantListAppendToken(NULL, &Y, -1);
A = tVariantListAppend(A, &Z, -1);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL);
diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h
index 28b9a60102d41a11fa02fee1ca51cbf8e263bf3b..32cbb56c62514001ea5314e7e5dce8549daddb97 100644
--- a/src/query/inc/tsqlfunction.h
+++ b/src/query/inc/tsqlfunction.h
@@ -128,7 +128,7 @@ typedef struct SArithmeticSupport {
SExprInfo *pArithExpr;
int32_t numOfCols;
SColumnInfo *colList;
- SArray* exprList; // client side used
+ void *exprList; // client side used
int32_t offset;
char** data;
} SArithmeticSupport;
@@ -145,15 +145,14 @@ typedef struct SInterpInfoDetail {
int8_t primaryCol;
} SInterpInfoDetail;
-typedef struct SResultInfo {
+typedef struct SResultRowCellInfo {
int8_t hasResult; // result generated, not NULL value
- bool initialized; // output buffer has been initialized
- bool complete; // query has completed
- bool superTableQ; // is super table query
- uint32_t bufLen; // buffer size
- uint64_t numOfRes; // num of output result in current buffer
- void* interResultBuf; // output result buffer
-} SResultInfo;
+ bool initialized; // output buffer has been initialized
+ bool complete; // query has completed
+ uint32_t numOfRes; // num of output result in current buffer
+} SResultRowCellInfo;
+
+#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo)))
struct SQLFunctionCtx;
@@ -175,9 +174,11 @@ typedef struct SQLFunctionCtx {
int16_t inputBytes;
int16_t outputType;
- int16_t outputBytes; // size of results, determined by function and input column data type
- bool hasNull; // null value exist in current block
+ int16_t outputBytes; // size of results, determined by function and input column data type
+ int32_t interBufBytes; // internal buffer size
+ bool hasNull; // null value exist in current block
bool requireNull; // require null in some function
+ bool stableQuery;
int16_t functionId; // function id
void * aInputElemBuf;
char * aOutputBuf; // final result output buffer, point to sdata->data
@@ -189,7 +190,8 @@ typedef struct SQLFunctionCtx {
void * ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
SQLPreAggVal preAggVals;
tVariant tag;
- SResultInfo *resultInfo;
+
+ SResultRowCellInfo *resultInfo;
SExtTagsInfo tagInfo;
} SQLFunctionCtx;
@@ -274,16 +276,16 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha
(_r)->initialized = false; \
} while (0)
-void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf);
+//void setResultInfoBuf(SResultRowCellInfo *pResInfo, char* buf);
-static FORCE_INLINE void initResultInfo(SResultInfo *pResInfo) {
+static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, uint32_t bufLen) {
pResInfo->initialized = true; // the this struct has been initialized flag
pResInfo->complete = false;
pResInfo->hasResult = false;
pResInfo->numOfRes = 0;
- memset(pResInfo->interResultBuf, 0, (size_t)pResInfo->bufLen);
+ memset(GET_ROWCELL_INTERBUF(pResInfo), 0, (size_t)bufLen);
}
#ifdef __cplusplus
diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c
index 893105e44ac4eb82843514cda20928d3e0dcdaf9..e813688d8408bd124149352b337f1c0bef71d2af 100644
--- a/src/query/src/qAst.c
+++ b/src/query/src/qAst.c
@@ -13,12 +13,10 @@
* along with this program. If not, see .
*/
-
#include "os.h"
#include "exception.h"
#include "qAst.h"
-#include "qSqlparser.h"
#include "qSyntaxtreefunction.h"
#include "taosdef.h"
#include "taosmsg.h"
@@ -30,201 +28,19 @@
#include "tskiplist.h"
#include "tsqlfunction.h"
#include "tstoken.h"
-#include "ttokendef.h"
-#include "tulog.h"
-#include "tutil.h"
-
-/*
- *
- * @date 2018-2-15
- * @version 0.2 operation for column filter
- *
- * @Description parse tag query expression to build ast
- * ver 0.2, filter the result on first column with high priority to limit the candidate set
- * ver 0.3, pipeline filter in the form of: (a+2)/9 > 14
- *
- */
-static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken);
-
-static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i);
-static void destroySyntaxTree(tExprNode *);
-
-static uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight);
-
-/*
- * Check the filter value type on the right hand side based on the column id on the left hand side,
- * the filter value type must be identical to field type for relational operation
- * As for binary arithmetic operation, it is not necessary to do so.
- */
-static void reviseBinaryExprIfNecessary(tExprNode **pLeft, tExprNode **pRight, uint8_t *optr) {
- if (*optr >= TSDB_RELATION_LESS && *optr <= TSDB_RELATION_LIKE) {
- // make sure that the type of data on both sides of relational comparision are identical
- if ((*pLeft)->nodeType == TSQL_NODE_VALUE) {
- tVariantTypeSetType((*pLeft)->pVal, (*pRight)->pSchema->type);
- } else if ((*pRight)->nodeType == TSQL_NODE_VALUE) {
- tVariantTypeSetType((*pRight)->pVal, (*pLeft)->pSchema->type);
- }
-
- } else if (*optr >= TSDB_BINARY_OP_ADD && *optr <= TSDB_BINARY_OP_REMAINDER) {
- if ((*pLeft)->nodeType == TSQL_NODE_VALUE) {
- /* convert to int/bigint may cause the precision loss */
- tVariantTypeSetType((*pLeft)->pVal, TSDB_DATA_TYPE_DOUBLE);
- } else if ((*pRight)->nodeType == TSQL_NODE_VALUE) {
- /* convert to int/bigint may cause the precision loss */
- tVariantTypeSetType((*pRight)->pVal, TSDB_DATA_TYPE_DOUBLE);
- }
- }
-
- /*
- * for expressions that are suitable for switch principle,
- * switch left and left and right hand side in expr if possible
- */
- if ((*pLeft)->nodeType == TSQL_NODE_VALUE && (*pRight)->nodeType == TSQL_NODE_COL) {
- if (*optr >= TSDB_RELATION_GREATER && *optr <= TSDB_RELATION_GREATER_EQUAL && *optr != TSDB_RELATION_EQUAL) {
- SWAP(*pLeft, *pRight, tExprNode *);
- }
-
- switch (*optr) {
- case TSDB_RELATION_GREATER:
- (*optr) = TSDB_RELATION_LESS;
- break;
- case TSDB_RELATION_LESS:
- (*optr) = TSDB_RELATION_GREATER;
- break;
- case TSDB_RELATION_GREATER_EQUAL:
- (*optr) = TSDB_RELATION_LESS_EQUAL;
- break;
- case TSDB_RELATION_LESS_EQUAL:
- (*optr) = TSDB_RELATION_GREATER_EQUAL;
- break;
- default:;
- // for other type of operations, do nothing
- }
- }
-}
-
-static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken) {
- /* if the token is not a value, return false */
- if (pToken->type == TK_RP || (pToken->type != TK_INTEGER && pToken->type != TK_FLOAT && pToken->type != TK_ID &&
- pToken->type != TK_TBNAME && pToken->type != TK_STRING && pToken->type != TK_BOOL)) {
- return NULL;
- }
-
- size_t nodeSize = sizeof(tExprNode);
- tExprNode *pNode = NULL;
-
- if (pToken->type == TK_ID || pToken->type == TK_TBNAME) {
- int32_t i = 0;
- if (pToken->type == TK_ID) {
- do {
- SStrToken tableToken = {0};
- extractTableNameFromToken(pToken, &tableToken);
-
- size_t len = strlen(pSchema[i].name);
- if (strncmp(pToken->z, pSchema[i].name, pToken->n) == 0 && pToken->n == len) break;
- } while (++i < numOfCols);
-
- if (i == numOfCols) { // column name is not valid, parse the expression failed
- return NULL;
- }
- }
-
- nodeSize += sizeof(SSchema);
-
- pNode = calloc(1, nodeSize);
- pNode->pSchema = (struct SSchema *)((char *)pNode + sizeof(tExprNode));
- pNode->nodeType = TSQL_NODE_COL;
-
- if (pToken->type == TK_ID) {
- memcpy(pNode->pSchema, &pSchema[i], sizeof(SSchema));
- } else {
- pNode->pSchema->type = TSDB_DATA_TYPE_BINARY;
- pNode->pSchema->bytes = TSDB_TABLE_NAME_LEN - 1;
- strcpy(pNode->pSchema->name, TSQL_TBNAME_L);
- pNode->pSchema->colId = -1;
- }
+#include "tschemautil.h"
- } else {
- nodeSize += sizeof(tVariant);
- pNode = calloc(1, nodeSize);
- pNode->pVal = (tVariant *)((char *)pNode + sizeof(tExprNode));
-
- toTSDBType(pToken->type);
- tVariantCreate(pNode->pVal, pToken);
- pNode->nodeType = TSQL_NODE_VALUE;
- }
-
- return pNode;
-}
-
-uint8_t getBinaryExprOptr(SStrToken *pToken) {
- switch (pToken->type) {
- case TK_LT:
- return TSDB_RELATION_LESS;
- case TK_LE:
- return TSDB_RELATION_LESS_EQUAL;
- case TK_GT:
- return TSDB_RELATION_GREATER;
- case TK_GE:
- return TSDB_RELATION_GREATER_EQUAL;
- case TK_NE:
- return TSDB_RELATION_NOT_EQUAL;
- case TK_AND:
- return TSDB_RELATION_AND;
- case TK_OR:
- return TSDB_RELATION_OR;
- case TK_EQ:
- return TSDB_RELATION_EQUAL;
- case TK_PLUS:
- return TSDB_BINARY_OP_ADD;
- case TK_MINUS:
- return TSDB_BINARY_OP_SUBTRACT;
- case TK_STAR:
- return TSDB_BINARY_OP_MULTIPLY;
- case TK_SLASH:
- case TK_DIVIDE:
- return TSDB_BINARY_OP_DIVIDE;
- case TK_REM:
- return TSDB_BINARY_OP_REMAINDER;
- case TK_LIKE:
- return TSDB_RELATION_LIKE;
- case TK_ISNULL:
- return TSDB_RELATION_ISNULL;
- case TK_NOTNULL:
- return TSDB_RELATION_NOTNULL;
- default: { return 0; }
- }
-}
-
-// previous generated expr is reduced as the left child
-static tExprNode *parseRemainStr(char *pstr, tExprNode *pExpr, SSchema *pSchema, int32_t optr,
- int32_t numOfCols, int32_t *i) {
- // set the previous generated node as the left child of new root
- pExpr->nodeType = TSQL_NODE_EXPR;
-
- // remain is the right child
- tExprNode *pRight = createSyntaxTree(pSchema, numOfCols, pstr, i);
- if (pRight == NULL || (pRight->nodeType == TSQL_NODE_COL && pExpr->nodeType != TSQL_NODE_VALUE) ||
- (pExpr->nodeType == TSQL_NODE_VALUE && pRight->nodeType != TSQL_NODE_COL)) {
- tExprNodeDestroy(pExpr, NULL);
- tExprNodeDestroy(pRight, NULL);
- return NULL;
- }
-
- tExprNode *pNewExpr = (tExprNode *)calloc(1, sizeof(tExprNode));
- uint8_t k = optr;
- reviseBinaryExprIfNecessary(&pExpr, &pRight, &k);
- pNewExpr->_node.pLeft = pExpr;
- pNewExpr->_node.pRight = pRight;
- pNewExpr->_node.optr = k;
-
- pNewExpr->_node.hasPK = isQueryOnPrimaryKey(pSchema[0].name, pExpr, pRight);
- pNewExpr->nodeType = TSQL_NODE_EXPR;
+typedef struct {
+ char* v;
+ int32_t optr;
+} SEndPoint;
- return pNewExpr;
-}
+typedef struct {
+ SEndPoint* start;
+ SEndPoint* end;
+} SQueryCond;
-uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
+static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
if (pLeft->nodeType == TSQL_NODE_COL) {
// if left node is the primary column,return true
return (strcmp(primaryColumnName, pLeft->pSchema->name) == 0) ? 1 : 0;
@@ -237,103 +53,6 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLef
}
}
-static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) {
- SStrToken t0 = tStrGetToken(str, i, false, 0, NULL);
- if (t0.n == 0) {
- return NULL;
- }
-
- tExprNode *pLeft = NULL;
- if (t0.type == TK_LP) { // start new left child branch
- pLeft = createSyntaxTree(pSchema, numOfCols, str, i);
- } else {
- if (t0.type == TK_RP) {
- return NULL;
- }
-
- pLeft = tExprNodeCreate(pSchema, numOfCols, &t0);
- }
-
- if (pLeft == NULL) {
- return NULL;
- }
-
- t0 = tStrGetToken(str, i, false, 0, NULL);
- if (t0.n == 0 || t0.type == TK_RP) {
- if (pLeft->nodeType != TSQL_NODE_EXPR) { // if left is not the expr, it is not a legal expr
- tExprNodeDestroy(pLeft, NULL);
- return NULL;
- }
-
- return pLeft;
- }
-
- // get the operator of expr
- uint8_t optr = getBinaryExprOptr(&t0);
- if (optr == 0) {
- uError("not support binary operator:%d", t0.type);
- tExprNodeDestroy(pLeft, NULL);
- return NULL;
- }
-
- assert(pLeft != NULL);
- tExprNode *pRight = NULL;
-
- if (t0.type == TK_AND || t0.type == TK_OR || t0.type == TK_LP) {
- pRight = createSyntaxTree(pSchema, numOfCols, str, i);
- } else {
- /*
- * In case that pLeft is a field identification,
- * we parse the value in expression according to queried field type,
- * if we do not get the information, in case of value of field presented first,
- * we revised the value after the binary expression is completed.
- */
- t0 = tStrGetToken(str, i, true, 0, NULL);
- if (t0.n == 0) {
- tExprNodeDestroy(pLeft, NULL); // illegal expression
- return NULL;
- }
-
- if (t0.type == TK_LP) {
- pRight = createSyntaxTree(pSchema, numOfCols, str, i);
- } else {
- pRight = tExprNodeCreate(pSchema, numOfCols, &t0);
- }
- }
-
- if (pRight == NULL) {
- tExprNodeDestroy(pLeft, NULL);
- return NULL;
- }
-
- /* create binary expr as the child of new parent node */
- tExprNode *pExpr = (tExprNode *)calloc(1, sizeof(tExprNode));
- reviseBinaryExprIfNecessary(&pLeft, &pRight, &optr);
-
- pExpr->_node.hasPK = isQueryOnPrimaryKey(pSchema[0].name, pLeft, pRight);
- pExpr->_node.pLeft = pLeft;
- pExpr->_node.pRight = pRight;
- pExpr->_node.optr = optr;
-
- t0 = tStrGetToken(str, i, true, 0, NULL);
-
- if (t0.n == 0 || t0.type == TK_RP) {
- pExpr->nodeType = TSQL_NODE_EXPR;
- return pExpr;
- } else {
- uint8_t localOptr = getBinaryExprOptr(&t0);
- if (localOptr == 0) {
- uError("not support binary operator:%d", t0.type);
- free(pExpr);
- return NULL;
- }
-
- return parseRemainStr(str, pExpr, pSchema, localOptr, numOfCols, i);
- }
-}
-
-static void UNUSED_FUNC destroySyntaxTree(tExprNode *pNode) { tExprNodeDestroy(pNode, NULL); }
-
void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) {
if (pNode == NULL) {
return;
@@ -373,16 +92,6 @@ void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) {
*pExpr = NULL;
}
-typedef struct {
- char* v;
- int32_t optr;
-} SEndPoint;
-
-typedef struct {
- SEndPoint* start;
- SEndPoint* end;
-} SQueryCond;
-
// todo check for malloc failure
static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
int32_t optr = queryColInfo->optr;
@@ -396,13 +105,10 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
pCond->end = calloc(1, sizeof(SEndPoint));
pCond->end->optr = queryColInfo->optr;
pCond->end->v = queryColInfo->q;
- } else if (optr == TSDB_RELATION_IN) {
- printf("relation is in\n");
- assert(0);
- } else if (optr == TSDB_RELATION_LIKE) {
- printf("relation is like\n");
+ } else if (optr == TSDB_RELATION_IN || optr == TSDB_RELATION_LIKE) {
assert(0);
}
+
return TSDB_CODE_SUCCESS;
}
@@ -415,9 +121,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
}
if (cond.start != NULL) {
- iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC);
+ iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC);
} else {
- iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->keyInfo.type, TSDB_ORDER_DESC);
+ iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC);
}
if (cond.start != NULL) {
@@ -432,7 +138,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
break;
}
- STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
} else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal
@@ -450,7 +156,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
if (ret == 0 && optr == TSDB_RELATION_GREATER) {
continue;
} else {
- STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
comp = false;
}
@@ -465,22 +171,22 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
continue;
}
- STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
tSkipListDestroyIter(iter);
comp = true;
- iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC);
+ iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC);
while(tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter);
comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0);
- if (comp) {
+ if (comp) {
continue;
}
- STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
@@ -504,7 +210,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
if (ret == 0 && optr == TSDB_RELATION_LESS) {
continue;
} else {
- STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
comp = false; // no need to compare anymore
}
@@ -518,7 +224,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type);
if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) ||
(pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) {
- STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
}
@@ -530,99 +236,6 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
tSkipListDestroyIter(iter);
}
-int32_t merge(SArray *pLeft, SArray *pRight, SArray *pFinalRes) {
-// assert(pFinalRes->pRes == 0);
-//
-// pFinalRes->pRes = calloc((size_t)(pLeft->num + pRight->num), POINTER_BYTES);
-// pFinalRes->num = 0;
-//
-// // sort according to address
-// tSkipListNode **pLeftNodes = (tSkipListNode **)pLeft->pRes;
-// qsort(pLeftNodes, pLeft->num, sizeof(pLeft->pRes[0]), compareByAddr);
-//
-// tSkipListNode **pRightNodes = (tSkipListNode **)pRight->pRes;
-// qsort(pRightNodes, pRight->num, sizeof(pRight->pRes[0]), compareByAddr);
-//
-// int32_t i = 0, j = 0;
-//
-// // merge two sorted arrays in O(n) time
-// while (i < pLeft->num && j < pRight->num) {
-// int64_t ret = (int64_t)pLeftNodes[i] - (int64_t)pRightNodes[j];
-//
-// if (ret < 0) {
-// pFinalRes->pRes[pFinalRes->num++] = pLeftNodes[i++];
-// } else if (ret > 0) {
-// pFinalRes->pRes[pFinalRes->num++] = pRightNodes[j++];
-// } else { // pNode->key > pkey[i]
-// pFinalRes->pRes[pFinalRes->num++] = pRightNodes[j++];
-// i++;
-// }
-// }
-//
-// while (i < pLeft->num) {
-// pFinalRes->pRes[pFinalRes->num++] = pLeftNodes[i++];
-// }
-//
-// while (j < pRight->num) {
-// pFinalRes->pRes[pFinalRes->num++] = pRightNodes[j++];
-// }
-//
-// return pFinalRes->num;
- return 0;
-}
-
-int32_t intersect(SArray *pLeft, SArray *pRight, SArray *pFinalRes) {
-// int64_t num = MIN(pLeft->num, pRight->num);
-//
-// assert(pFinalRes->pRes == 0);
-//
-// pFinalRes->pRes = calloc(num, POINTER_BYTES);
-// pFinalRes->num = 0;
-//
-// // sort according to address
-// tSkipListNode **pLeftNodes = (tSkipListNode **)pLeft->pRes;
-// qsort(pLeftNodes, pLeft->num, sizeof(pLeft->pRes[0]), compareByAddr);
-//
-// tSkipListNode **pRightNodes = (tSkipListNode **)pRight->pRes;
-// qsort(pRightNodes, pRight->num, sizeof(pRight->pRes[0]), compareByAddr);
-//
-// int32_t i = 0, j = 0;
-// // merge two sorted arrays in O(n) time
-// while (i < pLeft->num && j < pRight->num) {
-// int64_t ret = (int64_t)pLeftNodes[i] - (int64_t)pRightNodes[j];
-//
-// if (ret < 0) {
-// i++;
-// } else if (ret > 0) {
-// j++;
-// } else { // pNode->key > pkey[i]
-// pFinalRes->pRes[pFinalRes->num++] = pRightNodes[j];
-// i++;
-// j++;
-// }
-// }
-//
-// return pFinalRes->num;
- return 0;
-}
-
-/*
- * traverse the result and apply the function to each item to check if the item is qualified or not
- */
-static void tArrayTraverse(tExprNode *pExpr, __result_filter_fn_t fp, SArray *pResult) {
- assert(pExpr->_node.pLeft->nodeType == TSQL_NODE_COL && pExpr->_node.pRight->nodeType == TSQL_NODE_VALUE && fp != NULL);
-
- // scan the result array list and check for each item in the list
- for (int32_t i = 0; i < taosArrayGetSize(pResult); ++i) {
- void* item = taosArrayGet(pResult, i);
- if (fp(item, pExpr->_node.info)) {
- i++;
- } else {
- taosArrayRemove(pResult, i);
- }
- }
-}
-
static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) {
tExprNode *pLeft = pExpr->_node.pLeft;
tExprNode *pRight = pExpr->_node.pRight;
@@ -650,39 +263,13 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *p
return param->nodeFilterFn(pItem, pExpr->_node.info);
}
-/**
- * Apply the filter expression on non-indexed tag columns to each element in the result list, which is generated
- * by filtering on indexed tag column. So the whole result set only needs to be iterated once to generate
- * result that is satisfied to the filter expression, no matter how the filter expression consisting of.
- *
- * @param pExpr filter expression on non-indexed tag columns.
- * @param pResult results from filter on the indexed tag column, which is usually the first tag column
- * @param pSchema tag schemas
- * @param fp filter callback function
- */
-static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) {
- size_t size = taosArrayGetSize(pResult);
-
- SArray* array = taosArrayInit(size, POINTER_BYTES);
- for (int32_t i = 0; i < size; ++i) {
- void *pItem = taosArrayGetP(pResult, i);
-
- if (filterItem(pExpr, pItem, param)) {
- taosArrayPush(array, &pItem);
- }
- }
-
- taosArrayCopy(pResult, array);
- taosArrayDestroy(array);
-}
-
static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SExprTraverseSupp *param ) {
SSkipListIterator* iter = tSkipListCreateIter(pSkipList);
while (tSkipListIterNext(iter)) {
SSkipListNode *pNode = tSkipListIterGet(iter);
if (filterItem(pExpr, pNode, param)) {
- taosArrayPush(pResult, SL_GET_NODE_DATA(pNode));
+ taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode)));
}
}
tSkipListDestroyIter(iter);
@@ -697,7 +284,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
SSkipListNode *pNode = tSkipListIterGet(iter);
char * pData = SL_GET_NODE_DATA(pNode);
- tstr *name = (tstr*) tsdbGetTableName(*(void**) pData);
+ tstr *name = (tstr*) tsdbGetTableName((void*) pData);
// todo speed up by using hash
if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) {
@@ -711,7 +298,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
}
if (addToResult) {
- STableKeyInfo info = {.pTable = *(void**)pData, .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(res, &info);
}
}
@@ -733,10 +320,6 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY));
param->setupInfoFn(pExpr, param->pExtInfo);
- if (pSkipList == NULL) {
- tArrayTraverse(pExpr, param->nodeFilterFn, result);
- return;
- }
tQueryInfo *pQueryInfo = pExpr->_node.info;
if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) {
@@ -748,49 +331,14 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
return;
}
- // recursive traverse left child branch
+ // The value of hasPK is always 0.
uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK;
+ assert(weight == 0 && pSkipList != NULL && taosArrayGetSize(result) == 0);
- if (weight == 0 ) {
- if (taosArrayGetSize(result) > 0 && pSkipList == NULL) {
- /**
- * Perform the filter operation based on the initial filter result, which is obtained from filtering from index.
- * Since no index presented, the filter operation is done by scan all elements in the result set.
- *
- * if the query is a high selectivity filter, only small portion of meters are retrieved.
- */
- exprTreeTraverseImpl(pExpr, result, param);
- } else {
- /**
- * apply the hierarchical expression to every node in skiplist for find the qualified nodes
- */
- assert(taosArrayGetSize(result) == 0);
- tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param);
- }
-
- return;
- }
-
- if (weight == 2 || (weight == 1 && pExpr->_node.optr == TSDB_RELATION_OR)) {
- SArray* rLeft = taosArrayInit(10, POINTER_BYTES);
- SArray* rRight = taosArrayInit(10, POINTER_BYTES);
-
- tExprTreeTraverse(pLeft, pSkipList, rLeft, param);
- tExprTreeTraverse(pRight, pSkipList, rRight, param);
-
- if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS
- intersect(rLeft, rRight, result);
- } else if (pExpr->_node.optr == TSDB_RELATION_OR) { // or
- merge(rLeft, rRight, result);
- } else {
- assert(false);
- }
-
- taosArrayDestroy(rLeft);
- taosArrayDestroy(rRight);
- return;
- }
+ //apply the hierarchical expression to every node in skiplist for find the qualified nodes
+ tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param);
+#if 0
/*
* (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here
*
@@ -819,6 +367,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
* So, we do not set the skip list index as a parameter
*/
tExprTreeTraverse(pSecond, NULL, result, param);
+#endif
}
void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
@@ -1011,6 +560,7 @@ tExprNode* exprTreeFromBinary(const void* data, size_t size) {
if (size == 0) {
return NULL;
}
+
SBufferReader br = tbufInitReader(data, size, false);
return exprTreeFromBinaryImpl(&br);
}
@@ -1034,10 +584,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
SSchema* pSchema = exception_calloc(1, sizeof(SSchema));
left->pSchema = pSchema;
- pSchema->type = TSDB_DATA_TYPE_BINARY;
- pSchema->bytes = TSDB_TABLE_NAME_LEN - 1;
- strcpy(pSchema->name, TSQL_TBNAME_L);
- pSchema->colId = -1;
+ *pSchema = tscGetTbnameColumnSchema();
tExprNode* right = exception_calloc(1, sizeof(tExprNode));
expr->_node.pRight = right;
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index d46beab2cb88a275a15e3aa2c4de47cedf949175..869f57f117db9b349b6d28cd819c84aa36f82baf 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -170,17 +170,15 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) {
tw->ekey -= 1;
}
-#define GET_NEXT_TIMEWINDOW(_q, tw) getNextTimeWindow((_q), (tw))
-
#define SET_STABLE_QUERY_OVER(_q) ((_q)->tableIndex = (int32_t)((_q)->tableqinfoGroupInfo.numOfTables))
#define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (int32_t)((_q)->tableqinfoGroupInfo.numOfTables))
// todo move to utility
static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group);
-static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult);
-static void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult);
-static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo);
+static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult);
+static void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult);
+static void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow);
static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId);
static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo,
@@ -244,7 +242,7 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) {
int64_t maxOutput = 0;
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
- int32_t functionId = pQuery->pSelectExpr[j].base.functionId;
+ int32_t functionId = pQuery->pExpr1[j].base.functionId;
/*
* ts, tag, tagprj function can not decide the output number of current query
@@ -255,7 +253,7 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) {
continue;
}
- SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) {
maxOutput = pResInfo->numOfRes;
}
@@ -272,7 +270,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) {
SQuery *pQuery = pRuntimeEnv->pQuery;
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
- SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
int16_t functionId = pRuntimeEnv->pCtx[j].functionId;
if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ ||
@@ -339,7 +337,7 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) {
int32_t numOfSelectivity = 0;
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t functId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functId = pQuery->pExpr1[i].base.functionId;
if (functId == TSDB_FUNC_TAG_DUMMY || functId == TSDB_FUNC_TS_DUMMY) {
hasTags = true;
continue;
@@ -359,7 +357,7 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) {
bool isProjQuery(SQuery *pQuery) {
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t functId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functId = pQuery->pExpr1[i].base.functionId;
if (functId != TSDB_FUNC_PRJ && functId != TSDB_FUNC_TAGPRJ) {
return false;
}
@@ -368,7 +366,7 @@ bool isProjQuery(SQuery *pQuery) {
return true;
}
-bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].base.functionId == TSDB_FUNC_TS_COMP; }
+bool isTSCompQuery(SQuery *pQuery) { return pQuery->pExpr1[0].base.functionId == TSDB_FUNC_TS_COMP; }
static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) {
SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
@@ -389,7 +387,7 @@ static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) {
static bool isTopBottomQuery(SQuery *pQuery) {
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionId = pQuery->pExpr1[i].base.functionId;
if (functionId == TSDB_FUNC_TS) {
continue;
}
@@ -403,12 +401,12 @@ static bool isTopBottomQuery(SQuery *pQuery) {
}
static bool hasTagValOutput(SQuery* pQuery) {
- SExprInfo *pExprInfo = &pQuery->pSelectExpr[0];
+ SExprInfo *pExprInfo = &pQuery->pExpr1[0];
if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP) {
return true;
} else { // set tag value, by which the results are aggregated.
for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) {
- SExprInfo *pLocalExprInfo = &pQuery->pSelectExpr[idx];
+ SExprInfo *pLocalExprInfo = &pQuery->pExpr1[idx];
// ts_comp column required the tag value for join filter
if (TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) {
@@ -447,11 +445,11 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis
return true;
}
-static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData,
- int16_t bytes, bool masterscan) {
- SQuery *pQuery = pRuntimeEnv->pQuery;
-
- int32_t *p1 = (int32_t *) taosHashGet(pWindowResInfo->hashList, pData, bytes);
+static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData,
+ int16_t bytes, bool masterscan, uint64_t uid) {
+ SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid);
+ int32_t *p1 =
+ (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
if (p1 != NULL) {
pWindowResInfo->curIndex = *p1;
} else {
@@ -461,41 +459,37 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin
// more than the capacity, reallocate the resources
if (pWindowResInfo->size >= pWindowResInfo->capacity) {
- int64_t newCap = 0;
+ int64_t newCapacity = 0;
if (pWindowResInfo->capacity > 10000) {
- newCap = (int64_t)(pWindowResInfo->capacity * 1.25);
+ newCapacity = (int64_t)(pWindowResInfo->capacity * 1.25);
} else {
- newCap = (int64_t)(pWindowResInfo->capacity * 1.5);
+ newCapacity = (int64_t)(pWindowResInfo->capacity * 1.5);
}
- char *t = realloc(pWindowResInfo->pResult, (size_t)(newCap * sizeof(SWindowResult)));
- pRuntimeEnv->summary.internalSupSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult);
- pRuntimeEnv->summary.numOfTimeWindows += (newCap - pWindowResInfo->capacity);
-
+ char *t = realloc(pWindowResInfo->pResult, (size_t)(newCapacity * POINTER_BYTES));
if (t == NULL) {
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
- pWindowResInfo->pResult = (SWindowResult *)t;
+ pWindowResInfo->pResult = (SResultRow **)t;
- int32_t inc = (int32_t)newCap - pWindowResInfo->capacity;
- memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc);
+ int32_t inc = (int32_t)newCapacity - pWindowResInfo->capacity;
+ memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, POINTER_BYTES * inc);
- pRuntimeEnv->summary.internalSupSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc;
-
- for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) {
- int32_t ret = createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize);
- if (ret != TSDB_CODE_SUCCESS) {
- longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
- }
- }
+ pWindowResInfo->capacity = (int32_t)newCapacity;
+ }
- pWindowResInfo->capacity = (int32_t)newCap;
+ SResultRow *pResult = getNewResultRow(pRuntimeEnv->pool);
+ pWindowResInfo->pResult[pWindowResInfo->size] = pResult;
+ int32_t ret = initResultRow(pResult);
+ if (ret != TSDB_CODE_SUCCESS) {
+ longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
// add a new result set for a new group
pWindowResInfo->curIndex = pWindowResInfo->size++;
- taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t));
+ taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes),
+ (char *)&pWindowResInfo->curIndex, sizeof(int32_t));
}
// too many time window in query
@@ -503,7 +497,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
}
- return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex);
+ return getResultRow(pWindowResInfo, pWindowResInfo->curIndex);
}
// get the correct time window according to the handled timestamp
@@ -519,7 +513,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t
}
} else {
int32_t slot = curTimeWindowIndex(pWindowResInfo);
- SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot);
+ SResultRow* pWindowRes = getResultRow(pWindowResInfo, slot);
w = pWindowRes->win;
}
@@ -555,9 +549,9 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t
return w;
}
-static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t sid,
+static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid,
int32_t numOfRowsPerPage) {
- if (pWindowRes->pos.pageId != -1) {
+ if (pWindowRes->pageId != -1) {
return 0;
}
@@ -565,10 +559,10 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
// in the first scan, new space needed for results
int32_t pageId = -1;
- SIDList list = getDataBufPagesIdList(pResultBuf, sid);
+ SIDList list = getDataBufPagesIdList(pResultBuf, tid);
if (taosArrayGetSize(list) == 0) {
- pData = getNewDataBuf(pResultBuf, sid, &pageId);
+ pData = getNewDataBuf(pResultBuf, tid, &pageId);
} else {
SPageInfo* pi = getLastPageInfo(list);
pData = getResBufPage(pResultBuf, pi->pageId);
@@ -578,7 +572,7 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
// release current page first, and prepare the next one
releaseResBufPageInfo(pResultBuf, pi);
- pData = getNewDataBuf(pResultBuf, sid, &pageId);
+ pData = getNewDataBuf(pResultBuf, tid, &pageId);
if (pData != NULL) {
assert(pData->num == 0); // number of elements must be 0 for new allocated buffer
}
@@ -590,24 +584,23 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
}
// set the number of rows in current disk page
- if (pWindowRes->pos.pageId == -1) { // not allocated yet, allocate new buffer
- pWindowRes->pos.pageId = pageId;
- pWindowRes->pos.rowId = (int32_t)(pData->num++);
+ if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer
+ pWindowRes->pageId = pageId;
+ pWindowRes->rowId = (int32_t)(pData->num++);
- assert(pWindowRes->pos.pageId >= 0);
+ assert(pWindowRes->pageId >= 0);
}
return 0;
}
-static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t sid,
+static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, SDataBlockInfo* pBockInfo,
STimeWindow *win, bool masterscan, bool* newWind) {
assert(win->skey <= win->ekey);
SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
- SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey,
- TSDB_KEYSIZE, masterscan);
- if (pWindowRes == NULL) {
+ SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, pBockInfo->uid);
+ if (pResultRow == NULL) {
*newWind = false;
return masterscan? -1:0;
@@ -616,23 +609,22 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes
*newWind = true;
// not assign result buffer yet, add new result buffer
- if (pWindowRes->pos.pageId == -1) {
- int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, sid, pRuntimeEnv->numOfRowsPerPage);
+ if (pResultRow->pageId == -1) {
+ int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, pBockInfo->tid, pRuntimeEnv->numOfRowsPerPage);
if (ret != TSDB_CODE_SUCCESS) {
return -1;
}
}
// set time window for current result
- pWindowRes->win = (*win);
-
- setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes);
+ pResultRow->win = (*win);
+ setWindowResOutputBufInitCtx(pRuntimeEnv, pResultRow);
return TSDB_CODE_SUCCESS;
}
static bool getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) {
assert(slot >= 0 && slot < pWindowResInfo->size);
- return pWindowResInfo->pResult[slot].closed;
+ return pWindowResInfo->pResult[slot]->closed;
}
static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos,
@@ -691,7 +683,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe
int64_t skey = TSKEY_INITIAL_VAL;
for (i = 0; i < pWindowResInfo->size; ++i) {
- SWindowResult *pResult = &pWindowResInfo->pResult[i];
+ SResultRow *pResult = pWindowResInfo->pResult[i];
if (pResult->closed) {
numOfClosed += 1;
continue;
@@ -715,7 +707,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe
pWindowResInfo->curIndex = i;
}
- pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].win.skey;
+ pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey;
// the number of completed slots are larger than the threshold, return current generated results to client.
if (numOfClosed > pWindowResInfo->threshold) {
@@ -792,7 +784,7 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed
pCtx[k].size = forwardStep;
pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1);
- int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
+ int32_t functionId = pQuery->pExpr1[k].base.functionId;
if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
pCtx[k].ptsList = &tsCol[pCtx[k].startOffset];
}
@@ -821,7 +813,7 @@ static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed,
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
pCtx[k].nStartQueryTimestamp = pWin->skey;
- int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
+ int32_t functionId = pQuery->pExpr1[k].base.functionId;
if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
aAggs[functionId].xFunctionF(&pCtx[k], offset);
}
@@ -833,7 +825,7 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow
TSKEY *primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) {
SQuery *pQuery = pRuntimeEnv->pQuery;
- GET_NEXT_TIMEWINDOW(pQuery, pNext);
+ getNextTimeWindow(pQuery, pNext);
// next time window is not in current block
if ((pNext->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
@@ -930,9 +922,9 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
char *dataBlock = NULL;
SQuery *pQuery = pRuntimeEnv->pQuery;
- int32_t functionId = pQuery->pSelectExpr[col].base.functionId;
+ int32_t functionId = pQuery->pExpr1[col].base.functionId;
if (functionId == TSDB_FUNC_ARITHM) {
- sas->pArithExpr = &pQuery->pSelectExpr[col];
+ sas->pArithExpr = &pQuery->pExpr1[col];
sas->offset = 0;
sas->colList = pQuery->colList;
@@ -940,7 +932,6 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
sas->data = calloc(pQuery->numOfCols, POINTER_BYTES);
if (sas->data == NULL) {
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
@@ -963,9 +954,9 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
}
} else { // other type of query function
- SColIndex *pCol = &pQuery->pSelectExpr[col].base.colInfo;
+ SColIndex *pCol = &pQuery->pExpr1[col].base.colInfo;
if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) {
- SColIndex* pColIndex = &pQuery->pSelectExpr[col].base.colInfo;
+ SColIndex* pColIndex = &pQuery->pExpr1[col].base.colInfo;
SColumnInfoData *p = taosArrayGet(pDataBlock, pColIndex->colIndex);
assert(p->info.colId == pColIndex->colId);
@@ -1003,7 +994,6 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport));
if (sasArray == NULL) {
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
@@ -1026,9 +1016,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
bool hasTimeWindow = false;
STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
- if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow) !=
- TSDB_CODE_SUCCESS) {
- taosTFree(sasArray);
+ if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) {
+ tfree(sasArray);
return;
}
@@ -1055,8 +1044,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
// null data, failed to allocate more memory buffer
hasTimeWindow = false;
- if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan,
- &hasTimeWindow) != TSDB_CODE_SUCCESS) {
+ if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) {
break;
}
@@ -1079,7 +1067,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
* tag_prj function are changed to be TSDB_FUNC_TAG_DUMMY
*/
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
- int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
+ int32_t functionId = pQuery->pExpr1[k].base.functionId;
if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
aAggs[functionId].xFunction(&pCtx[k]);
}
@@ -1087,17 +1075,17 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
}
for(int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- if (pQuery->pSelectExpr[i].base.functionId != TSDB_FUNC_ARITHM) {
+ if (pQuery->pExpr1[i].base.functionId != TSDB_FUNC_ARITHM) {
continue;
}
- taosTFree(sasArray[i].data);
+ tfree(sasArray[i].data);
}
- taosTFree(sasArray);
+ tfree(sasArray);
}
-static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes) {
+static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) {
if (isNull(pData, type)) { // ignore the null value
return -1;
}
@@ -1114,13 +1102,14 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
len = varDataLen(pData);
} else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) {
SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
- qError("QInfo:%p group by not supported on double/float/binary/nchar columns, abort", pQInfo);
+ qError("QInfo:%p group by not supported on double/float columns, abort", pQInfo);
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR);
}
- SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true);
- if (pWindowRes == NULL) {
+ uint64_t uid = groupIndex; // uid is always set to be 0.
+ SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid);
+ if (pResultRow == NULL) {
return -1;
}
@@ -1134,23 +1123,21 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
}
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
- pWindowRes->key = malloc(varDataTLen(pData));
- varDataCopy(pWindowRes->key, pData);
+ pResultRow->key = malloc(varDataTLen(pData));
+ varDataCopy(pResultRow->key, pData);
} else {
- pWindowRes->win.skey = v;
- pWindowRes->win.ekey = v;
+ pResultRow->win.skey = v;
+ pResultRow->win.ekey = v;
}
- assert(pRuntimeEnv->windowResInfo.interval == 0);
-
- if (pWindowRes->pos.pageId == -1) {
- int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage);
+ if (pResultRow->pageId == -1) {
+ int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage);
if (ret != 0) {
return -1;
}
}
- setWindowResOutputBuf(pRuntimeEnv, pWindowRes);
+ setResultOutputBuf(pRuntimeEnv, pResultRow);
initCtxOutputBuf(pRuntimeEnv);
return TSDB_CODE_SUCCESS;
}
@@ -1233,7 +1220,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) {
}
static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
SQuery* pQuery = pRuntimeEnv->pQuery;
// in case of timestamp column, always generated results.
@@ -1277,7 +1264,6 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport));
if (sasArray == NULL) {
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
@@ -1338,7 +1324,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
bool hasTimeWindow = false;
- int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow);
+ int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow);
if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code
continue;
}
@@ -1354,9 +1340,9 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
int32_t index = pWindowResInfo->curIndex;
while (1) {
- GET_NEXT_TIMEWINDOW(pQuery, &nextWin);
+ getNextTimeWindow(pQuery, &nextWin);
if ((nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
- (nextWin.skey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) {
+ (nextWin.ekey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) {
break;
}
@@ -1366,7 +1352,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
// null data, failed to allocate more memory buffer
hasTimeWindow = false;
- if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) {
+ if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) {
break;
}
@@ -1382,14 +1368,14 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
if (groupbyColumnValue) {
char *val = groupbyColumnData + bytes * offset;
- int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes);
+ int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes, item->groupIndex);
if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code
continue;
}
}
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
- int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
+ int32_t functionId = pQuery->pExpr1[k].base.functionId;
if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
aAggs[functionId].xFunctionF(&pCtx[k], offset);
}
@@ -1418,11 +1404,11 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
// todo refactor: extract method
for(int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- if (pQuery->pSelectExpr[i].base.functionId != TSDB_FUNC_ARITHM) {
+ if (pQuery->pExpr1[i].base.functionId != TSDB_FUNC_ARITHM) {
continue;
}
- taosTFree(sasArray[i].data);
+ tfree(sasArray[i].data);
}
free(sasArray);
@@ -1478,11 +1464,11 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo,
SDataStatis *pStatis, void *param, int32_t colIndex, int32_t vgId) {
- int32_t functionId = pQuery->pSelectExpr[colIndex].base.functionId;
- int32_t colId = pQuery->pSelectExpr[colIndex].base.colInfo.colId;
+ int32_t functionId = pQuery->pExpr1[colIndex].base.functionId;
+ int32_t colId = pQuery->pExpr1[colIndex].base.colInfo.colId;
SDataStatis *tpField = NULL;
- pCtx->hasNull = hasNullValue(&pQuery->pSelectExpr[colIndex].base.colInfo, pStatis, &tpField);
+ pCtx->hasNull = hasNullValue(&pQuery->pExpr1[colIndex].base.colInfo, pStatis, &tpField);
pCtx->aInputElemBuf = inputData;
if (tpField != NULL) {
@@ -1515,13 +1501,14 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY
functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) {
/*
* least squares function needs two columns of input, currently, the x value of linear equation is set to
- * timestamp column, and the y-value is the column specified in pQuery->pSelectExpr[i].colIdxInBuffer
+ * timestamp column, and the y-value is the column specified in pQuery->pExpr1[i].colIdxInBuffer
*
* top/bottom function needs timestamp to indicate when the
* top/bottom values emerge, so does diff function
*/
if (functionId == TSDB_FUNC_TWA) {
- STwaInfo *pTWAInfo = GET_RES_INFO(pCtx)->interResultBuf;
+ SResultRowCellInfo* pInfo = GET_RES_INFO(pCtx);
+ STwaInfo *pTWAInfo = (STwaInfo*) GET_ROWCELL_INTERBUF(pInfo);
pTWAInfo->SKey = pQuery->window.skey;
pTWAInfo->EKey = pQuery->window.ekey;
}
@@ -1535,7 +1522,9 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY
pCtx->preAggVals.statis.max = pBlockInfo->window.ekey;
}
} else if (functionId == TSDB_FUNC_INTERP) {
- SInterpInfoDetail *pInterpInfo = GET_RES_INFO(pCtx)->interResultBuf;
+ SResultRowCellInfo* pInfo = GET_RES_INFO(pCtx);
+
+ SInterpInfoDetail *pInterpInfo = (SInterpInfoDetail *)GET_ROWCELL_INTERBUF(pInfo);
pInterpInfo->type = (int8_t)pQuery->fillType;
pInterpInfo->ts = pQuery->window.skey;
pInterpInfo->primaryCol = (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX);
@@ -1585,7 +1574,7 @@ static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx
}
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base;
+ SSqlFuncMsg *pSqlFuncMsg = &pQuery->pExpr1[i].base;
if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) {
tagLen += pCtx[i].outputBytes;
@@ -1605,39 +1594,28 @@ static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx
p->tagInfo.numOfTagCols = num;
p->tagInfo.tagsLen = tagLen;
} else {
- taosTFree(pTagCtx);
+ tfree(pTagCtx);
}
}
return TSDB_CODE_SUCCESS;
}
-static FORCE_INLINE void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery, char* buf) {
- char* p = buf;
- for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t size = pQuery->pSelectExpr[i].interBytes;
- setResultInfoBuf(&pResultInfo[i], size, isStableQuery, p);
-
- p += size;
- }
-}
-
static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order) {
qDebug("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv));
SQuery *pQuery = pRuntimeEnv->pQuery;
- size_t size = pRuntimeEnv->interBufSize + pQuery->numOfOutput * sizeof(SResultInfo);
-
- pRuntimeEnv->resultInfo = calloc(1, size);
pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx));
+ pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t));
+ pRuntimeEnv->pResultRow = getNewResultRow(pRuntimeEnv->pool);
- if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) {
+ if (pRuntimeEnv->pResultRow == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) {
goto _clean;
}
pRuntimeEnv->offset[0] = 0;
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base;
+ SSqlFuncMsg *pSqlFuncMsg = &pQuery->pExpr1[i].base;
SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
SColIndex* pIndex = &pSqlFuncMsg->colInfo;
@@ -1668,15 +1646,16 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
pCtx->inputType = pQuery->colList[index].type;
}
-
assert(isValidDataType(pCtx->inputType));
pCtx->ptsOutputBuf = NULL;
- pCtx->outputBytes = pQuery->pSelectExpr[i].bytes;
- pCtx->outputType = pQuery->pSelectExpr[i].type;
+ pCtx->outputBytes = pQuery->pExpr1[i].bytes;
+ pCtx->outputType = pQuery->pExpr1[i].type;
pCtx->order = pQuery->order.order;
pCtx->functionId = pSqlFuncMsg->functionId;
+ pCtx->stableQuery = pRuntimeEnv->stableQuery;
+ pCtx->interBufBytes = pQuery->pExpr1[i].interBytes;
pCtx->numOfParams = pSqlFuncMsg->numOfParams;
for (int32_t j = 0; j < pCtx->numOfParams; ++j) {
@@ -1693,7 +1672,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
int32_t functionId = pCtx->functionId;
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
- int32_t f = pQuery->pSelectExpr[0].base.functionId;
+ int32_t f = pQuery->pExpr1[0].base.functionId;
assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY);
pCtx->param[2].i64Key = order;
@@ -1706,16 +1685,14 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
if (i > 0) {
pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes;
+ pRuntimeEnv->rowCellInfoOffset[i] = pRuntimeEnv->rowCellInfoOffset[i - 1] + sizeof(SResultRowCellInfo) + pQuery->pExpr1[i - 1].interBytes;
}
- }
- char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultInfo) * pQuery->numOfOutput;
-
- // set the intermediate result output buffer
- setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf);
+ }
// if it is group by normal column, do not set output buffer, the output buffer is pResult
- if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery) {
+ // fixed output query/multi-output query for normal table
+ if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) {
resetCtxOutputBuf(pRuntimeEnv);
}
@@ -1727,12 +1704,24 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
return TSDB_CODE_SUCCESS;
_clean:
- taosTFree(pRuntimeEnv->resultInfo);
- taosTFree(pRuntimeEnv->pCtx);
+ tfree(pRuntimeEnv->pCtx);
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
+static void doFreeQueryHandle(SQInfo* pQInfo) {
+ SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
+
+ tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
+ tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
+
+ pRuntimeEnv->pQueryHandle = NULL;
+ pRuntimeEnv->pSecQueryHandle = NULL;
+
+ SMemRef* pMemRef = &pQInfo->memRef;
+ assert(pMemRef->ref == 0 && pMemRef->imem == NULL && pMemRef->mem == NULL);
+}
+
static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
if (pRuntimeEnv->pQuery == NULL) {
return;
@@ -1753,20 +1742,25 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
}
tVariantDestroy(&pCtx->tag);
- taosTFree(pCtx->tagInfo.pTagCtxList);
+ tfree(pCtx->tagInfo.pTagCtxList);
}
- taosTFree(pRuntimeEnv->resultInfo);
- taosTFree(pRuntimeEnv->pCtx);
+ tfree(pRuntimeEnv->pCtx);
}
- pRuntimeEnv->pFillInfo = taosDestoryFillInfo(pRuntimeEnv->pFillInfo);
+ pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo);
destroyResultBuf(pRuntimeEnv->pResultBuf);
- tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
- tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
+ doFreeQueryHandle(pQInfo);
pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf);
+ tfree(pRuntimeEnv->keyBuf);
+ tfree(pRuntimeEnv->rowCellInfoOffset);
+
+ taosHashCleanup(pRuntimeEnv->pResultRowHashTable);
+ pRuntimeEnv->pResultRowHashTable = NULL;
+
+ pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool);
}
#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED)
@@ -1785,7 +1779,7 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) {
}
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- SSqlFuncMsg *pExprMsg = &pQuery->pSelectExpr[i].base;
+ SSqlFuncMsg *pExprMsg = &pQuery->pExpr1[i].base;
// ignore the ts_comp function
if (i == 0 && pExprMsg->functionId == TSDB_FUNC_PRJ && pExprMsg->numOfParams == 1 &&
@@ -1808,7 +1802,7 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) {
// todo refactor with isLastRowQuery
static bool isPointInterpoQuery(SQuery *pQuery) {
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t functionID = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionID = pQuery->pExpr1[i].base.functionId;
if (functionID == TSDB_FUNC_INTERP) {
return true;
}
@@ -1820,7 +1814,7 @@ static bool isPointInterpoQuery(SQuery *pQuery) {
// TODO REFACTOR:MERGE WITH CLIENT-SIDE FUNCTION
static bool isSumAvgRateQuery(SQuery *pQuery) {
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionId = pQuery->pExpr1[i].base.functionId;
if (functionId == TSDB_FUNC_TS) {
continue;
}
@@ -1836,7 +1830,7 @@ static bool isSumAvgRateQuery(SQuery *pQuery) {
static bool isFirstLastRowQuery(SQuery *pQuery) {
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t functionID = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionID = pQuery->pExpr1[i].base.functionId;
if (functionID == TSDB_FUNC_LAST_ROW) {
return true;
}
@@ -1847,7 +1841,7 @@ static bool isFirstLastRowQuery(SQuery *pQuery) {
static bool needReverseScan(SQuery *pQuery) {
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionId = pQuery->pExpr1[i].base.functionId;
if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) {
continue;
}
@@ -1857,8 +1851,11 @@ static bool needReverseScan(SQuery *pQuery) {
}
if (functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) {
- int32_t order = (int32_t)pQuery->pSelectExpr[i].base.arg->argValue.i64;
- return order != pQuery->order.order;
+ // the scan order to acquire the last result of the specified column
+ int32_t order = (int32_t)pQuery->pExpr1[i].base.arg->argValue.i64;
+ if (order != pQuery->order.order) {
+ return true;
+ }
}
}
@@ -1871,7 +1868,7 @@ static bool needReverseScan(SQuery *pQuery) {
*/
static bool onlyQueryTags(SQuery* pQuery) {
for(int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- SExprInfo* pExprInfo = &pQuery->pSelectExpr[i];
+ SExprInfo* pExprInfo = &pQuery->pExpr1[i];
int32_t functionId = pExprInfo->base.functionId;
@@ -1914,7 +1911,7 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) {
} else {
bool hasMultioutput = false;
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- SSqlFuncMsg *pExprMsg = &pQuery->pSelectExpr[i].base;
+ SSqlFuncMsg *pExprMsg = &pQuery->pExpr1[i].base;
if (pExprMsg->functionId == TSDB_FUNC_TS || pExprMsg->functionId == TSDB_FUNC_TS_DUMMY) {
continue;
}
@@ -1948,7 +1945,7 @@ bool colIdCheck(SQuery *pQuery) {
// the scan order is not matter
static bool onlyOneQueryType(SQuery *pQuery, int32_t functId, int32_t functIdDst) {
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionId = pQuery->pExpr1[i].base.functionId;
if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG ||
functionId == TSDB_FUNC_TAG_DUMMY) {
@@ -2178,7 +2175,7 @@ static bool needToLoadDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDat
if (pRuntimeEnv->topBotQuery) {
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionId = pQuery->pExpr1[i].base.functionId;
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) {
return topbot_datablock_filter(&pCtx[i], functionId, (char *)&pDataStatis[i].min, (char *)&pDataStatis[i].max);
}
@@ -2203,7 +2200,7 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) {
}
while(1) {
- GET_NEXT_TIMEWINDOW(pQuery, &w);
+ getNextTimeWindow(pQuery, &w);
if (w.skey > pBlockInfo->window.ekey) {
break;
}
@@ -2222,7 +2219,7 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) {
}
while(1) {
- GET_NEXT_TIMEWINDOW(pQuery, &w);
+ getNextTimeWindow(pQuery, &w);
if (w.ekey < pBlockInfo->window.skey) {
break;
}
@@ -2262,14 +2259,14 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo * pW
TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey;
STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery);
- if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo->tid, &win, masterScan, &hasTimeWindow) !=
+ if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, masterScan, &hasTimeWindow) !=
TSDB_CODE_SUCCESS) {
// todo handle error in set result for timewindow
}
}
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- SSqlFuncMsg* pSqlFunc = &pQuery->pSelectExpr[i].base;
+ SSqlFuncMsg* pSqlFunc = &pQuery->pExpr1[i].base;
int32_t functionId = pSqlFunc->functionId;
int32_t colId = pSqlFunc->colInfo.colId;
@@ -2393,7 +2390,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa
}
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t bytes = pQuery->pSelectExpr[i].bytes;
+ int32_t bytes = pQuery->pExpr1[i].bytes;
assert(bytes > 0 && capacity > 0);
char *tmp = realloc(pQuery->sdata[i], bytes * capacity + sizeof(tFilePage));
@@ -2424,7 +2421,7 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB
int32_t newSize = (int32_t)(pRec->capacity + (pBlockInfo->rows - remain));
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t bytes = pQuery->pSelectExpr[i].bytes;
+ int32_t bytes = pQuery->pExpr1[i].bytes;
assert(bytes > 0 && newSize > 0);
char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(tFilePage));
@@ -2438,7 +2435,7 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB
// set the pCtx output buffer position
pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows * bytes;
- int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionId = pQuery->pExpr1[i].base.functionId;
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf;
}
@@ -2537,7 +2534,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
setQueryStatus(pQuery, QUERY_COMPLETED);
}
- if (QUERY_IS_INTERVAL_QUERY(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) {
+ if (QUERY_IS_INTERVAL_QUERY(pQuery) && (IS_MASTER_SCAN(pRuntimeEnv)|| pRuntimeEnv->scanFlag == REPEAT_SCAN)) {
if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
closeAllTimeWindow(&pRuntimeEnv->windowResInfo);
pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window
@@ -2602,7 +2599,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) {
SQuery *pQuery = pRuntimeEnv->pQuery;
SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
- SExprInfo *pExprInfo = &pQuery->pSelectExpr[0];
+ SExprInfo *pExprInfo = &pQuery->pExpr1[0];
if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP) {
assert(pExprInfo->base.numOfParams == 1);
@@ -2613,7 +2610,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) {
} else {
// set tag value, by which the results are aggregated.
for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) {
- SExprInfo* pLocalExprInfo = &pQuery->pSelectExpr[idx];
+ SExprInfo* pLocalExprInfo = &pQuery->pExpr1[idx];
// ts_comp column required the tag value for join filter
if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) {
@@ -2648,14 +2645,14 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) {
}
}
-static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowResult *pWindowRes, bool mergeFlag) {
+static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SResultRow *pWindowRes, bool mergeFlag) {
SQuery * pQuery = pRuntimeEnv->pQuery;
SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
- tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
+ tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId);
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionId = pQuery->pExpr1[i].base.functionId;
if (!mergeFlag) {
pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes;
pCtx[i].currentStage = FIRST_STAGE_MERGE;
@@ -2683,7 +2680,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes
}
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionId = pQuery->pExpr1[i].base.functionId;
if (functionId == TSDB_FUNC_TAG_DUMMY) {
continue;
}
@@ -2769,25 +2766,25 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim
for (int32_t j = 0; j < numOfRows; ++j) {
for (int32_t i = 0; i < numOfCols; ++i) {
- switch (pQuery->pSelectExpr[i].type) {
+ switch (pQuery->pExpr1[i].type) {
case TSDB_DATA_TYPE_BINARY: {
- int32_t type = pQuery->pSelectExpr[i].type;
- printBinaryData(pQuery->pSelectExpr[i].base.functionId, pdata[i]->data + pQuery->pSelectExpr[i].bytes * j,
+ int32_t type = pQuery->pExpr1[i].type;
+ printBinaryData(pQuery->pExpr1[i].base.functionId, pdata[i]->data + pQuery->pExpr1[i].bytes * j,
type);
break;
}
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT:
- printf("%" PRId64 "\t", *(int64_t *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j));
+ printf("%" PRId64 "\t", *(int64_t *)(pdata[i]->data + pQuery->pExpr1[i].bytes * j));
break;
case TSDB_DATA_TYPE_INT:
- printf("%d\t", *(int32_t *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j));
+ printf("%d\t", *(int32_t *)(pdata[i]->data + pQuery->pExpr1[i].bytes * j));
break;
case TSDB_DATA_TYPE_FLOAT:
- printf("%f\t", *(float *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j));
+ printf("%f\t", *(float *)(pdata[i]->data + pQuery->pExpr1[i].bytes * j));
break;
case TSDB_DATA_TYPE_DOUBLE:
- printf("%lf\t", *(double *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j));
+ printf("%lf\t", *(double *)(pdata[i]->data + pQuery->pExpr1[i].bytes * j));
break;
}
}
@@ -2822,15 +2819,15 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param)
}
SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo;
- SWindowResult * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos);
- tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pos.pageId);
+ SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos);
+ tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId);
char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1);
TSKEY leftTimestamp = GET_INT64_VAL(b1);
SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo;
- SWindowResult * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos);
- tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pos.pageId);
+ SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos);
+ tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId);
char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2);
TSKEY rightTimestamp = GET_INT64_VAL(b2);
@@ -2867,7 +2864,7 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) {
}
SGroupResInfo* info = &pQInfo->groupResInfo;
- if (pQInfo->groupIndex == numOfGroups && info->pos.pageId == info->numOfDataPages) {
+ if (pQInfo->groupIndex == numOfGroups && info->pageId == info->numOfDataPages) {
SET_STABLE_QUERY_OVER(pQInfo);
}
@@ -2883,10 +2880,10 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo;
// all results have been return to client, try next group
- if (pGroupResInfo->pos.pageId == pGroupResInfo->numOfDataPages) {
+ if (pGroupResInfo->pageId == pGroupResInfo->numOfDataPages) {
pGroupResInfo->numOfDataPages = 0;
- pGroupResInfo->pos.pageId = 0;
- pGroupResInfo->pos.rowId = 0;
+ pGroupResInfo->pageId = 0;
+ pGroupResInfo->rowId = 0;
// current results of group has been sent to client, try next group
if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) {
@@ -2914,22 +2911,22 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
assert(size == pGroupResInfo->numOfDataPages);
bool done = false;
- for (int32_t j = pGroupResInfo->pos.pageId; j < size; ++j) {
+ for (int32_t j = pGroupResInfo->pageId; j < size; ++j) {
SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j);
tFilePage* pData = getResBufPage(pResultBuf, pi->pageId);
- assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->pos.rowId < pData->num);
- int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->pos.rowId);
+ assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->rowId < pData->num);
+ int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->rowId);
if (numOfRes > pQuery->rec.capacity - offset) {
numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset);
- pGroupResInfo->pos.rowId += numOfCopiedRows;
+ pGroupResInfo->rowId += numOfCopiedRows;
done = true;
} else {
numOfCopiedRows = (int32_t)pData->num;
- pGroupResInfo->pos.pageId += 1;
- pGroupResInfo->pos.rowId = 0;
+ pGroupResInfo->pageId += 1;
+ pGroupResInfo->rowId = 0;
}
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
@@ -2950,9 +2947,11 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
pQuery->rec.rows += offset;
}
-int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) {
+int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) {
+ SQuery* pQuery = pRuntimeEnv->pQuery;
+
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
- int32_t functionId = pQuery->pSelectExpr[j].base.functionId;
+ int32_t functionId = pQuery->pExpr1[j].base.functionId;
/*
* ts, tag, tagprj function can not decide the output number of current query
@@ -2962,7 +2961,7 @@ int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) {
continue;
}
- SResultInfo *pResultInfo = &pWindowRes->resultInfo[j];
+ SResultRowCellInfo *pResultInfo = getResultCell(pRuntimeEnv, pResultRow, j);
assert(pResultInfo != NULL);
if (pResultInfo->numOfRes > 0) {
@@ -2984,8 +2983,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
STableQueryInfo **pTableList = malloc(POINTER_BYTES * size);
if (pTableList == NULL || posList == NULL) {
- taosTFree(posList);
- taosTFree(pTableList);
+ tfree(posList);
+ tfree(pTableList);
qError("QInfo:%p failed alloc memory", pQInfo);
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
@@ -3009,19 +3008,19 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
// there is no data in current group
if (numOfTables == 0) {
- taosTFree(posList);
- taosTFree(pTableList);
+ tfree(posList);
+ tfree(pTableList);
return 0;
} else if (numOfTables == 1) { // no need to merge results since only one table in each group
- taosTFree(posList);
- taosTFree(pTableList);
+ tfree(posList);
+ tfree(pTableList);
SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo;
pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList);
pGroupResInfo->groupId = tid;
- pGroupResInfo->pos.pageId = 0;
- pGroupResInfo->pos.rowId = 0;
+ pGroupResInfo->pageId = 0;
+ pGroupResInfo->rowId = 0;
return pGroupResInfo->numOfDataPages;
}
@@ -3031,18 +3030,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
SLoserTreeInfo *pTree = NULL;
tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn);
- SResultInfo *pResultInfo = calloc(pQuery->numOfOutput, sizeof(SResultInfo));
- if (pResultInfo == NULL) {
- longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
- }
-
- char* buf = calloc(1, pRuntimeEnv->interBufSize);
- if (buf == NULL) {
- longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
- }
-
- setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf);
- resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo);
+ SResultRow* pRow = getNewResultRow(pRuntimeEnv->pool);
+ resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow);
pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex);
@@ -3054,26 +3043,23 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
if (IS_QUERY_KILLED(pQInfo)) {
qDebug("QInfo:%p it is already killed, abort", pQInfo);
- taosTFree(pTableList);
- taosTFree(posList);
- taosTFree(pTree);
- taosTFree(pResultInfo);
- taosTFree(buf);
-
+ tfree(pTableList);
+ tfree(posList);
+ tfree(pTree);
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
int32_t pos = pTree->pNode[0].index;
SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo;
- SWindowResult *pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]);
- tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
+ SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.position[pos]);
+ tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId);
char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page);
TSKEY ts = GET_INT64_VAL(b);
assert(ts == pWindowRes->win.skey);
- int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes);
+ int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes);
if (num <= 0) {
cs.position[pos] += 1;
@@ -3094,7 +3080,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
return -1;
}
- resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo);
+ resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow);
}
doMerge(pRuntimeEnv, ts, pWindowRes, false);
@@ -3104,7 +3090,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
lastTimestamp = ts;
// move to the next element of current entry
- int32_t currentPageId = pWindowRes->pos.pageId;
+ int32_t currentPageId = pWindowRes->pageId;
cs.position[pos] += 1;
if (cs.position[pos] >= pWindowResInfo->size) {
@@ -3116,8 +3102,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
}
} else {
// current page is not needed anymore
- SWindowResult *pNextWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]);
- if (pNextWindowRes->pos.pageId != currentPageId) {
+ SResultRow *pNextWindowRes = getResultRow(pWindowResInfo, cs.position[pos]);
+ if (pNextWindowRes->pageId != currentPageId) {
releaseResBufPage(pRuntimeEnv->pResultBuf, page);
}
}
@@ -3130,11 +3116,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) {
qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo);
- taosTFree(pTree);
- taosTFree(pTableList);
- taosTFree(posList);
- taosTFree(pResultInfo);
-
+ tfree(pTree);
+ tfree(pTableList);
+ tfree(posList);
return -1;
}
}
@@ -3147,12 +3131,12 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt);
- taosTFree(pTableList);
- taosTFree(posList);
- taosTFree(pTree);
+ tfree(pTableList);
+ tfree(posList);
+ tfree(pTree);
- taosTFree(pResultInfo);
- taosTFree(buf);
+// tfree(pResultInfo);
+// tfree(buf);
return pQInfo->groupResInfo.numOfDataPages;
}
@@ -3195,12 +3179,14 @@ int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupR
return TSDB_CODE_SUCCESS;
}
-void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo) {
+void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow) {
+ SQuery* pQuery = pRuntimeEnv->pQuery;
+
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes;
pCtx[k].size = 1;
pCtx[k].startOffset = 0;
- pCtx[k].resultInfo = &pResultInfo[k];
+ pCtx[k].resultInfo = getResultCell(pRuntimeEnv, pRow, k);
pQuery->sdata[k]->num = 0;
}
@@ -3237,8 +3223,8 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *
pTableQueryInfo->windowResInfo.curIndex = pTableQueryInfo->windowResInfo.size - 1;
}
-static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindowResInfo, int32_t order) {
- SQuery* pQuery = pQInfo->runtimeEnv.pQuery;
+static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t order) {
+ SQuery* pQuery = pRuntimeEnv->pQuery;
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
bool closed = getTimeWindowResStatus(pWindowResInfo, i);
@@ -3246,17 +3232,18 @@ static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindow
continue;
}
- SWindowResult *buf = getWindowResult(pWindowResInfo, i);
+ SResultRow *pRow = getResultRow(pWindowResInfo, i);
// open/close the specified query for each group result
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
- int32_t functId = pQuery->pSelectExpr[j].base.functionId;
+ int32_t functId = pQuery->pExpr1[j].base.functionId;
+ SResultRowCellInfo* pInfo = getResultCell(pRuntimeEnv, pRow, j);
if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) ||
((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) {
- buf->resultInfo[j].complete = false;
+ pInfo->complete = false;
} else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) {
- buf->resultInfo[j].complete = true;
+ pInfo->complete = true;
}
}
}
@@ -3270,10 +3257,10 @@ void disableFuncInReverseScan(SQInfo *pQInfo) {
// group by normal columns and interval query on normal table
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) {
- disableFuncInReverseScanImpl(pQInfo, pWindowResInfo, order);
+ disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order);
} else { // for simple result of table query,
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor
- int32_t functId = pQuery->pSelectExpr[j].base.functionId;
+ int32_t functId = pQuery->pExpr1[j].base.functionId;
SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j];
if (pCtx->resultInfo == NULL) {
@@ -3320,26 +3307,16 @@ void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) {
}
}
-int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize) {
- int32_t numOfCols = pQuery->numOfOutput;
-
- size_t size = numOfCols * sizeof(SResultInfo) + interBufSize;
- pResultRow->resultInfo = calloc(1, size);
- if (pResultRow->resultInfo == NULL) {
- return TSDB_CODE_QRY_OUT_OF_MEMORY;
- }
-
- pResultRow->pos = (SPosInfo) {-1, -1};
-
- char* buf = (char*) pResultRow->resultInfo + numOfCols * sizeof(SResultInfo);
-
- // set the intermediate result output buffer
- setWindowResultInfo(pResultRow->resultInfo, pQuery, isSTableQuery, buf);
+int32_t initResultRow(SResultRow *pResultRow) {
+ pResultRow->pCellInfo = (SResultRowCellInfo*)((char*)pResultRow + sizeof(SResultRow));
+ pResultRow->pageId = -1;
+ pResultRow->rowId = -1;
return TSDB_CODE_SUCCESS;
}
void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) {
SQuery *pQuery = pRuntimeEnv->pQuery;
+ SResultRow* pRow = pRuntimeEnv->pResultRow;
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
@@ -3349,16 +3326,17 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) {
* set the output buffer information and intermediate buffer
* not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc.
*/
- RESET_RESULT_INFO(&pRuntimeEnv->resultInfo[i]);
- pCtx->resultInfo = &pRuntimeEnv->resultInfo[i];
+ SResultRowCellInfo* pCellInfo = getResultCell(pRuntimeEnv, pRow, i);
+ RESET_RESULT_INFO(pCellInfo);
+ pCtx->resultInfo = pCellInfo;
// set the timestamp output buffer for top/bottom/diff query
- int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionId = pQuery->pExpr1[i].base.functionId;
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf;
}
- memset(pQuery->sdata[i]->data, 0, (size_t)(pQuery->pSelectExpr[i].bytes * pQuery->rec.capacity));
+ memset(pQuery->sdata[i]->data, 0, (size_t)(pQuery->pExpr1[i].bytes * pQuery->rec.capacity));
}
initCtxOutputBuf(pRuntimeEnv);
@@ -3369,7 +3347,7 @@ void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) {
// reset the execution contexts
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
- int32_t functionId = pQuery->pSelectExpr[j].base.functionId;
+ int32_t functionId = pQuery->pExpr1[j].base.functionId;
assert(functionId != TSDB_FUNC_DIFF);
// set next output position
@@ -3396,10 +3374,10 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) {
SQuery *pQuery = pRuntimeEnv->pQuery;
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
- int32_t functionId = pQuery->pSelectExpr[j].base.functionId;
+ int32_t functionId = pQuery->pExpr1[j].base.functionId;
pRuntimeEnv->pCtx[j].currentStage = 0;
- SResultInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
+ SResultRowCellInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
if (pResInfo->initialized) {
continue;
}
@@ -3434,7 +3412,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) {
0, pQuery->rec.rows);
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionId = pQuery->pExpr1[i].base.functionId;
int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes;
memmove(pQuery->sdata[i]->data, (char*)pQuery->sdata[i]->data + bytes * numOfSkip, (size_t)(pQuery->rec.rows * bytes));
@@ -3468,34 +3446,34 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) {
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
- SWindowResult *pResult = getWindowResult(pWindowResInfo, i);
+ SResultRow *pResult = getResultRow(pWindowResInfo, i);
if (!pResult->closed) {
continue;
}
- setWindowResOutputBuf(pRuntimeEnv, pResult);
+ setResultOutputBuf(pRuntimeEnv, pResult);
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
- int16_t functId = pQuery->pSelectExpr[j].base.functionId;
+ int16_t functId = pQuery->pExpr1[j].base.functionId;
if (functId == TSDB_FUNC_TS) {
continue;
}
aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]);
- SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
toContinue |= (!pResInfo->complete);
}
}
} else {
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
- int16_t functId = pQuery->pSelectExpr[j].base.functionId;
+ int16_t functId = pQuery->pExpr1[j].base.functionId;
if (functId == TSDB_FUNC_TS) {
continue;
}
aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]);
- SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
toContinue |= (!pResInfo->complete);
}
@@ -3568,7 +3546,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI
tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
}
- pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+ pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
if (pRuntimeEnv->pSecQueryHandle == NULL) {
longjmp(pRuntimeEnv->env, terrno);
}
@@ -3652,7 +3630,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
}
restoreTimeWindow(&pQInfo->tableGroupInfo, &cond);
- pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+ pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
if (pRuntimeEnv->pSecQueryHandle == NULL) {
longjmp(pRuntimeEnv->env, terrno);
}
@@ -3666,7 +3644,6 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
// check if query is killed or not
if (IS_QUERY_KILLED(pQInfo)) {
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
}
@@ -3695,15 +3672,15 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) {
}
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
- SWindowResult *buf = &pWindowResInfo->pResult[i];
+ SResultRow *buf = pWindowResInfo->pResult[i];
if (!isWindowResClosed(pWindowResInfo, i)) {
continue;
}
- setWindowResOutputBuf(pRuntimeEnv, buf);
+ setResultOutputBuf(pRuntimeEnv, buf);
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
- aAggs[pQuery->pSelectExpr[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]);
+ aAggs[pQuery->pExpr1[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]);
}
/*
@@ -3715,14 +3692,14 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) {
} else {
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
- aAggs[pQuery->pSelectExpr[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]);
+ aAggs[pQuery->pExpr1[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]);
}
}
}
static bool hasMainOutput(SQuery *pQuery) {
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionId = pQuery->pExpr1[i].base.functionId;
if (functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TAGPRJ) {
return true;
@@ -3747,7 +3724,7 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void
if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) {
int32_t initialSize = 16;
int32_t initialThreshold = 100;
- int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, initialSize, initialThreshold, TSDB_DATA_TYPE_INT);
+ int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, initialSize, initialThreshold, TSDB_DATA_TYPE_INT);
if (code != TSDB_CODE_SUCCESS) {
return NULL;
}
@@ -3762,6 +3739,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) {
return;
}
+ tVariantDestroy(&pTableQueryInfo->tag);
cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo);
}
@@ -3786,9 +3764,10 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) {
return;
}
- SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex,
- sizeof(groupIndex), true);
- if (pWindowRes == NULL) {
+ uint64_t uid = 0; // uid is always set to be 0
+ SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex,
+ sizeof(groupIndex), true, uid);
+ if (pResultRow == NULL) {
return;
}
@@ -3796,8 +3775,8 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) {
* not assign result buffer yet, add new result buffer
* all group belong to one result set, and each group result has different group id so set the id to be one
*/
- if (pWindowRes->pos.pageId == -1) {
- if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) !=
+ if (pResultRow->pageId == -1) {
+ if (addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) !=
TSDB_CODE_SUCCESS) {
return;
}
@@ -3805,47 +3784,43 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) {
// record the current active group id
pRuntimeEnv->prevGroupId = groupIndex;
- setWindowResOutputBuf(pRuntimeEnv, pWindowRes);
+ setResultOutputBuf(pRuntimeEnv, pResultRow);
initCtxOutputBuf(pRuntimeEnv);
}
-void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) {
+void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) {
SQuery *pQuery = pRuntimeEnv->pQuery;
// Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
- tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
+ tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId);
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult, page);
- int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionId = pQuery->pExpr1[i].base.functionId;
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf;
}
/*
- * set the output buffer information and intermediate buffer
+ * set the output buffer information and intermediate buffer,
* not all queries require the interResultBuf, such as COUNT
*/
- pCtx->resultInfo = &pResult->resultInfo[i];
-
- // set super table query flag
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- pResInfo->superTableQ = pRuntimeEnv->stableQuery;
+ pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i);
}
}
-void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) {
+void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) {
SQuery *pQuery = pRuntimeEnv->pQuery;
// Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
- tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
+ tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId);
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
- pCtx->resultInfo = &pResult->resultInfo[i];
+ pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i);
if (pCtx->resultInfo->initialized && pCtx->resultInfo->complete) {
continue;
}
@@ -3858,12 +3833,6 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *
pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf;
}
- /*
- * set the output buffer information and intermediate buffer
- * not all queries require the interResultBuf, such as COUNT
- */
- pCtx->resultInfo->superTableQ = pRuntimeEnv->stableQuery; // set super table query flag
-
if (!pCtx->resultInfo->initialized) {
aAggs[functionId].init(pCtx);
}
@@ -3885,7 +3854,7 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ
STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pTableQueryInfo->tag);
// failed to find data with the specified tag value and vnodeId
- if (elem.vnode < 0) {
+ if (!tsBufIsValidElem(&elem)) {
if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz);
} else {
@@ -3972,7 +3941,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) {
bool requireTimestamp(SQuery *pQuery) {
for (int32_t i = 0; i < pQuery->numOfOutput; i++) {
- int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
+ int32_t functionId = pQuery->pExpr1[i].base.functionId;
if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_NEED_TS) != 0) {
return true;
}
@@ -4004,7 +3973,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_
qDebug("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo);
int32_t totalSet = numOfClosedTimeWindow(pResultInfo);
- SWindowResult* result = pResultInfo->pResult;
+ SResultRow** result = pResultInfo->pResult;
if (orderType == TSDB_ORDER_ASC) {
startIdx = pQInfo->groupIndex;
@@ -4017,14 +3986,14 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_
SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo;
for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) {
- if (result[i].numOfRows == 0) {
+ if (result[i]->numOfRows == 0) {
pQInfo->groupIndex += 1;
- pGroupResInfo->pos.rowId = 0;
+ pGroupResInfo->rowId = 0;
continue;
}
- int32_t numOfRowsToCopy = result[i].numOfRows - pGroupResInfo->pos.rowId;
- int32_t oldOffset = pGroupResInfo->pos.rowId;
+ int32_t numOfRowsToCopy = result[i]->numOfRows - pGroupResInfo->rowId;
+ int32_t oldOffset = pGroupResInfo->rowId;
/*
* current output space is not enough to accommodate all data of this page, only partial results
@@ -4032,19 +4001,19 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_
*/
if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) {
numOfRowsToCopy = (int32_t) pQuery->rec.capacity - numOfResult;
- pGroupResInfo->pos.rowId += numOfRowsToCopy;
+ pGroupResInfo->rowId += numOfRowsToCopy;
} else {
- pGroupResInfo->pos.rowId = 0;
+ pGroupResInfo->rowId = 0;
pQInfo->groupIndex += 1;
}
- tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i].pos.pageId);
+ tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i]->pageId);
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
int32_t size = pRuntimeEnv->pCtx[j].outputBytes;
char *out = pQuery->sdata[j]->data + numOfResult * size;
- char *in = getPosInResultPage(pRuntimeEnv, j, &result[i], page);
+ char *in = getPosInResultPage(pRuntimeEnv, j, result[i], page);
memcpy(out, in + oldOffset * size, size * numOfRowsToCopy);
}
@@ -4091,7 +4060,7 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) {
}
for (int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) {
- SWindowResult *pResult = &pRuntimeEnv->windowResInfo.pResult[i];
+ SResultRow *pResult = pRuntimeEnv->windowResInfo.pResult[i];
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
int32_t functionId = pRuntimeEnv->pCtx[j].functionId;
@@ -4099,7 +4068,8 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) {
continue;
}
- pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes));
+ SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j);
+ pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes));
}
}
}
@@ -4144,7 +4114,7 @@ bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) {
* first result row in the actual result set will fill nothing.
*/
if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
- int32_t numOfTotal = (int32_t)getFilledNumOfRes(pFillInfo, pQuery->window.ekey, (int32_t)pQuery->rec.capacity);
+ int32_t numOfTotal = (int32_t)getNumOfResWithFill(pFillInfo, pQuery->window.ekey, (int32_t)pQuery->rec.capacity);
return numOfTotal > 0;
}
@@ -4160,14 +4130,27 @@ bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) {
return false;
}
+static int16_t getNumOfFinalResCol(SQuery* pQuery) {
+ return pQuery->pExpr2 == NULL? pQuery->numOfOutput:pQuery->numOfExpr2;
+}
+
static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) {
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
- for (int32_t col = 0; col < pQuery->numOfOutput; ++col) {
- int32_t bytes = pQuery->pSelectExpr[col].bytes;
+ if (pQuery->pExpr2 == NULL) {
+ for (int32_t col = 0; col < pQuery->numOfOutput; ++col) {
+ int32_t bytes = pQuery->pExpr1[col].bytes;
- memmove(data, pQuery->sdata[col]->data, bytes * numOfRows);
- data += bytes * numOfRows;
+ memmove(data, pQuery->sdata[col]->data, bytes * numOfRows);
+ data += bytes * numOfRows;
+ }
+ } else {
+ for (int32_t col = 0; col < pQuery->numOfExpr2; ++col) {
+ int32_t bytes = pQuery->pExpr2[col].bytes;
+
+ memmove(data, pQuery->sdata[col]->data, bytes * numOfRows);
+ data += bytes * numOfRows;
+ }
}
int32_t numOfTables = (int32_t)taosArrayGetSize(pQInfo->arrTableIdInfo);
@@ -4202,7 +4185,7 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int
SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo;
while (1) {
- int32_t ret = (int32_t)taosGenerateDataBlock(pFillInfo, (tFilePage**)pQuery->sdata, (int32_t)pQuery->rec.capacity);
+ int32_t ret = (int32_t)taosFillResultDataBlock(pFillInfo, (tFilePage**)pQuery->sdata, (int32_t)pQuery->rec.capacity);
// todo apply limit output function
/* reached the start position of according to offset value, return immediately */
@@ -4217,10 +4200,9 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int
ret -= (int32_t)pQuery->limit.offset;
// todo !!!!there exactly number of interpo is not valid.
- // todo refactor move to the beginning of buffer
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].bytes * pQuery->limit.offset,
- ret * pQuery->pSelectExpr[i].bytes);
+ memmove(pDst[i]->data, pDst[i]->data + pQuery->pExpr1[i].bytes * pQuery->limit.offset,
+ ret * pQuery->pExpr1[i].bytes);
}
pQuery->limit.offset = 0;
@@ -4245,16 +4227,24 @@ static void queryCostStatis(SQInfo *pQInfo) {
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
SQueryCostInfo *pSummary = &pRuntimeEnv->summary;
+ uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable);
+ hashSize += taosHashGetMemSize(pQInfo->tableqinfoGroupInfo.map);
+ pSummary->hashSize = hashSize;
+
// add the merge time
pSummary->elapsedTime += pSummary->firstStageMergeTime;
+ SResultRowPool* p = pQInfo->runtimeEnv.pool;
+ pSummary->winInfoSize = getResultRowPoolMemSize(p);
+ pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p);
+
qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, "
"load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64,
pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis,
pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows);
- qDebug("QInfo:%p :cost summary: internal size:%"PRId64"B, numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize,
- pSummary->numOfTimeWindows);
+ qDebug("QInfo:%p :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo, pSummary->winInfoSize/1024.0,
+ pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0);
}
static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) {
@@ -4309,7 +4299,6 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
while (tsdbNextDataBlock(pQueryHandle)) {
if (IS_QUERY_KILLED(GET_QINFO_ADDR(pRuntimeEnv))) {
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
@@ -4333,6 +4322,56 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
}
}
+static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) {
+ SQuery *pQuery = pRuntimeEnv->pQuery;
+ SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+
+ assert(pQuery->limit.offset == 0);
+ STimeWindow tw = *win;
+ getNextTimeWindow(pQuery, &tw);
+
+ if ((tw.skey <= pBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
+ (tw.ekey >= pBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
+
+ // load the data block and check data remaining in current data block
+ // TODO optimize performance
+ SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL);
+ SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
+
+ tw = *win;
+ int32_t startPos =
+ getNextQualifiedWindow(pRuntimeEnv, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1);
+ assert(startPos >= 0);
+
+ // set the abort info
+ pQuery->pos = startPos;
+
+ // reset the query start timestamp
+ pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos];
+ pQuery->window.skey = pTableQueryInfo->win.skey;
+ TSKEY key = pTableQueryInfo->win.skey;
+
+ pWindowResInfo->prevSKey = tw.skey;
+ int32_t index = pRuntimeEnv->windowResInfo.curIndex;
+
+ int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock);
+ pRuntimeEnv->windowResInfo.curIndex = index; // restore the window index
+
+ qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64,
+ GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes,
+ pQuery->current->lastKey);
+
+ return key;
+ } else { // do nothing
+ pQuery->window.skey = tw.skey;
+ pWindowResInfo->prevSKey = tw.skey;
+
+ return tw.skey;
+ }
+
+ return true;
+}
+
static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
SQuery *pQuery = pRuntimeEnv->pQuery;
*start = pQuery->current->lastKey;
@@ -4375,52 +4414,19 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQuery);
while (pQuery->limit.offset > 0) {
+ STimeWindow tw = win;
+
if ((win.ekey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
(win.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
pQuery->limit.offset -= 1;
pWindowResInfo->prevSKey = win.skey;
}
- STimeWindow tw = win;
- GET_NEXT_TIMEWINDOW(pQuery, &tw);
-
+ // current window does not ended in current data block, try next data block
+ getNextTimeWindow(pQuery, &tw);
if (pQuery->limit.offset == 0) {
- if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
- (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
- // load the data block and check data remaining in current data block
- // TODO optimize performance
- SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL);
- SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
-
- tw = win;
- int32_t startPos =
- getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1);
- assert(startPos >= 0);
-
- // set the abort info
- pQuery->pos = startPos;
-
- // reset the query start timestamp
- pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos];
- pQuery->window.skey = pTableQueryInfo->win.skey;
- *start = pTableQueryInfo->win.skey;
-
- pWindowResInfo->prevSKey = tw.skey;
- int32_t index = pRuntimeEnv->windowResInfo.curIndex;
-
- int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, pDataBlock);
- pRuntimeEnv->windowResInfo.curIndex = index; // restore the window index
-
- qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64,
- GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey);
-
- return true;
- } else { // do nothing
- *start = tw.skey;
- pQuery->window.skey = tw.skey;
- pWindowResInfo->prevSKey = tw.skey;
- return true;
- }
+ *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo);
+ return true;
}
/*
@@ -4435,16 +4441,26 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL);
SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
- tw = win;
- int32_t startPos =
- getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1);
- assert(startPos >= 0);
+ if ((win.ekey > blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
+ (win.ekey < blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
+ pQuery->limit.offset -= 1;
+ }
+
+ if (pQuery->limit.offset == 0) {
+ *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo);
+ return true;
+ } else {
+ tw = win;
+ int32_t startPos =
+ getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1);
+ assert(startPos >= 0);
- // set the abort info
- pQuery->pos = startPos;
- pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos];
- pWindowResInfo->prevSKey = tw.skey;
- win = tw;
+ // set the abort info
+ pQuery->pos = startPos;
+ pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos];
+ pWindowResInfo->prevSKey = tw.skey;
+ win = tw;
+ }
} else {
break; // offset is not 0, and next time window begins or ends in the next block.
}
@@ -4495,7 +4511,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery)
terrno = TSDB_CODE_SUCCESS;
if (isFirstLastRowQuery(pQuery)) {
- pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+ pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
// update the query time window
pQuery->window = cond.twindow;
@@ -4517,16 +4533,16 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery)
}
}
} else if (isPointInterpoQuery(pQuery)) {
- pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+ pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
} else {
- pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+ pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
}
return terrno;
}
-static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) {
- int32_t numOfCols = pQuery->numOfOutput;
+static SFillColInfo* createFillColInfo(SQuery* pQuery) {
+ int32_t numOfCols = getNumOfFinalResCol(pQuery);
int32_t offset = 0;
SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo));
@@ -4534,12 +4550,14 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) {
return NULL;
}
+ // TODO refactor
for(int32_t i = 0; i < numOfCols; ++i) {
- SExprInfo* pExprInfo = &pQuery->pSelectExpr[i];
+ SExprInfo* pExprInfo = (pQuery->pExpr2 == NULL)? &pQuery->pExpr1[i]:&pQuery->pExpr2[i];
pFillCol[i].col.bytes = pExprInfo->bytes;
pFillCol[i].col.type = (int8_t)pExprInfo->type;
pFillCol[i].col.offset = offset;
+ pFillCol[i].tagIndex = -2;
pFillCol[i].flag = TSDB_COL_NORMAL; // always be ta normal column for table query
pFillCol[i].functionId = pExprInfo->base.functionId;
pFillCol[i].fillVal.i = pQuery->fillVal[i];
@@ -4553,7 +4571,6 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) {
int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery) {
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
- int32_t code = TSDB_CODE_SUCCESS;
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
pRuntimeEnv->topBotQuery = isTopBottomQuery(pQuery);
@@ -4561,7 +4578,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
setScanLimitationByResultBuffer(pQuery);
- code = setupQueryHandle(tsdb, pQInfo, isSTableQuery);
+ int32_t code = setupQueryHandle(tsdb, pQInfo, isSTableQuery);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@@ -4613,7 +4630,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
}
}
- code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 8, threshold, type);
+ code = initWindowResInfo(&pRuntimeEnv->windowResInfo, 8, threshold, type);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@@ -4633,23 +4650,24 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
type = TSDB_DATA_TYPE_TIMESTAMP;
}
- code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, numOfResultRows, 4096, type);
+ code = initWindowResInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, 1024, type);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) {
- SFillColInfo* pColInfo = taosCreateFillColInfo(pQuery);
+ SFillColInfo* pColInfo = createFillColInfo(pQuery);
STimeWindow w = TSWINDOW_INITIALIZER;
TSKEY sk = MIN(pQuery->window.skey, pQuery->window.ekey);
TSKEY ek = MAX(pQuery->window.skey, pQuery->window.ekey);
getAlignQueryTimeWindow(pQuery, pQuery->window.skey, sk, ek, &w);
- pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, w.skey, 0, (int32_t)pQuery->rec.capacity, pQuery->numOfOutput,
+ int32_t numOfCols = getNumOfFinalResCol(pQuery);
+ pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, w.skey, 0, (int32_t)pQuery->rec.capacity, numOfCols,
pQuery->interval.sliding, pQuery->interval.slidingUnit, (int8_t)pQuery->precision,
- pQuery->fillType, pColInfo);
+ pQuery->fillType, pColInfo, pQInfo);
}
setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
@@ -4660,7 +4678,7 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) {
SQuery *pQuery = pRuntimeEnv->pQuery;
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]);
if (pResInfo != NULL) {
pResInfo->complete = false;
}
@@ -4798,7 +4816,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
pRuntimeEnv->pQueryHandle = NULL;
}
- pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
+ pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef);
taosArrayDestroy(tx);
taosArrayDestroy(g1);
if (pRuntimeEnv->pQueryHandle == NULL) {
@@ -4811,7 +4829,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
if (pRuntimeEnv->cur.vgroupIndex == -1) {
STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag);
// failed to find data with the specified tag value and vnodeId
- if (elem.vnode < 0) {
+ if (!tsBufIsValidElem(&elem)) {
if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz);
} else {
@@ -4836,7 +4854,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag);
// failed to find data with the specified tag value and vnodeId
- if (elem1.vnode < 0) {
+ if (!tsBufIsValidElem(&elem1)) {
if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz);
} else {
@@ -4883,7 +4901,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo);
- if (isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery)) {
+ if (isPointInterpoQuery(pQuery)) {
resetCtxOutputBuf(pRuntimeEnv);
assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0);
@@ -4913,11 +4931,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
pRuntimeEnv->pQueryHandle = NULL;
}
- if (isFirstLastRowQuery(pQuery)) {
- assert(0); // last_row query switch to other routine to handle
- } else {
- pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo);
- }
+ pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef);
taosArrayDestroy(tx);
taosArrayDestroy(g1);
@@ -4931,10 +4945,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
assert(taosArrayGetSize(s) >= 1);
setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb);
- if (isFirstLastRowQuery(pQuery)) {
- assert(taosArrayGetSize(s) == 1);
- }
-
taosArrayDestroy(s);
// here we simply set the first table as current table
@@ -4987,7 +4997,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
}
// no need to update the lastkey for each table
- pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
+ pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef);
taosArrayDestroy(g1);
taosArrayDestroy(tx);
@@ -5013,11 +5023,12 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
}
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
- pWindowResInfo->pResult[i].closed = true; // enable return all results for group by normal columns
+ pWindowResInfo->pResult[i]->closed = true; // enable return all results for group by normal columns
- SWindowResult *pResult = &pWindowResInfo->pResult[i];
+ SResultRow *pResult = pWindowResInfo->pResult[i];
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
- pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes));
+ SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j);
+ pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes));
}
}
@@ -5173,7 +5184,7 @@ static void doSaveContext(SQInfo *pQInfo) {
SWITCH_ORDER(pQuery->order.order);
if (pRuntimeEnv->pTSBuf != NULL) {
- pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order;
+ SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order);
}
STsdbQueryCond cond = {
@@ -5195,7 +5206,7 @@ static void doSaveContext(SQInfo *pQInfo) {
setupQueryRangeForReverseScan(pQInfo);
pRuntimeEnv->prevGroupId = INT32_MIN;
- pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+ pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
if (pRuntimeEnv->pSecQueryHandle == NULL) {
longjmp(pRuntimeEnv->env, terrno);
}
@@ -5266,7 +5277,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
// query error occurred or query is killed, abort current execution
if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) {
qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code));
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
@@ -5288,7 +5298,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) {
qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code));
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
+ //TODO finalizeQueryResult may cause SEGSEV, since the memory may not allocated yet, add a cleanup function instead
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
@@ -5308,6 +5318,74 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
qDebug("QInfo:%p points returned:%" PRId64 ", total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows);
}
+
+static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) {
+ SArithmeticSupport *pSupport = (SArithmeticSupport *) param;
+ SExprInfo* pExprInfo = (SExprInfo*) pSupport->exprList;
+
+ int32_t index = -1;
+ for (int32_t i = 0; i < pSupport->numOfCols; ++i) {
+ if (colId == pExprInfo[i].base.resColId) {
+ index = i;
+ break;
+ }
+ }
+
+ assert(index >= 0 && index < pSupport->numOfCols);
+ return pSupport->data[index] + pSupport->offset * pExprInfo[index].bytes;
+}
+
+static void doSecondaryArithmeticProcess(SQuery* pQuery) {
+ SArithmeticSupport arithSup = {0};
+
+ tFilePage **data = calloc(pQuery->numOfExpr2, POINTER_BYTES);
+ for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) {
+ int32_t bytes = pQuery->pExpr2[i].bytes;
+ data[i] = (tFilePage *)malloc(bytes * pQuery->rec.rows + sizeof(tFilePage));
+ }
+
+ arithSup.offset = 0;
+ arithSup.numOfCols = (int32_t)pQuery->numOfOutput;
+ arithSup.exprList = pQuery->pExpr1;
+ arithSup.data = calloc(arithSup.numOfCols, POINTER_BYTES);
+
+ for (int32_t k = 0; k < arithSup.numOfCols; ++k) {
+ arithSup.data[k] = pQuery->sdata[k]->data;
+ }
+
+ for (int i = 0; i < pQuery->numOfExpr2; ++i) {
+ SExprInfo *pExpr = &pQuery->pExpr2[i];
+
+ // calculate the result from several other columns
+ SSqlFuncMsg* pSqlFunc = &pExpr->base;
+ if (pSqlFunc->functionId != TSDB_FUNC_ARITHM) {
+
+ for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
+ if (pSqlFunc->functionId == pQuery->pExpr1[j].base.functionId &&
+ pSqlFunc->colInfo.colId == pQuery->pExpr1[j].base.colInfo.colId) {
+ memcpy(data[i]->data, pQuery->sdata[j]->data, pQuery->pExpr1[j].bytes * pQuery->rec.rows);
+ break;
+ }
+ }
+ } else {
+ arithSup.pArithExpr = pExpr;
+ tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t)pQuery->rec.rows, data[i]->data, &arithSup, TSDB_ORDER_ASC,
+ getArithemicInputSrc);
+ }
+ }
+
+ for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) {
+ memcpy(pQuery->sdata[i]->data, data[i]->data, pQuery->pExpr2[i].bytes * pQuery->rec.rows);
+ }
+
+ for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) {
+ tfree(data[i]);
+ }
+
+ tfree(data);
+ tfree(arithSup.data);
+}
+
/*
* in each query, this function will be called only once, no retry for further result.
*
@@ -5327,14 +5405,14 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
scanOneTableDataBlocks(pRuntimeEnv, pTableInfo->lastKey);
finalizeQueryResult(pRuntimeEnv);
+ // since the numOfRows must be identical for all sql functions that are allowed to be executed simutaneously.
+ pQuery->rec.rows = getNumOfResult(pRuntimeEnv);
+ doSecondaryArithmeticProcess(pQuery);
+
if (IS_QUERY_KILLED(pQInfo)) {
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
- // since the numOfRows must be identical for all sql functions that are allowed to be executed simutaneously.
- pQuery->rec.rows = getNumOfResult(pRuntimeEnv);
-
skipResults(pRuntimeEnv);
limitResults(pRuntimeEnv);
}
@@ -5454,13 +5532,20 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex);
}
+ // no result generated, abort
+ if (pQuery->rec.rows == 0) {
+ break;
+ }
+
+ doSecondaryArithmeticProcess(pQuery);
+
// the offset is handled at prepare stage if no interpolation involved
- if (pQuery->fillType == TSDB_FILL_NONE || pQuery->rec.rows == 0) {
+ if (pQuery->fillType == TSDB_FILL_NONE) {
limitResults(pRuntimeEnv);
break;
} else {
taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey);
- taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata);
+ taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage**) pQuery->sdata);
numOfFilled = 0;
pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled);
@@ -5678,7 +5763,7 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p
* @param pExpr
* @return
*/
-static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr,
+static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr, SSqlFuncMsg ***pSecStageExpr,
char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols) {
int32_t code = TSDB_CODE_SUCCESS;
@@ -5709,6 +5794,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks);
pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder);
pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags);
+ pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput);
// query msg safety check
if (!validateQueryMsg(pQueryMsg)) {
@@ -5778,9 +5864,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex);
pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId);
- pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag);
- pExprMsg->functionId = htons(pExprMsg->functionId);
- pExprMsg->numOfParams = htons(pExprMsg->numOfParams);
+ pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag);
+ pExprMsg->functionId = htons(pExprMsg->functionId);
+ pExprMsg->numOfParams = htons(pExprMsg->numOfParams);
+ pExprMsg->resColId = htons(pExprMsg->resColId);
pMsg += sizeof(SSqlFuncMsg);
@@ -5816,6 +5903,49 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
goto _cleanup;
}
+ if (pQueryMsg->secondStageOutput) {
+ pExprMsg = (SSqlFuncMsg *)pMsg;
+ *pSecStageExpr = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES);
+
+ for (int32_t i = 0; i < pQueryMsg->secondStageOutput; ++i) {
+ (*pSecStageExpr)[i] = pExprMsg;
+
+ pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex);
+ pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId);
+ pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag);
+ pExprMsg->functionId = htons(pExprMsg->functionId);
+ pExprMsg->numOfParams = htons(pExprMsg->numOfParams);
+
+ pMsg += sizeof(SSqlFuncMsg);
+
+ for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) {
+ pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType);
+ pExprMsg->arg[j].argBytes = htons(pExprMsg->arg[j].argBytes);
+
+ if (pExprMsg->arg[j].argType == TSDB_DATA_TYPE_BINARY) {
+ pExprMsg->arg[j].argValue.pz = pMsg;
+ pMsg += pExprMsg->arg[j].argBytes; // one more for the string terminated char.
+ } else {
+ pExprMsg->arg[j].argValue.i64 = htobe64(pExprMsg->arg[j].argValue.i64);
+ }
+ }
+
+ int16_t functionId = pExprMsg->functionId;
+ if (functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG_DUMMY) {
+ if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression.
+ code = TSDB_CODE_QRY_INVALID_MSG;
+ goto _cleanup;
+ }
+ } else {
+// if (!validateExprColumnInfo(pQueryMsg, pExprMsg)) {
+// return TSDB_CODE_QRY_INVALID_MSG;
+// }
+ }
+
+ pExprMsg = (SSqlFuncMsg *)pMsg;
+ }
+ }
+
pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList);
if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns
@@ -5910,19 +6040,19 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
return TSDB_CODE_SUCCESS;
_cleanup:
- taosTFree(*pExpr);
+ tfree(*pExpr);
taosArrayDestroy(*pTableIdList);
*pTableIdList = NULL;
- taosTFree(*tbnameCond);
- taosTFree(*groupbyCols);
- taosTFree(*tagCols);
- taosTFree(*tagCond);
+ tfree(*tbnameCond);
+ tfree(*groupbyCols);
+ tfree(*tagCols);
+ tfree(*tagCond);
return code;
}
-static int32_t buildAirthmeticExprFromMsg(SExprInfo *pArithExprInfo, SQueryTableMsg *pQueryMsg) {
- qDebug("qmsg:%p create arithmetic expr from binary string: %s", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz);
+static int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, SQueryTableMsg *pQueryMsg) {
+ qDebug("qmsg:%p create arithmetic expr from binary", pQueryMsg);
tExprNode* pExprNode = NULL;
TRY(TSDB_MAX_TAG_CONDITIONS) {
@@ -5942,7 +6072,7 @@ static int32_t buildAirthmeticExprFromMsg(SExprInfo *pArithExprInfo, SQueryTable
return TSDB_CODE_SUCCESS;
}
-static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo **pExprInfo, SSqlFuncMsg **pExprMsg,
+static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, SSqlFuncMsg **pExprMsg,
SColumnInfo* pTagCols) {
*pExprInfo = NULL;
int32_t code = TSDB_CODE_SUCCESS;
@@ -5955,7 +6085,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType);
int16_t tagLen = 0;
- for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) {
+ for (int32_t i = 0; i < numOfOutput; ++i) {
pExprs[i].base = *pExprMsg[i];
pExprs[i].bytes = 0;
@@ -5964,10 +6094,10 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
// parse the arithmetic expression
if (pExprs[i].base.functionId == TSDB_FUNC_ARITHM) {
- code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg);
+ code = buildArithmeticExprFromMsg(&pExprs[i], pQueryMsg);
if (code != TSDB_CODE_SUCCESS) {
- taosTFree(pExprs);
+ tfree(pExprs);
return code;
}
@@ -6006,7 +6136,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
int32_t param = (int32_t)pExprs[i].base.arg[0].argValue.i64;
if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].type, &pExprs[i].bytes,
&pExprs[i].interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) {
- taosTFree(pExprs);
+ tfree(pExprs);
return TSDB_CODE_QRY_INVALID_MSG;
}
@@ -6017,7 +6147,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
}
// TODO refactor
- for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) {
+ for (int32_t i = 0; i < numOfOutput; ++i) {
pExprs[i].base = *pExprMsg[i];
int16_t functId = pExprs[i].base.functionId;
@@ -6157,10 +6287,10 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) {
}
static void doUpdateExprColumnIndex(SQuery *pQuery) {
- assert(pQuery->pSelectExpr != NULL && pQuery != NULL);
+ assert(pQuery->pExpr1 != NULL && pQuery != NULL);
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
- SSqlFuncMsg *pSqlExprMsg = &pQuery->pSelectExpr[k].base;
+ SSqlFuncMsg *pSqlExprMsg = &pQuery->pExpr1[k].base;
if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM) {
continue;
}
@@ -6215,7 +6345,7 @@ static void calResultBufSize(SQuery* pQuery) {
}
static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
- STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery) {
+ SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery) {
int16_t numOfCols = pQueryMsg->numOfCols;
int16_t numOfOutput = pQueryMsg->numOfOutput;
@@ -6241,7 +6371,9 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
pQuery->limit.offset = pQueryMsg->offset;
pQuery->order.order = pQueryMsg->order;
pQuery->order.orderColId = pQueryMsg->orderColId;
- pQuery->pSelectExpr = pExprs;
+ pQuery->pExpr1 = pExprs;
+ pQuery->pExpr2 = pSecExprs;
+ pQuery->numOfExpr2 = pQueryMsg->secondStageOutput;
pQuery->pGroupbyExpr = pGroupbyExpr;
memcpy(&pQuery->interval, &pQueryMsg->interval, sizeof(pQuery->interval));
pQuery->fillType = pQueryMsg->fillType;
@@ -6280,10 +6412,16 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
calResultBufSize(pQuery);
for (int32_t col = 0; col < pQuery->numOfOutput; ++col) {
- assert(pExprs[col].interBytes >= pExprs[col].bytes);
-
// allocate additional memory for interResults that are usually larger then final results
- size_t size = (size_t)((pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interBytes + sizeof(tFilePage));
+ // TODO refactor
+ int16_t bytes = 0;
+ if (pQuery->pExpr2 == NULL || col > pQuery->numOfExpr2) {
+ bytes = pExprs[col].bytes;
+ } else {
+ bytes = MAX(pQuery->pExpr2[col].bytes, pExprs[col].bytes);
+ }
+
+ size_t size = (size_t)((pQuery->rec.capacity + 1) * bytes + pExprs[col].interBytes + sizeof(tFilePage));
pQuery->sdata[col] = (tFilePage *)calloc(1, size);
if (pQuery->sdata[col] == NULL) {
goto _cleanup;
@@ -6307,12 +6445,18 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
pQInfo->tableqinfoGroupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES);
pQInfo->tableqinfoGroupInfo.numOfTables = pTableGroupInfo->numOfTables;
pQInfo->tableqinfoGroupInfo.map = taosHashInit(pTableGroupInfo->numOfTables,
- taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
+ taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
}
int tableIndex = 0;
pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery);
+ pQInfo->runtimeEnv.summary.tableInfoSize += (pTableGroupInfo->numOfTables * sizeof(STableQueryInfo));
+
+ pQInfo->runtimeEnv.pResultRowHashTable = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+ pQInfo->runtimeEnv.keyBuf = malloc(TSDB_MAX_BYTES_PER_ROW);
+ pQInfo->runtimeEnv.pool = initResultRowPool(getWindowResultSize(&pQInfo->runtimeEnv));
+
pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo));
if (pQInfo->pBuf == NULL) {
goto _cleanup;
@@ -6347,7 +6491,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
for(int32_t j = 0; j < s; ++j) {
STableKeyInfo* info = taosArrayGet(pa, j);
- void* buf = (char*)pQInfo->pBuf + index * sizeof(STableQueryInfo);
+ void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo);
window.skey = info->lastKey;
STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf);
@@ -6378,7 +6522,7 @@ _cleanup_query:
free(pGroupbyExpr);
}
- taosTFree(pTagCols);
+ tfree(pTagCols);
for (int32_t i = 0; i < numOfOutput; ++i) {
SExprInfo* pExprInfo = &pExprs[i];
if (pExprInfo->pExpr != NULL) {
@@ -6386,7 +6530,7 @@ _cleanup_query:
}
}
- taosTFree(pExprs);
+ tfree(pExprs);
_cleanup:
freeQInfo(pQInfo);
@@ -6412,12 +6556,13 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
STSBuf *pTSBuf = NULL;
- if (pQueryMsg->tsLen > 0) { // open new file to save the result
+ if (pQueryMsg->tsLen > 0) { // open new file to save the result
char *tsBlock = (char *) pQueryMsg + pQueryMsg->tsOffset;
pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder, vgId);
tsBufResetPos(pTSBuf);
bool ret = tsBufNextPos(pTSBuf);
+
UNUSED(ret);
}
@@ -6489,6 +6634,22 @@ static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo) {
pTableqinfoGroupInfo->numOfTables = 0;
}
+static void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr) {
+ if (pExprInfo == NULL) {
+ assert(numOfExpr == 0);
+ return NULL;
+ }
+
+ for (int32_t i = 0; i < numOfExpr; ++i) {
+ if (pExprInfo[i].pExpr != NULL) {
+ tExprNodeDestroy(pExprInfo[i].pExpr, NULL);
+ }
+ }
+
+ tfree(pExprInfo);
+ return NULL;
+}
+
static void freeQInfo(SQInfo *pQInfo) {
if (!isValidQInfo(pQInfo)) {
return;
@@ -6504,56 +6665,47 @@ static void freeQInfo(SQInfo *pQInfo) {
if (pQuery != NULL) {
if (pQuery->sdata != NULL) {
for (int32_t col = 0; col < pQuery->numOfOutput; ++col) {
- taosTFree(pQuery->sdata[col]);
+ tfree(pQuery->sdata[col]);
}
- taosTFree(pQuery->sdata);
+ tfree(pQuery->sdata);
}
if (pQuery->fillVal != NULL) {
- taosTFree(pQuery->fillVal);
+ tfree(pQuery->fillVal);
}
for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) {
SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i];
if (pColFilter->numOfFilters > 0) {
- taosTFree(pColFilter->pFilters);
+ tfree(pColFilter->pFilters);
}
}
- if (pQuery->pSelectExpr != NULL) {
- for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- SExprInfo *pExprInfo = &pQuery->pSelectExpr[i];
-
- if (pExprInfo->pExpr != NULL) {
- tExprTreeDestroy(&pExprInfo->pExpr, NULL);
- }
- }
-
- taosTFree(pQuery->pSelectExpr);
- }
-
- if (pQuery->pGroupbyExpr != NULL) {
- taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo);
- taosTFree(pQuery->pGroupbyExpr);
- }
+ pQuery->pExpr1 = destroyQueryFuncExpr(pQuery->pExpr1, pQuery->numOfOutput);
+ pQuery->pExpr2 = destroyQueryFuncExpr(pQuery->pExpr2, pQuery->numOfExpr2);
- taosTFree(pQuery->tagColList);
- taosTFree(pQuery->pFilterInfo);
+ tfree(pQuery->tagColList);
+ tfree(pQuery->pFilterInfo);
if (pQuery->colList != NULL) {
for (int32_t i = 0; i < pQuery->numOfCols; i++) {
SColumnInfo *column = pQuery->colList + i;
freeColumnFilterInfo(column->filters, column->numOfFilters);
}
- taosTFree(pQuery->colList);
+ tfree(pQuery->colList);
}
- taosTFree(pQuery);
+ if (pQuery->pGroupbyExpr != NULL) {
+ taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo);
+ tfree(pQuery->pGroupbyExpr);
+ }
+
+ tfree(pQuery);
}
doDestroyTableQueryInfo(&pQInfo->tableqinfoGroupInfo);
- taosTFree(pQInfo->pBuf);
+ tfree(pQInfo->pBuf);
tsdbDestroyTableGroup(&pQInfo->tableGroupInfo);
taosArrayDestroy(pQInfo->arrTableIdInfo);
@@ -6561,7 +6713,7 @@ static void freeQInfo(SQInfo *pQInfo) {
qDebug("QInfo:%p QInfo is freed", pQInfo);
- taosTFree(pQInfo);
+ tfree(pQInfo);
}
static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) {
@@ -6650,16 +6802,19 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
int32_t code = TSDB_CODE_SUCCESS;
- char *tagCond = NULL;
- char *tbnameCond = NULL;
+ char *tagCond = NULL;
+ char *tbnameCond = NULL;
SArray *pTableIdList = NULL;
- SSqlFuncMsg **pExprMsg = NULL;
- SExprInfo *pExprs = NULL;
+ SSqlFuncMsg **pExprMsg = NULL;
+ SSqlFuncMsg **pSecExprMsg = NULL;
+ SExprInfo *pExprs = NULL;
+ SExprInfo *pSecExprs = NULL;
+
SColIndex *pGroupColIndex = NULL;
SColumnInfo *pTagColumnInfo = NULL;
SSqlGroupbyExpr *pGroupbyExpr = NULL;
- code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo);
+ code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &pSecExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _over;
}
@@ -6676,10 +6831,16 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
goto _over;
}
- if ((code = createQFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) {
+ if ((code = createQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->numOfOutput, &pExprs, pExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}
+ if (pSecExprMsg != NULL) {
+ if ((code = createQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, &pSecExprs, pSecExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) {
+ goto _over;
+ }
+ }
+
pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, pGroupColIndex, &code);
if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) {
goto _over;
@@ -6737,8 +6898,10 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
goto _over;
}
- (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &tableGroupInfo, pTagColumnInfo, isSTableQuery);
+ (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, pSecExprs, &tableGroupInfo, pTagColumnInfo, isSTableQuery);
+
pExprs = NULL;
+ pSecExprs = NULL;
pGroupbyExpr = NULL;
pTagColumnInfo = NULL;
@@ -6753,13 +6916,19 @@ _over:
free(tagCond);
free(tbnameCond);
free(pGroupColIndex);
+
if (pGroupbyExpr != NULL) {
taosArrayDestroy(pGroupbyExpr->columnInfo);
free(pGroupbyExpr);
}
+
free(pTagColumnInfo);
free(pExprs);
+ free(pSecExprs);
+
free(pExprMsg);
+ free(pSecExprMsg);
+
taosArrayDestroy(pTableIdList);
for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) {
@@ -6878,7 +7047,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex
int32_t code = TSDB_CODE_SUCCESS;
-#if 0
+#if _NON_BLOCKING_RETRIEVE
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
pthread_mutex_lock(&pQInfo->lock);
@@ -6950,6 +7119,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
pQInfo->dataReady = QUERY_RESULT_NOT_READY;
if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
+ // here current thread hold the refcount, so it is safe to free tsdbQueryHandle.
*continueExec = false;
(*pRsp)->completed = 1; // notify no more result to client
} else {
@@ -7022,11 +7192,11 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
assert(num == pQInfo->tableqinfoGroupInfo.numOfTables);
int32_t count = 0;
- int32_t functionId = pQuery->pSelectExpr[0].base.functionId;
+ int32_t functionId = pQuery->pExpr1[0].base.functionId;
if (functionId == TSDB_FUNC_TID_TAG) { // return the tags & table Id
assert(pQuery->numOfOutput == 1);
- SExprInfo* pExprInfo = &pQuery->pSelectExpr[0];
+ SExprInfo* pExprInfo = &pQuery->pExpr1[0];
int32_t rsize = pExprInfo->bytes;
count = 0;
@@ -7100,7 +7270,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
continue;
}
- SExprInfo* pExprInfo = pQuery->pSelectExpr;
+ SExprInfo* pExprInfo = pQuery->pExpr1;
STableQueryInfo* item = taosArrayGetP(pa, i);
char *data = NULL, *dst = NULL;
@@ -7254,9 +7424,9 @@ void qCleanupQueryMgmt(void* pQMgmt) {
taosCacheCleanup(pqinfoPool);
pthread_mutex_destroy(&pQueryMgmt->lock);
- taosTFree(pQueryMgmt);
+ tfree(pQueryMgmt);
- qDebug("vgId:%d queryMgmt cleanup completed", vgId);
+ qDebug("vgId:%d, queryMgmt cleanup completed", vgId);
}
void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) {
diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c
index 17be294531e51982924d84700fc21653ad231224..fa3fe285a857cc0d146e25f52710d0ebc0a69193 100644
--- a/src/query/src/qExtbuffer.c
+++ b/src/query/src/qExtbuffer.c
@@ -64,7 +64,7 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) {
// release flush out info link
SExtFileInfo *pFileMeta = &pMemBuffer->fileMeta;
if (pFileMeta->flushoutData.nAllocSize != 0 && pFileMeta->flushoutData.pFlushoutInfo != NULL) {
- taosTFree(pFileMeta->flushoutData.pFlushoutInfo);
+ tfree(pFileMeta->flushoutData.pFlushoutInfo);
}
// release all in-memory buffer pages
@@ -72,7 +72,7 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) {
while (pFilePages != NULL) {
tFilePagesItem *pTmp = pFilePages;
pFilePages = pFilePages->pNext;
- taosTFree(pTmp);
+ tfree(pTmp);
}
// close temp file
@@ -87,8 +87,8 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) {
destroyColumnModel(pMemBuffer->pColumnModel);
- taosTFree(pMemBuffer->path);
- taosTFree(pMemBuffer);
+ tfree(pMemBuffer->path);
+ tfree(pMemBuffer);
return NULL;
}
@@ -275,7 +275,7 @@ int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) {
tFilePagesItem *ptmp = first;
first = first->pNext;
- taosTFree(ptmp); // release all data in memory buffer
+ tfree(ptmp); // release all data in memory buffer
}
fflush(pMemBuffer->file); // flush to disk
@@ -300,7 +300,7 @@ void tExtMemBufferClear(tExtMemBuffer *pMemBuffer) {
while (first != NULL) {
tFilePagesItem *ptmp = first;
first = first->pNext;
- taosTFree(ptmp);
+ tfree(ptmp);
}
pMemBuffer->fileMeta.numOfElemsInFile = 0;
@@ -802,7 +802,7 @@ void destroyColumnModel(SColumnModel *pModel) {
return;
}
- taosTFree(pModel);
+ tfree(pModel);
}
static void printBinaryData(char *data, int32_t len) {
@@ -1087,5 +1087,5 @@ void tOrderDescDestroy(tOrderDescriptor *pDesc) {
}
destroyColumnModel(pDesc->pColumnModel);
- taosTFree(pDesc);
+ tfree(pDesc);
}
diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c
index f186726c0120f2e2e34580fec0da00c2083e5da9..17642892196d2aa6efa64cc674746b0284424c3a 100644
--- a/src/query/src/qFill.c
+++ b/src/query/src/qFill.c
@@ -13,46 +13,23 @@
* along with this program. If not, see .
*/
-#include "qFill.h"
#include "os.h"
-#include "qExtbuffer.h"
+
#include "taosdef.h"
#include "taosmsg.h"
#include "tsqlfunction.h"
+#include "ttype.h"
-#define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC)
-
-SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
- int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, SFillColInfo* pFillCol) {
- if (fillType == TSDB_FILL_NONE) {
- return NULL;
- }
-
- SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo));
-
- taosResetFillInfo(pFillInfo, skey);
-
- pFillInfo->order = order;
- pFillInfo->fillType = fillType;
- pFillInfo->pFillCol = pFillCol;
- pFillInfo->numOfTags = numOfTags;
- pFillInfo->numOfCols = numOfCols;
- pFillInfo->precision = precision;
-
- pFillInfo->interval.interval = slidingTime;
- pFillInfo->interval.intervalUnit = slidingUnit;
- pFillInfo->interval.sliding = slidingTime;
- pFillInfo->interval.slidingUnit = slidingUnit;
+#include "qFill.h"
+#include "qExtbuffer.h"
+#include "queryLog.h"
- pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
- if (numOfTags > 0) {
- pFillInfo->pTags = calloc(pFillInfo->numOfTags, sizeof(SFillTagColInfo));
- for(int32_t i = 0; i < numOfTags; ++i) {
- pFillInfo->pTags[i].col.colId = -2;
- }
- }
+#define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC)
+// there are no duplicated tags in the SFillTagColInfo list
+static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t capacity) {
int32_t rowsize = 0;
+
int32_t k = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
SFillColInfo* pColInfo = &pFillInfo->pFillCol[i];
@@ -60,72 +37,120 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_
if (TSDB_COL_IS_TAG(pColInfo->flag)) {
bool exists = false;
- for(int32_t j = 0; j < k; ++j) {
+ int32_t index = -1;
+ for (int32_t j = 0; j < k; ++j) {
if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) {
exists = true;
+ index = j;
break;
}
}
if (!exists) {
- pFillInfo->pTags[k].col.colId = pColInfo->col.colId;
+ SSchema* pSchema = &pFillInfo->pTags[k].col;
+ pSchema->colId = pColInfo->col.colId;
+ pSchema->type = pColInfo->col.type;
+ pSchema->bytes = pColInfo->col.bytes;
+
pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes);
+ pColInfo->tagIndex = k;
k += 1;
+ } else {
+ pColInfo->tagIndex = index;
}
}
+
rowsize += pColInfo->col.bytes;
}
- pFillInfo->rowSize = rowsize;
- pFillInfo->capacityInRows = capacity;
-
+ assert(k <= pFillInfo->numOfTags);
+ return rowsize;
+}
+
+SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
+ int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
+ SFillColInfo* pCol, void* handle) {
+ if (fillType == TSDB_FILL_NONE) {
+ return NULL;
+ }
+
+ SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo));
+
+ taosResetFillInfo(pFillInfo, skey);
+
+ pFillInfo->order = order;
+ pFillInfo->type = fillType;
+ pFillInfo->pFillCol = pCol;
+ pFillInfo->numOfTags = numOfTags;
+ pFillInfo->numOfCols = numOfCols;
+ pFillInfo->precision = precision;
+ pFillInfo->alloc = capacity;
+ pFillInfo->handle = handle;
+
+ pFillInfo->interval.interval = slidingTime;
+ pFillInfo->interval.intervalUnit = slidingUnit;
+ pFillInfo->interval.sliding = slidingTime;
+ pFillInfo->interval.slidingUnit = slidingUnit;
+
+ pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
+ if (numOfTags > 0) {
+ pFillInfo->pTags = calloc(pFillInfo->numOfTags, sizeof(SFillTagColInfo));
+ for (int32_t i = 0; i < numOfTags; ++i) {
+ pFillInfo->pTags[i].col.colId = -2; // TODO
+ }
+ }
+
+ pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc);
+ assert(pFillInfo->rowSize > 0);
+
return pFillInfo;
}
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
pFillInfo->start = startTimestamp;
- pFillInfo->rowIdx = -1;
+ pFillInfo->currentKey = startTimestamp;
+ pFillInfo->index = -1;
pFillInfo->numOfRows = 0;
pFillInfo->numOfCurrent = 0;
pFillInfo->numOfTotal = 0;
}
-void* taosDestoryFillInfo(SFillInfo* pFillInfo) {
+void* taosDestroyFillInfo(SFillInfo* pFillInfo) {
if (pFillInfo == NULL) {
return NULL;
}
- taosTFree(pFillInfo->prevValues);
- taosTFree(pFillInfo->nextValues);
- taosTFree(pFillInfo->pTags);
+ tfree(pFillInfo->prevValues);
+ tfree(pFillInfo->nextValues);
+ tfree(pFillInfo->pTags);
for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
- taosTFree(pFillInfo->pData[i]);
+ tfree(pFillInfo->pData[i]);
}
- taosTFree(pFillInfo->pData);
- taosTFree(pFillInfo->pFillCol);
+ tfree(pFillInfo->pData);
+ tfree(pFillInfo->pFillCol);
- taosTFree(pFillInfo);
+ tfree(pFillInfo);
return NULL;
}
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) {
- if (pFillInfo->fillType == TSDB_FILL_NONE) {
+ if (pFillInfo->type == TSDB_FILL_NONE) {
return;
}
- pFillInfo->endKey = endKey;
- if (pFillInfo->order != TSDB_ORDER_ASC) {
- pFillInfo->endKey = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision);
+ pFillInfo->end = endKey;
+ if (!FILL_IS_ASC_FILL(pFillInfo)) {
+ pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision);
}
- pFillInfo->rowIdx = 0;
+ pFillInfo->index = 0;
pFillInfo->numOfRows = numOfRows;
// ensure the space
- if (pFillInfo->capacityInRows < numOfRows) {
+ if (pFillInfo->alloc < numOfRows) {
for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
char* tmp = realloc(pFillInfo->pData[i], numOfRows*pFillInfo->pFillCol[i].col.bytes);
assert(tmp != NULL); // todo handle error
@@ -136,42 +161,38 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey)
}
}
-void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput) {
- // copy the data into source data buffer
+// copy the data into source data buffer
+void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput) {
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
memcpy(pFillInfo->pData[i], pInput[i]->data, pFillInfo->numOfRows * pFillInfo->pFillCol[i].col.bytes);
}
}
-void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput) {
+void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput) {
assert(pFillInfo->numOfRows == pInput->num);
for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
- char* data = pInput->data + pCol->col.offset * pInput->num;
+ const char* data = pInput->data + pCol->col.offset * pInput->num;
memcpy(pFillInfo->pData[i], data, (size_t)(pInput->num * pCol->col.bytes));
if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer
- for (int32_t j = 0; j < pFillInfo->numOfTags; ++j) {
- SFillTagColInfo* pTag = &pFillInfo->pTags[j];
- if (pTag->col.colId == pCol->col.colId) {
- memcpy(pTag->tagVal, data, pCol->col.bytes);
- break;
- }
- }
+ SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
+ assert (pTag->col.colId == pCol->col.colId);
+ memcpy(pTag->tagVal, data, pCol->col.bytes);
}
}
}
-int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) {
+int64_t getNumOfResWithFill(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) {
int64_t* tsList = (int64_t*) pFillInfo->pData[0];
int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
TSKEY ekey1 = ekey;
- if (pFillInfo->order != TSDB_ORDER_ASC) {
- pFillInfo->endKey = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
+ if (!FILL_IS_ASC_FILL(pFillInfo)) {
+ pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
}
int64_t numOfRes = -1;
@@ -179,20 +200,20 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows
TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];
numOfRes = taosTimeCountInterval(
lastKey,
- pFillInfo->start,
+ pFillInfo->currentKey,
pFillInfo->interval.sliding,
pFillInfo->interval.slidingUnit,
pFillInfo->precision);
numOfRes += 1;
assert(numOfRes >= numOfRows);
} else { // reach the end of data
- if ((ekey1 < pFillInfo->start && FILL_IS_ASC_FILL(pFillInfo)) ||
- (ekey1 > pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo))) {
+ if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) ||
+ (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) {
return 0;
}
numOfRes = taosTimeCountInterval(
ekey1,
- pFillInfo->start,
+ pFillInfo->currentKey,
pFillInfo->interval.sliding,
pFillInfo->interval.slidingUnit,
pFillInfo->precision);
@@ -203,315 +224,283 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows
}
int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) {
- if (pFillInfo->rowIdx == -1 || pFillInfo->numOfRows == 0) {
+ if (pFillInfo->numOfRows == 0 || (pFillInfo->numOfRows > 0 && pFillInfo->index >= pFillInfo->numOfRows)) {
return 0;
}
- return pFillInfo->numOfRows - pFillInfo->rowIdx;
+ return pFillInfo->numOfRows - pFillInfo->index;
}
-// todo: refactor
-static double linearInterpolationImpl(double v1, double v2, double k1, double k2, double k) {
- return v1 + (v2 - v1) * (k - k1) / (k2 - k1);
-}
+#define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1))))
-int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) {
- switch (type) {
- case TSDB_DATA_TYPE_INT: {
- *(int32_t*)point->val = (int32_t)linearInterpolationImpl(*(int32_t*)point1->val, *(int32_t*)point2->val, (double)point1->key,
- (double)point2->key, (double)point->key);
- break;
- }
- case TSDB_DATA_TYPE_FLOAT: {
- *(float*)point->val = (float)
- linearInterpolationImpl(*(float*)point1->val, *(float*)point2->val, (double)point1->key, (double)point2->key, (double)point->key);
- break;
- };
- case TSDB_DATA_TYPE_DOUBLE: {
- *(double*)point->val =
- linearInterpolationImpl(*(double*)point1->val, *(double*)point2->val, (double)point1->key, (double)point2->key, (double)point->key);
- break;
- };
- case TSDB_DATA_TYPE_TIMESTAMP:
- case TSDB_DATA_TYPE_BIGINT: {
- *(int64_t*)point->val = (int64_t)linearInterpolationImpl((double)(*(int64_t*)point1->val), (double)(*(int64_t*)point2->val), (double)point1->key,
- (double)point2->key, (double)point->key);
- break;
- };
- case TSDB_DATA_TYPE_SMALLINT: {
- *(int16_t*)point->val = (int16_t)linearInterpolationImpl(*(int16_t*)point1->val, *(int16_t*)point2->val, (double)point1->key,
- (double)point2->key, (double)point->key);
- break;
- };
- case TSDB_DATA_TYPE_TINYINT: {
- *(int8_t*) point->val = (int8_t)
- linearInterpolationImpl(*(int8_t*)point1->val, *(int8_t*)point2->val, (double)point1->key, (double)point2->key, (double)point->key);
- break;
- };
- default: {
- // TODO: Deal with interpolation with bool and strings and timestamp
- return -1;
- }
+int32_t taosGetLinearInterpolationVal(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) {
+ double v1 = -1;
+ double v2 = -1;
+
+ GET_TYPED_DATA(v1, double, type, point1->val);
+ GET_TYPED_DATA(v2, double, type, point2->val);
+
+ double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key);
+
+ switch(type) {
+ case TSDB_DATA_TYPE_TINYINT: *(int8_t*) point->val = (int8_t) r;break;
+ case TSDB_DATA_TYPE_SMALLINT: *(int16_t*) point->val = (int16_t) r;break;
+ case TSDB_DATA_TYPE_INT: *(int32_t*) point->val = (int32_t) r;break;
+ case TSDB_DATA_TYPE_BIGINT: *(int64_t*) point->val = (int64_t) r;break;
+ case TSDB_DATA_TYPE_DOUBLE: *(double*) point->val = (double) r;break;
+ case TSDB_DATA_TYPE_FLOAT: *(float*) point->val = (float) r;break;
+ default:
+ assert(0);
}
- return 0;
+ return TSDB_CODE_SUCCESS;
}
-static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t num) {
+static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t genRows) {
for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) {
SFillColInfo* pCol = &pFillInfo->pFillCol[j];
if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) {
continue;
}
- char* val1 = elePtrAt(data[j]->data, pCol->col.bytes, num);
+ char* val1 = elePtrAt(data[j]->data, pCol->col.bytes, genRows);
- for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) {
- SFillTagColInfo* pTag = &pFillInfo->pTags[i];
- if (pTag->col.colId == pCol->col.colId) {
- assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type);
- break;
- }
- }
+ assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags);
+ SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
+
+ assert (pTag->col.colId == pCol->col.colId);
+ assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type);
}
}
-static void doFillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t* num, char** srcData, int64_t ts,
- bool outOfBound) {
- char* prevValues = pFillInfo->prevValues;
- char* nextValues = pFillInfo->nextValues;
+static void setNullValueForRow(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfCol, int32_t rowIndex) {
+ // the first are always the timestamp column, so start from the second column.
+ for (int32_t i = 1; i < numOfCol; ++i) {
+ SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+
+ char* output = elePtrAt(data[i]->data, pCol->col.bytes, rowIndex);
+ setNull(output, pCol->col.type, pCol->col.bytes);
+ }
+}
+
+static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** srcData, int64_t ts, bool outOfBound) {
+ char* prev = pFillInfo->prevValues;
+ char* next = pFillInfo->nextValues;
SPoint point1, point2, point;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
- char* val = elePtrAt(data[0]->data, TSDB_KEYSIZE, *num);
- *(TSKEY*) val = pFillInfo->start;
-
- int32_t numOfValCols = pFillInfo->numOfCols - pFillInfo->numOfTags;
+ // set the primary timestamp column value
+ int32_t index = pFillInfo->numOfCurrent;
+ char* val = elePtrAt(data[0]->data, TSDB_KEYSIZE, index);
+ *(TSKEY*) val = pFillInfo->currentKey;
// set the other values
- if (pFillInfo->fillType == TSDB_FILL_PREV) {
- char* p = FILL_IS_ASC_FILL(pFillInfo) ? prevValues : nextValues;
+ if (pFillInfo->type == TSDB_FILL_PREV) {
+ char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next;
if (p != NULL) {
- for (int32_t i = 1; i < numOfValCols; ++i) {
+ for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
-
- char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
- if (isNull(p + pCol->col.offset, pCol->col.type)) {
- if (pCol->col.type == TSDB_DATA_TYPE_BINARY || pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
- setVardataNull(val1, pCol->col.type);
- } else {
- setNull(val1, pCol->col.type, pCol->col.bytes);
- }
- } else {
- assignVal(val1, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
+ if (TSDB_COL_IS_TAG(pCol->flag)) {
+ continue;
}
- }
- } else { // no prev value yet, set the value for NULL
- for (int32_t i = 1; i < numOfValCols; ++i) {
- SFillColInfo* pCol = &pFillInfo->pFillCol[i];
- char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
- if (pCol->col.type == TSDB_DATA_TYPE_BINARY||pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
- setVardataNull(val1, pCol->col.type);
- } else {
- setNull(val1, pCol->col.type, pCol->col.bytes);
- }
+ char* output = elePtrAt(data[i]->data, pCol->col.bytes, index);
+ assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
}
+ } else { // no prev value yet, set the value for NULL
+ setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
}
-
- setTagsValue(pFillInfo, data, *num);
- } else if (pFillInfo->fillType == TSDB_FILL_LINEAR) {
+ } else if (pFillInfo->type == TSDB_FILL_LINEAR) {
// TODO : linear interpolation supports NULL value
- if (prevValues != NULL && !outOfBound) {
- for (int32_t i = 1; i < numOfValCols; ++i) {
+ if (prev != NULL && !outOfBound) {
+ for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
-
+ if (TSDB_COL_IS_TAG(pCol->flag)) {
+ continue;
+ }
+
int16_t type = pCol->col.type;
int16_t bytes = pCol->col.bytes;
- char *val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
- if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR) {
- setVardataNull(val1, pCol->col.type);
- continue;
- } else if (type == TSDB_DATA_TYPE_BOOL) {
+ char *val1 = elePtrAt(data[i]->data, pCol->col.bytes, index);
+ if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
setNull(val1, pCol->col.type, bytes);
continue;
}
- point1 = (SPoint){.key = *(TSKEY*)(prevValues), .val = prevValues + pCol->col.offset};
- point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->rowIdx * bytes};
+ point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->col.offset};
+ point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes};
point = (SPoint){.key = pFillInfo->start, .val = val1};
- taosDoLinearInterpolation(type, &point1, &point2, &point);
+ taosGetLinearInterpolationVal(type, &point1, &point2, &point);
}
-
- setTagsValue(pFillInfo, data, *num);
-
} else {
- for (int32_t i = 1; i < numOfValCols; ++i) {
- SFillColInfo* pCol = &pFillInfo->pFillCol[i];
-
- char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
-
- if (pCol->col.type == TSDB_DATA_TYPE_BINARY || pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
- setVardataNull(val1, pCol->col.type);
- } else {
- setNull(val1, pCol->col.type, pCol->col.bytes);
- }
- }
-
- setTagsValue(pFillInfo, data, *num);
-
+ setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
}
} else { /* fill the default value */
- for (int32_t i = 1; i < numOfValCols; ++i) {
+ for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
-
- char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
+ if (TSDB_COL_IS_TAG(pCol->flag)) {
+ continue;
+ }
+
+ char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, index);
assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
}
-
- setTagsValue(pFillInfo, data, *num);
}
- pFillInfo->start = taosTimeAdd(pFillInfo->start, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
+ setTagsValue(pFillInfo, data, index);
+ pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
pFillInfo->numOfCurrent++;
-
- (*num) += 1;
}
-static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** nextValues) {
- if (*nextValues != NULL) {
+static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) {
+ if (*next != NULL) {
return;
}
- *nextValues = calloc(1, pFillInfo->rowSize);
+ *next = calloc(1, pFillInfo->rowSize);
for (int i = 1; i < pFillInfo->numOfCols; i++) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
-
- if (pCol->col.type == TSDB_DATA_TYPE_BINARY||pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
- setVardataNull(*nextValues + pCol->col.offset, pCol->col.type);
- } else {
- setNull(*nextValues + pCol->col.offset, pCol->col.type, pCol->col.bytes);
- }
+ setNull(*next + pCol->col.offset, pCol->col.type, pCol->col.bytes);
+ }
+}
+
+static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* buf) {
+ int32_t rowIndex = pFillInfo->index;
+ for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
+ SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+ memcpy(buf + pCol->col.offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes);
}
}
-int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfRows, int32_t outputRows, char** srcData) {
- int32_t num = 0;
+static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t outputRows) {
pFillInfo->numOfCurrent = 0;
- char** prevValues = &pFillInfo->prevValues;
- char** nextValues = &pFillInfo->nextValues;
+ char** srcData = pFillInfo->pData;
+ char** prev = &pFillInfo->prevValues;
+ char** next = &pFillInfo->nextValues;
- int32_t numOfTags = pFillInfo->numOfTags;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
- if (numOfRows == 0) {
- /*
- * These data are generated according to fill strategy, since the current timestamp is out of time window of
- * real result set. Note that we need to keep the direct previous result rows, to generated the filled data.
- */
- while (num < outputRows) {
- doFillResultImpl(pFillInfo, data, &num, srcData, pFillInfo->start, true);
- }
-
- pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
- return outputRows;
+ if (FILL_IS_ASC_FILL(pFillInfo)) {
+ assert(pFillInfo->currentKey >= pFillInfo->start);
} else {
- while (1) {
- int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->rowIdx];
+ assert(pFillInfo->currentKey <= pFillInfo->start);
+ }
- if ((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
- (pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) {
- /* set the next value for interpolation */
- initBeforeAfterDataBuf(pFillInfo, nextValues);
-
- int32_t offset = pFillInfo->rowIdx;
- for (int32_t i = 0; i < pFillInfo->numOfCols - numOfTags; ++i) {
- SFillColInfo* pCol = &pFillInfo->pFillCol[i];
- memcpy(*nextValues + pCol->col.offset, srcData[i] + offset * pCol->col.bytes, pCol->col.bytes);
- }
+ while (pFillInfo->numOfCurrent < outputRows) {
+ int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index];
+
+ if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
+ (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) {
+ /* set the next value for interpolation */
+ initBeforeAfterDataBuf(pFillInfo, next);
+ copyCurrentRowIntoBuf(pFillInfo, srcData, *next);
+ }
+
+ if (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
+ pFillInfo->numOfCurrent < outputRows) {
+
+ // fill the gap between two actual input rows
+ while (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
+ (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
+ pFillInfo->numOfCurrent < outputRows) {
+ doFillOneRowResult(pFillInfo, data, srcData, ts, false);
}
- if (((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
- (pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) && num < outputRows) {
-
- while (((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
- (pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) && num < outputRows) {
- doFillResultImpl(pFillInfo, data, &num, srcData, ts, false);
- }
+ // output buffer is full, abort
+ if (pFillInfo->numOfCurrent == outputRows) {
+ pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
+ return outputRows;
+ }
+ } else {
+ assert(pFillInfo->currentKey == ts);
+ initBeforeAfterDataBuf(pFillInfo, prev);
- /* output buffer is full, abort */
- if ((num == outputRows && FILL_IS_ASC_FILL(pFillInfo)) || (num < 0 && !FILL_IS_ASC_FILL(pFillInfo))) {
- pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
- return outputRows;
+ // assign rows to dst buffer
+ for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
+ SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+ if (TSDB_COL_IS_TAG(pCol->flag)) {
+ continue;
}
- } else {
- assert(pFillInfo->start == ts);
- initBeforeAfterDataBuf(pFillInfo, prevValues);
-
- // assign rows to dst buffer
- for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
- SFillColInfo* pCol = &pFillInfo->pFillCol[i];
- if (TSDB_COL_IS_TAG(pCol->flag)) {
- continue;
- }
- char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, num);
- char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->rowIdx);
-
- if (i == 0 ||
- (pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) ||
- (pCol->functionId == TSDB_FUNC_COUNT && GET_INT64_VAL(src) != 0)) {
- assignVal(val1, src, pCol->col.bytes, pCol->col.type);
- memcpy(*prevValues + pCol->col.offset, src, pCol->col.bytes);
- } else { // i > 0 and data is null , do interpolation
- if (pFillInfo->fillType == TSDB_FILL_PREV) {
- assignVal(val1, *prevValues + pCol->col.offset, pCol->col.bytes, pCol->col.type);
- } else if (pFillInfo->fillType == TSDB_FILL_LINEAR) {
- assignVal(val1, src, pCol->col.bytes, pCol->col.type);
- memcpy(*prevValues + pCol->col.offset, src, pCol->col.bytes);
- } else {
- assignVal(val1, (char*) &pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
- }
+ char* output = elePtrAt(data[i]->data, pCol->col.bytes, pFillInfo->numOfCurrent);
+ char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index);
+
+ if (i == 0 || (pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) ||
+ (pCol->functionId == TSDB_FUNC_COUNT && GET_INT64_VAL(src) != 0)) {
+ assignVal(output, src, pCol->col.bytes, pCol->col.type);
+ memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
+ } else { // i > 0 and data is null , do interpolation
+ if (pFillInfo->type == TSDB_FILL_PREV) {
+ assignVal(output, *prev + pCol->col.offset, pCol->col.bytes, pCol->col.type);
+ } else if (pFillInfo->type == TSDB_FILL_LINEAR) {
+ assignVal(output, src, pCol->col.bytes, pCol->col.type);
+ memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
+ } else {
+ assignVal(output, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
}
}
+ }
- // set the tag value for final result
- setTagsValue(pFillInfo, data, num);
+ // set the tag value for final result
+ setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent);
- pFillInfo->start = taosTimeAdd(pFillInfo->start, pFillInfo->interval.sliding*step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
- pFillInfo->rowIdx += 1;
+ pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step,
+ pFillInfo->interval.slidingUnit, pFillInfo->precision);
+ pFillInfo->index += 1;
+ pFillInfo->numOfCurrent += 1;
+ }
- pFillInfo->numOfCurrent +=1;
- num += 1;
+ if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) {
+ /* the raw data block is exhausted, next value does not exists */
+ if (pFillInfo->index >= pFillInfo->numOfRows) {
+ tfree(*next);
}
- if ((pFillInfo->rowIdx >= pFillInfo->numOfRows && FILL_IS_ASC_FILL(pFillInfo)) ||
- (pFillInfo->rowIdx < 0 && !FILL_IS_ASC_FILL(pFillInfo)) || num >= outputRows) {
- if (pFillInfo->rowIdx >= pFillInfo->numOfRows || pFillInfo->rowIdx < 0) {
- pFillInfo->rowIdx = -1;
- pFillInfo->numOfRows = 0;
+ pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
+ return pFillInfo->numOfCurrent;
+ }
+ }
- /* the raw data block is exhausted, next value does not exists */
- taosTFree(*nextValues);
- }
+ return pFillInfo->numOfCurrent;
+}
- pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
- return num;
- }
- }
+static int64_t fillExternalResults(SFillInfo* pFillInfo, tFilePage** output, int64_t resultCapacity) {
+ /*
+ * These data are generated according to fill strategy, since the current timestamp is out of the time window of
+ * real result set. Note that we need to keep the direct previous result rows, to generated the filled data.
+ */
+ pFillInfo->numOfCurrent = 0;
+ while (pFillInfo->numOfCurrent < resultCapacity) {
+ doFillOneRowResult(pFillInfo, output, pFillInfo->pData, pFillInfo->start, true);
}
+
+ pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
+
+ assert(pFillInfo->numOfCurrent == resultCapacity);
+ return resultCapacity;
}
-int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) {
- int32_t remain = taosNumOfRemainRows(pFillInfo); // todo use iterator?
+int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) {
+ int32_t remain = taosNumOfRemainRows(pFillInfo);
+
+ int64_t numOfRes = getNumOfResWithFill(pFillInfo, pFillInfo->end, capacity);
+ assert(numOfRes <= capacity);
+
+ // no data existed for fill operation now, append result according to the fill strategy
+ if (remain == 0) {
+ fillExternalResults(pFillInfo, output, numOfRes);
+ } else {
+ fillResultImpl(pFillInfo, output, (int32_t) numOfRes);
+ assert(numOfRes == pFillInfo->numOfCurrent);
+ }
+
+ qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%"PRId64"-%"PRId64", currentKey:%"PRId64", current:%d, total:%d, %p",
+ pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, pFillInfo->numOfCurrent,
+ pFillInfo->numOfTotal, pFillInfo->handle);
- int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, pFillInfo->endKey, capacity);
- int32_t numOfRes = generateDataBlockImpl(pFillInfo, output, remain, rows, pFillInfo->pData);
- assert(numOfRes == rows);
-
return numOfRes;
}
diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c
index 703ee2c52129a067d1070cfbe567087ec48a7a6e..35e5906d1f797c2f53f3906bf93c6797ac766c3e 100644
--- a/src/query/src/qHistogram.c
+++ b/src/query/src/qHistogram.c
@@ -168,7 +168,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
(*pHisto)->numOfEntries += 1;
}
} else { /* insert a new slot */
- if ((*pHisto)->numOfElems > 1 && idx < (*pHisto)->numOfEntries) {
+ if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) {
if (idx > 0) {
assert((*pHisto)->elems[idx - 1].val <= val);
}
@@ -661,4 +661,4 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2
free(pHistoBins);
return pResHistogram;
-}
\ No newline at end of file
+}
diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c
index 7e8128f200658af282bab89bd73dc2c27ddb25c1..4481ff5e8fbbf0244ec664e4c699e4747ebd7e92 100644
--- a/src/query/src/qParserImpl.c
+++ b/src/query/src/qParserImpl.c
@@ -15,11 +15,9 @@
#include "os.h"
#include "qSqlparser.h"
-#include "queryLog.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "tcmdtype.h"
-#include "tglobal.h"
#include "tstoken.h"
#include "tstrbuild.h"
#include "ttokendef.h"
@@ -227,13 +225,12 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) {
tSQLExprDestroy(pLeft);
tSQLExprDestroy(pRight);
- } else if ((pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE && pRight->val.nType == TSDB_DATA_TYPE_BIGINT) ||
- (pRight->val.nType == TSDB_DATA_TYPE_DOUBLE && pLeft->val.nType == TSDB_DATA_TYPE_BIGINT)) {
+ } else if ((pLeft->nSQLOptr == TK_FLOAT && pRight->nSQLOptr == TK_INTEGER) || (pLeft->nSQLOptr == TK_INTEGER && pRight->nSQLOptr == TK_FLOAT)) {
pExpr->val.nType = TSDB_DATA_TYPE_DOUBLE;
- pExpr->nSQLOptr = TK_FLOAT;
+ pExpr->nSQLOptr = TK_FLOAT;
- double left = pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE ? pLeft->val.dKey : pLeft->val.i64Key;
- double right = pRight->val.nType == TSDB_DATA_TYPE_DOUBLE ? pRight->val.dKey : pRight->val.i64Key;
+ double left = (pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->val.dKey : pLeft->val.i64Key;
+ double right = (pRight->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->val.dKey : pRight->val.i64Key;
switch (optrType) {
case TK_PLUS: {
@@ -314,130 +311,57 @@ void tSQLExprDestroy(tSQLExpr *pExpr) {
tSQLExprNodeDestroy(pExpr);
}
-static void *tVariantListExpand(tVariantList *pList) {
- if (pList->nAlloc <= pList->nExpr) { //
- int32_t newSize = (pList->nAlloc << 1) + 4;
-
- void *ptr = realloc(pList->a, newSize * sizeof(pList->a[0]));
- if (ptr == 0) {
- return NULL;
- }
-
- pList->nAlloc = newSize;
- pList->a = ptr;
- }
-
- assert(pList->a != 0);
- return pList;
-}
-
-tVariantList *tVariantListAppend(tVariantList *pList, tVariant *pVar, uint8_t sortOrder) {
+SArray *tVariantListAppendToken(SArray *pList, SStrToken *pToken, uint8_t order) {
if (pList == NULL) {
- pList = calloc(1, sizeof(tVariantList));
- }
-
- if (tVariantListExpand(pList) == NULL) {
- return pList;
+ pList = taosArrayInit(4, sizeof(tVariantListItem));
}
- if (pVar) {
- tVariantListItem *pItem = &pList->a[pList->nExpr++];
- /*
- * Here we do not employ the assign function, since we need the pz attribute of structure
- * , which is the point to char string, to free it!
- *
- * Otherwise, the original pointer may be lost, which causes memory leak.
- */
- memcpy(pItem, pVar, sizeof(tVariant));
- pItem->sortOrder = sortOrder;
- }
- return pList;
-}
-
-tVariantList *tVariantListInsert(tVariantList *pList, tVariant *pVar, uint8_t sortOrder, int32_t index) {
- if (pList == NULL || index >= pList->nExpr) {
- return tVariantListAppend(NULL, pVar, sortOrder);
- }
-
- if (tVariantListExpand(pList) == NULL) {
- return pList;
- }
-
- if (pVar) {
- memmove(&pList->a[index + 1], &pList->a[index], sizeof(tVariantListItem) * (pList->nExpr - index));
-
- tVariantListItem *pItem = &pList->a[index];
- /*
- * Here we do not employ the assign function, since we need the pz attribute of structure
- * , which is the point to char string, to free it!
- *
- * Otherwise, the original pointer may be lost, which causes memory leak.
- */
- memcpy(pItem, pVar, sizeof(tVariant));
- pItem->sortOrder = sortOrder;
+ if (pToken) {
+ tVariantListItem item;
+ tVariantCreate(&item.pVar, pToken);
+ item.sortOrder = order;
- pList->nExpr++;
+ taosArrayPush(pList, &item);
}
return pList;
}
-void tVariantListDestroy(tVariantList *pList) {
- if (pList == NULL) return;
-
- for (int32_t i = 0; i < pList->nExpr; ++i) {
- tVariantDestroy(&pList->a[i].pVar);
- }
-
- free(pList->a);
- free(pList);
-}
-
-tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pToken, uint8_t sortOrder) {
+SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder) {
if (pList == NULL) {
- pList = calloc(1, sizeof(tVariantList));
+ pList = taosArrayInit(4, sizeof(tVariantListItem));
}
- if (tVariantListExpand(pList) == NULL) {
+ if (pVar == NULL) {
return pList;
}
- if (pToken) {
- tVariant t = {0};
- tVariantCreate(&t, pToken);
+ /*
+ * Here we do not employ the assign function, since we need the pz attribute of structure
+ * , which is the point to char string, to free it!
+ *
+ * Otherwise, the original pointer may be lost, which causes memory leak.
+ */
+ tVariantListItem item;
+ item.pVar = *pVar;
+ item.sortOrder = sortOrder;
- tVariantListItem *pItem = &pList->a[pList->nExpr++];
- memcpy(pItem, &t, sizeof(tVariant));
- pItem->sortOrder = sortOrder;
- }
+ taosArrayPush(pList, &item);
return pList;
}
-tFieldList *tFieldListAppend(tFieldList *pList, TAOS_FIELD *pField) {
- if (pList == NULL) pList = calloc(1, sizeof(tFieldList));
-
- if (pList->nAlloc <= pList->nField) { //
- pList->nAlloc = (pList->nAlloc << 1) + 4;
- pList->p = realloc(pList->p, pList->nAlloc * sizeof(pList->p[0]));
- if (pList->p == 0) {
- pList->nField = pList->nAlloc = 0;
- return pList;
- }
+SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index) {
+ if (pList == NULL || pVar == NULL || index >= taosArrayGetSize(pList)) {
+ return tVariantListAppend(NULL, pVar, sortOrder);
}
- assert(pList->p != 0);
- if (pField) {
- struct TAOS_FIELD *pItem = (struct TAOS_FIELD *)&pList->p[pList->nField++];
- memcpy(pItem, pField, sizeof(TAOS_FIELD));
- }
- return pList;
-}
+ tVariantListItem item;
-void tFieldListDestroy(tFieldList *pList) {
- if (pList == NULL) return;
+ item.pVar = *pVar;
+ item.sortOrder = sortOrder;
- free(pList->p);
- free(pList);
+ taosArrayInsert(pList, index, &item);
+ return pList;
}
void setDBName(SStrToken *pCpxName, SStrToken *pDB) {
@@ -464,8 +388,6 @@ void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType)
void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
pField->type = -1;
- int32_t LENGTH_SIZE_OF_STR = 2; // in case of nchar and binary, there two bytes to keep the length of binary|nchar.
-
for (int8_t i = 0; i < tListLen(tDataTypeDesc); ++i) {
if ((strncasecmp(type->z, tDataTypeDesc[i].aName, tDataTypeDesc[i].nameLen) == 0) &&
(type->n == tDataTypeDesc[i].nameLen)) {
@@ -481,14 +403,14 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
if (type->type == 0) {
pField->bytes = 0;
} else {
- pField->bytes = -(int32_t)type->type * TSDB_NCHAR_SIZE + LENGTH_SIZE_OF_STR;
+ pField->bytes = (int16_t)(-(int32_t)type->type * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
}
} else if (i == TSDB_DATA_TYPE_BINARY) {
/* for binary, the TOKENTYPE is the length of binary */
if (type->type == 0) {
pField->bytes = 0;
} else {
- pField->bytes = -(int32_t) type->type + LENGTH_SIZE_OF_STR;
+ pField->bytes = (int16_t) (-(int32_t) type->type + VARSTR_HEADER_SIZE);
}
}
break;
@@ -499,9 +421,9 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
/*
* extract the select info out of sql string
*/
-SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere,
- tVariantList *pGroupby, tVariantList *pSortOrder, SIntervalVal *pInterval,
- SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) {
+SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere,
+ SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval,
+ SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) {
assert(pSelection != NULL);
SQuerySQL *pQuery = calloc(1, sizeof(SQuerySQL));
@@ -535,6 +457,11 @@ SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection,
return pQuery;
}
+void freeVariant(void *pItem) {
+ tVariantListItem* p = (tVariantListItem*) pItem;
+ tVariantDestroy(&p->pVar);
+}
+
void doDestroyQuerySql(SQuerySQL *pQuerySql) {
if (pQuerySql == NULL) {
return;
@@ -547,17 +474,18 @@ void doDestroyQuerySql(SQuerySQL *pQuerySql) {
tSQLExprDestroy(pQuerySql->pWhere);
pQuerySql->pWhere = NULL;
- tVariantListDestroy(pQuerySql->pSortOrder);
+ taosArrayDestroyEx(pQuerySql->pSortOrder, freeVariant);
pQuerySql->pSortOrder = NULL;
-
- tVariantListDestroy(pQuerySql->pGroupby);
+
+ taosArrayDestroyEx(pQuerySql->pGroupby, freeVariant);
pQuerySql->pGroupby = NULL;
-
- tVariantListDestroy(pQuerySql->from);
+
+ taosArrayDestroyEx(pQuerySql->from, freeVariant);
pQuerySql->from = NULL;
-
- tVariantListDestroy(pQuerySql->fillType);
-
+
+ taosArrayDestroyEx(pQuerySql->fillType, freeVariant);
+ pQuerySql->fillType = NULL;
+
free(pQuerySql);
}
@@ -571,11 +499,11 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) {
doDestroyQuerySql(pQuerySql);
}
- taosTFree(pClause->pClause);
+ tfree(pClause->pClause);
}
-SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pStableName,
- tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type) {
+SCreateTableSQL *tSetCreateSQLElems(SArray *pCols, SArray *pTags, SStrToken *pStableName,
+ SArray *pTagVals, SQuerySQL *pSelect, int32_t type) {
SCreateTableSQL *pCreate = calloc(1, sizeof(SCreateTableSQL));
switch (type) {
@@ -607,7 +535,7 @@ SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrTo
return pCreate;
}
-SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type) {
+SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, SArray *pCols, SArray *pVals, int32_t type) {
SAlterTableSQL *pAlterTable = calloc(1, sizeof(SAlterTableSQL));
pAlterTable->name = *pMeterName;
@@ -637,26 +565,26 @@ void SQLInfoDestroy(SSqlInfo *pInfo) {
SCreateTableSQL *pCreateTableInfo = pInfo->pCreateTableInfo;
doDestroyQuerySql(pCreateTableInfo->pSelect);
- tFieldListDestroy(pCreateTableInfo->colInfo.pColumns);
- tFieldListDestroy(pCreateTableInfo->colInfo.pTagColumns);
+ taosArrayDestroy(pCreateTableInfo->colInfo.pColumns);
+ taosArrayDestroy(pCreateTableInfo->colInfo.pTagColumns);
- tVariantListDestroy(pCreateTableInfo->usingInfo.pTagVals);
- taosTFree(pInfo->pCreateTableInfo);
+ taosArrayDestroyEx(pCreateTableInfo->usingInfo.pTagVals, freeVariant);
+ tfree(pInfo->pCreateTableInfo);
} else if (pInfo->type == TSDB_SQL_ALTER_TABLE) {
- tVariantListDestroy(pInfo->pAlterInfo->varList);
- tFieldListDestroy(pInfo->pAlterInfo->pAddColumns);
+ taosArrayDestroyEx(pInfo->pAlterInfo->varList, freeVariant);
+ taosArrayDestroy(pInfo->pAlterInfo->pAddColumns);
- taosTFree(pInfo->pAlterInfo);
+ tfree(pInfo->pAlterInfo);
} else {
if (pInfo->pDCLInfo != NULL && pInfo->pDCLInfo->nAlloc > 0) {
free(pInfo->pDCLInfo->a);
}
if (pInfo->pDCLInfo != NULL && pInfo->type == TSDB_SQL_CREATE_DB) {
- tVariantListDestroy(pInfo->pDCLInfo->dbOpt.keep);
+ taosArrayDestroyEx(pInfo->pDCLInfo->dbOpt.keep, freeVariant);
}
- taosTFree(pInfo->pDCLInfo);
+ tfree(pInfo->pDCLInfo);
}
}
@@ -872,5 +800,6 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) {
pDBInfo->quorum = -1;
pDBInfo->keep = NULL;
+ pDBInfo->update = -1;
memset(&pDBInfo->precision, 0, sizeof(SStrToken));
}
diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c
index ab9ffb7bcb458129b7f170e7020cee904d2dfda6..3bdc0d477f777faa71a8e5d2264021932dddeea4 100644
--- a/src/query/src/qPercentile.c
+++ b/src/query/src/qPercentile.c
@@ -361,8 +361,8 @@ void tMemBucketDestroy(tMemBucket *pBucket) {
}
destroyResultBuf(pBucket->pBuffer);
- taosTFree(pBucket->pSlots);
- taosTFree(pBucket);
+ tfree(pBucket->pSlots);
+ tfree(pBucket);
}
void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) {
@@ -680,7 +680,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
}
double val = (1 - fraction) * td + fraction * nd;
- taosTFree(buffer);
+ tfree(buffer);
return val;
} else { // incur a second round bucket split
diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c
index b3e97459d36c68d48c04447b159473bd179a5dbb..2645cff678b7fe91f62dda0f3fe65f3a968f9c11 100644
--- a/src/query/src/qResultbuf.c
+++ b/src/query/src/qResultbuf.c
@@ -267,7 +267,7 @@ static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) {
assert(d->pn == pn);
d->pn = NULL;
- taosTFree(pn);
+ tfree(pn);
bufPage = flushPageToDisk(pResultBuf, d);
}
@@ -407,18 +407,18 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
}
if (pResultBuf->file != NULL) {
- qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, inmem size:%dbytes, file size:%"PRId64" bytes",
- pResultBuf->handle, pResultBuf->totalBufSize, listNEles(pResultBuf->lruList) * pResultBuf->pageSize,
- pResultBuf->fileSize);
+ qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f",
+ pResultBuf->handle, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0,
+ pResultBuf->fileSize/1024.0);
fclose(pResultBuf->file);
} else {
- qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle,
- pResultBuf->totalBufSize);
+ qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, no file created", pResultBuf->handle,
+ pResultBuf->totalBufSize/1024.0);
}
unlink(pResultBuf->path);
- taosTFree(pResultBuf->path);
+ tfree(pResultBuf->path);
SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet);
while(taosHashIterNext(iter)) {
@@ -426,8 +426,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
size_t n = taosArrayGetSize(*p);
for(int32_t i = 0; i < n; ++i) {
SPageInfo* pi = taosArrayGetP(*p, i);
- taosTFree(pi->pData);
- taosTFree(pi);
+ tfree(pi->pData);
+ tfree(pi);
}
taosArrayDestroy(*p);
@@ -440,8 +440,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
taosHashCleanup(pResultBuf->groupSet);
taosHashCleanup(pResultBuf->all);
- taosTFree(pResultBuf->assistBuf);
- taosTFree(pResultBuf);
+ tfree(pResultBuf->assistBuf);
+ tfree(pResultBuf);
}
SPageInfo* getLastPageInfo(SIDList pList) {
diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c
index 0c9f92786fdf837e2453fe70f594c10e7a093421..98545c8ef3930587b6f7bd2c5b8d40ded0066d52 100644
--- a/src/query/src/qTokenizer.c
+++ b/src/query/src/qTokenizer.c
@@ -155,6 +155,7 @@ static SKeyword keywordTable[] = {
{"INSERT", TK_INSERT},
{"INTO", TK_INTO},
{"VALUES", TK_VALUES},
+ {"UPDATE", TK_UPDATE},
{"RESET", TK_RESET},
{"QUERY", TK_QUERY},
{"ADD", TK_ADD},
@@ -251,16 +252,16 @@ static const char isIdChar[] = {
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
};
-static void* KeywordHashTable = NULL;
+static void* keywordHashTable = NULL;
static void doInitKeywordsTable(void) {
int numOfEntries = tListLen(keywordTable);
- KeywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false);
+ keywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false);
for (int32_t i = 0; i < numOfEntries; i++) {
keywordTable[i].len = (uint8_t)strlen(keywordTable[i].name);
void* ptr = &keywordTable[i];
- taosHashPut(KeywordHashTable, keywordTable[i].name, keywordTable[i].len, (void*)&ptr, POINTER_BYTES);
+ taosHashPut(keywordHashTable, keywordTable[i].name, keywordTable[i].len, (void*)&ptr, POINTER_BYTES);
}
}
@@ -282,7 +283,7 @@ int tSQLKeywordCode(const char* z, int n) {
}
}
- SKeyword** pKey = (SKeyword**)taosHashGet(KeywordHashTable, key, n);
+ SKeyword** pKey = (SKeyword**)taosHashGet(keywordHashTable, key, n);
return (pKey != NULL)? (*pKey)->type:TK_ID;
}
@@ -660,5 +661,8 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn
bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); }
void taosCleanupKeywordsTable() {
- taosHashCleanup(KeywordHashTable);
-}
\ No newline at end of file
+ void* m = keywordHashTable;
+ if (m != NULL && atomic_val_compare_exchange_ptr(&keywordHashTable, m, 0) == m) {
+ taosHashCleanup(m);
+ }
+}
diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c
index ad29cef5c290f4a6bcd45f8a79828b0c14727dc0..8a5a87baab5abbfd04f1c97dbe91a7226606263e 100644
--- a/src/query/src/qTsbuf.c
+++ b/src/query/src/qTsbuf.c
@@ -4,7 +4,7 @@
#include "tutil.h"
static int32_t getDataStartOffset();
-static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo);
+static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo);
static STSBuf* allocResForTSBuf(STSBuf* pTSBuf);
static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader);
@@ -32,7 +32,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
}
// update the header info
- STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = TSDB_ORDER_ASC};
+ STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = TSDB_ORDER_ASC};
STSBufUpdateHeader(pTSBuf, &header);
tsBufResetPos(pTSBuf);
@@ -75,9 +75,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
return NULL;
}
- if (header.numOfVnode > pTSBuf->numOfAlloc) {
- pTSBuf->numOfAlloc = header.numOfVnode;
- STSVnodeBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * pTSBuf->numOfAlloc);
+ if (header.numOfGroup > pTSBuf->numOfAlloc) {
+ pTSBuf->numOfAlloc = header.numOfGroup;
+ STSGroupBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc);
if (tmp == NULL) {
tsBufDestroy(pTSBuf);
return NULL;
@@ -86,7 +86,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
pTSBuf->pData = tmp;
}
- pTSBuf->numOfVnodes = header.numOfVnode;
+ pTSBuf->numOfGroups = header.numOfGroup;
// check the ts order
pTSBuf->tsOrder = header.tsOrder;
@@ -96,9 +96,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
return NULL;
}
- size_t infoSize = sizeof(STSVnodeBlockInfo) * pTSBuf->numOfVnodes;
+ size_t infoSize = sizeof(STSGroupBlockInfo) * pTSBuf->numOfGroups;
- STSVnodeBlockInfo* buf = (STSVnodeBlockInfo*)calloc(1, infoSize);
+ STSGroupBlockInfo* buf = (STSGroupBlockInfo*)calloc(1, infoSize);
if (buf == NULL) {
tsBufDestroy(pTSBuf);
return NULL;
@@ -109,9 +109,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
UNUSED(sz);
// the length value for each vnode is not kept in file, so does not set the length value
- for (int32_t i = 0; i < pTSBuf->numOfVnodes; ++i) {
- STSVnodeBlockInfoEx* pBlockList = &pTSBuf->pData[i];
- memcpy(&pBlockList->info, &buf[i], sizeof(STSVnodeBlockInfo));
+ for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) {
+ STSGroupBlockInfoEx* pBlockList = &pTSBuf->pData[i];
+ memcpy(&pBlockList->info, &buf[i], sizeof(STSGroupBlockInfo));
}
free(buf);
@@ -131,8 +131,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
pTSBuf->cur.order = TSDB_ORDER_ASC;
pTSBuf->autoDelete = autoDelete;
-// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfVnode:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f),
-// pTSBuf->fileSize, pTSBuf->numOfVnodes, pTSBuf->autoDelete);
+// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f),
+// pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete);
return pTSBuf;
}
@@ -142,11 +142,11 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
return NULL;
}
- taosTFree(pTSBuf->assistBuf);
- taosTFree(pTSBuf->tsData.rawBuf);
+ tfree(pTSBuf->assistBuf);
+ tfree(pTSBuf->tsData.rawBuf);
- taosTFree(pTSBuf->pData);
- taosTFree(pTSBuf->block.payload);
+ tfree(pTSBuf->pData);
+ tfree(pTSBuf->block.payload);
fclose(pTSBuf->f);
@@ -156,58 +156,59 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
} else {
// tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path);
}
-
+
+ tVariantDestroy(&pTSBuf->block.tag);
free(pTSBuf);
return NULL;
}
-static STSVnodeBlockInfoEx* tsBufGetLastVnodeInfo(STSBuf* pTSBuf) {
- int32_t last = pTSBuf->numOfVnodes - 1;
+static STSGroupBlockInfoEx* tsBufGetLastGroupInfo(STSBuf* pTSBuf) {
+ int32_t last = pTSBuf->numOfGroups - 1;
assert(last >= 0);
return &pTSBuf->pData[last];
}
-static STSVnodeBlockInfoEx* addOneVnodeInfo(STSBuf* pTSBuf, int32_t vnodeId) {
- if (pTSBuf->numOfAlloc <= pTSBuf->numOfVnodes) {
+static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) {
+ if (pTSBuf->numOfAlloc <= pTSBuf->numOfGroups) {
uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5);
assert((int32_t)newSize > pTSBuf->numOfAlloc);
- STSVnodeBlockInfoEx* tmp = (STSVnodeBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * newSize);
+ STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize);
if (tmp == NULL) {
return NULL;
}
pTSBuf->pData = tmp;
pTSBuf->numOfAlloc = newSize;
- memset(&pTSBuf->pData[pTSBuf->numOfVnodes], 0, sizeof(STSVnodeBlockInfoEx) * (newSize - pTSBuf->numOfVnodes));
+ memset(&pTSBuf->pData[pTSBuf->numOfGroups], 0, sizeof(STSGroupBlockInfoEx) * (newSize - pTSBuf->numOfGroups));
}
- if (pTSBuf->numOfVnodes > 0) {
- STSVnodeBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastVnodeInfo(pTSBuf);
+ if (pTSBuf->numOfGroups > 0) {
+ STSGroupBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf);
// update prev vnode length info in file
- TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, &pPrevBlockInfoEx->info);
+ TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pPrevBlockInfoEx->info);
}
// set initial value for vnode block
- STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfVnodes].info;
- pBlockInfo->vnode = vnodeId;
+ STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfGroups].info;
+ pBlockInfo->id = id;
pBlockInfo->offset = pTSBuf->fileSize;
assert(pBlockInfo->offset >= getDataStartOffset());
// update vnode info in file
- TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes, pBlockInfo);
+ TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups, pBlockInfo);
// add one vnode info
- pTSBuf->numOfVnodes += 1;
+ pTSBuf->numOfGroups += 1;
// update the header info
STSBufFileHeader header = {
- .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder};
+ .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder};
STSBufUpdateHeader(pTSBuf, &header);
- return tsBufGetLastVnodeInfo(pTSBuf);
+ return tsBufGetLastGroupInfo(pTSBuf);
}
static void shrinkBuffer(STSList* ptsData) {
@@ -218,6 +219,15 @@ static void shrinkBuffer(STSList* ptsData) {
}
}
+static int32_t getTagAreaLength(tVariant* pa) {
+ int32_t t = sizeof(pa->nLen) * 2 + sizeof(pa->nType);
+ if (pa->nType != TSDB_DATA_TYPE_NULL) {
+ t += pa->nLen;
+ }
+
+ return t;
+}
+
static void writeDataToDisk(STSBuf* pTSBuf) {
if (pTSBuf->tsData.len == 0) {
return;
@@ -243,28 +253,36 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
*/
int32_t metaLen = 0;
metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f);
- metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+ int32_t trueLen = pBlock->tag.nLen;
if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) {
+ metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
- metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f);
+ metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+ metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
+ } else {
+ trueLen = 0;
+ metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
}
fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
-
+
+ metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+ assert(metaLen == getTagAreaLength(&pBlock->tag));
+
int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen;
pTSBuf->fileSize += blockSize;
pTSBuf->tsData.len = 0;
- STSVnodeBlockInfoEx* pVnodeBlockInfoEx = tsBufGetLastVnodeInfo(pTSBuf);
+ STSGroupBlockInfoEx* pGroupBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf);
- pVnodeBlockInfoEx->info.compLen += blockSize;
- pVnodeBlockInfoEx->info.numOfBlocks += 1;
+ pGroupBlockInfoEx->info.compLen += blockSize;
+ pGroupBlockInfoEx->info.numOfBlocks += 1;
shrinkBuffer(&pTSBuf->tsData);
}
@@ -284,23 +302,28 @@ static void expandBuffer(STSList* ptsData, int32_t inputSize) {
STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
STSBlock* pBlock = &pTSBuf->block;
-
+
// clear the memory buffer
- void* tmp = pBlock->payload;
- memset(pBlock, 0, sizeof(STSBlock));
- pBlock->payload = tmp;
-
+ pBlock->compLen = 0;
+ pBlock->padding = 0;
+ pBlock->numOfElem = 0;
+
+ int32_t offset = -1;
+
if (order == TSDB_ORDER_DESC) {
/*
* set the right position for the reversed traverse, the reversed traverse is started from
* the end of each comp data block
*/
- int32_t ret = fseek(pTSBuf->f, -(int32_t)(sizeof(pBlock->padding)), SEEK_CUR);
- size_t sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
+ int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen));
+ int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR);
+ size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f);
+ sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
UNUSED(sz);
-
+
pBlock->compLen = pBlock->padding;
- int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
+
+ offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
ret = fseek(pTSBuf->f, -offset, SEEK_CUR);
UNUSED(ret);
}
@@ -319,7 +342,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
- sz = fread(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f);
+ sz = fread(&pBlock->tag.i64Key, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
}
sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
@@ -327,8 +350,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
UNUSED(sz);
sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
- UNUSED(sz);
-
+
if (decomp) {
pTSBuf->tsData.len =
tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf,
@@ -337,11 +359,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
// read the comp length at the length of comp block
sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
+ assert(pBlock->padding == pBlock->compLen);
+
+ int32_t n = 0;
+ sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
+ if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) {
+ assert(n == 0);
+ } else {
+ assert(n == pBlock->tag.nLen);
+ }
+
UNUSED(sz);
// for backwards traverse, set the start position at the end of previous block
if (order == TSDB_ORDER_DESC) {
- int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR);
UNUSED(r);
}
@@ -382,20 +413,20 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) {
return TSDB_CODE_SUCCESS;
}
-void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len) {
- STSVnodeBlockInfoEx* pBlockInfo = NULL;
+void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len) {
+ STSGroupBlockInfoEx* pBlockInfo = NULL;
STSList* ptsData = &pTSBuf->tsData;
- if (pTSBuf->numOfVnodes == 0 || tsBufGetLastVnodeInfo(pTSBuf)->info.vnode != vnodeId) {
+ if (pTSBuf->numOfGroups == 0 || tsBufGetLastGroupInfo(pTSBuf)->info.id != id) {
writeDataToDisk(pTSBuf);
shrinkBuffer(ptsData);
- pBlockInfo = addOneVnodeInfo(pTSBuf, vnodeId);
+ pBlockInfo = addOneGroupInfo(pTSBuf, id);
} else {
- pBlockInfo = tsBufGetLastVnodeInfo(pTSBuf);
+ pBlockInfo = tsBufGetLastGroupInfo(pTSBuf);
}
- assert(pBlockInfo->info.vnode == vnodeId);
+ assert(pBlockInfo->info.id == id);
if ((tVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) {
// new arrived data with different tags value, save current value into disk first
@@ -433,23 +464,23 @@ void tsBufFlush(STSBuf* pTSBuf) {
writeDataToDisk(pTSBuf);
shrinkBuffer(&pTSBuf->tsData);
- STSVnodeBlockInfoEx* pBlockInfoEx = tsBufGetLastVnodeInfo(pTSBuf);
+ STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf);
// update prev vnode length info in file
- TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, &pBlockInfoEx->info);
+ TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pBlockInfoEx->info);
// save the ts order into header
STSBufFileHeader header = {
- .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder};
+ .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder};
STSBufUpdateHeader(pTSBuf, &header);
fsync(fileno(pTSBuf->f));
}
-static int32_t tsBufFindVnodeIndexFromId(STSVnodeBlockInfoEx* pVnodeInfoEx, int32_t numOfVnodes, int32_t vnodeId) {
+static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t numOfGroups, int32_t id) {
int32_t j = -1;
- for (int32_t i = 0; i < numOfVnodes; ++i) {
- if (pVnodeInfoEx[i].info.vnode == vnodeId) {
+ for (int32_t i = 0; i < numOfGroups; ++i) {
+ if (pGroupInfoEx[i].info.id == id) {
j = i;
break;
}
@@ -459,7 +490,7 @@ static int32_t tsBufFindVnodeIndexFromId(STSVnodeBlockInfoEx* pVnodeInfoEx, int3
}
// todo opt performance by cache blocks info
-static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int32_t blockIndex) {
+static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) {
if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
return -1;
}
@@ -478,7 +509,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int
if (pTSBuf->cur.order == TSDB_ORDER_DESC) {
STSBlock* pBlock = &pTSBuf->block;
int32_t compBlockSize =
- pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
+ pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR);
UNUSED(ret);
}
@@ -486,7 +517,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int
return 0;
}
-static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, tVariant* tag) {
+static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, tVariant* tag) {
bool decomp = false;
int64_t offset = 0;
@@ -506,21 +537,21 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo
}
if (tVariantCompare(&pTSBuf->block.tag, tag) == 0) {
- return i;
+ return (pTSBuf->cur.order == TSDB_ORDER_ASC)? i: (pBlockInfo->numOfBlocks - (i + 1));
}
}
return -1;
}
-static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex) {
- STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[vnodeIndex].info;
+static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex) {
+ STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info;
if (pBlockInfo->numOfBlocks <= blockIndex) {
assert(false);
}
STSCursor* pCur = &pTSBuf->cur;
- if (pCur->vgroupIndex == vnodeIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) ||
+ if (pCur->vgroupIndex == groupIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) ||
(pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) {
int32_t i = 0;
bool decomp = false;
@@ -555,13 +586,13 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex
assert((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len));
- pCur->vgroupIndex = vnodeIndex;
+ pCur->vgroupIndex = groupIndex;
pCur->blockIndex = blockIndex;
pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1;
}
-static int32_t doUpdateVnodeInfo(STSBuf* pTSBuf, int64_t offset, STSVnodeBlockInfo* pVInfo) {
+static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockInfo* pVInfo) {
if (offset < 0 || offset >= getDataStartOffset()) {
return -1;
}
@@ -570,12 +601,12 @@ static int32_t doUpdateVnodeInfo(STSBuf* pTSBuf, int64_t offset, STSVnodeBlockIn
return -1;
}
- fwrite(pVInfo, sizeof(STSVnodeBlockInfo), 1, pTSBuf->f);
+ fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f);
return 0;
}
-STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) {
- int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId);
+STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) {
+ int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id);
if (j == -1) {
return NULL;
}
@@ -584,7 +615,7 @@ STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) {
}
int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
- if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfVnode == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
+ if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
return -1;
}
@@ -600,7 +631,7 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
}
bool tsBufNextPos(STSBuf* pTSBuf) {
- if (pTSBuf == NULL || pTSBuf->numOfVnodes == 0) {
+ if (pTSBuf == NULL || pTSBuf->numOfGroups == 0) {
return false;
}
@@ -619,16 +650,16 @@ bool tsBufNextPos(STSBuf* pTSBuf) {
}
} else { // get the last timestamp record in the last block of the last vnode
- assert(pTSBuf->numOfVnodes > 0);
+ assert(pTSBuf->numOfGroups > 0);
- int32_t vnodeIndex = pTSBuf->numOfVnodes - 1;
- pCur->vgroupIndex = vnodeIndex;
+ int32_t groupIndex = pTSBuf->numOfGroups - 1;
+ pCur->vgroupIndex = groupIndex;
- int32_t vnodeId = pTSBuf->pData[pCur->vgroupIndex].info.vnode;
- STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId);
+ int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id;
+ STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id);
int32_t blockIndex = pBlockInfo->numOfBlocks - 1;
- tsBufGetBlock(pTSBuf, vnodeIndex, blockIndex);
+ tsBufGetBlock(pTSBuf, groupIndex, blockIndex);
pCur->tsIndex = pTSBuf->block.numOfElem - 1;
if (pTSBuf->block.numOfElem == 0) {
@@ -647,12 +678,12 @@ bool tsBufNextPos(STSBuf* pTSBuf) {
if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) ||
(pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) {
- int32_t vnodeId = pTSBuf->pData[pCur->vgroupIndex].info.vnode;
+ int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id;
- STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId);
+ STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id);
if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) ||
(pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) {
- if ((pCur->vgroupIndex >= pTSBuf->numOfVnodes - 1 && pCur->order == TSDB_ORDER_ASC) ||
+ if ((pCur->vgroupIndex >= pTSBuf->numOfGroups - 1 && pCur->order == TSDB_ORDER_ASC) ||
(pCur->vgroupIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) {
pCur->vgroupIndex = -1;
return false;
@@ -688,7 +719,7 @@ void tsBufResetPos(STSBuf* pTSBuf) {
}
STSElem tsBufGetElem(STSBuf* pTSBuf) {
- STSElem elem1 = {.vnode = -1};
+ STSElem elem1 = {.id = -1};
if (pTSBuf == NULL) {
return elem1;
}
@@ -700,7 +731,7 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) {
STSBlock* pBlock = &pTSBuf->block;
- elem1.vnode = pTSBuf->pData[pCur->vgroupIndex].info.vnode;
+ elem1.id = pTSBuf->pData[pCur->vgroupIndex].info.id;
elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE);
elem1.tag = &pBlock->tag;
@@ -711,34 +742,34 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) {
* current only support ts comp data from two vnode merge
* @param pDestBuf
* @param pSrcBuf
- * @param vnodeId
+ * @param id
* @return
*/
int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
- if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfVnodes <= 0) {
+ if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfGroups <= 0) {
return 0;
}
- if (pDestBuf->numOfVnodes + pSrcBuf->numOfVnodes > TS_COMP_FILE_VNODE_MAX) {
+ if (pDestBuf->numOfGroups + pSrcBuf->numOfGroups > TS_COMP_FILE_GROUP_MAX) {
return -1;
}
// src can only have one vnode index
- assert(pSrcBuf->numOfVnodes == 1);
+ assert(pSrcBuf->numOfGroups == 1);
// there are data in buffer, flush to disk first
tsBufFlush(pDestBuf);
// compared with the last vnode id
- int32_t vnodeId = tsBufGetLastVnodeInfo((STSBuf*) pSrcBuf)->info.vnode;
- if (vnodeId != tsBufGetLastVnodeInfo(pDestBuf)->info.vnode) {
- int32_t oldSize = pDestBuf->numOfVnodes;
- int32_t newSize = oldSize + pSrcBuf->numOfVnodes;
+ int32_t id = tsBufGetLastGroupInfo((STSBuf*) pSrcBuf)->info.id;
+ if (id != tsBufGetLastGroupInfo(pDestBuf)->info.id) {
+ int32_t oldSize = pDestBuf->numOfGroups;
+ int32_t newSize = oldSize + pSrcBuf->numOfGroups;
if (pDestBuf->numOfAlloc < newSize) {
pDestBuf->numOfAlloc = newSize;
- STSVnodeBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSVnodeBlockInfoEx) * newSize);
+ STSGroupBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize);
if (tmp == NULL) {
return -1;
}
@@ -747,23 +778,23 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
}
// directly copy the vnode index information
- memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfVnodes * sizeof(STSVnodeBlockInfoEx));
+ memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfGroups * sizeof(STSGroupBlockInfoEx));
// set the new offset value
- for (int32_t i = 0; i < pSrcBuf->numOfVnodes; ++i) {
- STSVnodeBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize];
+ for (int32_t i = 0; i < pSrcBuf->numOfGroups; ++i) {
+ STSGroupBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize];
pBlockInfoEx->info.offset = (pSrcBuf->pData[i].info.offset - getDataStartOffset()) + pDestBuf->fileSize;
- pBlockInfoEx->info.vnode = vnodeId;
+ pBlockInfoEx->info.id = id;
}
- pDestBuf->numOfVnodes = newSize;
+ pDestBuf->numOfGroups = newSize;
} else {
- STSVnodeBlockInfoEx* pBlockInfoEx = tsBufGetLastVnodeInfo(pDestBuf);
+ STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pDestBuf);
pBlockInfoEx->len += pSrcBuf->pData[0].len;
pBlockInfoEx->info.numOfBlocks += pSrcBuf->pData[0].info.numOfBlocks;
pBlockInfoEx->info.compLen += pSrcBuf->pData[0].info.compLen;
- pBlockInfoEx->info.vnode = vnodeId;
+ pBlockInfoEx->info.id = id;
}
int32_t r = fseek(pDestBuf->f, 0, SEEK_END);
@@ -772,7 +803,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
int64_t offset = getDataStartOffset();
int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset;
- ssize_t rc = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size);
+ int64_t rc = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size);
if (rc == -1) {
// tscError("failed to merge tsBuf from:%s to %s, reason:%s\n", pSrcBuf->path, pDestBuf->path, strerror(errno));
@@ -796,23 +827,23 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
assert(pDestBuf->fileSize == oldSize + size);
-// tscDebug("tsBuf merge success, %p, path:%s, fd:%d, file size:%d, numOfVnode:%d, autoDelete:%d", pDestBuf,
-// pDestBuf->path, fileno(pDestBuf->f), pDestBuf->fileSize, pDestBuf->numOfVnodes, pDestBuf->autoDelete);
+// tscDebug("tsBuf merge success, %p, path:%s, fd:%d, file size:%d, numOfGroups:%d, autoDelete:%d", pDestBuf,
+// pDestBuf->path, fileno(pDestBuf->f), pDestBuf->fileSize, pDestBuf->numOfGroups, pDestBuf->autoDelete);
return 0;
}
-STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order, int32_t vnodeId) {
+STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order, int32_t id) {
STSBuf* pTSBuf = tsBufCreate(true, order);
- STSVnodeBlockInfo* pBlockInfo = &(addOneVnodeInfo(pTSBuf, 0)->info);
+ STSGroupBlockInfo* pBlockInfo = &(addOneGroupInfo(pTSBuf, 0)->info);
pBlockInfo->numOfBlocks = numOfBlocks;
pBlockInfo->compLen = len;
pBlockInfo->offset = getDataStartOffset();
- pBlockInfo->vnode = vnodeId;
+ pBlockInfo->id = id;
// update prev vnode length info in file
- TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, pBlockInfo);
+ TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo);
int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET);
UNUSED(ret);
@@ -824,7 +855,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);
STSBufFileHeader header = {
- .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder};
+ .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder};
STSBufUpdateHeader(pTSBuf, &header);
fsync(fileno(pTSBuf->f));
@@ -832,14 +863,14 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
return pTSBuf;
}
-STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) {
- STSElem elem = {.vnode = -1};
+STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag) {
+ STSElem elem = {.id = -1};
if (pTSBuf == NULL) {
return elem;
}
- int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId);
+ int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id);
if (j == -1) {
return elem;
}
@@ -848,7 +879,7 @@ STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) {
// tsBufDisplay(pTSBuf);
STSCursor* pCur = &pTSBuf->cur;
- STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[j].info;
+ STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[j].info;
int32_t blockIndex = tsBufFindBlockByTag(pTSBuf, pBlockInfo, tag);
if (blockIndex < 0) {
@@ -904,7 +935,7 @@ STSBuf* tsBufClone(STSBuf* pTSBuf) {
void tsBufDisplay(STSBuf* pTSBuf) {
printf("-------start of ts comp file-------\n");
- printf("number of vnode:%d\n", pTSBuf->numOfVnodes);
+ printf("number of vnode:%d\n", pTSBuf->numOfGroups);
int32_t old = pTSBuf->cur.order;
pTSBuf->cur.order = TSDB_ORDER_ASC;
@@ -914,7 +945,7 @@ void tsBufDisplay(STSBuf* pTSBuf) {
while (tsBufNextPos(pTSBuf)) {
STSElem elem = tsBufGetElem(pTSBuf);
if (elem.tag->nType == TSDB_DATA_TYPE_BIGINT) {
- printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag->i64Key, elem.ts);
+ printf("%d-%" PRId64 "-%" PRId64 "\n", elem.id, elem.tag->i64Key, elem.ts);
}
}
@@ -923,20 +954,20 @@ void tsBufDisplay(STSBuf* pTSBuf) {
}
static int32_t getDataStartOffset() {
- return sizeof(STSBufFileHeader) + TS_COMP_FILE_VNODE_MAX * sizeof(STSVnodeBlockInfo);
+ return sizeof(STSBufFileHeader) + TS_COMP_FILE_GROUP_MAX * sizeof(STSGroupBlockInfo);
}
// update prev vnode length info in file
-static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo) {
- int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSVnodeBlockInfo);
- doUpdateVnodeInfo(pTSBuf, offset, pBlockInfo);
+static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo) {
+ int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSGroupBlockInfo);
+ doUpdateGroupInfo(pTSBuf, offset, pBlockInfo);
}
static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) {
- const int32_t INITIAL_VNODEINFO_SIZE = 4;
+ const int32_t INITIAL_GROUPINFO_SIZE = 4;
- pTSBuf->numOfAlloc = INITIAL_VNODEINFO_SIZE;
- pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSVnodeBlockInfoEx));
+ pTSBuf->numOfAlloc = INITIAL_GROUPINFO_SIZE;
+ pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx));
if (pTSBuf->pData == NULL) {
tsBufDestroy(pTSBuf);
return NULL;
@@ -968,28 +999,71 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) {
return pTSBuf;
}
-int32_t tsBufGetNumOfVnodes(STSBuf* pTSBuf) {
+int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf) {
if (pTSBuf == NULL) {
return 0;
}
- return pTSBuf->numOfVnodes;
+ return pTSBuf->numOfGroups;
}
-void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId) {
- int32_t size = tsBufGetNumOfVnodes(pTSBuf);
+void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) {
+ int32_t size = tsBufGetNumOfGroup(pTSBuf);
if (num != NULL) {
*num = size;
}
- *vnodeId = NULL;
+ *id = NULL;
if (size == 0) {
return;
}
- (*vnodeId) = malloc(tsBufGetNumOfVnodes(pTSBuf) * sizeof(int32_t));
+ (*id) = malloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t));
for(int32_t i = 0; i < size; ++i) {
- (*vnodeId)[i] = pTSBuf->pData[i].info.vnode;
+ (*id)[i] = pTSBuf->pData[i].info.id;
+ }
+}
+
+int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, int32_t* len, int32_t* numOfBlocks) {
+ assert(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups);
+ STSGroupBlockInfo *pBlockInfo = &pTSBuf->pData[groupIndex].info;
+
+ *len = 0;
+ *numOfBlocks = 0;
+
+ if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
+ int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
+// qError("%p: fseek failed: %s", pSql, tstrerror(code));
+ return code;
+ }
+
+ size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f);
+ if (s != pBlockInfo->compLen) {
+ int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
+// tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
+ return code;
+ }
+
+ *len = pBlockInfo->compLen;
+ *numOfBlocks = pBlockInfo->numOfBlocks;
+
+ return TSDB_CODE_SUCCESS;
+}
+
+STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag) {
+ STSElem el = {.id = -1};
+
+ for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) {
+ el = tsBufGetElemStartPos(pTSBuf, pTSBuf->pData[i].info.id, pTag);
+ if (el.id == pTSBuf->pData[i].info.id) {
+ return el;
+ }
}
-}
\ No newline at end of file
+
+ return el;
+}
+
+bool tsBufIsValidElem(STSElem* pElem) {
+ return pElem->id >= 0;
+}
diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c
index 2bd92c74a456c16fe946288968d25d797907a390..f4f89a8709a16d0d84ff8333cc6ef3d4463b6c64 100644
--- a/src/query/src/qUtil.c
+++ b/src/query/src/qUtil.c
@@ -14,8 +14,9 @@
*/
#include "os.h"
-#include "hash.h"
#include "taosmsg.h"
+#include "hash.h"
+
#include "qExecutor.h"
#include "qUtil.h"
@@ -23,78 +24,40 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) {
int32_t size = 0;
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- size += pQuery->pSelectExpr[i].interBytes;
+ size += pQuery->pExpr1[i].interBytes;
}
- assert(size > 0);
+ assert(size >= 0);
return size;
}
-int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t size,
- int32_t threshold, int16_t type) {
+int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int32_t threshold, int16_t type) {
pWindowResInfo->capacity = size;
pWindowResInfo->threshold = threshold;
pWindowResInfo->type = type;
- _hash_fn_t fn = taosGetDefaultHashFunction(type);
- pWindowResInfo->hashList = taosHashInit(threshold, fn, true, false);
- if (pWindowResInfo->hashList == NULL) {
- return TSDB_CODE_QRY_OUT_OF_MEMORY;
- }
-
pWindowResInfo->curIndex = -1;
pWindowResInfo->size = 0;
pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
- SQueryCostInfo* pSummary = &pRuntimeEnv->summary;
-
- // use the pointer arraylist
- pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult));
+ pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, POINTER_BYTES);
if (pWindowResInfo->pResult == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
- pWindowResInfo->interval = pRuntimeEnv->pQuery->interval.interval;
-
- pSummary->internalSupSize += sizeof(SWindowResult) * threshold;
- pSummary->internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity;
- pSummary->numOfTimeWindows = threshold;
-
- for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
- int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
- }
-
return TSDB_CODE_SUCCESS;
}
-void destroyTimeWindowRes(SWindowResult *pWindowRes) {
- if (pWindowRes == NULL) {
- return;
- }
-
- free(pWindowRes->resultInfo);
-}
-
void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) {
if (pWindowResInfo == NULL) {
return;
}
if (pWindowResInfo->capacity == 0) {
- assert(pWindowResInfo->hashList == NULL && pWindowResInfo->pResult == NULL);
+ assert(/*pWindowResInfo->hashList == NULL && */pWindowResInfo->pResult == NULL);
return;
}
- if (pWindowResInfo->pResult != NULL) {
- for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
- destroyTimeWindowRes(&pWindowResInfo->pResult[i]);
- }
- }
-
- taosHashCleanup(pWindowResInfo->hashList);
- taosTFree(pWindowResInfo->pResult);
+ tfree(pWindowResInfo->pResult);
}
void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo) {
@@ -103,17 +66,13 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR
}
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
- SWindowResult *pWindowRes = &pWindowResInfo->pResult[i];
- clearTimeWindowResBuf(pRuntimeEnv, pWindowRes);
+ SResultRow *pWindowRes = pWindowResInfo->pResult[i];
+ clearResultRow(pRuntimeEnv, pWindowRes);
}
pWindowResInfo->curIndex = -1;
- taosHashCleanup(pWindowResInfo->hashList);
pWindowResInfo->size = 0;
- _hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type);
- pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, true, false);
-
pWindowResInfo->startTime = TSKEY_INITIAL_VAL;
pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
}
@@ -127,13 +86,13 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo);
assert(num >= 0 && num <= numOfClosed);
- int16_t type = pWindowResInfo->type;
-
- char *key = NULL;
- int16_t bytes = -1;
+ int16_t type = pWindowResInfo->type;
+ STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); // uid is always set to be 0.
+ char *key = NULL;
+ int16_t bytes = -1;
for (int32_t i = 0; i < num; ++i) {
- SWindowResult *pResult = &pWindowResInfo->pResult[i];
+ SResultRow *pResult = pWindowResInfo->pResult[i];
if (pResult->closed) { // remove the window slot from hash table
// todo refactor
@@ -145,7 +104,8 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
}
- taosHashRemove(pWindowResInfo->hashList, (const char *)key, bytes);
+ SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
+ taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
} else {
break;
}
@@ -155,19 +115,19 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
// clear all the closed windows from the window list
for (int32_t k = 0; k < remain; ++k) {
- copyTimeWindowResBuf(pRuntimeEnv, &pWindowResInfo->pResult[k], &pWindowResInfo->pResult[num + k]);
+ copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k]);
}
// move the unclosed window in the front of the window list
for (int32_t k = remain; k < pWindowResInfo->size; ++k) {
- SWindowResult *pWindowRes = &pWindowResInfo->pResult[k];
- clearTimeWindowResBuf(pRuntimeEnv, pWindowRes);
+ SResultRow *pWindowRes = pWindowResInfo->pResult[k];
+ clearResultRow(pRuntimeEnv, pWindowRes);
}
pWindowResInfo->size = remain;
for (int32_t k = 0; k < pWindowResInfo->size; ++k) {
- SWindowResult *pResult = &pWindowResInfo->pResult[k];
+ SResultRow *pResult = pWindowResInfo->pResult[k];
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
key = varDataVal(pResult->key);
@@ -177,12 +137,15 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
}
- int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)key, bytes);
+ SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
+ int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
assert(p != NULL);
int32_t v = (*p - num);
assert(v >= 0 && v <= pWindowResInfo->size);
- taosHashPut(pWindowResInfo->hashList, (char *)key, bytes, (char *)&v, sizeof(int32_t));
+
+ SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
+ taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t));
}
pWindowResInfo->curIndex = -1;
@@ -200,7 +163,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) {
int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) {
int32_t i = 0;
- while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].closed) {
+ while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) {
++i;
}
@@ -211,11 +174,11 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) {
assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size);
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
- if (pWindowResInfo->pResult[i].closed) {
+ if (pWindowResInfo->pResult[i]->closed) {
continue;
}
- pWindowResInfo->pResult[i].closed = true;
+ pWindowResInfo->pResult[i]->closed = true;
}
}
@@ -231,19 +194,19 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_
}
// get the result order
- int32_t resultOrder = (pWindowResInfo->pResult[0].win.skey < pWindowResInfo->pResult[1].win.skey)? 1:-1;
+ int32_t resultOrder = (pWindowResInfo->pResult[0]->win.skey < pWindowResInfo->pResult[1]->win.skey)? 1:-1;
if (order != resultOrder) {
return;
}
int32_t i = 0;
if (order == QUERY_ASC_FORWARD_STEP) {
- TSKEY ekey = pWindowResInfo->pResult[i].win.ekey;
+ TSKEY ekey = pWindowResInfo->pResult[i]->win.ekey;
while (i < pWindowResInfo->size && (ekey < lastKey)) {
++i;
}
} else if (order == QUERY_DESC_FORWARD_STEP) {
- while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].win.skey > lastKey)) {
+ while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i]->win.skey > lastKey)) {
++i;
}
}
@@ -254,32 +217,36 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_
}
bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) {
- return (getWindowResult(pWindowResInfo, slot)->closed == true);
+ return (getResultRow(pWindowResInfo, slot)->closed == true);
}
void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) {
- getWindowResult(pWindowResInfo, slot)->closed = true;
+ getResultRow(pWindowResInfo, slot)->closed = true;
}
-void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) {
+void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) {
if (pWindowRes == NULL) {
return;
}
- tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
+ // the result does not put into the SDiskbasedResultBuf, ignore it.
+ if (pWindowRes->pageId >= 0) {
+ tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId);
- for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) {
- SResultInfo *pResultInfo = &pWindowRes->resultInfo[i];
-
- char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
- size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
- memset(s, 0, size);
-
- RESET_RESULT_INFO(pResultInfo);
+ for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) {
+ SResultRowCellInfo *pResultInfo = &pWindowRes->pCellInfo[i];
+
+ char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
+ size_t size = pRuntimeEnv->pQuery->pExpr1[i].bytes;
+ memset(s, 0, size);
+
+ RESET_RESULT_INFO(pResultInfo);
+ }
}
-
+
pWindowRes->numOfRows = 0;
- pWindowRes->pos = (SPosInfo){-1, -1};
+ pWindowRes->pageId = -1;
+ pWindowRes->rowId = -1;
pWindowRes->closed = false;
pWindowRes->win = TSWINDOW_INITIALIZER;
}
@@ -289,7 +256,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow
* since the attribute of "Pos" is bound to each window result when the window result is created in the
* disk-based result buffer.
*/
-void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) {
+void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src) {
dst->numOfRows = src->numOfRows;
dst->win = src->win;
dst->closed = src->closed;
@@ -297,25 +264,105 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con
int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput;
for (int32_t i = 0; i < nOutputCols; ++i) {
- SResultInfo *pDst = &dst->resultInfo[i];
- SResultInfo *pSrc = &src->resultInfo[i];
+ SResultRowCellInfo *pDst = getResultCell(pRuntimeEnv, dst, i);
+ SResultRowCellInfo *pSrc = getResultCell(pRuntimeEnv, src, i);
- char *buf = pDst->interResultBuf;
- memcpy(pDst, pSrc, sizeof(SResultInfo));
- pDst->interResultBuf = buf; // restore the allocated buffer
+// char *buf = pDst->interResultBuf;
+ memcpy(pDst, pSrc, sizeof(SResultRowCellInfo) + pRuntimeEnv->pCtx[i].interBufBytes);
+// pDst->interResultBuf = buf; // restore the allocated buffer
// copy the result info struct
- memcpy(pDst->interResultBuf, pSrc->interResultBuf, pDst->bufLen);
+// memcpy(pDst->interResultBuf, pSrc->interResultBuf, pRuntimeEnv->pCtx[i].interBufBytes);
// copy the output buffer data from src to dst, the position info keep unchanged
- tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pos.pageId);
+ tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pageId);
char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage);
- tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pos.pageId);
- char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src, srcpage);
- size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
+ tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pageId);
+ char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SResultRow *)src, srcpage);
+ size_t s = pRuntimeEnv->pQuery->pExpr1[i].bytes;
memcpy(dstBuf, srcBuf, s);
}
}
+SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index) {
+ assert(index >= 0 && index < pRuntimeEnv->pQuery->numOfOutput);
+ return (SResultRowCellInfo*)((char*) pRow->pCellInfo + pRuntimeEnv->rowCellInfoOffset[index]);
+}
+
+size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv) {
+ return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pRuntimeEnv->interBufSize + sizeof(SResultRow);
+}
+
+SResultRowPool* initResultRowPool(size_t size) {
+ SResultRowPool* p = calloc(1, sizeof(SResultRowPool));
+ if (p == NULL) {
+ return NULL;
+ }
+
+ p->numOfElemPerBlock = 128;
+
+ p->elemSize = (int32_t) size;
+ p->blockSize = p->numOfElemPerBlock * p->elemSize;
+ p->position.pos = 0;
+
+ p->pData = taosArrayInit(8, POINTER_BYTES);
+ return p;
+}
+
+SResultRow* getNewResultRow(SResultRowPool* p) {
+ if (p == NULL) {
+ return NULL;
+ }
+
+ void* ptr = NULL;
+ if (p->position.pos == 0) {
+ ptr = calloc(1, p->blockSize);
+ taosArrayPush(p->pData, &ptr);
+
+ } else {
+ size_t last = taosArrayGetSize(p->pData);
+
+ void** pBlock = taosArrayGet(p->pData, last - 1);
+ ptr = ((char*) (*pBlock)) + p->elemSize * p->position.pos;
+ }
+
+ p->position.pos = (p->position.pos + 1)%p->numOfElemPerBlock;
+ initResultRow(ptr);
+
+ return ptr;
+}
+
+int64_t getResultRowPoolMemSize(SResultRowPool* p) {
+ if (p == NULL) {
+ return 0;
+ }
+
+ return taosArrayGetSize(p->pData) * p->blockSize;
+}
+
+int32_t getNumOfAllocatedResultRows(SResultRowPool* p) {
+ return (int32_t) taosArrayGetSize(p->pData) * p->numOfElemPerBlock;
+}
+
+int32_t getNumOfUsedResultRows(SResultRowPool* p) {
+ return getNumOfAllocatedResultRows(p) - p->numOfElemPerBlock + p->position.pos;
+}
+
+void* destroyResultRowPool(SResultRowPool* p) {
+ if (p == NULL) {
+ return NULL;
+ }
+
+ size_t size = taosArrayGetSize(p->pData);
+ for(int32_t i = 0; i < size; ++i) {
+ void** ptr = taosArrayGet(p->pData, i);
+ tfree(*ptr);
+ }
+
+ taosArrayDestroy(p->pData);
+
+ tfree(p);
+ return NULL;
+}
diff --git a/src/query/src/sql.c b/src/query/src/sql.c
index a18efdeb744ba039c685d5fc1067c95e3ca15d86..da2c56ee9e4ec3b4bb1ea9c323bdd4da39821048 100644
--- a/src/query/src/sql.c
+++ b/src/query/src/sql.c
@@ -97,27 +97,26 @@
#endif
/************* Begin control #defines *****************************************/
#define YYCODETYPE unsigned short int
-#define YYNOCODE 272
+#define YYNOCODE 274
#define YYACTIONTYPE unsigned short int
#define ParseTOKENTYPE SStrToken
typedef union {
int yyinit;
ParseTOKENTYPE yy0;
- SSubclauseInfo* yy25;
- tSQLExpr* yy66;
- SCreateAcctSQL yy73;
- int yy82;
- SQuerySQL* yy150;
- SCreateDBInfo yy158;
- TAOS_FIELD yy181;
- SLimitVal yy188;
- tSQLExprList* yy224;
- int64_t yy271;
- tVariant yy312;
- SIntervalVal yy314;
- SCreateTableSQL* yy374;
- tFieldList* yy449;
- tVariantList* yy494;
+ int yy46;
+ tSQLExpr* yy64;
+ tVariant yy134;
+ SCreateAcctSQL yy149;
+ SArray* yy165;
+ int64_t yy207;
+ SLimitVal yy216;
+ TAOS_FIELD yy223;
+ SSubclauseInfo* yy231;
+ SCreateDBInfo yy268;
+ tSQLExprList* yy290;
+ SQuerySQL* yy414;
+ SCreateTableSQL* yy470;
+ SIntervalVal yy532;
} YYMINORTYPE;
#ifndef YYSTACKDEPTH
#define YYSTACKDEPTH 100
@@ -127,17 +126,17 @@ typedef union {
#define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo
#define ParseARG_STORE yypParser->pInfo = pInfo
#define YYFALLBACK 1
-#define YYNSTATE 252
-#define YYNRULE 230
-#define YYNTOKEN 206
-#define YY_MAX_SHIFT 251
-#define YY_MIN_SHIFTREDUCE 416
-#define YY_MAX_SHIFTREDUCE 645
-#define YY_ERROR_ACTION 646
-#define YY_ACCEPT_ACTION 647
-#define YY_NO_ACTION 648
-#define YY_MIN_REDUCE 649
-#define YY_MAX_REDUCE 878
+#define YYNSTATE 253
+#define YYNRULE 233
+#define YYNTOKEN 207
+#define YY_MAX_SHIFT 252
+#define YY_MIN_SHIFTREDUCE 420
+#define YY_MAX_SHIFTREDUCE 652
+#define YY_ERROR_ACTION 653
+#define YY_ACCEPT_ACTION 654
+#define YY_NO_ACTION 655
+#define YY_MIN_REDUCE 656
+#define YY_MAX_REDUCE 888
/************* End control #defines *******************************************/
/* Define the yytestcase() macro to be a no-op if is not already defined
@@ -203,223 +202,224 @@ typedef union {
** yy_default[] Default action for each state.
**
*********** Begin parsing tables **********************************************/
-#define YY_ACTTAB_COUNT (566)
+#define YY_ACTTAB_COUNT (571)
static const YYACTIONTYPE yy_action[] = {
- /* 0 */ 751, 459, 11, 749, 750, 647, 251, 459, 752, 460,
- /* 10 */ 754, 755, 753, 35, 36, 460, 37, 38, 159, 249,
- /* 20 */ 170, 29, 141, 459, 206, 41, 39, 43, 40, 140,
- /* 30 */ 145, 460, 865, 34, 33, 862, 141, 32, 31, 30,
- /* 40 */ 35, 36, 781, 37, 38, 165, 866, 170, 29, 141,
- /* 50 */ 62, 206, 41, 39, 43, 40, 191, 525, 164, 866,
- /* 60 */ 34, 33, 27, 21, 32, 31, 30, 417, 418, 419,
- /* 70 */ 420, 421, 422, 423, 424, 425, 426, 427, 428, 250,
- /* 80 */ 35, 36, 181, 37, 38, 227, 226, 170, 29, 781,
- /* 90 */ 176, 206, 41, 39, 43, 40, 174, 162, 767, 792,
- /* 100 */ 34, 33, 56, 160, 32, 31, 30, 21, 36, 8,
- /* 110 */ 37, 38, 63, 118, 170, 29, 770, 108, 206, 41,
- /* 120 */ 39, 43, 40, 32, 31, 30, 599, 34, 33, 78,
- /* 130 */ 875, 32, 31, 30, 238, 37, 38, 108, 238, 170,
- /* 140 */ 29, 184, 766, 206, 41, 39, 43, 40, 188, 187,
- /* 150 */ 789, 177, 34, 33, 224, 223, 32, 31, 30, 16,
- /* 160 */ 218, 244, 243, 217, 216, 215, 242, 214, 241, 240,
- /* 170 */ 239, 213, 747, 818, 735, 736, 737, 738, 739, 740,
- /* 180 */ 741, 742, 743, 744, 745, 746, 169, 612, 103, 12,
- /* 190 */ 603, 17, 606, 819, 609, 201, 169, 612, 26, 108,
- /* 200 */ 603, 108, 606, 861, 609, 153, 169, 612, 173, 567,
- /* 210 */ 603, 154, 606, 105, 609, 90, 89, 148, 166, 167,
- /* 220 */ 34, 33, 205, 102, 32, 31, 30, 770, 166, 167,
- /* 230 */ 26, 21, 557, 41, 39, 43, 40, 549, 166, 167,
- /* 240 */ 194, 34, 33, 17, 193, 32, 31, 30, 860, 16,
- /* 250 */ 26, 244, 243, 203, 21, 60, 242, 61, 241, 240,
- /* 260 */ 239, 248, 247, 96, 175, 229, 767, 76, 80, 245,
- /* 270 */ 190, 554, 21, 85, 88, 79, 18, 156, 121, 122,
- /* 280 */ 605, 82, 608, 42, 70, 66, 69, 225, 770, 767,
- /* 290 */ 135, 133, 601, 42, 611, 768, 93, 92, 91, 690,
- /* 300 */ 168, 207, 131, 42, 611, 230, 545, 767, 546, 610,
- /* 310 */ 699, 157, 691, 131, 611, 131, 604, 541, 607, 610,
- /* 320 */ 538, 571, 539, 47, 540, 46, 580, 581, 602, 610,
- /* 330 */ 572, 631, 613, 50, 14, 13, 13, 531, 543, 3,
- /* 340 */ 544, 46, 48, 530, 75, 74, 811, 22, 178, 179,
- /* 350 */ 51, 211, 10, 9, 829, 22, 87, 86, 101, 99,
- /* 360 */ 158, 143, 144, 146, 147, 151, 152, 150, 139, 149,
- /* 370 */ 769, 142, 828, 171, 825, 824, 172, 791, 761, 796,
- /* 380 */ 228, 783, 798, 104, 810, 119, 120, 701, 117, 212,
- /* 390 */ 615, 137, 24, 221, 698, 26, 222, 192, 874, 72,
- /* 400 */ 873, 871, 123, 719, 25, 100, 23, 138, 566, 688,
- /* 410 */ 81, 686, 83, 84, 684, 195, 780, 683, 161, 542,
- /* 420 */ 180, 199, 132, 681, 680, 679, 52, 49, 678, 677,
- /* 430 */ 109, 134, 44, 675, 204, 673, 671, 669, 667, 202,
- /* 440 */ 200, 198, 196, 28, 136, 220, 57, 58, 812, 77,
- /* 450 */ 231, 232, 233, 234, 235, 236, 237, 246, 209, 645,
- /* 460 */ 53, 182, 183, 644, 110, 64, 67, 155, 186, 185,
- /* 470 */ 682, 643, 94, 636, 676, 189, 126, 125, 720, 124,
- /* 480 */ 127, 128, 130, 129, 95, 668, 1, 551, 193, 765,
- /* 490 */ 2, 55, 113, 111, 114, 112, 115, 116, 59, 568,
- /* 500 */ 163, 106, 197, 5, 573, 107, 6, 65, 614, 19,
- /* 510 */ 4, 20, 15, 208, 616, 7, 210, 500, 496, 494,
- /* 520 */ 493, 492, 489, 463, 219, 68, 45, 71, 73, 22,
- /* 530 */ 527, 526, 524, 54, 484, 482, 474, 480, 476, 478,
- /* 540 */ 472, 470, 499, 498, 497, 495, 491, 490, 46, 461,
- /* 550 */ 432, 430, 649, 648, 648, 648, 648, 648, 648, 648,
- /* 560 */ 648, 648, 648, 648, 97, 98,
+ /* 0 */ 108, 463, 141, 11, 654, 252, 802, 463, 140, 464,
+ /* 10 */ 162, 165, 876, 35, 36, 464, 37, 38, 159, 250,
+ /* 20 */ 170, 29, 141, 463, 206, 41, 39, 43, 40, 173,
+ /* 30 */ 780, 464, 875, 34, 33, 145, 141, 32, 31, 30,
+ /* 40 */ 35, 36, 791, 37, 38, 164, 876, 170, 29, 780,
+ /* 50 */ 21, 206, 41, 39, 43, 40, 191, 829, 799, 201,
+ /* 60 */ 34, 33, 21, 21, 32, 31, 30, 421, 422, 423,
+ /* 70 */ 424, 425, 426, 427, 428, 429, 430, 431, 432, 251,
+ /* 80 */ 35, 36, 181, 37, 38, 532, 776, 170, 29, 238,
+ /* 90 */ 246, 206, 41, 39, 43, 40, 174, 175, 777, 777,
+ /* 100 */ 34, 33, 872, 56, 32, 31, 30, 176, 871, 36,
+ /* 110 */ 780, 37, 38, 227, 226, 170, 29, 791, 17, 206,
+ /* 120 */ 41, 39, 43, 40, 108, 26, 870, 606, 34, 33,
+ /* 130 */ 78, 160, 32, 31, 30, 238, 157, 16, 218, 245,
+ /* 140 */ 244, 217, 216, 215, 243, 214, 242, 241, 240, 213,
+ /* 150 */ 239, 755, 103, 743, 744, 745, 746, 747, 748, 749,
+ /* 160 */ 750, 751, 752, 753, 754, 756, 37, 38, 229, 177,
+ /* 170 */ 170, 29, 224, 223, 206, 41, 39, 43, 40, 203,
+ /* 180 */ 62, 60, 8, 34, 33, 63, 118, 32, 31, 30,
+ /* 190 */ 169, 619, 27, 12, 610, 184, 613, 158, 616, 778,
+ /* 200 */ 169, 619, 188, 187, 610, 194, 613, 108, 616, 153,
+ /* 210 */ 169, 619, 561, 108, 610, 154, 613, 18, 616, 90,
+ /* 220 */ 89, 148, 166, 167, 34, 33, 205, 143, 32, 31,
+ /* 230 */ 30, 697, 166, 167, 131, 144, 564, 41, 39, 43,
+ /* 240 */ 40, 706, 166, 167, 131, 34, 33, 146, 17, 32,
+ /* 250 */ 31, 30, 32, 31, 30, 26, 16, 207, 245, 244,
+ /* 260 */ 21, 587, 588, 243, 828, 242, 241, 240, 698, 239,
+ /* 270 */ 61, 131, 76, 80, 147, 190, 102, 151, 85, 88,
+ /* 280 */ 79, 760, 156, 26, 758, 759, 82, 21, 42, 761,
+ /* 290 */ 556, 763, 764, 762, 225, 765, 777, 193, 42, 618,
+ /* 300 */ 249, 248, 96, 574, 121, 122, 608, 105, 42, 618,
+ /* 310 */ 70, 66, 69, 578, 617, 168, 579, 46, 152, 618,
+ /* 320 */ 14, 230, 548, 777, 617, 545, 638, 546, 150, 547,
+ /* 330 */ 13, 135, 133, 612, 617, 615, 139, 93, 92, 91,
+ /* 340 */ 620, 611, 609, 614, 13, 47, 538, 622, 50, 552,
+ /* 350 */ 46, 553, 537, 178, 179, 3, 22, 211, 75, 74,
+ /* 360 */ 149, 22, 10, 9, 48, 51, 142, 550, 885, 551,
+ /* 370 */ 87, 86, 101, 99, 779, 839, 838, 171, 835, 834,
+ /* 380 */ 172, 801, 771, 228, 806, 793, 808, 104, 821, 119,
+ /* 390 */ 820, 117, 120, 708, 212, 137, 24, 221, 705, 222,
+ /* 400 */ 26, 192, 100, 884, 72, 883, 881, 123, 726, 25,
+ /* 410 */ 573, 23, 138, 695, 49, 81, 693, 83, 84, 691,
+ /* 420 */ 790, 690, 195, 161, 199, 549, 57, 52, 180, 132,
+ /* 430 */ 688, 687, 686, 685, 684, 134, 682, 109, 680, 678,
+ /* 440 */ 44, 676, 674, 136, 204, 202, 58, 822, 200, 198,
+ /* 450 */ 196, 220, 77, 28, 231, 232, 233, 235, 652, 234,
+ /* 460 */ 236, 237, 247, 209, 183, 53, 651, 182, 185, 186,
+ /* 470 */ 64, 67, 155, 650, 643, 189, 193, 689, 558, 94,
+ /* 480 */ 683, 675, 126, 125, 727, 129, 124, 127, 128, 95,
+ /* 490 */ 130, 1, 114, 110, 111, 775, 2, 55, 59, 116,
+ /* 500 */ 112, 113, 115, 575, 106, 163, 197, 5, 580, 107,
+ /* 510 */ 6, 65, 621, 19, 4, 20, 15, 208, 623, 7,
+ /* 520 */ 210, 504, 500, 498, 497, 496, 493, 467, 219, 68,
+ /* 530 */ 45, 71, 73, 22, 534, 533, 531, 488, 54, 486,
+ /* 540 */ 478, 484, 480, 482, 476, 474, 505, 503, 502, 501,
+ /* 550 */ 499, 495, 494, 46, 465, 436, 434, 656, 655, 655,
+ /* 560 */ 655, 655, 655, 655, 655, 655, 655, 655, 655, 97,
+ /* 570 */ 98,
};
static const YYCODETYPE yy_lookahead[] = {
- /* 0 */ 226, 1, 260, 229, 230, 207, 208, 1, 234, 9,
- /* 10 */ 236, 237, 238, 13, 14, 9, 16, 17, 209, 210,
- /* 20 */ 20, 21, 260, 1, 24, 25, 26, 27, 28, 260,
- /* 30 */ 260, 9, 270, 33, 34, 260, 260, 37, 38, 39,
- /* 40 */ 13, 14, 244, 16, 17, 269, 270, 20, 21, 260,
- /* 50 */ 247, 24, 25, 26, 27, 28, 258, 5, 269, 270,
- /* 60 */ 33, 34, 259, 210, 37, 38, 39, 45, 46, 47,
+ /* 0 */ 211, 1, 262, 262, 208, 209, 211, 1, 262, 9,
+ /* 10 */ 228, 271, 272, 13, 14, 9, 16, 17, 210, 211,
+ /* 20 */ 20, 21, 262, 1, 24, 25, 26, 27, 28, 228,
+ /* 30 */ 248, 9, 272, 33, 34, 262, 262, 37, 38, 39,
+ /* 40 */ 13, 14, 246, 16, 17, 271, 272, 20, 21, 248,
+ /* 50 */ 211, 24, 25, 26, 27, 28, 260, 268, 263, 270,
+ /* 60 */ 33, 34, 211, 211, 37, 38, 39, 45, 46, 47,
/* 70 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57,
- /* 80 */ 13, 14, 60, 16, 17, 33, 34, 20, 21, 244,
- /* 90 */ 66, 24, 25, 26, 27, 28, 243, 227, 245, 210,
- /* 100 */ 33, 34, 102, 258, 37, 38, 39, 210, 14, 98,
- /* 110 */ 16, 17, 101, 102, 20, 21, 246, 210, 24, 25,
- /* 120 */ 26, 27, 28, 37, 38, 39, 99, 33, 34, 73,
- /* 130 */ 246, 37, 38, 39, 78, 16, 17, 210, 78, 20,
- /* 140 */ 21, 126, 245, 24, 25, 26, 27, 28, 133, 134,
- /* 150 */ 261, 127, 33, 34, 130, 131, 37, 38, 39, 85,
- /* 160 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95,
- /* 170 */ 96, 97, 226, 266, 228, 229, 230, 231, 232, 233,
- /* 180 */ 234, 235, 236, 237, 238, 239, 1, 2, 210, 44,
- /* 190 */ 5, 98, 7, 266, 9, 268, 1, 2, 105, 210,
- /* 200 */ 5, 210, 7, 260, 9, 60, 1, 2, 227, 99,
- /* 210 */ 5, 66, 7, 103, 9, 70, 71, 72, 33, 34,
- /* 220 */ 33, 34, 37, 98, 37, 38, 39, 246, 33, 34,
- /* 230 */ 105, 210, 37, 25, 26, 27, 28, 99, 33, 34,
- /* 240 */ 262, 33, 34, 98, 106, 37, 38, 39, 260, 85,
- /* 250 */ 105, 87, 88, 264, 210, 266, 92, 266, 94, 95,
- /* 260 */ 96, 63, 64, 65, 243, 210, 245, 61, 62, 227,
- /* 270 */ 125, 103, 210, 67, 68, 69, 108, 132, 61, 62,
- /* 280 */ 5, 75, 7, 98, 67, 68, 69, 243, 246, 245,
- /* 290 */ 61, 62, 1, 98, 109, 240, 67, 68, 69, 214,
- /* 300 */ 59, 15, 217, 98, 109, 243, 5, 245, 7, 124,
- /* 310 */ 214, 260, 214, 217, 109, 217, 5, 2, 7, 124,
- /* 320 */ 5, 99, 7, 103, 9, 103, 115, 116, 37, 124,
- /* 330 */ 99, 99, 99, 103, 103, 103, 103, 99, 5, 98,
- /* 340 */ 7, 103, 122, 99, 128, 129, 267, 103, 33, 34,
- /* 350 */ 120, 99, 128, 129, 241, 103, 73, 74, 61, 62,
- /* 360 */ 260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
- /* 370 */ 246, 260, 241, 241, 241, 241, 241, 210, 242, 210,
- /* 380 */ 241, 244, 210, 210, 267, 210, 210, 210, 248, 210,
- /* 390 */ 104, 210, 210, 210, 210, 105, 210, 244, 210, 210,
- /* 400 */ 210, 210, 210, 210, 210, 59, 210, 210, 109, 210,
- /* 410 */ 210, 210, 210, 210, 210, 263, 257, 210, 263, 104,
- /* 420 */ 210, 263, 210, 210, 210, 210, 119, 121, 210, 210,
- /* 430 */ 256, 210, 118, 210, 113, 210, 210, 210, 210, 117,
- /* 440 */ 112, 111, 110, 123, 210, 76, 211, 211, 211, 84,
- /* 450 */ 83, 49, 80, 82, 53, 81, 79, 76, 211, 5,
- /* 460 */ 211, 135, 5, 5, 255, 215, 215, 211, 5, 135,
- /* 470 */ 211, 5, 212, 86, 211, 126, 219, 223, 225, 224,
- /* 480 */ 222, 220, 218, 221, 212, 211, 216, 99, 106, 244,
- /* 490 */ 213, 107, 252, 254, 251, 253, 250, 249, 103, 99,
- /* 500 */ 1, 98, 98, 114, 99, 98, 114, 73, 99, 103,
- /* 510 */ 98, 103, 98, 100, 104, 98, 100, 9, 5, 5,
- /* 520 */ 5, 5, 5, 77, 15, 73, 16, 129, 129, 103,
- /* 530 */ 5, 5, 99, 98, 5, 5, 5, 5, 5, 5,
- /* 540 */ 5, 5, 5, 5, 5, 5, 5, 5, 103, 77,
- /* 550 */ 59, 58, 0, 271, 271, 271, 271, 271, 271, 271,
- /* 560 */ 271, 271, 271, 271, 21, 21, 271, 271, 271, 271,
- /* 570 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 580 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 590 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 600 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 610 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 620 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 630 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 640 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 650 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 660 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 670 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 680 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 690 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 700 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 710 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 720 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 730 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 740 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 750 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 760 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 770 */ 271, 271,
+ /* 80 */ 13, 14, 60, 16, 17, 5, 247, 20, 21, 78,
+ /* 90 */ 228, 24, 25, 26, 27, 28, 245, 245, 247, 247,
+ /* 100 */ 33, 34, 262, 103, 37, 38, 39, 66, 262, 14,
+ /* 110 */ 248, 16, 17, 33, 34, 20, 21, 246, 99, 24,
+ /* 120 */ 25, 26, 27, 28, 211, 106, 262, 100, 33, 34,
+ /* 130 */ 73, 260, 37, 38, 39, 78, 262, 85, 86, 87,
+ /* 140 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 97,
+ /* 150 */ 98, 227, 211, 229, 230, 231, 232, 233, 234, 235,
+ /* 160 */ 236, 237, 238, 239, 240, 241, 16, 17, 211, 128,
+ /* 170 */ 20, 21, 131, 132, 24, 25, 26, 27, 28, 266,
+ /* 180 */ 249, 268, 99, 33, 34, 102, 103, 37, 38, 39,
+ /* 190 */ 1, 2, 261, 44, 5, 127, 7, 262, 9, 242,
+ /* 200 */ 1, 2, 134, 135, 5, 264, 7, 211, 9, 60,
+ /* 210 */ 1, 2, 104, 211, 5, 66, 7, 109, 9, 70,
+ /* 220 */ 71, 72, 33, 34, 33, 34, 37, 262, 37, 38,
+ /* 230 */ 39, 215, 33, 34, 218, 262, 37, 25, 26, 27,
+ /* 240 */ 28, 215, 33, 34, 218, 33, 34, 262, 99, 37,
+ /* 250 */ 38, 39, 37, 38, 39, 106, 85, 15, 87, 88,
+ /* 260 */ 211, 116, 117, 92, 268, 94, 95, 96, 215, 98,
+ /* 270 */ 268, 218, 61, 62, 262, 126, 99, 262, 67, 68,
+ /* 280 */ 69, 227, 133, 106, 230, 231, 75, 211, 99, 235,
+ /* 290 */ 100, 237, 238, 239, 245, 241, 247, 107, 99, 110,
+ /* 300 */ 63, 64, 65, 100, 61, 62, 1, 104, 99, 110,
+ /* 310 */ 67, 68, 69, 100, 125, 59, 100, 104, 262, 110,
+ /* 320 */ 104, 245, 2, 247, 125, 5, 100, 7, 262, 9,
+ /* 330 */ 104, 61, 62, 5, 125, 7, 262, 67, 68, 69,
+ /* 340 */ 100, 5, 37, 7, 104, 104, 100, 105, 104, 5,
+ /* 350 */ 104, 7, 100, 33, 34, 99, 104, 100, 129, 130,
+ /* 360 */ 262, 104, 129, 130, 123, 121, 262, 5, 248, 7,
+ /* 370 */ 73, 74, 61, 62, 248, 243, 243, 243, 243, 243,
+ /* 380 */ 243, 211, 244, 243, 211, 246, 211, 211, 269, 211,
+ /* 390 */ 269, 250, 211, 211, 211, 211, 211, 211, 211, 211,
+ /* 400 */ 106, 246, 59, 211, 211, 211, 211, 211, 211, 211,
+ /* 410 */ 110, 211, 211, 211, 122, 211, 211, 211, 211, 211,
+ /* 420 */ 259, 211, 265, 265, 265, 105, 212, 120, 211, 211,
+ /* 430 */ 211, 211, 211, 211, 211, 211, 211, 258, 211, 211,
+ /* 440 */ 119, 211, 211, 211, 114, 118, 212, 212, 113, 112,
+ /* 450 */ 111, 76, 84, 124, 83, 49, 80, 53, 5, 82,
+ /* 460 */ 81, 79, 76, 212, 5, 212, 5, 136, 136, 5,
+ /* 470 */ 216, 216, 212, 5, 86, 127, 107, 212, 100, 213,
+ /* 480 */ 212, 212, 220, 224, 226, 222, 225, 223, 221, 213,
+ /* 490 */ 219, 217, 253, 257, 256, 246, 214, 108, 104, 251,
+ /* 500 */ 255, 254, 252, 100, 99, 1, 99, 115, 100, 99,
+ /* 510 */ 115, 73, 100, 104, 99, 104, 99, 101, 105, 99,
+ /* 520 */ 101, 9, 5, 5, 5, 5, 5, 77, 15, 73,
+ /* 530 */ 16, 130, 130, 104, 5, 5, 100, 5, 99, 5,
+ /* 540 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
+ /* 550 */ 5, 5, 5, 104, 77, 59, 58, 0, 273, 273,
+ /* 560 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 21,
+ /* 570 */ 21, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 580 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 590 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 600 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 610 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 620 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 630 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 640 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 650 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 660 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 670 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 680 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 690 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 700 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 710 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 720 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 730 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 740 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 750 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 760 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 770 */ 273, 273, 273, 273, 273, 273, 273, 273,
};
-#define YY_SHIFT_COUNT (251)
+#define YY_SHIFT_COUNT (252)
#define YY_SHIFT_MIN (0)
-#define YY_SHIFT_MAX (552)
+#define YY_SHIFT_MAX (557)
static const unsigned short int yy_shift_ofst[] = {
- /* 0 */ 145, 74, 164, 185, 205, 6, 6, 6, 6, 6,
- /* 10 */ 6, 0, 22, 205, 315, 315, 315, 93, 6, 6,
- /* 20 */ 6, 6, 6, 56, 60, 60, 566, 195, 205, 205,
- /* 30 */ 205, 205, 205, 205, 205, 205, 205, 205, 205, 205,
- /* 40 */ 205, 205, 205, 205, 205, 315, 315, 52, 52, 52,
- /* 50 */ 52, 52, 52, 11, 52, 125, 6, 6, 6, 6,
- /* 60 */ 211, 211, 168, 6, 6, 6, 6, 6, 6, 6,
+ /* 0 */ 149, 52, 171, 189, 209, 6, 6, 6, 6, 6,
+ /* 10 */ 6, 0, 22, 209, 320, 320, 320, 19, 6, 6,
+ /* 20 */ 6, 6, 6, 57, 11, 11, 571, 199, 209, 209,
+ /* 30 */ 209, 209, 209, 209, 209, 209, 209, 209, 209, 209,
+ /* 40 */ 209, 209, 209, 209, 209, 320, 320, 80, 80, 80,
+ /* 50 */ 80, 80, 80, 83, 80, 177, 6, 6, 6, 6,
+ /* 60 */ 145, 145, 108, 6, 6, 6, 6, 6, 6, 6,
/* 70 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
/* 80 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
/* 90 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
- /* 100 */ 6, 6, 290, 346, 346, 299, 299, 299, 346, 307,
- /* 110 */ 306, 314, 321, 322, 328, 330, 332, 320, 290, 346,
- /* 120 */ 346, 369, 369, 346, 365, 367, 402, 372, 371, 401,
- /* 130 */ 374, 377, 346, 381, 346, 381, 346, 566, 566, 27,
- /* 140 */ 67, 67, 67, 94, 119, 208, 208, 208, 206, 187,
- /* 150 */ 187, 187, 187, 217, 229, 24, 15, 86, 86, 198,
- /* 160 */ 138, 110, 222, 231, 232, 233, 275, 311, 291, 241,
- /* 170 */ 286, 220, 230, 238, 244, 252, 216, 224, 301, 333,
- /* 180 */ 283, 297, 454, 326, 457, 458, 334, 463, 466, 387,
- /* 190 */ 349, 382, 388, 384, 395, 400, 403, 499, 404, 405,
- /* 200 */ 407, 406, 389, 408, 392, 409, 412, 410, 414, 413,
- /* 210 */ 417, 416, 434, 508, 513, 514, 515, 516, 517, 446,
- /* 220 */ 509, 452, 510, 398, 399, 426, 525, 526, 433, 435,
- /* 230 */ 426, 529, 530, 531, 532, 533, 534, 535, 536, 537,
- /* 240 */ 538, 539, 540, 541, 542, 445, 472, 543, 544, 491,
- /* 250 */ 493, 552,
+ /* 100 */ 6, 6, 294, 343, 343, 300, 300, 300, 343, 307,
+ /* 110 */ 292, 321, 330, 327, 335, 337, 339, 329, 294, 343,
+ /* 120 */ 343, 375, 375, 343, 368, 371, 406, 376, 377, 404,
+ /* 130 */ 379, 382, 343, 386, 343, 386, 343, 571, 571, 27,
+ /* 140 */ 67, 67, 67, 95, 150, 212, 212, 212, 211, 191,
+ /* 150 */ 191, 191, 191, 243, 270, 41, 68, 215, 215, 237,
+ /* 160 */ 190, 203, 213, 216, 226, 240, 328, 336, 305, 256,
+ /* 170 */ 242, 241, 244, 246, 252, 257, 229, 233, 344, 362,
+ /* 180 */ 297, 311, 453, 331, 459, 461, 332, 464, 468, 388,
+ /* 190 */ 348, 369, 378, 389, 394, 403, 405, 504, 407, 408,
+ /* 200 */ 410, 409, 392, 411, 395, 412, 415, 413, 417, 416,
+ /* 210 */ 420, 419, 438, 512, 517, 518, 519, 520, 521, 450,
+ /* 220 */ 513, 456, 514, 401, 402, 429, 529, 530, 436, 439,
+ /* 230 */ 429, 532, 534, 535, 536, 537, 538, 539, 540, 541,
+ /* 240 */ 542, 543, 544, 545, 546, 547, 449, 477, 548, 549,
+ /* 250 */ 496, 498, 557,
};
#define YY_REDUCE_COUNT (138)
-#define YY_REDUCE_MIN (-258)
-#define YY_REDUCE_MAX (277)
+#define YY_REDUCE_MIN (-260)
+#define YY_REDUCE_MAX (282)
static const short yy_reduce_ofst[] = {
- /* 0 */ -202, -54, -226, -224, -211, -73, -11, -147, 21, 44,
- /* 10 */ 62, -111, -191, -238, -130, -19, 42, -155, -22, -93,
- /* 20 */ -9, 55, -103, 85, 96, 98, -197, -258, -231, -230,
- /* 30 */ -225, -57, -12, 51, 100, 101, 102, 103, 104, 105,
- /* 40 */ 106, 107, 108, 109, 111, -116, 124, 113, 131, 132,
- /* 50 */ 133, 134, 135, 136, 139, 137, 167, 169, 172, 173,
- /* 60 */ 79, 117, 140, 175, 176, 177, 179, 181, 182, 183,
- /* 70 */ 184, 186, 188, 189, 190, 191, 192, 193, 194, 196,
- /* 80 */ 197, 199, 200, 201, 202, 203, 204, 207, 210, 212,
- /* 90 */ 213, 214, 215, 218, 219, 221, 223, 225, 226, 227,
- /* 100 */ 228, 234, 153, 235, 236, 152, 155, 158, 237, 159,
- /* 110 */ 174, 209, 239, 242, 240, 243, 246, 248, 245, 247,
- /* 120 */ 249, 250, 251, 256, 253, 255, 254, 257, 258, 261,
- /* 130 */ 262, 264, 259, 260, 263, 272, 274, 270, 277,
+ /* 0 */ -204, -76, 54, -260, -226, -211, -87, -149, -148, 49,
+ /* 10 */ 76, -205, -192, -240, -218, -199, -138, -129, -59, -4,
+ /* 20 */ 2, -43, -161, 16, 26, 53, -69, -259, -254, -227,
+ /* 30 */ -160, -154, -136, -126, -65, -35, -27, -15, 12, 15,
+ /* 40 */ 56, 66, 74, 98, 104, 120, 126, 132, 133, 134,
+ /* 50 */ 135, 136, 137, 138, 140, 139, 170, 173, 175, 176,
+ /* 60 */ 119, 121, 141, 178, 181, 182, 183, 184, 185, 186,
+ /* 70 */ 187, 188, 192, 193, 194, 195, 196, 197, 198, 200,
+ /* 80 */ 201, 202, 204, 205, 206, 207, 208, 210, 217, 218,
+ /* 90 */ 219, 220, 221, 222, 223, 224, 225, 227, 228, 230,
+ /* 100 */ 231, 232, 155, 214, 234, 157, 158, 159, 235, 161,
+ /* 110 */ 179, 236, 238, 245, 247, 239, 250, 248, 249, 251,
+ /* 120 */ 253, 254, 255, 260, 258, 261, 259, 262, 264, 267,
+ /* 130 */ 263, 271, 265, 266, 268, 276, 269, 274, 282,
};
static const YYACTIONTYPE yy_default[] = {
- /* 0 */ 646, 700, 689, 868, 868, 646, 646, 646, 646, 646,
- /* 10 */ 646, 793, 664, 868, 646, 646, 646, 646, 646, 646,
- /* 20 */ 646, 646, 646, 702, 702, 702, 788, 646, 646, 646,
- /* 30 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
- /* 40 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
- /* 50 */ 646, 646, 646, 646, 646, 646, 646, 795, 797, 646,
- /* 60 */ 815, 815, 786, 646, 646, 646, 646, 646, 646, 646,
- /* 70 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
- /* 80 */ 646, 687, 646, 685, 646, 646, 646, 646, 646, 646,
- /* 90 */ 646, 646, 646, 646, 646, 646, 674, 646, 646, 646,
- /* 100 */ 646, 646, 646, 666, 666, 646, 646, 646, 666, 822,
- /* 110 */ 826, 820, 808, 816, 807, 803, 802, 830, 646, 666,
- /* 120 */ 666, 697, 697, 666, 718, 716, 714, 706, 712, 708,
- /* 130 */ 710, 704, 666, 695, 666, 695, 666, 734, 748, 646,
- /* 140 */ 831, 867, 821, 857, 856, 863, 855, 854, 646, 850,
- /* 150 */ 851, 853, 852, 646, 646, 646, 646, 859, 858, 646,
- /* 160 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 833,
- /* 170 */ 646, 827, 823, 646, 646, 646, 646, 646, 646, 646,
- /* 180 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
- /* 190 */ 646, 785, 646, 646, 794, 646, 646, 646, 646, 646,
- /* 200 */ 646, 817, 646, 809, 646, 646, 646, 646, 646, 646,
- /* 210 */ 646, 762, 646, 646, 646, 646, 646, 646, 646, 646,
- /* 220 */ 646, 646, 646, 646, 646, 872, 646, 646, 646, 756,
- /* 230 */ 870, 646, 646, 646, 646, 646, 646, 646, 646, 646,
- /* 240 */ 646, 646, 646, 646, 646, 721, 646, 672, 670, 646,
- /* 250 */ 662, 646,
+ /* 0 */ 653, 707, 696, 878, 878, 653, 653, 653, 653, 653,
+ /* 10 */ 653, 803, 671, 878, 653, 653, 653, 653, 653, 653,
+ /* 20 */ 653, 653, 653, 709, 709, 709, 798, 653, 653, 653,
+ /* 30 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653,
+ /* 40 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653,
+ /* 50 */ 653, 653, 653, 653, 653, 653, 653, 805, 807, 653,
+ /* 60 */ 825, 825, 796, 653, 653, 653, 653, 653, 653, 653,
+ /* 70 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653,
+ /* 80 */ 653, 694, 653, 692, 653, 653, 653, 653, 653, 653,
+ /* 90 */ 653, 653, 653, 653, 653, 653, 681, 653, 653, 653,
+ /* 100 */ 653, 653, 653, 673, 673, 653, 653, 653, 673, 832,
+ /* 110 */ 836, 830, 818, 826, 817, 813, 812, 840, 653, 673,
+ /* 120 */ 673, 704, 704, 673, 725, 723, 721, 713, 719, 715,
+ /* 130 */ 717, 711, 673, 702, 673, 702, 673, 742, 757, 653,
+ /* 140 */ 841, 877, 831, 867, 866, 873, 865, 864, 653, 860,
+ /* 150 */ 861, 863, 862, 653, 653, 653, 653, 869, 868, 653,
+ /* 160 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 843,
+ /* 170 */ 653, 837, 833, 653, 653, 653, 653, 653, 653, 653,
+ /* 180 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653,
+ /* 190 */ 653, 795, 653, 653, 804, 653, 653, 653, 653, 653,
+ /* 200 */ 653, 827, 653, 819, 653, 653, 653, 653, 653, 653,
+ /* 210 */ 653, 772, 653, 653, 653, 653, 653, 653, 653, 653,
+ /* 220 */ 653, 653, 653, 653, 653, 882, 653, 653, 653, 766,
+ /* 230 */ 880, 653, 653, 653, 653, 653, 653, 653, 653, 653,
+ /* 240 */ 653, 653, 653, 653, 653, 653, 728, 653, 679, 677,
+ /* 250 */ 653, 669, 653,
};
/********** End of lemon-generated parsing tables *****************************/
@@ -537,6 +537,7 @@ static const YYCODETYPE yyFallback[] = {
0, /* FSYNC => nothing */
0, /* COMP => nothing */
0, /* PRECISION => nothing */
+ 0, /* UPDATE => nothing */
0, /* LP => nothing */
0, /* RP => nothing */
0, /* TAGS => nothing */
@@ -829,179 +830,181 @@ static const char *const yyTokenName[] = {
/* 95 */ "FSYNC",
/* 96 */ "COMP",
/* 97 */ "PRECISION",
- /* 98 */ "LP",
- /* 99 */ "RP",
- /* 100 */ "TAGS",
- /* 101 */ "USING",
- /* 102 */ "AS",
- /* 103 */ "COMMA",
- /* 104 */ "NULL",
- /* 105 */ "SELECT",
- /* 106 */ "UNION",
- /* 107 */ "ALL",
- /* 108 */ "FROM",
- /* 109 */ "VARIABLE",
- /* 110 */ "INTERVAL",
- /* 111 */ "FILL",
- /* 112 */ "SLIDING",
- /* 113 */ "ORDER",
- /* 114 */ "BY",
- /* 115 */ "ASC",
- /* 116 */ "DESC",
- /* 117 */ "GROUP",
- /* 118 */ "HAVING",
- /* 119 */ "LIMIT",
- /* 120 */ "OFFSET",
- /* 121 */ "SLIMIT",
- /* 122 */ "SOFFSET",
- /* 123 */ "WHERE",
- /* 124 */ "NOW",
- /* 125 */ "RESET",
- /* 126 */ "QUERY",
- /* 127 */ "ADD",
- /* 128 */ "COLUMN",
- /* 129 */ "TAG",
- /* 130 */ "CHANGE",
- /* 131 */ "SET",
- /* 132 */ "KILL",
- /* 133 */ "CONNECTION",
- /* 134 */ "STREAM",
- /* 135 */ "COLON",
- /* 136 */ "ABORT",
- /* 137 */ "AFTER",
- /* 138 */ "ATTACH",
- /* 139 */ "BEFORE",
- /* 140 */ "BEGIN",
- /* 141 */ "CASCADE",
- /* 142 */ "CLUSTER",
- /* 143 */ "CONFLICT",
- /* 144 */ "COPY",
- /* 145 */ "DEFERRED",
- /* 146 */ "DELIMITERS",
- /* 147 */ "DETACH",
- /* 148 */ "EACH",
- /* 149 */ "END",
- /* 150 */ "EXPLAIN",
- /* 151 */ "FAIL",
- /* 152 */ "FOR",
- /* 153 */ "IGNORE",
- /* 154 */ "IMMEDIATE",
- /* 155 */ "INITIALLY",
- /* 156 */ "INSTEAD",
- /* 157 */ "MATCH",
- /* 158 */ "KEY",
- /* 159 */ "OF",
- /* 160 */ "RAISE",
- /* 161 */ "REPLACE",
- /* 162 */ "RESTRICT",
- /* 163 */ "ROW",
- /* 164 */ "STATEMENT",
- /* 165 */ "TRIGGER",
- /* 166 */ "VIEW",
- /* 167 */ "COUNT",
- /* 168 */ "SUM",
- /* 169 */ "AVG",
- /* 170 */ "MIN",
- /* 171 */ "MAX",
- /* 172 */ "FIRST",
- /* 173 */ "LAST",
- /* 174 */ "TOP",
- /* 175 */ "BOTTOM",
- /* 176 */ "STDDEV",
- /* 177 */ "PERCENTILE",
- /* 178 */ "APERCENTILE",
- /* 179 */ "LEASTSQUARES",
- /* 180 */ "HISTOGRAM",
- /* 181 */ "DIFF",
- /* 182 */ "SPREAD",
- /* 183 */ "TWA",
- /* 184 */ "INTERP",
- /* 185 */ "LAST_ROW",
- /* 186 */ "RATE",
- /* 187 */ "IRATE",
- /* 188 */ "SUM_RATE",
- /* 189 */ "SUM_IRATE",
- /* 190 */ "AVG_RATE",
- /* 191 */ "AVG_IRATE",
- /* 192 */ "TBID",
- /* 193 */ "SEMI",
- /* 194 */ "NONE",
- /* 195 */ "PREV",
- /* 196 */ "LINEAR",
- /* 197 */ "IMPORT",
- /* 198 */ "METRIC",
- /* 199 */ "TBNAME",
- /* 200 */ "JOIN",
- /* 201 */ "METRICS",
- /* 202 */ "STABLE",
- /* 203 */ "INSERT",
- /* 204 */ "INTO",
- /* 205 */ "VALUES",
- /* 206 */ "error",
- /* 207 */ "program",
- /* 208 */ "cmd",
- /* 209 */ "dbPrefix",
- /* 210 */ "ids",
- /* 211 */ "cpxName",
- /* 212 */ "ifexists",
- /* 213 */ "alter_db_optr",
- /* 214 */ "acct_optr",
- /* 215 */ "ifnotexists",
- /* 216 */ "db_optr",
- /* 217 */ "pps",
- /* 218 */ "tseries",
- /* 219 */ "dbs",
- /* 220 */ "streams",
- /* 221 */ "storage",
- /* 222 */ "qtime",
- /* 223 */ "users",
- /* 224 */ "conns",
- /* 225 */ "state",
- /* 226 */ "keep",
- /* 227 */ "tagitemlist",
- /* 228 */ "cache",
- /* 229 */ "replica",
- /* 230 */ "quorum",
- /* 231 */ "days",
- /* 232 */ "minrows",
- /* 233 */ "maxrows",
- /* 234 */ "blocks",
- /* 235 */ "ctime",
- /* 236 */ "wal",
- /* 237 */ "fsync",
- /* 238 */ "comp",
- /* 239 */ "prec",
- /* 240 */ "typename",
- /* 241 */ "signed",
- /* 242 */ "create_table_args",
- /* 243 */ "columnlist",
- /* 244 */ "select",
- /* 245 */ "column",
- /* 246 */ "tagitem",
- /* 247 */ "selcollist",
- /* 248 */ "from",
- /* 249 */ "where_opt",
- /* 250 */ "interval_opt",
- /* 251 */ "fill_opt",
- /* 252 */ "sliding_opt",
- /* 253 */ "groupby_opt",
- /* 254 */ "orderby_opt",
- /* 255 */ "having_opt",
- /* 256 */ "slimit_opt",
- /* 257 */ "limit_opt",
- /* 258 */ "union",
- /* 259 */ "sclp",
- /* 260 */ "expr",
- /* 261 */ "as",
- /* 262 */ "tablelist",
- /* 263 */ "tmvar",
- /* 264 */ "sortlist",
- /* 265 */ "sortitem",
- /* 266 */ "item",
- /* 267 */ "sortorder",
- /* 268 */ "grouplist",
- /* 269 */ "exprlist",
- /* 270 */ "expritem",
+ /* 98 */ "UPDATE",
+ /* 99 */ "LP",
+ /* 100 */ "RP",
+ /* 101 */ "TAGS",
+ /* 102 */ "USING",
+ /* 103 */ "AS",
+ /* 104 */ "COMMA",
+ /* 105 */ "NULL",
+ /* 106 */ "SELECT",
+ /* 107 */ "UNION",
+ /* 108 */ "ALL",
+ /* 109 */ "FROM",
+ /* 110 */ "VARIABLE",
+ /* 111 */ "INTERVAL",
+ /* 112 */ "FILL",
+ /* 113 */ "SLIDING",
+ /* 114 */ "ORDER",
+ /* 115 */ "BY",
+ /* 116 */ "ASC",
+ /* 117 */ "DESC",
+ /* 118 */ "GROUP",
+ /* 119 */ "HAVING",
+ /* 120 */ "LIMIT",
+ /* 121 */ "OFFSET",
+ /* 122 */ "SLIMIT",
+ /* 123 */ "SOFFSET",
+ /* 124 */ "WHERE",
+ /* 125 */ "NOW",
+ /* 126 */ "RESET",
+ /* 127 */ "QUERY",
+ /* 128 */ "ADD",
+ /* 129 */ "COLUMN",
+ /* 130 */ "TAG",
+ /* 131 */ "CHANGE",
+ /* 132 */ "SET",
+ /* 133 */ "KILL",
+ /* 134 */ "CONNECTION",
+ /* 135 */ "STREAM",
+ /* 136 */ "COLON",
+ /* 137 */ "ABORT",
+ /* 138 */ "AFTER",
+ /* 139 */ "ATTACH",
+ /* 140 */ "BEFORE",
+ /* 141 */ "BEGIN",
+ /* 142 */ "CASCADE",
+ /* 143 */ "CLUSTER",
+ /* 144 */ "CONFLICT",
+ /* 145 */ "COPY",
+ /* 146 */ "DEFERRED",
+ /* 147 */ "DELIMITERS",
+ /* 148 */ "DETACH",
+ /* 149 */ "EACH",
+ /* 150 */ "END",
+ /* 151 */ "EXPLAIN",
+ /* 152 */ "FAIL",
+ /* 153 */ "FOR",
+ /* 154 */ "IGNORE",
+ /* 155 */ "IMMEDIATE",
+ /* 156 */ "INITIALLY",
+ /* 157 */ "INSTEAD",
+ /* 158 */ "MATCH",
+ /* 159 */ "KEY",
+ /* 160 */ "OF",
+ /* 161 */ "RAISE",
+ /* 162 */ "REPLACE",
+ /* 163 */ "RESTRICT",
+ /* 164 */ "ROW",
+ /* 165 */ "STATEMENT",
+ /* 166 */ "TRIGGER",
+ /* 167 */ "VIEW",
+ /* 168 */ "COUNT",
+ /* 169 */ "SUM",
+ /* 170 */ "AVG",
+ /* 171 */ "MIN",
+ /* 172 */ "MAX",
+ /* 173 */ "FIRST",
+ /* 174 */ "LAST",
+ /* 175 */ "TOP",
+ /* 176 */ "BOTTOM",
+ /* 177 */ "STDDEV",
+ /* 178 */ "PERCENTILE",
+ /* 179 */ "APERCENTILE",
+ /* 180 */ "LEASTSQUARES",
+ /* 181 */ "HISTOGRAM",
+ /* 182 */ "DIFF",
+ /* 183 */ "SPREAD",
+ /* 184 */ "TWA",
+ /* 185 */ "INTERP",
+ /* 186 */ "LAST_ROW",
+ /* 187 */ "RATE",
+ /* 188 */ "IRATE",
+ /* 189 */ "SUM_RATE",
+ /* 190 */ "SUM_IRATE",
+ /* 191 */ "AVG_RATE",
+ /* 192 */ "AVG_IRATE",
+ /* 193 */ "TBID",
+ /* 194 */ "SEMI",
+ /* 195 */ "NONE",
+ /* 196 */ "PREV",
+ /* 197 */ "LINEAR",
+ /* 198 */ "IMPORT",
+ /* 199 */ "METRIC",
+ /* 200 */ "TBNAME",
+ /* 201 */ "JOIN",
+ /* 202 */ "METRICS",
+ /* 203 */ "STABLE",
+ /* 204 */ "INSERT",
+ /* 205 */ "INTO",
+ /* 206 */ "VALUES",
+ /* 207 */ "error",
+ /* 208 */ "program",
+ /* 209 */ "cmd",
+ /* 210 */ "dbPrefix",
+ /* 211 */ "ids",
+ /* 212 */ "cpxName",
+ /* 213 */ "ifexists",
+ /* 214 */ "alter_db_optr",
+ /* 215 */ "acct_optr",
+ /* 216 */ "ifnotexists",
+ /* 217 */ "db_optr",
+ /* 218 */ "pps",
+ /* 219 */ "tseries",
+ /* 220 */ "dbs",
+ /* 221 */ "streams",
+ /* 222 */ "storage",
+ /* 223 */ "qtime",
+ /* 224 */ "users",
+ /* 225 */ "conns",
+ /* 226 */ "state",
+ /* 227 */ "keep",
+ /* 228 */ "tagitemlist",
+ /* 229 */ "cache",
+ /* 230 */ "replica",
+ /* 231 */ "quorum",
+ /* 232 */ "days",
+ /* 233 */ "minrows",
+ /* 234 */ "maxrows",
+ /* 235 */ "blocks",
+ /* 236 */ "ctime",
+ /* 237 */ "wal",
+ /* 238 */ "fsync",
+ /* 239 */ "comp",
+ /* 240 */ "prec",
+ /* 241 */ "update",
+ /* 242 */ "typename",
+ /* 243 */ "signed",
+ /* 244 */ "create_table_args",
+ /* 245 */ "columnlist",
+ /* 246 */ "select",
+ /* 247 */ "column",
+ /* 248 */ "tagitem",
+ /* 249 */ "selcollist",
+ /* 250 */ "from",
+ /* 251 */ "where_opt",
+ /* 252 */ "interval_opt",
+ /* 253 */ "fill_opt",
+ /* 254 */ "sliding_opt",
+ /* 255 */ "groupby_opt",
+ /* 256 */ "orderby_opt",
+ /* 257 */ "having_opt",
+ /* 258 */ "slimit_opt",
+ /* 259 */ "limit_opt",
+ /* 260 */ "union",
+ /* 261 */ "sclp",
+ /* 262 */ "expr",
+ /* 263 */ "as",
+ /* 264 */ "tablelist",
+ /* 265 */ "tmvar",
+ /* 266 */ "sortlist",
+ /* 267 */ "sortitem",
+ /* 268 */ "item",
+ /* 269 */ "sortorder",
+ /* 270 */ "grouplist",
+ /* 271 */ "exprlist",
+ /* 272 */ "expritem",
};
#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */
@@ -1094,151 +1097,154 @@ static const char *const yyRuleName[] = {
/* 82 */ "fsync ::= FSYNC INTEGER",
/* 83 */ "comp ::= COMP INTEGER",
/* 84 */ "prec ::= PRECISION STRING",
- /* 85 */ "db_optr ::=",
- /* 86 */ "db_optr ::= db_optr cache",
- /* 87 */ "db_optr ::= db_optr replica",
- /* 88 */ "db_optr ::= db_optr quorum",
- /* 89 */ "db_optr ::= db_optr days",
- /* 90 */ "db_optr ::= db_optr minrows",
- /* 91 */ "db_optr ::= db_optr maxrows",
- /* 92 */ "db_optr ::= db_optr blocks",
- /* 93 */ "db_optr ::= db_optr ctime",
- /* 94 */ "db_optr ::= db_optr wal",
- /* 95 */ "db_optr ::= db_optr fsync",
- /* 96 */ "db_optr ::= db_optr comp",
- /* 97 */ "db_optr ::= db_optr prec",
- /* 98 */ "db_optr ::= db_optr keep",
- /* 99 */ "alter_db_optr ::=",
- /* 100 */ "alter_db_optr ::= alter_db_optr replica",
- /* 101 */ "alter_db_optr ::= alter_db_optr quorum",
- /* 102 */ "alter_db_optr ::= alter_db_optr keep",
- /* 103 */ "alter_db_optr ::= alter_db_optr blocks",
- /* 104 */ "alter_db_optr ::= alter_db_optr comp",
- /* 105 */ "alter_db_optr ::= alter_db_optr wal",
- /* 106 */ "alter_db_optr ::= alter_db_optr fsync",
- /* 107 */ "typename ::= ids",
- /* 108 */ "typename ::= ids LP signed RP",
- /* 109 */ "signed ::= INTEGER",
- /* 110 */ "signed ::= PLUS INTEGER",
- /* 111 */ "signed ::= MINUS INTEGER",
- /* 112 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args",
- /* 113 */ "create_table_args ::= LP columnlist RP",
- /* 114 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP",
- /* 115 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP",
- /* 116 */ "create_table_args ::= AS select",
- /* 117 */ "columnlist ::= columnlist COMMA column",
- /* 118 */ "columnlist ::= column",
- /* 119 */ "column ::= ids typename",
- /* 120 */ "tagitemlist ::= tagitemlist COMMA tagitem",
- /* 121 */ "tagitemlist ::= tagitem",
- /* 122 */ "tagitem ::= INTEGER",
- /* 123 */ "tagitem ::= FLOAT",
- /* 124 */ "tagitem ::= STRING",
- /* 125 */ "tagitem ::= BOOL",
- /* 126 */ "tagitem ::= NULL",
- /* 127 */ "tagitem ::= MINUS INTEGER",
- /* 128 */ "tagitem ::= MINUS FLOAT",
- /* 129 */ "tagitem ::= PLUS INTEGER",
- /* 130 */ "tagitem ::= PLUS FLOAT",
- /* 131 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt",
- /* 132 */ "union ::= select",
- /* 133 */ "union ::= LP union RP",
- /* 134 */ "union ::= union UNION ALL select",
- /* 135 */ "union ::= union UNION ALL LP select RP",
- /* 136 */ "cmd ::= union",
- /* 137 */ "select ::= SELECT selcollist",
- /* 138 */ "sclp ::= selcollist COMMA",
- /* 139 */ "sclp ::=",
- /* 140 */ "selcollist ::= sclp expr as",
- /* 141 */ "selcollist ::= sclp STAR",
- /* 142 */ "as ::= AS ids",
- /* 143 */ "as ::= ids",
- /* 144 */ "as ::=",
- /* 145 */ "from ::= FROM tablelist",
- /* 146 */ "tablelist ::= ids cpxName",
- /* 147 */ "tablelist ::= ids cpxName ids",
- /* 148 */ "tablelist ::= tablelist COMMA ids cpxName",
- /* 149 */ "tablelist ::= tablelist COMMA ids cpxName ids",
- /* 150 */ "tmvar ::= VARIABLE",
- /* 151 */ "interval_opt ::= INTERVAL LP tmvar RP",
- /* 152 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP",
- /* 153 */ "interval_opt ::=",
- /* 154 */ "fill_opt ::=",
- /* 155 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP",
- /* 156 */ "fill_opt ::= FILL LP ID RP",
- /* 157 */ "sliding_opt ::= SLIDING LP tmvar RP",
- /* 158 */ "sliding_opt ::=",
- /* 159 */ "orderby_opt ::=",
- /* 160 */ "orderby_opt ::= ORDER BY sortlist",
- /* 161 */ "sortlist ::= sortlist COMMA item sortorder",
- /* 162 */ "sortlist ::= item sortorder",
- /* 163 */ "item ::= ids cpxName",
- /* 164 */ "sortorder ::= ASC",
- /* 165 */ "sortorder ::= DESC",
- /* 166 */ "sortorder ::=",
- /* 167 */ "groupby_opt ::=",
- /* 168 */ "groupby_opt ::= GROUP BY grouplist",
- /* 169 */ "grouplist ::= grouplist COMMA item",
- /* 170 */ "grouplist ::= item",
- /* 171 */ "having_opt ::=",
- /* 172 */ "having_opt ::= HAVING expr",
- /* 173 */ "limit_opt ::=",
- /* 174 */ "limit_opt ::= LIMIT signed",
- /* 175 */ "limit_opt ::= LIMIT signed OFFSET signed",
- /* 176 */ "limit_opt ::= LIMIT signed COMMA signed",
- /* 177 */ "slimit_opt ::=",
- /* 178 */ "slimit_opt ::= SLIMIT signed",
- /* 179 */ "slimit_opt ::= SLIMIT signed SOFFSET signed",
- /* 180 */ "slimit_opt ::= SLIMIT signed COMMA signed",
- /* 181 */ "where_opt ::=",
- /* 182 */ "where_opt ::= WHERE expr",
- /* 183 */ "expr ::= LP expr RP",
- /* 184 */ "expr ::= ID",
- /* 185 */ "expr ::= ID DOT ID",
- /* 186 */ "expr ::= ID DOT STAR",
- /* 187 */ "expr ::= INTEGER",
- /* 188 */ "expr ::= MINUS INTEGER",
- /* 189 */ "expr ::= PLUS INTEGER",
- /* 190 */ "expr ::= FLOAT",
- /* 191 */ "expr ::= MINUS FLOAT",
- /* 192 */ "expr ::= PLUS FLOAT",
- /* 193 */ "expr ::= STRING",
- /* 194 */ "expr ::= NOW",
- /* 195 */ "expr ::= VARIABLE",
- /* 196 */ "expr ::= BOOL",
- /* 197 */ "expr ::= ID LP exprlist RP",
- /* 198 */ "expr ::= ID LP STAR RP",
- /* 199 */ "expr ::= expr IS NULL",
- /* 200 */ "expr ::= expr IS NOT NULL",
- /* 201 */ "expr ::= expr LT expr",
- /* 202 */ "expr ::= expr GT expr",
- /* 203 */ "expr ::= expr LE expr",
- /* 204 */ "expr ::= expr GE expr",
- /* 205 */ "expr ::= expr NE expr",
- /* 206 */ "expr ::= expr EQ expr",
- /* 207 */ "expr ::= expr AND expr",
- /* 208 */ "expr ::= expr OR expr",
- /* 209 */ "expr ::= expr PLUS expr",
- /* 210 */ "expr ::= expr MINUS expr",
- /* 211 */ "expr ::= expr STAR expr",
- /* 212 */ "expr ::= expr SLASH expr",
- /* 213 */ "expr ::= expr REM expr",
- /* 214 */ "expr ::= expr LIKE expr",
- /* 215 */ "expr ::= expr IN LP exprlist RP",
- /* 216 */ "exprlist ::= exprlist COMMA expritem",
- /* 217 */ "exprlist ::= expritem",
- /* 218 */ "expritem ::= expr",
- /* 219 */ "expritem ::=",
- /* 220 */ "cmd ::= RESET QUERY CACHE",
- /* 221 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist",
- /* 222 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids",
- /* 223 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist",
- /* 224 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids",
- /* 225 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids",
- /* 226 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem",
- /* 227 */ "cmd ::= KILL CONNECTION INTEGER",
- /* 228 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER",
- /* 229 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER",
+ /* 85 */ "update ::= UPDATE INTEGER",
+ /* 86 */ "db_optr ::=",
+ /* 87 */ "db_optr ::= db_optr cache",
+ /* 88 */ "db_optr ::= db_optr replica",
+ /* 89 */ "db_optr ::= db_optr quorum",
+ /* 90 */ "db_optr ::= db_optr days",
+ /* 91 */ "db_optr ::= db_optr minrows",
+ /* 92 */ "db_optr ::= db_optr maxrows",
+ /* 93 */ "db_optr ::= db_optr blocks",
+ /* 94 */ "db_optr ::= db_optr ctime",
+ /* 95 */ "db_optr ::= db_optr wal",
+ /* 96 */ "db_optr ::= db_optr fsync",
+ /* 97 */ "db_optr ::= db_optr comp",
+ /* 98 */ "db_optr ::= db_optr prec",
+ /* 99 */ "db_optr ::= db_optr keep",
+ /* 100 */ "db_optr ::= db_optr update",
+ /* 101 */ "alter_db_optr ::=",
+ /* 102 */ "alter_db_optr ::= alter_db_optr replica",
+ /* 103 */ "alter_db_optr ::= alter_db_optr quorum",
+ /* 104 */ "alter_db_optr ::= alter_db_optr keep",
+ /* 105 */ "alter_db_optr ::= alter_db_optr blocks",
+ /* 106 */ "alter_db_optr ::= alter_db_optr comp",
+ /* 107 */ "alter_db_optr ::= alter_db_optr wal",
+ /* 108 */ "alter_db_optr ::= alter_db_optr fsync",
+ /* 109 */ "alter_db_optr ::= alter_db_optr update",
+ /* 110 */ "typename ::= ids",
+ /* 111 */ "typename ::= ids LP signed RP",
+ /* 112 */ "signed ::= INTEGER",
+ /* 113 */ "signed ::= PLUS INTEGER",
+ /* 114 */ "signed ::= MINUS INTEGER",
+ /* 115 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args",
+ /* 116 */ "create_table_args ::= LP columnlist RP",
+ /* 117 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP",
+ /* 118 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP",
+ /* 119 */ "create_table_args ::= AS select",
+ /* 120 */ "columnlist ::= columnlist COMMA column",
+ /* 121 */ "columnlist ::= column",
+ /* 122 */ "column ::= ids typename",
+ /* 123 */ "tagitemlist ::= tagitemlist COMMA tagitem",
+ /* 124 */ "tagitemlist ::= tagitem",
+ /* 125 */ "tagitem ::= INTEGER",
+ /* 126 */ "tagitem ::= FLOAT",
+ /* 127 */ "tagitem ::= STRING",
+ /* 128 */ "tagitem ::= BOOL",
+ /* 129 */ "tagitem ::= NULL",
+ /* 130 */ "tagitem ::= MINUS INTEGER",
+ /* 131 */ "tagitem ::= MINUS FLOAT",
+ /* 132 */ "tagitem ::= PLUS INTEGER",
+ /* 133 */ "tagitem ::= PLUS FLOAT",
+ /* 134 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt",
+ /* 135 */ "union ::= select",
+ /* 136 */ "union ::= LP union RP",
+ /* 137 */ "union ::= union UNION ALL select",
+ /* 138 */ "union ::= union UNION ALL LP select RP",
+ /* 139 */ "cmd ::= union",
+ /* 140 */ "select ::= SELECT selcollist",
+ /* 141 */ "sclp ::= selcollist COMMA",
+ /* 142 */ "sclp ::=",
+ /* 143 */ "selcollist ::= sclp expr as",
+ /* 144 */ "selcollist ::= sclp STAR",
+ /* 145 */ "as ::= AS ids",
+ /* 146 */ "as ::= ids",
+ /* 147 */ "as ::=",
+ /* 148 */ "from ::= FROM tablelist",
+ /* 149 */ "tablelist ::= ids cpxName",
+ /* 150 */ "tablelist ::= ids cpxName ids",
+ /* 151 */ "tablelist ::= tablelist COMMA ids cpxName",
+ /* 152 */ "tablelist ::= tablelist COMMA ids cpxName ids",
+ /* 153 */ "tmvar ::= VARIABLE",
+ /* 154 */ "interval_opt ::= INTERVAL LP tmvar RP",
+ /* 155 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP",
+ /* 156 */ "interval_opt ::=",
+ /* 157 */ "fill_opt ::=",
+ /* 158 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP",
+ /* 159 */ "fill_opt ::= FILL LP ID RP",
+ /* 160 */ "sliding_opt ::= SLIDING LP tmvar RP",
+ /* 161 */ "sliding_opt ::=",
+ /* 162 */ "orderby_opt ::=",
+ /* 163 */ "orderby_opt ::= ORDER BY sortlist",
+ /* 164 */ "sortlist ::= sortlist COMMA item sortorder",
+ /* 165 */ "sortlist ::= item sortorder",
+ /* 166 */ "item ::= ids cpxName",
+ /* 167 */ "sortorder ::= ASC",
+ /* 168 */ "sortorder ::= DESC",
+ /* 169 */ "sortorder ::=",
+ /* 170 */ "groupby_opt ::=",
+ /* 171 */ "groupby_opt ::= GROUP BY grouplist",
+ /* 172 */ "grouplist ::= grouplist COMMA item",
+ /* 173 */ "grouplist ::= item",
+ /* 174 */ "having_opt ::=",
+ /* 175 */ "having_opt ::= HAVING expr",
+ /* 176 */ "limit_opt ::=",
+ /* 177 */ "limit_opt ::= LIMIT signed",
+ /* 178 */ "limit_opt ::= LIMIT signed OFFSET signed",
+ /* 179 */ "limit_opt ::= LIMIT signed COMMA signed",
+ /* 180 */ "slimit_opt ::=",
+ /* 181 */ "slimit_opt ::= SLIMIT signed",
+ /* 182 */ "slimit_opt ::= SLIMIT signed SOFFSET signed",
+ /* 183 */ "slimit_opt ::= SLIMIT signed COMMA signed",
+ /* 184 */ "where_opt ::=",
+ /* 185 */ "where_opt ::= WHERE expr",
+ /* 186 */ "expr ::= LP expr RP",
+ /* 187 */ "expr ::= ID",
+ /* 188 */ "expr ::= ID DOT ID",
+ /* 189 */ "expr ::= ID DOT STAR",
+ /* 190 */ "expr ::= INTEGER",
+ /* 191 */ "expr ::= MINUS INTEGER",
+ /* 192 */ "expr ::= PLUS INTEGER",
+ /* 193 */ "expr ::= FLOAT",
+ /* 194 */ "expr ::= MINUS FLOAT",
+ /* 195 */ "expr ::= PLUS FLOAT",
+ /* 196 */ "expr ::= STRING",
+ /* 197 */ "expr ::= NOW",
+ /* 198 */ "expr ::= VARIABLE",
+ /* 199 */ "expr ::= BOOL",
+ /* 200 */ "expr ::= ID LP exprlist RP",
+ /* 201 */ "expr ::= ID LP STAR RP",
+ /* 202 */ "expr ::= expr IS NULL",
+ /* 203 */ "expr ::= expr IS NOT NULL",
+ /* 204 */ "expr ::= expr LT expr",
+ /* 205 */ "expr ::= expr GT expr",
+ /* 206 */ "expr ::= expr LE expr",
+ /* 207 */ "expr ::= expr GE expr",
+ /* 208 */ "expr ::= expr NE expr",
+ /* 209 */ "expr ::= expr EQ expr",
+ /* 210 */ "expr ::= expr AND expr",
+ /* 211 */ "expr ::= expr OR expr",
+ /* 212 */ "expr ::= expr PLUS expr",
+ /* 213 */ "expr ::= expr MINUS expr",
+ /* 214 */ "expr ::= expr STAR expr",
+ /* 215 */ "expr ::= expr SLASH expr",
+ /* 216 */ "expr ::= expr REM expr",
+ /* 217 */ "expr ::= expr LIKE expr",
+ /* 218 */ "expr ::= expr IN LP exprlist RP",
+ /* 219 */ "exprlist ::= exprlist COMMA expritem",
+ /* 220 */ "exprlist ::= expritem",
+ /* 221 */ "expritem ::= expr",
+ /* 222 */ "expritem ::=",
+ /* 223 */ "cmd ::= RESET QUERY CACHE",
+ /* 224 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist",
+ /* 225 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids",
+ /* 226 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist",
+ /* 227 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids",
+ /* 228 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids",
+ /* 229 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem",
+ /* 230 */ "cmd ::= KILL CONNECTION INTEGER",
+ /* 231 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER",
+ /* 232 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER",
};
#endif /* NDEBUG */
@@ -1359,50 +1365,46 @@ static void yy_destructor(
** inside the C code.
*/
/********* Begin destructor definitions ***************************************/
- case 226: /* keep */
- case 227: /* tagitemlist */
- case 251: /* fill_opt */
- case 253: /* groupby_opt */
- case 254: /* orderby_opt */
- case 264: /* sortlist */
- case 268: /* grouplist */
+ case 227: /* keep */
+ case 228: /* tagitemlist */
+ case 245: /* columnlist */
+ case 253: /* fill_opt */
+ case 255: /* groupby_opt */
+ case 256: /* orderby_opt */
+ case 266: /* sortlist */
+ case 270: /* grouplist */
{
-tVariantListDestroy((yypminor->yy494));
+taosArrayDestroy((yypminor->yy165));
}
break;
- case 243: /* columnlist */
+ case 246: /* select */
{
-tFieldListDestroy((yypminor->yy449));
+doDestroyQuerySql((yypminor->yy414));
}
break;
- case 244: /* select */
+ case 249: /* selcollist */
+ case 261: /* sclp */
+ case 271: /* exprlist */
{
-doDestroyQuerySql((yypminor->yy150));
+tSQLExprListDestroy((yypminor->yy290));
}
break;
- case 247: /* selcollist */
- case 259: /* sclp */
- case 269: /* exprlist */
+ case 251: /* where_opt */
+ case 257: /* having_opt */
+ case 262: /* expr */
+ case 272: /* expritem */
{
-tSQLExprListDestroy((yypminor->yy224));
+tSQLExprDestroy((yypminor->yy64));
}
break;
- case 249: /* where_opt */
- case 255: /* having_opt */
- case 260: /* expr */
- case 270: /* expritem */
+ case 260: /* union */
{
-tSQLExprDestroy((yypminor->yy66));
+destroyAllSelectClause((yypminor->yy231));
}
break;
- case 258: /* union */
+ case 267: /* sortitem */
{
-destroyAllSelectClause((yypminor->yy25));
-}
- break;
- case 265: /* sortitem */
-{
-tVariantDestroy(&(yypminor->yy312));
+tVariantDestroy(&(yypminor->yy134));
}
break;
/********* End destructor definitions *****************************************/
@@ -1696,236 +1698,239 @@ static const struct {
YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */
signed char nrhs; /* Negative of the number of RHS symbols in the rule */
} yyRuleInfo[] = {
- { 207, -1 }, /* (0) program ::= cmd */
- { 208, -2 }, /* (1) cmd ::= SHOW DATABASES */
- { 208, -2 }, /* (2) cmd ::= SHOW MNODES */
- { 208, -2 }, /* (3) cmd ::= SHOW DNODES */
- { 208, -2 }, /* (4) cmd ::= SHOW ACCOUNTS */
- { 208, -2 }, /* (5) cmd ::= SHOW USERS */
- { 208, -2 }, /* (6) cmd ::= SHOW MODULES */
- { 208, -2 }, /* (7) cmd ::= SHOW QUERIES */
- { 208, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */
- { 208, -2 }, /* (9) cmd ::= SHOW STREAMS */
- { 208, -2 }, /* (10) cmd ::= SHOW VARIABLES */
- { 208, -2 }, /* (11) cmd ::= SHOW SCORES */
- { 208, -2 }, /* (12) cmd ::= SHOW GRANTS */
- { 208, -2 }, /* (13) cmd ::= SHOW VNODES */
- { 208, -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */
- { 209, 0 }, /* (15) dbPrefix ::= */
- { 209, -2 }, /* (16) dbPrefix ::= ids DOT */
- { 211, 0 }, /* (17) cpxName ::= */
- { 211, -2 }, /* (18) cpxName ::= DOT ids */
- { 208, -5 }, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */
- { 208, -4 }, /* (20) cmd ::= SHOW CREATE DATABASE ids */
- { 208, -3 }, /* (21) cmd ::= SHOW dbPrefix TABLES */
- { 208, -5 }, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */
- { 208, -3 }, /* (23) cmd ::= SHOW dbPrefix STABLES */
- { 208, -5 }, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */
- { 208, -3 }, /* (25) cmd ::= SHOW dbPrefix VGROUPS */
- { 208, -4 }, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */
- { 208, -5 }, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */
- { 208, -4 }, /* (28) cmd ::= DROP DATABASE ifexists ids */
- { 208, -3 }, /* (29) cmd ::= DROP DNODE ids */
- { 208, -3 }, /* (30) cmd ::= DROP USER ids */
- { 208, -3 }, /* (31) cmd ::= DROP ACCOUNT ids */
- { 208, -2 }, /* (32) cmd ::= USE ids */
- { 208, -3 }, /* (33) cmd ::= DESCRIBE ids cpxName */
- { 208, -5 }, /* (34) cmd ::= ALTER USER ids PASS ids */
- { 208, -5 }, /* (35) cmd ::= ALTER USER ids PRIVILEGE ids */
- { 208, -4 }, /* (36) cmd ::= ALTER DNODE ids ids */
- { 208, -5 }, /* (37) cmd ::= ALTER DNODE ids ids ids */
- { 208, -3 }, /* (38) cmd ::= ALTER LOCAL ids */
- { 208, -4 }, /* (39) cmd ::= ALTER LOCAL ids ids */
- { 208, -4 }, /* (40) cmd ::= ALTER DATABASE ids alter_db_optr */
- { 208, -4 }, /* (41) cmd ::= ALTER ACCOUNT ids acct_optr */
- { 208, -6 }, /* (42) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */
- { 210, -1 }, /* (43) ids ::= ID */
- { 210, -1 }, /* (44) ids ::= STRING */
- { 212, -2 }, /* (45) ifexists ::= IF EXISTS */
- { 212, 0 }, /* (46) ifexists ::= */
- { 215, -3 }, /* (47) ifnotexists ::= IF NOT EXISTS */
- { 215, 0 }, /* (48) ifnotexists ::= */
- { 208, -3 }, /* (49) cmd ::= CREATE DNODE ids */
- { 208, -6 }, /* (50) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */
- { 208, -5 }, /* (51) cmd ::= CREATE DATABASE ifnotexists ids db_optr */
- { 208, -5 }, /* (52) cmd ::= CREATE USER ids PASS ids */
- { 217, 0 }, /* (53) pps ::= */
- { 217, -2 }, /* (54) pps ::= PPS INTEGER */
- { 218, 0 }, /* (55) tseries ::= */
- { 218, -2 }, /* (56) tseries ::= TSERIES INTEGER */
- { 219, 0 }, /* (57) dbs ::= */
- { 219, -2 }, /* (58) dbs ::= DBS INTEGER */
- { 220, 0 }, /* (59) streams ::= */
- { 220, -2 }, /* (60) streams ::= STREAMS INTEGER */
- { 221, 0 }, /* (61) storage ::= */
- { 221, -2 }, /* (62) storage ::= STORAGE INTEGER */
- { 222, 0 }, /* (63) qtime ::= */
- { 222, -2 }, /* (64) qtime ::= QTIME INTEGER */
- { 223, 0 }, /* (65) users ::= */
- { 223, -2 }, /* (66) users ::= USERS INTEGER */
- { 224, 0 }, /* (67) conns ::= */
- { 224, -2 }, /* (68) conns ::= CONNS INTEGER */
- { 225, 0 }, /* (69) state ::= */
- { 225, -2 }, /* (70) state ::= STATE ids */
- { 214, -9 }, /* (71) acct_optr ::= pps tseries storage streams qtime dbs users conns state */
- { 226, -2 }, /* (72) keep ::= KEEP tagitemlist */
- { 228, -2 }, /* (73) cache ::= CACHE INTEGER */
- { 229, -2 }, /* (74) replica ::= REPLICA INTEGER */
- { 230, -2 }, /* (75) quorum ::= QUORUM INTEGER */
- { 231, -2 }, /* (76) days ::= DAYS INTEGER */
- { 232, -2 }, /* (77) minrows ::= MINROWS INTEGER */
- { 233, -2 }, /* (78) maxrows ::= MAXROWS INTEGER */
- { 234, -2 }, /* (79) blocks ::= BLOCKS INTEGER */
- { 235, -2 }, /* (80) ctime ::= CTIME INTEGER */
- { 236, -2 }, /* (81) wal ::= WAL INTEGER */
- { 237, -2 }, /* (82) fsync ::= FSYNC INTEGER */
- { 238, -2 }, /* (83) comp ::= COMP INTEGER */
- { 239, -2 }, /* (84) prec ::= PRECISION STRING */
- { 216, 0 }, /* (85) db_optr ::= */
- { 216, -2 }, /* (86) db_optr ::= db_optr cache */
- { 216, -2 }, /* (87) db_optr ::= db_optr replica */
- { 216, -2 }, /* (88) db_optr ::= db_optr quorum */
- { 216, -2 }, /* (89) db_optr ::= db_optr days */
- { 216, -2 }, /* (90) db_optr ::= db_optr minrows */
- { 216, -2 }, /* (91) db_optr ::= db_optr maxrows */
- { 216, -2 }, /* (92) db_optr ::= db_optr blocks */
- { 216, -2 }, /* (93) db_optr ::= db_optr ctime */
- { 216, -2 }, /* (94) db_optr ::= db_optr wal */
- { 216, -2 }, /* (95) db_optr ::= db_optr fsync */
- { 216, -2 }, /* (96) db_optr ::= db_optr comp */
- { 216, -2 }, /* (97) db_optr ::= db_optr prec */
- { 216, -2 }, /* (98) db_optr ::= db_optr keep */
- { 213, 0 }, /* (99) alter_db_optr ::= */
- { 213, -2 }, /* (100) alter_db_optr ::= alter_db_optr replica */
- { 213, -2 }, /* (101) alter_db_optr ::= alter_db_optr quorum */
- { 213, -2 }, /* (102) alter_db_optr ::= alter_db_optr keep */
- { 213, -2 }, /* (103) alter_db_optr ::= alter_db_optr blocks */
- { 213, -2 }, /* (104) alter_db_optr ::= alter_db_optr comp */
- { 213, -2 }, /* (105) alter_db_optr ::= alter_db_optr wal */
- { 213, -2 }, /* (106) alter_db_optr ::= alter_db_optr fsync */
- { 240, -1 }, /* (107) typename ::= ids */
- { 240, -4 }, /* (108) typename ::= ids LP signed RP */
- { 241, -1 }, /* (109) signed ::= INTEGER */
- { 241, -2 }, /* (110) signed ::= PLUS INTEGER */
- { 241, -2 }, /* (111) signed ::= MINUS INTEGER */
- { 208, -6 }, /* (112) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
- { 242, -3 }, /* (113) create_table_args ::= LP columnlist RP */
- { 242, -7 }, /* (114) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
- { 242, -7 }, /* (115) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
- { 242, -2 }, /* (116) create_table_args ::= AS select */
- { 243, -3 }, /* (117) columnlist ::= columnlist COMMA column */
- { 243, -1 }, /* (118) columnlist ::= column */
- { 245, -2 }, /* (119) column ::= ids typename */
- { 227, -3 }, /* (120) tagitemlist ::= tagitemlist COMMA tagitem */
- { 227, -1 }, /* (121) tagitemlist ::= tagitem */
- { 246, -1 }, /* (122) tagitem ::= INTEGER */
- { 246, -1 }, /* (123) tagitem ::= FLOAT */
- { 246, -1 }, /* (124) tagitem ::= STRING */
- { 246, -1 }, /* (125) tagitem ::= BOOL */
- { 246, -1 }, /* (126) tagitem ::= NULL */
- { 246, -2 }, /* (127) tagitem ::= MINUS INTEGER */
- { 246, -2 }, /* (128) tagitem ::= MINUS FLOAT */
- { 246, -2 }, /* (129) tagitem ::= PLUS INTEGER */
- { 246, -2 }, /* (130) tagitem ::= PLUS FLOAT */
- { 244, -12 }, /* (131) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
- { 258, -1 }, /* (132) union ::= select */
- { 258, -3 }, /* (133) union ::= LP union RP */
- { 258, -4 }, /* (134) union ::= union UNION ALL select */
- { 258, -6 }, /* (135) union ::= union UNION ALL LP select RP */
- { 208, -1 }, /* (136) cmd ::= union */
- { 244, -2 }, /* (137) select ::= SELECT selcollist */
- { 259, -2 }, /* (138) sclp ::= selcollist COMMA */
- { 259, 0 }, /* (139) sclp ::= */
- { 247, -3 }, /* (140) selcollist ::= sclp expr as */
- { 247, -2 }, /* (141) selcollist ::= sclp STAR */
- { 261, -2 }, /* (142) as ::= AS ids */
- { 261, -1 }, /* (143) as ::= ids */
- { 261, 0 }, /* (144) as ::= */
- { 248, -2 }, /* (145) from ::= FROM tablelist */
- { 262, -2 }, /* (146) tablelist ::= ids cpxName */
- { 262, -3 }, /* (147) tablelist ::= ids cpxName ids */
- { 262, -4 }, /* (148) tablelist ::= tablelist COMMA ids cpxName */
- { 262, -5 }, /* (149) tablelist ::= tablelist COMMA ids cpxName ids */
- { 263, -1 }, /* (150) tmvar ::= VARIABLE */
- { 250, -4 }, /* (151) interval_opt ::= INTERVAL LP tmvar RP */
- { 250, -6 }, /* (152) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */
- { 250, 0 }, /* (153) interval_opt ::= */
- { 251, 0 }, /* (154) fill_opt ::= */
- { 251, -6 }, /* (155) fill_opt ::= FILL LP ID COMMA tagitemlist RP */
- { 251, -4 }, /* (156) fill_opt ::= FILL LP ID RP */
- { 252, -4 }, /* (157) sliding_opt ::= SLIDING LP tmvar RP */
- { 252, 0 }, /* (158) sliding_opt ::= */
- { 254, 0 }, /* (159) orderby_opt ::= */
- { 254, -3 }, /* (160) orderby_opt ::= ORDER BY sortlist */
- { 264, -4 }, /* (161) sortlist ::= sortlist COMMA item sortorder */
- { 264, -2 }, /* (162) sortlist ::= item sortorder */
- { 266, -2 }, /* (163) item ::= ids cpxName */
- { 267, -1 }, /* (164) sortorder ::= ASC */
- { 267, -1 }, /* (165) sortorder ::= DESC */
- { 267, 0 }, /* (166) sortorder ::= */
- { 253, 0 }, /* (167) groupby_opt ::= */
- { 253, -3 }, /* (168) groupby_opt ::= GROUP BY grouplist */
- { 268, -3 }, /* (169) grouplist ::= grouplist COMMA item */
- { 268, -1 }, /* (170) grouplist ::= item */
- { 255, 0 }, /* (171) having_opt ::= */
- { 255, -2 }, /* (172) having_opt ::= HAVING expr */
- { 257, 0 }, /* (173) limit_opt ::= */
- { 257, -2 }, /* (174) limit_opt ::= LIMIT signed */
- { 257, -4 }, /* (175) limit_opt ::= LIMIT signed OFFSET signed */
- { 257, -4 }, /* (176) limit_opt ::= LIMIT signed COMMA signed */
- { 256, 0 }, /* (177) slimit_opt ::= */
- { 256, -2 }, /* (178) slimit_opt ::= SLIMIT signed */
- { 256, -4 }, /* (179) slimit_opt ::= SLIMIT signed SOFFSET signed */
- { 256, -4 }, /* (180) slimit_opt ::= SLIMIT signed COMMA signed */
- { 249, 0 }, /* (181) where_opt ::= */
- { 249, -2 }, /* (182) where_opt ::= WHERE expr */
- { 260, -3 }, /* (183) expr ::= LP expr RP */
- { 260, -1 }, /* (184) expr ::= ID */
- { 260, -3 }, /* (185) expr ::= ID DOT ID */
- { 260, -3 }, /* (186) expr ::= ID DOT STAR */
- { 260, -1 }, /* (187) expr ::= INTEGER */
- { 260, -2 }, /* (188) expr ::= MINUS INTEGER */
- { 260, -2 }, /* (189) expr ::= PLUS INTEGER */
- { 260, -1 }, /* (190) expr ::= FLOAT */
- { 260, -2 }, /* (191) expr ::= MINUS FLOAT */
- { 260, -2 }, /* (192) expr ::= PLUS FLOAT */
- { 260, -1 }, /* (193) expr ::= STRING */
- { 260, -1 }, /* (194) expr ::= NOW */
- { 260, -1 }, /* (195) expr ::= VARIABLE */
- { 260, -1 }, /* (196) expr ::= BOOL */
- { 260, -4 }, /* (197) expr ::= ID LP exprlist RP */
- { 260, -4 }, /* (198) expr ::= ID LP STAR RP */
- { 260, -3 }, /* (199) expr ::= expr IS NULL */
- { 260, -4 }, /* (200) expr ::= expr IS NOT NULL */
- { 260, -3 }, /* (201) expr ::= expr LT expr */
- { 260, -3 }, /* (202) expr ::= expr GT expr */
- { 260, -3 }, /* (203) expr ::= expr LE expr */
- { 260, -3 }, /* (204) expr ::= expr GE expr */
- { 260, -3 }, /* (205) expr ::= expr NE expr */
- { 260, -3 }, /* (206) expr ::= expr EQ expr */
- { 260, -3 }, /* (207) expr ::= expr AND expr */
- { 260, -3 }, /* (208) expr ::= expr OR expr */
- { 260, -3 }, /* (209) expr ::= expr PLUS expr */
- { 260, -3 }, /* (210) expr ::= expr MINUS expr */
- { 260, -3 }, /* (211) expr ::= expr STAR expr */
- { 260, -3 }, /* (212) expr ::= expr SLASH expr */
- { 260, -3 }, /* (213) expr ::= expr REM expr */
- { 260, -3 }, /* (214) expr ::= expr LIKE expr */
- { 260, -5 }, /* (215) expr ::= expr IN LP exprlist RP */
- { 269, -3 }, /* (216) exprlist ::= exprlist COMMA expritem */
- { 269, -1 }, /* (217) exprlist ::= expritem */
- { 270, -1 }, /* (218) expritem ::= expr */
- { 270, 0 }, /* (219) expritem ::= */
- { 208, -3 }, /* (220) cmd ::= RESET QUERY CACHE */
- { 208, -7 }, /* (221) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
- { 208, -7 }, /* (222) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
- { 208, -7 }, /* (223) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
- { 208, -7 }, /* (224) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
- { 208, -8 }, /* (225) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
- { 208, -9 }, /* (226) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
- { 208, -3 }, /* (227) cmd ::= KILL CONNECTION INTEGER */
- { 208, -5 }, /* (228) cmd ::= KILL STREAM INTEGER COLON INTEGER */
- { 208, -5 }, /* (229) cmd ::= KILL QUERY INTEGER COLON INTEGER */
+ { 208, -1 }, /* (0) program ::= cmd */
+ { 209, -2 }, /* (1) cmd ::= SHOW DATABASES */
+ { 209, -2 }, /* (2) cmd ::= SHOW MNODES */
+ { 209, -2 }, /* (3) cmd ::= SHOW DNODES */
+ { 209, -2 }, /* (4) cmd ::= SHOW ACCOUNTS */
+ { 209, -2 }, /* (5) cmd ::= SHOW USERS */
+ { 209, -2 }, /* (6) cmd ::= SHOW MODULES */
+ { 209, -2 }, /* (7) cmd ::= SHOW QUERIES */
+ { 209, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */
+ { 209, -2 }, /* (9) cmd ::= SHOW STREAMS */
+ { 209, -2 }, /* (10) cmd ::= SHOW VARIABLES */
+ { 209, -2 }, /* (11) cmd ::= SHOW SCORES */
+ { 209, -2 }, /* (12) cmd ::= SHOW GRANTS */
+ { 209, -2 }, /* (13) cmd ::= SHOW VNODES */
+ { 209, -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */
+ { 210, 0 }, /* (15) dbPrefix ::= */
+ { 210, -2 }, /* (16) dbPrefix ::= ids DOT */
+ { 212, 0 }, /* (17) cpxName ::= */
+ { 212, -2 }, /* (18) cpxName ::= DOT ids */
+ { 209, -5 }, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */
+ { 209, -4 }, /* (20) cmd ::= SHOW CREATE DATABASE ids */
+ { 209, -3 }, /* (21) cmd ::= SHOW dbPrefix TABLES */
+ { 209, -5 }, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */
+ { 209, -3 }, /* (23) cmd ::= SHOW dbPrefix STABLES */
+ { 209, -5 }, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */
+ { 209, -3 }, /* (25) cmd ::= SHOW dbPrefix VGROUPS */
+ { 209, -4 }, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */
+ { 209, -5 }, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */
+ { 209, -4 }, /* (28) cmd ::= DROP DATABASE ifexists ids */
+ { 209, -3 }, /* (29) cmd ::= DROP DNODE ids */
+ { 209, -3 }, /* (30) cmd ::= DROP USER ids */
+ { 209, -3 }, /* (31) cmd ::= DROP ACCOUNT ids */
+ { 209, -2 }, /* (32) cmd ::= USE ids */
+ { 209, -3 }, /* (33) cmd ::= DESCRIBE ids cpxName */
+ { 209, -5 }, /* (34) cmd ::= ALTER USER ids PASS ids */
+ { 209, -5 }, /* (35) cmd ::= ALTER USER ids PRIVILEGE ids */
+ { 209, -4 }, /* (36) cmd ::= ALTER DNODE ids ids */
+ { 209, -5 }, /* (37) cmd ::= ALTER DNODE ids ids ids */
+ { 209, -3 }, /* (38) cmd ::= ALTER LOCAL ids */
+ { 209, -4 }, /* (39) cmd ::= ALTER LOCAL ids ids */
+ { 209, -4 }, /* (40) cmd ::= ALTER DATABASE ids alter_db_optr */
+ { 209, -4 }, /* (41) cmd ::= ALTER ACCOUNT ids acct_optr */
+ { 209, -6 }, /* (42) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */
+ { 211, -1 }, /* (43) ids ::= ID */
+ { 211, -1 }, /* (44) ids ::= STRING */
+ { 213, -2 }, /* (45) ifexists ::= IF EXISTS */
+ { 213, 0 }, /* (46) ifexists ::= */
+ { 216, -3 }, /* (47) ifnotexists ::= IF NOT EXISTS */
+ { 216, 0 }, /* (48) ifnotexists ::= */
+ { 209, -3 }, /* (49) cmd ::= CREATE DNODE ids */
+ { 209, -6 }, /* (50) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */
+ { 209, -5 }, /* (51) cmd ::= CREATE DATABASE ifnotexists ids db_optr */
+ { 209, -5 }, /* (52) cmd ::= CREATE USER ids PASS ids */
+ { 218, 0 }, /* (53) pps ::= */
+ { 218, -2 }, /* (54) pps ::= PPS INTEGER */
+ { 219, 0 }, /* (55) tseries ::= */
+ { 219, -2 }, /* (56) tseries ::= TSERIES INTEGER */
+ { 220, 0 }, /* (57) dbs ::= */
+ { 220, -2 }, /* (58) dbs ::= DBS INTEGER */
+ { 221, 0 }, /* (59) streams ::= */
+ { 221, -2 }, /* (60) streams ::= STREAMS INTEGER */
+ { 222, 0 }, /* (61) storage ::= */
+ { 222, -2 }, /* (62) storage ::= STORAGE INTEGER */
+ { 223, 0 }, /* (63) qtime ::= */
+ { 223, -2 }, /* (64) qtime ::= QTIME INTEGER */
+ { 224, 0 }, /* (65) users ::= */
+ { 224, -2 }, /* (66) users ::= USERS INTEGER */
+ { 225, 0 }, /* (67) conns ::= */
+ { 225, -2 }, /* (68) conns ::= CONNS INTEGER */
+ { 226, 0 }, /* (69) state ::= */
+ { 226, -2 }, /* (70) state ::= STATE ids */
+ { 215, -9 }, /* (71) acct_optr ::= pps tseries storage streams qtime dbs users conns state */
+ { 227, -2 }, /* (72) keep ::= KEEP tagitemlist */
+ { 229, -2 }, /* (73) cache ::= CACHE INTEGER */
+ { 230, -2 }, /* (74) replica ::= REPLICA INTEGER */
+ { 231, -2 }, /* (75) quorum ::= QUORUM INTEGER */
+ { 232, -2 }, /* (76) days ::= DAYS INTEGER */
+ { 233, -2 }, /* (77) minrows ::= MINROWS INTEGER */
+ { 234, -2 }, /* (78) maxrows ::= MAXROWS INTEGER */
+ { 235, -2 }, /* (79) blocks ::= BLOCKS INTEGER */
+ { 236, -2 }, /* (80) ctime ::= CTIME INTEGER */
+ { 237, -2 }, /* (81) wal ::= WAL INTEGER */
+ { 238, -2 }, /* (82) fsync ::= FSYNC INTEGER */
+ { 239, -2 }, /* (83) comp ::= COMP INTEGER */
+ { 240, -2 }, /* (84) prec ::= PRECISION STRING */
+ { 241, -2 }, /* (85) update ::= UPDATE INTEGER */
+ { 217, 0 }, /* (86) db_optr ::= */
+ { 217, -2 }, /* (87) db_optr ::= db_optr cache */
+ { 217, -2 }, /* (88) db_optr ::= db_optr replica */
+ { 217, -2 }, /* (89) db_optr ::= db_optr quorum */
+ { 217, -2 }, /* (90) db_optr ::= db_optr days */
+ { 217, -2 }, /* (91) db_optr ::= db_optr minrows */
+ { 217, -2 }, /* (92) db_optr ::= db_optr maxrows */
+ { 217, -2 }, /* (93) db_optr ::= db_optr blocks */
+ { 217, -2 }, /* (94) db_optr ::= db_optr ctime */
+ { 217, -2 }, /* (95) db_optr ::= db_optr wal */
+ { 217, -2 }, /* (96) db_optr ::= db_optr fsync */
+ { 217, -2 }, /* (97) db_optr ::= db_optr comp */
+ { 217, -2 }, /* (98) db_optr ::= db_optr prec */
+ { 217, -2 }, /* (99) db_optr ::= db_optr keep */
+ { 217, -2 }, /* (100) db_optr ::= db_optr update */
+ { 214, 0 }, /* (101) alter_db_optr ::= */
+ { 214, -2 }, /* (102) alter_db_optr ::= alter_db_optr replica */
+ { 214, -2 }, /* (103) alter_db_optr ::= alter_db_optr quorum */
+ { 214, -2 }, /* (104) alter_db_optr ::= alter_db_optr keep */
+ { 214, -2 }, /* (105) alter_db_optr ::= alter_db_optr blocks */
+ { 214, -2 }, /* (106) alter_db_optr ::= alter_db_optr comp */
+ { 214, -2 }, /* (107) alter_db_optr ::= alter_db_optr wal */
+ { 214, -2 }, /* (108) alter_db_optr ::= alter_db_optr fsync */
+ { 214, -2 }, /* (109) alter_db_optr ::= alter_db_optr update */
+ { 242, -1 }, /* (110) typename ::= ids */
+ { 242, -4 }, /* (111) typename ::= ids LP signed RP */
+ { 243, -1 }, /* (112) signed ::= INTEGER */
+ { 243, -2 }, /* (113) signed ::= PLUS INTEGER */
+ { 243, -2 }, /* (114) signed ::= MINUS INTEGER */
+ { 209, -6 }, /* (115) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
+ { 244, -3 }, /* (116) create_table_args ::= LP columnlist RP */
+ { 244, -7 }, /* (117) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
+ { 244, -7 }, /* (118) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
+ { 244, -2 }, /* (119) create_table_args ::= AS select */
+ { 245, -3 }, /* (120) columnlist ::= columnlist COMMA column */
+ { 245, -1 }, /* (121) columnlist ::= column */
+ { 247, -2 }, /* (122) column ::= ids typename */
+ { 228, -3 }, /* (123) tagitemlist ::= tagitemlist COMMA tagitem */
+ { 228, -1 }, /* (124) tagitemlist ::= tagitem */
+ { 248, -1 }, /* (125) tagitem ::= INTEGER */
+ { 248, -1 }, /* (126) tagitem ::= FLOAT */
+ { 248, -1 }, /* (127) tagitem ::= STRING */
+ { 248, -1 }, /* (128) tagitem ::= BOOL */
+ { 248, -1 }, /* (129) tagitem ::= NULL */
+ { 248, -2 }, /* (130) tagitem ::= MINUS INTEGER */
+ { 248, -2 }, /* (131) tagitem ::= MINUS FLOAT */
+ { 248, -2 }, /* (132) tagitem ::= PLUS INTEGER */
+ { 248, -2 }, /* (133) tagitem ::= PLUS FLOAT */
+ { 246, -12 }, /* (134) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
+ { 260, -1 }, /* (135) union ::= select */
+ { 260, -3 }, /* (136) union ::= LP union RP */
+ { 260, -4 }, /* (137) union ::= union UNION ALL select */
+ { 260, -6 }, /* (138) union ::= union UNION ALL LP select RP */
+ { 209, -1 }, /* (139) cmd ::= union */
+ { 246, -2 }, /* (140) select ::= SELECT selcollist */
+ { 261, -2 }, /* (141) sclp ::= selcollist COMMA */
+ { 261, 0 }, /* (142) sclp ::= */
+ { 249, -3 }, /* (143) selcollist ::= sclp expr as */
+ { 249, -2 }, /* (144) selcollist ::= sclp STAR */
+ { 263, -2 }, /* (145) as ::= AS ids */
+ { 263, -1 }, /* (146) as ::= ids */
+ { 263, 0 }, /* (147) as ::= */
+ { 250, -2 }, /* (148) from ::= FROM tablelist */
+ { 264, -2 }, /* (149) tablelist ::= ids cpxName */
+ { 264, -3 }, /* (150) tablelist ::= ids cpxName ids */
+ { 264, -4 }, /* (151) tablelist ::= tablelist COMMA ids cpxName */
+ { 264, -5 }, /* (152) tablelist ::= tablelist COMMA ids cpxName ids */
+ { 265, -1 }, /* (153) tmvar ::= VARIABLE */
+ { 252, -4 }, /* (154) interval_opt ::= INTERVAL LP tmvar RP */
+ { 252, -6 }, /* (155) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */
+ { 252, 0 }, /* (156) interval_opt ::= */
+ { 253, 0 }, /* (157) fill_opt ::= */
+ { 253, -6 }, /* (158) fill_opt ::= FILL LP ID COMMA tagitemlist RP */
+ { 253, -4 }, /* (159) fill_opt ::= FILL LP ID RP */
+ { 254, -4 }, /* (160) sliding_opt ::= SLIDING LP tmvar RP */
+ { 254, 0 }, /* (161) sliding_opt ::= */
+ { 256, 0 }, /* (162) orderby_opt ::= */
+ { 256, -3 }, /* (163) orderby_opt ::= ORDER BY sortlist */
+ { 266, -4 }, /* (164) sortlist ::= sortlist COMMA item sortorder */
+ { 266, -2 }, /* (165) sortlist ::= item sortorder */
+ { 268, -2 }, /* (166) item ::= ids cpxName */
+ { 269, -1 }, /* (167) sortorder ::= ASC */
+ { 269, -1 }, /* (168) sortorder ::= DESC */
+ { 269, 0 }, /* (169) sortorder ::= */
+ { 255, 0 }, /* (170) groupby_opt ::= */
+ { 255, -3 }, /* (171) groupby_opt ::= GROUP BY grouplist */
+ { 270, -3 }, /* (172) grouplist ::= grouplist COMMA item */
+ { 270, -1 }, /* (173) grouplist ::= item */
+ { 257, 0 }, /* (174) having_opt ::= */
+ { 257, -2 }, /* (175) having_opt ::= HAVING expr */
+ { 259, 0 }, /* (176) limit_opt ::= */
+ { 259, -2 }, /* (177) limit_opt ::= LIMIT signed */
+ { 259, -4 }, /* (178) limit_opt ::= LIMIT signed OFFSET signed */
+ { 259, -4 }, /* (179) limit_opt ::= LIMIT signed COMMA signed */
+ { 258, 0 }, /* (180) slimit_opt ::= */
+ { 258, -2 }, /* (181) slimit_opt ::= SLIMIT signed */
+ { 258, -4 }, /* (182) slimit_opt ::= SLIMIT signed SOFFSET signed */
+ { 258, -4 }, /* (183) slimit_opt ::= SLIMIT signed COMMA signed */
+ { 251, 0 }, /* (184) where_opt ::= */
+ { 251, -2 }, /* (185) where_opt ::= WHERE expr */
+ { 262, -3 }, /* (186) expr ::= LP expr RP */
+ { 262, -1 }, /* (187) expr ::= ID */
+ { 262, -3 }, /* (188) expr ::= ID DOT ID */
+ { 262, -3 }, /* (189) expr ::= ID DOT STAR */
+ { 262, -1 }, /* (190) expr ::= INTEGER */
+ { 262, -2 }, /* (191) expr ::= MINUS INTEGER */
+ { 262, -2 }, /* (192) expr ::= PLUS INTEGER */
+ { 262, -1 }, /* (193) expr ::= FLOAT */
+ { 262, -2 }, /* (194) expr ::= MINUS FLOAT */
+ { 262, -2 }, /* (195) expr ::= PLUS FLOAT */
+ { 262, -1 }, /* (196) expr ::= STRING */
+ { 262, -1 }, /* (197) expr ::= NOW */
+ { 262, -1 }, /* (198) expr ::= VARIABLE */
+ { 262, -1 }, /* (199) expr ::= BOOL */
+ { 262, -4 }, /* (200) expr ::= ID LP exprlist RP */
+ { 262, -4 }, /* (201) expr ::= ID LP STAR RP */
+ { 262, -3 }, /* (202) expr ::= expr IS NULL */
+ { 262, -4 }, /* (203) expr ::= expr IS NOT NULL */
+ { 262, -3 }, /* (204) expr ::= expr LT expr */
+ { 262, -3 }, /* (205) expr ::= expr GT expr */
+ { 262, -3 }, /* (206) expr ::= expr LE expr */
+ { 262, -3 }, /* (207) expr ::= expr GE expr */
+ { 262, -3 }, /* (208) expr ::= expr NE expr */
+ { 262, -3 }, /* (209) expr ::= expr EQ expr */
+ { 262, -3 }, /* (210) expr ::= expr AND expr */
+ { 262, -3 }, /* (211) expr ::= expr OR expr */
+ { 262, -3 }, /* (212) expr ::= expr PLUS expr */
+ { 262, -3 }, /* (213) expr ::= expr MINUS expr */
+ { 262, -3 }, /* (214) expr ::= expr STAR expr */
+ { 262, -3 }, /* (215) expr ::= expr SLASH expr */
+ { 262, -3 }, /* (216) expr ::= expr REM expr */
+ { 262, -3 }, /* (217) expr ::= expr LIKE expr */
+ { 262, -5 }, /* (218) expr ::= expr IN LP exprlist RP */
+ { 271, -3 }, /* (219) exprlist ::= exprlist COMMA expritem */
+ { 271, -1 }, /* (220) exprlist ::= expritem */
+ { 272, -1 }, /* (221) expritem ::= expr */
+ { 272, 0 }, /* (222) expritem ::= */
+ { 209, -3 }, /* (223) cmd ::= RESET QUERY CACHE */
+ { 209, -7 }, /* (224) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
+ { 209, -7 }, /* (225) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
+ { 209, -7 }, /* (226) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
+ { 209, -7 }, /* (227) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
+ { 209, -8 }, /* (228) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
+ { 209, -9 }, /* (229) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
+ { 209, -3 }, /* (230) cmd ::= KILL CONNECTION INTEGER */
+ { 209, -5 }, /* (231) cmd ::= KILL STREAM INTEGER COLON INTEGER */
+ { 209, -5 }, /* (232) cmd ::= KILL QUERY INTEGER COLON INTEGER */
};
static void yy_accept(yyParser*); /* Forward Declaration */
@@ -2156,13 +2161,13 @@ static void yy_reduce(
{ setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); }
break;
case 40: /* cmd ::= ALTER DATABASE ids alter_db_optr */
-{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy158, &t);}
+{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);}
break;
case 41: /* cmd ::= ALTER ACCOUNT ids acct_optr */
-{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy73);}
+{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);}
break;
case 42: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */
-{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy73);}
+{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);}
break;
case 43: /* ids ::= ID */
case 44: /* ids ::= STRING */ yytestcase(yyruleno==44);
@@ -2183,10 +2188,10 @@ static void yy_reduce(
{ setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);}
break;
case 50: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */
-{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy73);}
+{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);}
break;
case 51: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */
-{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy158, &yymsp[-2].minor.yy0);}
+{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &yymsp[-2].minor.yy0);}
break;
case 52: /* cmd ::= CREATE USER ids PASS ids */
{ setCreateUserSQL(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);}
@@ -2215,20 +2220,20 @@ static void yy_reduce(
break;
case 71: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */
{
- yylhsminor.yy73.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1;
- yylhsminor.yy73.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1;
- yylhsminor.yy73.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1;
- yylhsminor.yy73.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1;
- yylhsminor.yy73.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1;
- yylhsminor.yy73.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1;
- yylhsminor.yy73.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1;
- yylhsminor.yy73.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1;
- yylhsminor.yy73.stat = yymsp[0].minor.yy0;
-}
- yymsp[-8].minor.yy73 = yylhsminor.yy73;
+ yylhsminor.yy149.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1;
+ yylhsminor.yy149.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1;
+ yylhsminor.yy149.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1;
+ yylhsminor.yy149.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1;
+ yylhsminor.yy149.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1;
+ yylhsminor.yy149.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1;
+ yylhsminor.yy149.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1;
+ yylhsminor.yy149.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1;
+ yylhsminor.yy149.stat = yymsp[0].minor.yy0;
+}
+ yymsp[-8].minor.yy149 = yylhsminor.yy149;
break;
case 72: /* keep ::= KEEP tagitemlist */
-{ yymsp[-1].minor.yy494 = yymsp[0].minor.yy494; }
+{ yymsp[-1].minor.yy165 = yymsp[0].minor.yy165; }
break;
case 73: /* cache ::= CACHE INTEGER */
case 74: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==74);
@@ -2242,574 +2247,586 @@ static void yy_reduce(
case 82: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==82);
case 83: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==83);
case 84: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==84);
+ case 85: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==85);
{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; }
break;
- case 85: /* db_optr ::= */
-{setDefaultCreateDbOption(&yymsp[1].minor.yy158);}
- break;
- case 86: /* db_optr ::= db_optr cache */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 87: /* db_optr ::= db_optr replica */
- case 100: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==100);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 88: /* db_optr ::= db_optr quorum */
- case 101: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==101);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 89: /* db_optr ::= db_optr days */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 90: /* db_optr ::= db_optr minrows */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 91: /* db_optr ::= db_optr maxrows */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 92: /* db_optr ::= db_optr blocks */
- case 103: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==103);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 93: /* db_optr ::= db_optr ctime */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 94: /* db_optr ::= db_optr wal */
- case 105: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==105);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 95: /* db_optr ::= db_optr fsync */
- case 106: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==106);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 96: /* db_optr ::= db_optr comp */
- case 104: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==104);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 97: /* db_optr ::= db_optr prec */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.precision = yymsp[0].minor.yy0; }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 98: /* db_optr ::= db_optr keep */
- case 102: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==102);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.keep = yymsp[0].minor.yy494; }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 99: /* alter_db_optr ::= */
-{ setDefaultCreateDbOption(&yymsp[1].minor.yy158);}
- break;
- case 107: /* typename ::= ids */
+ case 86: /* db_optr ::= */
+{setDefaultCreateDbOption(&yymsp[1].minor.yy268);}
+ break;
+ case 87: /* db_optr ::= db_optr cache */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 88: /* db_optr ::= db_optr replica */
+ case 102: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==102);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 89: /* db_optr ::= db_optr quorum */
+ case 103: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==103);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 90: /* db_optr ::= db_optr days */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 91: /* db_optr ::= db_optr minrows */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 92: /* db_optr ::= db_optr maxrows */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 93: /* db_optr ::= db_optr blocks */
+ case 105: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==105);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 94: /* db_optr ::= db_optr ctime */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 95: /* db_optr ::= db_optr wal */
+ case 107: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==107);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 96: /* db_optr ::= db_optr fsync */
+ case 108: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==108);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 97: /* db_optr ::= db_optr comp */
+ case 106: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==106);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 98: /* db_optr ::= db_optr prec */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.precision = yymsp[0].minor.yy0; }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 99: /* db_optr ::= db_optr keep */
+ case 104: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==104);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.keep = yymsp[0].minor.yy165; }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 100: /* db_optr ::= db_optr update */
+ case 109: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==109);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 101: /* alter_db_optr ::= */
+{ setDefaultCreateDbOption(&yymsp[1].minor.yy268);}
+ break;
+ case 110: /* typename ::= ids */
{
yymsp[0].minor.yy0.type = 0;
- tSQLSetColumnType (&yylhsminor.yy181, &yymsp[0].minor.yy0);
+ tSQLSetColumnType (&yylhsminor.yy223, &yymsp[0].minor.yy0);
}
- yymsp[0].minor.yy181 = yylhsminor.yy181;
+ yymsp[0].minor.yy223 = yylhsminor.yy223;
break;
- case 108: /* typename ::= ids LP signed RP */
+ case 111: /* typename ::= ids LP signed RP */
{
- if (yymsp[-1].minor.yy271 <= 0) {
+ if (yymsp[-1].minor.yy207 <= 0) {
yymsp[-3].minor.yy0.type = 0;
- tSQLSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0);
+ tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0);
} else {
- yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy271; // negative value of name length
- tSQLSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0);
+ yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy207; // negative value of name length
+ tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0);
}
}
- yymsp[-3].minor.yy181 = yylhsminor.yy181;
+ yymsp[-3].minor.yy223 = yylhsminor.yy223;
break;
- case 109: /* signed ::= INTEGER */
-{ yylhsminor.yy271 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[0].minor.yy271 = yylhsminor.yy271;
+ case 112: /* signed ::= INTEGER */
+{ yylhsminor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[0].minor.yy207 = yylhsminor.yy207;
break;
- case 110: /* signed ::= PLUS INTEGER */
-{ yymsp[-1].minor.yy271 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ case 113: /* signed ::= PLUS INTEGER */
+{ yymsp[-1].minor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
break;
- case 111: /* signed ::= MINUS INTEGER */
-{ yymsp[-1].minor.yy271 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);}
+ case 114: /* signed ::= MINUS INTEGER */
+{ yymsp[-1].minor.yy207 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);}
break;
- case 112: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
+ case 115: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
{
yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n;
setCreatedTableName(pInfo, &yymsp[-2].minor.yy0, &yymsp[-3].minor.yy0);
}
break;
- case 113: /* create_table_args ::= LP columnlist RP */
+ case 116: /* create_table_args ::= LP columnlist RP */
{
- yymsp[-2].minor.yy374 = tSetCreateSQLElems(yymsp[-1].minor.yy449, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE);
- setSQLInfo(pInfo, yymsp[-2].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+ yymsp[-2].minor.yy470 = tSetCreateSQLElems(yymsp[-1].minor.yy165, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE);
+ setSQLInfo(pInfo, yymsp[-2].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
}
break;
- case 114: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
+ case 117: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
{
- yymsp[-6].minor.yy374 = tSetCreateSQLElems(yymsp[-5].minor.yy449, yymsp[-1].minor.yy449, NULL, NULL, NULL, TSQL_CREATE_STABLE);
- setSQLInfo(pInfo, yymsp[-6].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+ yymsp[-6].minor.yy470 = tSetCreateSQLElems(yymsp[-5].minor.yy165, yymsp[-1].minor.yy165, NULL, NULL, NULL, TSQL_CREATE_STABLE);
+ setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
}
break;
- case 115: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
+ case 118: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
{
yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n;
- yymsp[-6].minor.yy374 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy494, NULL, TSQL_CREATE_TABLE_FROM_STABLE);
- setSQLInfo(pInfo, yymsp[-6].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+ yymsp[-6].minor.yy470 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy165, NULL, TSQL_CREATE_TABLE_FROM_STABLE);
+ setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
}
break;
- case 116: /* create_table_args ::= AS select */
+ case 119: /* create_table_args ::= AS select */
{
- yymsp[-1].minor.yy374 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy150, TSQL_CREATE_STREAM);
- setSQLInfo(pInfo, yymsp[-1].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+ yymsp[-1].minor.yy470 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy414, TSQL_CREATE_STREAM);
+ setSQLInfo(pInfo, yymsp[-1].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
}
break;
- case 117: /* columnlist ::= columnlist COMMA column */
-{yylhsminor.yy449 = tFieldListAppend(yymsp[-2].minor.yy449, &yymsp[0].minor.yy181); }
- yymsp[-2].minor.yy449 = yylhsminor.yy449;
+ case 120: /* columnlist ::= columnlist COMMA column */
+{taosArrayPush(yymsp[-2].minor.yy165, &yymsp[0].minor.yy223); yylhsminor.yy165 = yymsp[-2].minor.yy165; }
+ yymsp[-2].minor.yy165 = yylhsminor.yy165;
break;
- case 118: /* columnlist ::= column */
-{yylhsminor.yy449 = tFieldListAppend(NULL, &yymsp[0].minor.yy181);}
- yymsp[0].minor.yy449 = yylhsminor.yy449;
+ case 121: /* columnlist ::= column */
+{yylhsminor.yy165 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy165, &yymsp[0].minor.yy223);}
+ yymsp[0].minor.yy165 = yylhsminor.yy165;
break;
- case 119: /* column ::= ids typename */
+ case 122: /* column ::= ids typename */
{
- tSQLSetColumnInfo(&yylhsminor.yy181, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy181);
+ tSQLSetColumnInfo(&yylhsminor.yy223, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy223);
}
- yymsp[-1].minor.yy181 = yylhsminor.yy181;
+ yymsp[-1].minor.yy223 = yylhsminor.yy223;
break;
- case 120: /* tagitemlist ::= tagitemlist COMMA tagitem */
-{ yylhsminor.yy494 = tVariantListAppend(yymsp[-2].minor.yy494, &yymsp[0].minor.yy312, -1); }
- yymsp[-2].minor.yy494 = yylhsminor.yy494;
+ case 123: /* tagitemlist ::= tagitemlist COMMA tagitem */
+{ yylhsminor.yy165 = tVariantListAppend(yymsp[-2].minor.yy165, &yymsp[0].minor.yy134, -1); }
+ yymsp[-2].minor.yy165 = yylhsminor.yy165;
break;
- case 121: /* tagitemlist ::= tagitem */
-{ yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[0].minor.yy312, -1); }
- yymsp[0].minor.yy494 = yylhsminor.yy494;
+ case 124: /* tagitemlist ::= tagitem */
+{ yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); }
+ yymsp[0].minor.yy165 = yylhsminor.yy165;
break;
- case 122: /* tagitem ::= INTEGER */
- case 123: /* tagitem ::= FLOAT */ yytestcase(yyruleno==123);
- case 124: /* tagitem ::= STRING */ yytestcase(yyruleno==124);
- case 125: /* tagitem ::= BOOL */ yytestcase(yyruleno==125);
-{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy312, &yymsp[0].minor.yy0); }
- yymsp[0].minor.yy312 = yylhsminor.yy312;
+ case 125: /* tagitem ::= INTEGER */
+ case 126: /* tagitem ::= FLOAT */ yytestcase(yyruleno==126);
+ case 127: /* tagitem ::= STRING */ yytestcase(yyruleno==127);
+ case 128: /* tagitem ::= BOOL */ yytestcase(yyruleno==128);
+{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); }
+ yymsp[0].minor.yy134 = yylhsminor.yy134;
break;
- case 126: /* tagitem ::= NULL */
-{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy312, &yymsp[0].minor.yy0); }
- yymsp[0].minor.yy312 = yylhsminor.yy312;
+ case 129: /* tagitem ::= NULL */
+{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); }
+ yymsp[0].minor.yy134 = yylhsminor.yy134;
break;
- case 127: /* tagitem ::= MINUS INTEGER */
- case 128: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==128);
- case 129: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==129);
- case 130: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==130);
+ case 130: /* tagitem ::= MINUS INTEGER */
+ case 131: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==131);
+ case 132: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==132);
+ case 133: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==133);
{
yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type;
toTSDBType(yymsp[-1].minor.yy0.type);
- tVariantCreate(&yylhsminor.yy312, &yymsp[-1].minor.yy0);
+ tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0);
}
- yymsp[-1].minor.yy312 = yylhsminor.yy312;
+ yymsp[-1].minor.yy134 = yylhsminor.yy134;
break;
- case 131: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
+ case 134: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
{
- yylhsminor.yy150 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy224, yymsp[-9].minor.yy494, yymsp[-8].minor.yy66, yymsp[-4].minor.yy494, yymsp[-3].minor.yy494, &yymsp[-7].minor.yy314, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy494, &yymsp[0].minor.yy188, &yymsp[-1].minor.yy188);
+ yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy290, yymsp[-9].minor.yy165, yymsp[-8].minor.yy64, yymsp[-4].minor.yy165, yymsp[-3].minor.yy165, &yymsp[-7].minor.yy532, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy165, &yymsp[0].minor.yy216, &yymsp[-1].minor.yy216);
}
- yymsp[-11].minor.yy150 = yylhsminor.yy150;
+ yymsp[-11].minor.yy414 = yylhsminor.yy414;
break;
- case 132: /* union ::= select */
-{ yylhsminor.yy25 = setSubclause(NULL, yymsp[0].minor.yy150); }
- yymsp[0].minor.yy25 = yylhsminor.yy25;
+ case 135: /* union ::= select */
+{ yylhsminor.yy231 = setSubclause(NULL, yymsp[0].minor.yy414); }
+ yymsp[0].minor.yy231 = yylhsminor.yy231;
break;
- case 133: /* union ::= LP union RP */
-{ yymsp[-2].minor.yy25 = yymsp[-1].minor.yy25; }
+ case 136: /* union ::= LP union RP */
+{ yymsp[-2].minor.yy231 = yymsp[-1].minor.yy231; }
break;
- case 134: /* union ::= union UNION ALL select */
-{ yylhsminor.yy25 = appendSelectClause(yymsp[-3].minor.yy25, yymsp[0].minor.yy150); }
- yymsp[-3].minor.yy25 = yylhsminor.yy25;
+ case 137: /* union ::= union UNION ALL select */
+{ yylhsminor.yy231 = appendSelectClause(yymsp[-3].minor.yy231, yymsp[0].minor.yy414); }
+ yymsp[-3].minor.yy231 = yylhsminor.yy231;
break;
- case 135: /* union ::= union UNION ALL LP select RP */
-{ yylhsminor.yy25 = appendSelectClause(yymsp[-5].minor.yy25, yymsp[-1].minor.yy150); }
- yymsp[-5].minor.yy25 = yylhsminor.yy25;
+ case 138: /* union ::= union UNION ALL LP select RP */
+{ yylhsminor.yy231 = appendSelectClause(yymsp[-5].minor.yy231, yymsp[-1].minor.yy414); }
+ yymsp[-5].minor.yy231 = yylhsminor.yy231;
break;
- case 136: /* cmd ::= union */
-{ setSQLInfo(pInfo, yymsp[0].minor.yy25, NULL, TSDB_SQL_SELECT); }
+ case 139: /* cmd ::= union */
+{ setSQLInfo(pInfo, yymsp[0].minor.yy231, NULL, TSDB_SQL_SELECT); }
break;
- case 137: /* select ::= SELECT selcollist */
+ case 140: /* select ::= SELECT selcollist */
{
- yylhsminor.yy150 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy224, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+ yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy290, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
}
- yymsp[-1].minor.yy150 = yylhsminor.yy150;
+ yymsp[-1].minor.yy414 = yylhsminor.yy414;
break;
- case 138: /* sclp ::= selcollist COMMA */
-{yylhsminor.yy224 = yymsp[-1].minor.yy224;}
- yymsp[-1].minor.yy224 = yylhsminor.yy224;
+ case 141: /* sclp ::= selcollist COMMA */
+{yylhsminor.yy290 = yymsp[-1].minor.yy290;}
+ yymsp[-1].minor.yy290 = yylhsminor.yy290;
break;
- case 139: /* sclp ::= */
-{yymsp[1].minor.yy224 = 0;}
+ case 142: /* sclp ::= */
+{yymsp[1].minor.yy290 = 0;}
break;
- case 140: /* selcollist ::= sclp expr as */
+ case 143: /* selcollist ::= sclp expr as */
{
- yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224, yymsp[-1].minor.yy66, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0);
+ yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290, yymsp[-1].minor.yy64, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0);
}
- yymsp[-2].minor.yy224 = yylhsminor.yy224;
+ yymsp[-2].minor.yy290 = yylhsminor.yy290;
break;
- case 141: /* selcollist ::= sclp STAR */
+ case 144: /* selcollist ::= sclp STAR */
{
tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL);
- yylhsminor.yy224 = tSQLExprListAppend(yymsp[-1].minor.yy224, pNode, 0);
+ yylhsminor.yy290 = tSQLExprListAppend(yymsp[-1].minor.yy290, pNode, 0);
}
- yymsp[-1].minor.yy224 = yylhsminor.yy224;
+ yymsp[-1].minor.yy290 = yylhsminor.yy290;
break;
- case 142: /* as ::= AS ids */
+ case 145: /* as ::= AS ids */
{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; }
break;
- case 143: /* as ::= ids */
+ case 146: /* as ::= ids */
{ yylhsminor.yy0 = yymsp[0].minor.yy0; }
yymsp[0].minor.yy0 = yylhsminor.yy0;
break;
- case 144: /* as ::= */
+ case 147: /* as ::= */
{ yymsp[1].minor.yy0.n = 0; }
break;
- case 145: /* from ::= FROM tablelist */
-{yymsp[-1].minor.yy494 = yymsp[0].minor.yy494;}
+ case 148: /* from ::= FROM tablelist */
+{yymsp[-1].minor.yy165 = yymsp[0].minor.yy165;}
break;
- case 146: /* tablelist ::= ids cpxName */
+ case 149: /* tablelist ::= ids cpxName */
{
toTSDBType(yymsp[-1].minor.yy0.type);
yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
- yylhsminor.yy494 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
- yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[-1].minor.yy0, -1); // table alias name
+ yylhsminor.yy165 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
+ yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[-1].minor.yy0, -1); // table alias name
}
- yymsp[-1].minor.yy494 = yylhsminor.yy494;
+ yymsp[-1].minor.yy165 = yylhsminor.yy165;
break;
- case 147: /* tablelist ::= ids cpxName ids */
+ case 150: /* tablelist ::= ids cpxName ids */
{
toTSDBType(yymsp[-2].minor.yy0.type);
toTSDBType(yymsp[0].minor.yy0.type);
yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n;
- yylhsminor.yy494 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
- yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[0].minor.yy0, -1);
+ yylhsminor.yy165 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
+ yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[0].minor.yy0, -1);
}
- yymsp[-2].minor.yy494 = yylhsminor.yy494;
+ yymsp[-2].minor.yy165 = yylhsminor.yy165;
break;
- case 148: /* tablelist ::= tablelist COMMA ids cpxName */
+ case 151: /* tablelist ::= tablelist COMMA ids cpxName */
{
toTSDBType(yymsp[-1].minor.yy0.type);
yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
- yylhsminor.yy494 = tVariantListAppendToken(yymsp[-3].minor.yy494, &yymsp[-1].minor.yy0, -1);
- yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[-1].minor.yy0, -1);
+ yylhsminor.yy165 = tVariantListAppendToken(yymsp[-3].minor.yy165, &yymsp[-1].minor.yy0, -1);
+ yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[-1].minor.yy0, -1);
}
- yymsp[-3].minor.yy494 = yylhsminor.yy494;
+ yymsp[-3].minor.yy165 = yylhsminor.yy165;
break;
- case 149: /* tablelist ::= tablelist COMMA ids cpxName ids */
+ case 152: /* tablelist ::= tablelist COMMA ids cpxName ids */
{
toTSDBType(yymsp[-2].minor.yy0.type);
toTSDBType(yymsp[0].minor.yy0.type);
yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n;
- yylhsminor.yy494 = tVariantListAppendToken(yymsp[-4].minor.yy494, &yymsp[-2].minor.yy0, -1);
- yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[0].minor.yy0, -1);
+ yylhsminor.yy165 = tVariantListAppendToken(yymsp[-4].minor.yy165, &yymsp[-2].minor.yy0, -1);
+ yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[0].minor.yy0, -1);
}
- yymsp[-4].minor.yy494 = yylhsminor.yy494;
+ yymsp[-4].minor.yy165 = yylhsminor.yy165;
break;
- case 150: /* tmvar ::= VARIABLE */
+ case 153: /* tmvar ::= VARIABLE */
{yylhsminor.yy0 = yymsp[0].minor.yy0;}
yymsp[0].minor.yy0 = yylhsminor.yy0;
break;
- case 151: /* interval_opt ::= INTERVAL LP tmvar RP */
-{yymsp[-3].minor.yy314.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy314.offset.n = 0; yymsp[-3].minor.yy314.offset.z = NULL; yymsp[-3].minor.yy314.offset.type = 0;}
+ case 154: /* interval_opt ::= INTERVAL LP tmvar RP */
+{yymsp[-3].minor.yy532.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy532.offset.n = 0; yymsp[-3].minor.yy532.offset.z = NULL; yymsp[-3].minor.yy532.offset.type = 0;}
break;
- case 152: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */
-{yymsp[-5].minor.yy314.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy314.offset = yymsp[-1].minor.yy0;}
+ case 155: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */
+{yymsp[-5].minor.yy532.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy532.offset = yymsp[-1].minor.yy0;}
break;
- case 153: /* interval_opt ::= */
-{memset(&yymsp[1].minor.yy314, 0, sizeof(yymsp[1].minor.yy314));}
+ case 156: /* interval_opt ::= */
+{memset(&yymsp[1].minor.yy532, 0, sizeof(yymsp[1].minor.yy532));}
break;
- case 154: /* fill_opt ::= */
-{yymsp[1].minor.yy494 = 0; }
+ case 157: /* fill_opt ::= */
+{yymsp[1].minor.yy165 = 0; }
break;
- case 155: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */
+ case 158: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */
{
tVariant A = {0};
toTSDBType(yymsp[-3].minor.yy0.type);
tVariantCreate(&A, &yymsp[-3].minor.yy0);
- tVariantListInsert(yymsp[-1].minor.yy494, &A, -1, 0);
- yymsp[-5].minor.yy494 = yymsp[-1].minor.yy494;
+ tVariantListInsert(yymsp[-1].minor.yy165, &A, -1, 0);
+ yymsp[-5].minor.yy165 = yymsp[-1].minor.yy165;
}
break;
- case 156: /* fill_opt ::= FILL LP ID RP */
+ case 159: /* fill_opt ::= FILL LP ID RP */
{
toTSDBType(yymsp[-1].minor.yy0.type);
- yymsp[-3].minor.yy494 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
+ yymsp[-3].minor.yy165 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
}
break;
- case 157: /* sliding_opt ::= SLIDING LP tmvar RP */
+ case 160: /* sliding_opt ::= SLIDING LP tmvar RP */
{yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; }
break;
- case 158: /* sliding_opt ::= */
+ case 161: /* sliding_opt ::= */
{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; }
break;
- case 159: /* orderby_opt ::= */
- case 167: /* groupby_opt ::= */ yytestcase(yyruleno==167);
-{yymsp[1].minor.yy494 = 0;}
+ case 162: /* orderby_opt ::= */
+ case 170: /* groupby_opt ::= */ yytestcase(yyruleno==170);
+{yymsp[1].minor.yy165 = 0;}
break;
- case 160: /* orderby_opt ::= ORDER BY sortlist */
- case 168: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==168);
-{yymsp[-2].minor.yy494 = yymsp[0].minor.yy494;}
+ case 163: /* orderby_opt ::= ORDER BY sortlist */
+ case 171: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==171);
+{yymsp[-2].minor.yy165 = yymsp[0].minor.yy165;}
break;
- case 161: /* sortlist ::= sortlist COMMA item sortorder */
+ case 164: /* sortlist ::= sortlist COMMA item sortorder */
{
- yylhsminor.yy494 = tVariantListAppend(yymsp[-3].minor.yy494, &yymsp[-1].minor.yy312, yymsp[0].minor.yy82);
+ yylhsminor.yy165 = tVariantListAppend(yymsp[-3].minor.yy165, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
}
- yymsp[-3].minor.yy494 = yylhsminor.yy494;
+ yymsp[-3].minor.yy165 = yylhsminor.yy165;
break;
- case 162: /* sortlist ::= item sortorder */
+ case 165: /* sortlist ::= item sortorder */
{
- yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[-1].minor.yy312, yymsp[0].minor.yy82);
+ yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
}
- yymsp[-1].minor.yy494 = yylhsminor.yy494;
+ yymsp[-1].minor.yy165 = yylhsminor.yy165;
break;
- case 163: /* item ::= ids cpxName */
+ case 166: /* item ::= ids cpxName */
{
toTSDBType(yymsp[-1].minor.yy0.type);
yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
- tVariantCreate(&yylhsminor.yy312, &yymsp[-1].minor.yy0);
+ tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0);
}
- yymsp[-1].minor.yy312 = yylhsminor.yy312;
+ yymsp[-1].minor.yy134 = yylhsminor.yy134;
break;
- case 164: /* sortorder ::= ASC */
-{yymsp[0].minor.yy82 = TSDB_ORDER_ASC; }
+ case 167: /* sortorder ::= ASC */
+{yymsp[0].minor.yy46 = TSDB_ORDER_ASC; }
break;
- case 165: /* sortorder ::= DESC */
-{yymsp[0].minor.yy82 = TSDB_ORDER_DESC;}
+ case 168: /* sortorder ::= DESC */
+{yymsp[0].minor.yy46 = TSDB_ORDER_DESC;}
break;
- case 166: /* sortorder ::= */
-{yymsp[1].minor.yy82 = TSDB_ORDER_ASC;}
+ case 169: /* sortorder ::= */
+{yymsp[1].minor.yy46 = TSDB_ORDER_ASC;}
break;
- case 169: /* grouplist ::= grouplist COMMA item */
+ case 172: /* grouplist ::= grouplist COMMA item */
{
- yylhsminor.yy494 = tVariantListAppend(yymsp[-2].minor.yy494, &yymsp[0].minor.yy312, -1);
+ yylhsminor.yy165 = tVariantListAppend(yymsp[-2].minor.yy165, &yymsp[0].minor.yy134, -1);
}
- yymsp[-2].minor.yy494 = yylhsminor.yy494;
+ yymsp[-2].minor.yy165 = yylhsminor.yy165;
break;
- case 170: /* grouplist ::= item */
+ case 173: /* grouplist ::= item */
{
- yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[0].minor.yy312, -1);
+ yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1);
}
- yymsp[0].minor.yy494 = yylhsminor.yy494;
+ yymsp[0].minor.yy165 = yylhsminor.yy165;
+ break;
+ case 174: /* having_opt ::= */
+ case 184: /* where_opt ::= */ yytestcase(yyruleno==184);
+ case 222: /* expritem ::= */ yytestcase(yyruleno==222);
+{yymsp[1].minor.yy64 = 0;}
+ break;
+ case 175: /* having_opt ::= HAVING expr */
+ case 185: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==185);
+{yymsp[-1].minor.yy64 = yymsp[0].minor.yy64;}
+ break;
+ case 176: /* limit_opt ::= */
+ case 180: /* slimit_opt ::= */ yytestcase(yyruleno==180);
+{yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;}
break;
- case 171: /* having_opt ::= */
- case 181: /* where_opt ::= */ yytestcase(yyruleno==181);
- case 219: /* expritem ::= */ yytestcase(yyruleno==219);
-{yymsp[1].minor.yy66 = 0;}
+ case 177: /* limit_opt ::= LIMIT signed */
+{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;}
break;
- case 172: /* having_opt ::= HAVING expr */
- case 182: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==182);
-{yymsp[-1].minor.yy66 = yymsp[0].minor.yy66;}
+ case 178: /* limit_opt ::= LIMIT signed OFFSET signed */
+{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
break;
- case 173: /* limit_opt ::= */
- case 177: /* slimit_opt ::= */ yytestcase(yyruleno==177);
-{yymsp[1].minor.yy188.limit = -1; yymsp[1].minor.yy188.offset = 0;}
+ case 179: /* limit_opt ::= LIMIT signed COMMA signed */
+{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
break;
- case 174: /* limit_opt ::= LIMIT signed */
- case 178: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==178);
-{yymsp[-1].minor.yy188.limit = yymsp[0].minor.yy271; yymsp[-1].minor.yy188.offset = 0;}
+ case 181: /* slimit_opt ::= SLIMIT signed */
+{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;}
break;
- case 175: /* limit_opt ::= LIMIT signed OFFSET signed */
- case 179: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==179);
-{yymsp[-3].minor.yy188.limit = yymsp[-2].minor.yy271; yymsp[-3].minor.yy188.offset = yymsp[0].minor.yy271;}
+ case 182: /* slimit_opt ::= SLIMIT signed SOFFSET signed */
+{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
break;
- case 176: /* limit_opt ::= LIMIT signed COMMA signed */
- case 180: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==180);
-{yymsp[-3].minor.yy188.limit = yymsp[0].minor.yy271; yymsp[-3].minor.yy188.offset = yymsp[-2].minor.yy271;}
+ case 183: /* slimit_opt ::= SLIMIT signed COMMA signed */
+{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
break;
- case 183: /* expr ::= LP expr RP */
-{yymsp[-2].minor.yy66 = yymsp[-1].minor.yy66; }
+ case 186: /* expr ::= LP expr RP */
+{yymsp[-2].minor.yy64 = yymsp[-1].minor.yy64; }
break;
- case 184: /* expr ::= ID */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 187: /* expr ::= ID */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 185: /* expr ::= ID DOT ID */
-{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 188: /* expr ::= ID DOT ID */
+{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 186: /* expr ::= ID DOT STAR */
-{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 189: /* expr ::= ID DOT STAR */
+{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 187: /* expr ::= INTEGER */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 190: /* expr ::= INTEGER */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 188: /* expr ::= MINUS INTEGER */
- case 189: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==189);
-{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);}
- yymsp[-1].minor.yy66 = yylhsminor.yy66;
+ case 191: /* expr ::= MINUS INTEGER */
+ case 192: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==192);
+{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);}
+ yymsp[-1].minor.yy64 = yylhsminor.yy64;
break;
- case 190: /* expr ::= FLOAT */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 193: /* expr ::= FLOAT */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 191: /* expr ::= MINUS FLOAT */
- case 192: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==192);
-{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);}
- yymsp[-1].minor.yy66 = yylhsminor.yy66;
+ case 194: /* expr ::= MINUS FLOAT */
+ case 195: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==195);
+{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);}
+ yymsp[-1].minor.yy64 = yylhsminor.yy64;
break;
- case 193: /* expr ::= STRING */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 196: /* expr ::= STRING */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 194: /* expr ::= NOW */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); }
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 197: /* expr ::= NOW */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); }
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 195: /* expr ::= VARIABLE */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 198: /* expr ::= VARIABLE */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 196: /* expr ::= BOOL */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 199: /* expr ::= BOOL */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 197: /* expr ::= ID LP exprlist RP */
-{ yylhsminor.yy66 = tSQLExprCreateFunction(yymsp[-1].minor.yy224, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
- yymsp[-3].minor.yy66 = yylhsminor.yy66;
+ case 200: /* expr ::= ID LP exprlist RP */
+{ yylhsminor.yy64 = tSQLExprCreateFunction(yymsp[-1].minor.yy290, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
+ yymsp[-3].minor.yy64 = yylhsminor.yy64;
break;
- case 198: /* expr ::= ID LP STAR RP */
-{ yylhsminor.yy66 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
- yymsp[-3].minor.yy66 = yylhsminor.yy66;
+ case 201: /* expr ::= ID LP STAR RP */
+{ yylhsminor.yy64 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
+ yymsp[-3].minor.yy64 = yylhsminor.yy64;
break;
- case 199: /* expr ::= expr IS NULL */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, NULL, TK_ISNULL);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 202: /* expr ::= expr IS NULL */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, NULL, TK_ISNULL);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 200: /* expr ::= expr IS NOT NULL */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-3].minor.yy66, NULL, TK_NOTNULL);}
- yymsp[-3].minor.yy66 = yylhsminor.yy66;
+ case 203: /* expr ::= expr IS NOT NULL */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-3].minor.yy64, NULL, TK_NOTNULL);}
+ yymsp[-3].minor.yy64 = yylhsminor.yy64;
break;
- case 201: /* expr ::= expr LT expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LT);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 204: /* expr ::= expr LT expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LT);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 202: /* expr ::= expr GT expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_GT);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 205: /* expr ::= expr GT expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GT);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 203: /* expr ::= expr LE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LE);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 206: /* expr ::= expr LE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LE);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 204: /* expr ::= expr GE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_GE);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 207: /* expr ::= expr GE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GE);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 205: /* expr ::= expr NE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_NE);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 208: /* expr ::= expr NE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_NE);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 206: /* expr ::= expr EQ expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_EQ);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 209: /* expr ::= expr EQ expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_EQ);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 207: /* expr ::= expr AND expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_AND);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 210: /* expr ::= expr AND expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_AND);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 208: /* expr ::= expr OR expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_OR); }
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 211: /* expr ::= expr OR expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_OR); }
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 209: /* expr ::= expr PLUS expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_PLUS); }
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 212: /* expr ::= expr PLUS expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_PLUS); }
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 210: /* expr ::= expr MINUS expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_MINUS); }
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 213: /* expr ::= expr MINUS expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_MINUS); }
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 211: /* expr ::= expr STAR expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_STAR); }
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 214: /* expr ::= expr STAR expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_STAR); }
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 212: /* expr ::= expr SLASH expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_DIVIDE);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 215: /* expr ::= expr SLASH expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_DIVIDE);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 213: /* expr ::= expr REM expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_REM); }
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 216: /* expr ::= expr REM expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_REM); }
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 214: /* expr ::= expr LIKE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LIKE); }
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 217: /* expr ::= expr LIKE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LIKE); }
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 215: /* expr ::= expr IN LP exprlist RP */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-4].minor.yy66, (tSQLExpr*)yymsp[-1].minor.yy224, TK_IN); }
- yymsp[-4].minor.yy66 = yylhsminor.yy66;
+ case 218: /* expr ::= expr IN LP exprlist RP */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-4].minor.yy64, (tSQLExpr*)yymsp[-1].minor.yy290, TK_IN); }
+ yymsp[-4].minor.yy64 = yylhsminor.yy64;
break;
- case 216: /* exprlist ::= exprlist COMMA expritem */
-{yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224,yymsp[0].minor.yy66,0);}
- yymsp[-2].minor.yy224 = yylhsminor.yy224;
+ case 219: /* exprlist ::= exprlist COMMA expritem */
+{yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290,yymsp[0].minor.yy64,0);}
+ yymsp[-2].minor.yy290 = yylhsminor.yy290;
break;
- case 217: /* exprlist ::= expritem */
-{yylhsminor.yy224 = tSQLExprListAppend(0,yymsp[0].minor.yy66,0);}
- yymsp[0].minor.yy224 = yylhsminor.yy224;
+ case 220: /* exprlist ::= expritem */
+{yylhsminor.yy290 = tSQLExprListAppend(0,yymsp[0].minor.yy64,0);}
+ yymsp[0].minor.yy290 = yylhsminor.yy290;
break;
- case 218: /* expritem ::= expr */
-{yylhsminor.yy66 = yymsp[0].minor.yy66;}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 221: /* expritem ::= expr */
+{yylhsminor.yy64 = yymsp[0].minor.yy64;}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 220: /* cmd ::= RESET QUERY CACHE */
+ case 223: /* cmd ::= RESET QUERY CACHE */
{ setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);}
break;
- case 221: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
+ case 224: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
{
yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
- SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_COLUMN);
+ SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
- case 222: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
+ case 225: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
{
yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
toTSDBType(yymsp[0].minor.yy0.type);
- tVariantList* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1);
+ SArray* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
- case 223: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
+ case 226: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
{
yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
- SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN);
+ SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
- case 224: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
+ case 227: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
{
yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
toTSDBType(yymsp[0].minor.yy0.type);
- tVariantList* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1);
+ SArray* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
- case 225: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
+ case 228: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
{
yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n;
toTSDBType(yymsp[-1].minor.yy0.type);
- tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
+ SArray* A = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
toTSDBType(yymsp[0].minor.yy0.type);
A = tVariantListAppendToken(A, &yymsp[0].minor.yy0, -1);
@@ -2818,25 +2835,25 @@ static void yy_reduce(
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
- case 226: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
+ case 229: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
{
yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n;
toTSDBType(yymsp[-2].minor.yy0.type);
- tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
- A = tVariantListAppend(A, &yymsp[0].minor.yy312, -1);
+ SArray* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
+ A = tVariantListAppend(A, &yymsp[0].minor.yy134, -1);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
- case 227: /* cmd ::= KILL CONNECTION INTEGER */
+ case 230: /* cmd ::= KILL CONNECTION INTEGER */
{setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);}
break;
- case 228: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */
+ case 231: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */
{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);}
break;
- case 229: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */
+ case 232: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */
{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);}
break;
default:
diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp
index 8cd3a9cbef0bbf268db074a935a7b25cb389944e..bada3194bd90e30de465dceaef07742cf6e8ac07 100644
--- a/src/query/tests/tsBufTest.cpp
+++ b/src/query/tests/tsBufTest.cpp
@@ -42,7 +42,7 @@ void simpleTest() {
EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num);
EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0);
- EXPECT_EQ(pTSBuf->numOfVnodes, 1);
+ EXPECT_EQ(pTSBuf->numOfGroups, 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
@@ -69,7 +69,7 @@ void largeTSTest() {
// the data has been flush to disk, no data in cache
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0);
- EXPECT_EQ(pTSBuf->numOfVnodes, 1);
+ EXPECT_EQ(pTSBuf->numOfGroups, 1);
EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC);
tsBufFlush(pTSBuf);
@@ -105,7 +105,7 @@ void multiTagsTest() {
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1);
- EXPECT_EQ(pTSBuf->numOfVnodes, 1);
+ EXPECT_EQ(pTSBuf->numOfGroups, 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
@@ -139,7 +139,7 @@ void multiVnodeTagsTest() {
start += step * num;
}
- EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
+ EXPECT_EQ(pTSBuf->numOfGroups, j + 1);
}
EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC);
@@ -184,7 +184,7 @@ void loadDataTest() {
start += step * num;
}
- EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
+ EXPECT_EQ(pTSBuf->numOfGroups, j + 1);
}
EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC);
@@ -203,7 +203,7 @@ void loadDataTest() {
// create from exists file
STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false);
EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder);
- EXPECT_EQ(pNewBuf->numOfVnodes, numOfVnode);
+ EXPECT_EQ(pNewBuf->numOfGroups, numOfVnode);
EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize);
EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset);
@@ -269,7 +269,7 @@ void TSTraverse() {
start += step * num;
}
- EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
+ EXPECT_EQ(pTSBuf->numOfGroups, j + 1);
}
tsBufResetPos(pTSBuf);
@@ -304,7 +304,7 @@ void TSTraverse() {
int32_t totalOutput = 10;
while (1) {
STSElem elem = tsBufGetElem(pTSBuf);
- printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag->i64Key, elem.ts);
+ printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts);
if (!tsBufNextPos(pTSBuf)) {
break;
@@ -352,7 +352,7 @@ void TSTraverse() {
totalOutput = 10;
while (1) {
STSElem elem = tsBufGetElem(pTSBuf);
- printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag->i64Key, elem.ts);
+ printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts);
if (!tsBufNextPos(pTSBuf)) {
break;
@@ -427,7 +427,7 @@ void mergeDiffVnodeBufferTest() {
tsBufFlush(pTSBuf2);
tsBufMerge(pTSBuf1, pTSBuf2);
- EXPECT_EQ(pTSBuf1->numOfVnodes, 2);
+ EXPECT_EQ(pTSBuf1->numOfGroups, 2);
EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);
tsBufDisplay(pTSBuf1);
@@ -472,15 +472,22 @@ void mergeIdenticalVnodeBufferTest() {
tsBufFlush(pTSBuf2);
tsBufMerge(pTSBuf1, pTSBuf2);
- EXPECT_EQ(pTSBuf1->numOfVnodes, 1);
+ EXPECT_EQ(pTSBuf1->numOfGroups, 2);
EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);
tsBufResetPos(pTSBuf1);
+
+ int32_t count = 0;
while (tsBufNextPos(pTSBuf1)) {
STSElem elem = tsBufGetElem(pTSBuf1);
- EXPECT_EQ(elem.vnode, 12);
- printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag->i64Key, elem.ts);
+ if (count++ < numOfTags * num) {
+ EXPECT_EQ(elem.id, 12);
+ } else {
+ EXPECT_EQ(elem.id, 77);
+ }
+
+ printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts);
}
tsBufDestroy(pTSBuf1);
diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c
index 46b0d4e3bb9428f98c1022556f2f57c6ec0e9a14..09d8f3bff1faad5596d85e931adea7f83670a48a 100644
--- a/src/rpc/src/rpcCache.c
+++ b/src/rpc/src/rpcCache.c
@@ -101,9 +101,9 @@ void rpcCloseConnCache(void *handle) {
if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool);
- taosTFree(pCache->connHashList);
- taosTFree(pCache->count);
- taosTFree(pCache->lockedBy);
+ tfree(pCache->connHashList);
+ tfree(pCache->count);
+ tfree(pCache->lockedBy);
pthread_mutex_unlock(&pCache->mutex);
diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index 6e9088d9fb82a80bb28857d563ad51f19594e6b7..acceaf9d7a63f3378afbc8d8f485f14f77b50af6 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -20,6 +20,7 @@
#include "ttimer.h"
#include "tutil.h"
#include "lz4.h"
+#include "tref.h"
#include "taoserror.h"
#include "tsocket.h"
#include "tglobal.h"
@@ -72,7 +73,6 @@ typedef struct {
SRpcInfo *pRpc; // associated SRpcInfo
SRpcEpSet epSet; // ip list provided by app
void *ahandle; // handle provided by app
- void *signature; // for validation
struct SRpcConn *pConn; // pConn allocated
char msgType; // message type
uint8_t *pCont; // content provided by app
@@ -82,6 +82,7 @@ typedef struct {
int8_t oldInUse; // server EP inUse passed by app
int8_t redirect; // flag to indicate redirect
int8_t connType; // connection type
+ int64_t rid; // refId returned by taosAddRef
SRpcMsg *pRsp; // for synchronous API
tsem_t *pSem; // for synchronous API
SRpcEpSet *pSet; // for synchronous API
@@ -132,6 +133,10 @@ int tsRpcMaxRetry;
int tsRpcHeadSize;
int tsRpcOverhead;
+static int tsRpcRefId = -1;
+static int32_t tsRpcNum = 0;
+//static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT;
+
// server:0 client:1 tcp:2 udp:0
#define RPC_CONN_UDPS 0
#define RPC_CONN_UDPC 1
@@ -211,14 +216,32 @@ static void rpcUnlockConn(SRpcConn *pConn);
static void rpcAddRef(SRpcInfo *pRpc);
static void rpcDecRef(SRpcInfo *pRpc);
-void *rpcOpen(const SRpcInit *pInit) {
- SRpcInfo *pRpc;
+static void rpcFree(void *p) {
+ tTrace("free mem: %p", p);
+ free(p);
+}
+int32_t rpcInit(void) {
tsProgressTimer = tsRpcTimer/2;
tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer;
tsRpcHeadSize = RPC_MSG_OVERHEAD;
tsRpcOverhead = sizeof(SRpcReqContext);
+ tsRpcRefId = taosOpenRef(200, rpcFree);
+
+ return 0;
+}
+
+void rpcCleanup(void) {
+ taosCloseRef(tsRpcRefId);
+ tsRpcRefId = -1;
+}
+
+void *rpcOpen(const SRpcInit *pInit) {
+ SRpcInfo *pRpc;
+
+ //pthread_once(&tsRpcInit, rpcInit);
+
pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo));
if (pRpc == NULL) return NULL;
@@ -237,6 +260,8 @@ void *rpcOpen(const SRpcInit *pInit) {
pRpc->afp = pInit->afp;
pRpc->refCount = 1;
+ atomic_add_fetch_32(&tsRpcNum, 1);
+
size_t size = sizeof(SRpcConn) * pRpc->sessions;
pRpc->connList = (SRpcConn *)calloc(1, size);
if (pRpc->connList == NULL) {
@@ -323,7 +348,7 @@ void *rpcMallocCont(int contLen) {
tError("failed to malloc msg, size:%d", size);
return NULL;
} else {
- tTrace("malloc mem: %p", start);
+ tTrace("malloc mem:%p size:%d", start, size);
}
return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
@@ -356,14 +381,13 @@ void *rpcReallocCont(void *ptr, int contLen) {
return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
}
-void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
+void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) {
SRpcInfo *pRpc = (SRpcInfo *)shandle;
SRpcReqContext *pContext;
int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen);
pContext = (SRpcReqContext *) ((char*)pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext));
pContext->ahandle = pMsg->ahandle;
- pContext->signature = pContext;
pContext->pRpc = (SRpcInfo *)shandle;
pContext->epSet = *pEpSet;
pContext->contLen = contLen;
@@ -383,12 +407,10 @@ void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
|| type == TSDB_MSG_TYPE_CM_SHOW )
pContext->connType = RPC_CONN_TCPC;
- // set the handle to pContext, so app can cancel the request
- if (pMsg->handle) *((void **)pMsg->handle) = pContext;
+ pContext->rid = taosAddRef(tsRpcRefId, pContext);
+ if (pRid) *pRid = pContext->rid;
rpcSendReqToServer(pRpc, pContext);
-
- return;
}
void rpcSendResponse(const SRpcMsg *pRsp) {
@@ -504,7 +526,7 @@ void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp)
pContext->pRsp = pRsp;
pContext->pSet = pEpSet;
- rpcSendRequest(shandle, pEpSet, pMsg);
+ rpcSendRequest(shandle, pEpSet, pMsg, NULL);
tsem_wait(&sem);
tsem_destroy(&sem);
@@ -533,17 +555,14 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) {
return code;
}
-void rpcCancelRequest(void *handle) {
- SRpcReqContext *pContext = handle;
+void rpcCancelRequest(int64_t rid) {
- // signature is used to check if pContext is freed.
- // pContext may have been released just before app calls the rpcCancelRequest
- if (pContext == NULL || pContext->signature != pContext) return;
+ SRpcReqContext *pContext = taosAcquireRef(tsRpcRefId, rid);
+ if (pContext == NULL) return;
- if (pContext->pConn) {
- tDebug("%s, app tries to cancel request", pContext->pConn->info);
- rpcCloseConn(pContext->pConn);
- }
+ rpcCloseConn(pContext->pConn);
+
+ taosReleaseRef(tsRpcRefId, rid);
}
static void rpcFreeMsg(void *msg) {
@@ -612,7 +631,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
// if there is an outgoing message, free it
if (pConn->outType && pConn->pReqMsg) {
if (pConn->pContext) pConn->pContext->pConn = NULL;
- rpcFreeMsg(pConn->pReqMsg);
+ taosRemoveRef(tsRpcRefId, pConn->pContext->rid);
}
}
@@ -636,6 +655,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
static void rpcCloseConn(void *thandle) {
SRpcConn *pConn = (SRpcConn *)thandle;
+ if (pConn == NULL) return;
rpcLockConn(pConn);
@@ -1007,6 +1027,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) {
if (pConn->outType) {
SRpcReqContext *pContext = pConn->pContext;
pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+ pContext->pConn = NULL;
pConn->pReqMsg = NULL;
taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl);
}
@@ -1057,6 +1078,13 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
}
} else { // msg is passed to app only parsing is ok
+
+ if (pHead->msgType == TSDB_MSG_TYPE_NETWORK_TEST) {
+ rpcSendQuickRsp(pConn, TSDB_CODE_SUCCESS);
+ rpcFreeMsg(pRecv->msg);
+ return pConn;
+ }
+
rpcProcessIncomingMsg(pConn, pHead, pContext);
}
}
@@ -1068,7 +1096,6 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
SRpcInfo *pRpc = pContext->pRpc;
- pContext->signature = NULL;
pContext->pConn = NULL;
if (pContext->pRsp) {
// for synchronous API
@@ -1085,7 +1112,7 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
}
// free the request message
- rpcFreeCont(pContext->pCont);
+ taosRemoveRef(tsRpcRefId, pContext->rid);
}
static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) {
@@ -1110,6 +1137,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte
// it's a response
rpcMsg.handle = pContext;
rpcMsg.ahandle = pContext->ahandle;
+ pContext->pConn = NULL;
// for UDP, port may be changed by server, the port in epSet shall be used for cache
if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) {
@@ -1345,6 +1373,7 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) {
tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort);
if (pConn->pContext) {
pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+ pConn->pContext->pConn = NULL;
pConn->pReqMsg = NULL;
taosTmrStart(rpcProcessConnError, 0, pConn->pContext, pRpc->tmrCtrl);
rpcReleaseConn(pConn);
@@ -1453,7 +1482,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) {
pNewHead->msgLen = rpcMsgLenFromCont(origLen);
rpcFreeMsg(pHead); // free the compressed message buffer
pHead = pNewHead;
- tTrace("decomp malloc mem: %p", temp);
+ tTrace("decomp malloc mem:%p", temp);
} else {
tError("failed to allocate memory to decompress msg, contLen:%d", contLen);
}
@@ -1589,10 +1618,12 @@ static void rpcDecRef(SRpcInfo *pRpc)
taosTmrCleanUp(pRpc->tmrCtrl);
taosIdPoolCleanUp(pRpc->idPool);
- taosTFree(pRpc->connList);
+ tfree(pRpc->connList);
pthread_mutex_destroy(&pRpc->mutex);
tDebug("%s rpc resources are released", pRpc->label);
- taosTFree(pRpc);
+ tfree(pRpc);
+
+ atomic_sub_fetch_32(&tsRpcNum, 1);
}
}
diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c
index bc8d360d39509ce9a2fdbe6a9dd883c5c5c99190..7b8cf3cda2f98137ac68bacec257d5b7d14a2a4c 100644
--- a/src/rpc/src/rpcTcp.c
+++ b/src/rpc/src/rpcTcp.c
@@ -171,40 +171,17 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
}
static void taosStopTcpThread(SThreadObj* pThreadObj) {
- pThreadObj->stop = true;
- eventfd_t fd = -1;
-
- // save thread into local variable since pThreadObj is freed when thread exits
+ // save thread into local variable and signal thread to stop
pthread_t thread = pThreadObj->thread;
-
- if (taosComparePthread(pThreadObj->thread, pthread_self())) {
- pthread_detach(pthread_self());
+ if (!taosCheckPthreadValid(thread)) {
return;
}
-
- if (taosCheckPthreadValid(pThreadObj->thread)) {
- // signal the thread to stop, try graceful method first,
- // and use pthread_cancel when failed
- struct epoll_event event = { .events = EPOLLIN };
- fd = eventfd(1, 0);
- if (fd == -1) {
- // failed to create eventfd, call pthread_cancel instead, which may result in data corruption:
- tError("%s, failed to create eventfd(%s)", pThreadObj->label, strerror(errno));
- pThreadObj->stop = true;
- pthread_cancel(pThreadObj->thread);
- } else if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
- // failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption:
- tError("%s, failed to call epoll_ctl(%s)", pThreadObj->label, strerror(errno));
- pthread_cancel(pThreadObj->thread);
- }
- }
-
- // at this step, pThreadObj has already been released
- if (taosCheckPthreadValid(thread)) {
- pthread_join(thread, NULL);
+ pThreadObj->stop = true;
+ if (taosComparePthread(thread, pthread_self())) {
+ pthread_detach(pthread_self());
+ return;
}
-
- if (fd != -1) taosCloseSocket(fd);
+ pthread_join(thread, NULL);
}
void taosStopTcpServer(void *handle) {
@@ -236,8 +213,8 @@ void taosCleanUpTcpServer(void *handle) {
tDebug("%s TCP server is cleaned up", pServerObj->label);
- taosTFree(pServerObj->pThreadObj);
- taosTFree(pServerObj);
+ tfree(pServerObj->pThreadObj);
+ tfree(pServerObj);
}
static void *taosAcceptTcpConnection(void *arg) {
@@ -373,8 +350,8 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d",
pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds);
} else {
- taosCloseSocket(fd);
tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno));
+ taosCloseSocket(fd);
}
return pFdObj;
@@ -437,12 +414,13 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
}
msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
- buffer = malloc(msgLen + tsRpcOverhead);
+ int32_t size = msgLen + tsRpcOverhead;
+ buffer = malloc(size);
if (NULL == buffer) {
tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
return -1;
} else {
- tTrace("TCP malloc mem: %p", buffer);
+ tTrace("TCP malloc mem:%p size:%d", buffer, size);
}
msg = buffer + tsRpcOverhead;
@@ -534,7 +512,7 @@ static void *taosProcessTcpData(void *param) {
pthread_mutex_destroy(&(pThreadObj->mutex));
tDebug("%s TCP thread exits ...", pThreadObj->label);
- taosTFree(pThreadObj);
+ tfree(pThreadObj);
return NULL;
}
@@ -555,7 +533,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) {
event.events = EPOLLIN | EPOLLRDHUP;
event.data.ptr = pFdObj;
if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
- taosTFree(pFdObj);
+ tfree(pFdObj);
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
@@ -608,5 +586,5 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d",
pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds);
- taosTFree(pFdObj);
+ tfree(pFdObj);
}
diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c
index ad97c410834989e46d2f1af4aebc06d10d949bcc..311676e7d3531234118a8103bdfeba66716996d0 100644
--- a/src/rpc/src/rpcUdp.c
+++ b/src/rpc/src/rpcUdp.c
@@ -146,7 +146,7 @@ void taosStopUdpConnection(void *handle) {
if (taosCheckPthreadValid(pConn->thread)) {
pthread_join(pConn->thread, NULL);
}
- taosTFree(pConn->buffer);
+ tfree(pConn->buffer);
// tTrace("%s UDP thread is closed, index:%d", pConn->label, i);
}
@@ -165,7 +165,7 @@ void taosCleanUpUdpConnection(void *handle) {
}
tDebug("%s UDP is cleaned up", pSet->label);
- taosTFree(pSet);
+ tfree(pSet);
}
void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) {
@@ -208,12 +208,13 @@ static void *taosRecvUdpData(void *param) {
continue;
}
- char *tmsg = malloc(dataLen + tsRpcOverhead);
+ int32_t size = dataLen + tsRpcOverhead;
+ char *tmsg = malloc(size);
if (NULL == tmsg) {
tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen);
continue;
} else {
- tTrace("UDP malloc mem: %p", tmsg);
+ tTrace("UDP malloc mem:%p size:%d", tmsg, size);
}
tmsg += tsRpcOverhead; // overhead for SRpcReqContext
diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c
index 7a963e9ce47f7c89edc0204d6502d548dbdb40eb..5721525adee3fc847a1ba2476ccb0995fb50a65c 100644
--- a/src/rpc/test/rclient.c
+++ b/src/rpc/test/rclient.c
@@ -57,7 +57,7 @@ static void *sendRequest(void *param) {
rpcMsg.ahandle = pInfo;
rpcMsg.msgType = 1;
tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
- rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg);
+ rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL);
if ( pInfo->num % 20000 == 0 )
tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
tsem_wait(&pInfo->rspSem);
diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h
index f6818106462cb9c9d694d59c7cb9012cd1a54c8b..7156a2d08a83fc4a36f82be9898b794032639de8 100644
--- a/src/sync/inc/syncInt.h
+++ b/src/sync/inc/syncInt.h
@@ -35,6 +35,9 @@ extern "C" {
#define TAOS_SMSG_SYNC_MUST 6
#define TAOS_SMSG_STATUS 7
+#define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16)
+#define SYNC_RECV_BUFFER_SIZE (5*1024*1024)
+
#define nodeRole pNode->peerInfo[pNode->selfIndex]->role
#define nodeVersion pNode->peerInfo[pNode->selfIndex]->version
#define nodeSStatus pNode->peerInfo[pNode->selfIndex]->sstatus
@@ -89,11 +92,11 @@ typedef struct {
#pragma pack(pop)
typedef struct {
- char *buffer;
- int bufferSize;
- char *offset;
- int forwards;
- int code;
+ char * buffer;
+ int32_t bufferSize;
+ char * offset;
+ int32_t forwards;
+ int32_t code;
} SRecvBuffer;
typedef struct {
@@ -107,10 +110,10 @@ typedef struct {
} SFwdInfo;
typedef struct {
- int first;
- int last;
- int fwds; // number of forwards
- SFwdInfo fwdInfo[];
+ int32_t first;
+ int32_t last;
+ int32_t fwds; // number of forwards
+ SFwdInfo fwdInfo[];
} SSyncFwds;
typedef struct SsyncPeer {
@@ -123,15 +126,15 @@ typedef struct SsyncPeer {
int8_t sstatus; // sync status
uint64_t version;
uint64_t sversion; // track the peer version in retrieve process
- int syncFd;
- int peerFd; // forward FD
- int numOfRetrieves; // number of retrieves tried
- int fileChanged; // a flag to indicate file is changed during retrieving process
+ int32_t syncFd;
+ int32_t peerFd; // forward FD
+ int32_t numOfRetrieves; // number of retrieves tried
+ int32_t fileChanged; // a flag to indicate file is changed during retrieving process
void * timer;
void * pConn;
- int notifyFd;
- int watchNum;
- int * watchFd;
+ int32_t notifyFd;
+ int32_t watchNum;
+ int32_t *watchFd;
int8_t refCount; // reference count
struct SSyncNode *pSyncNode;
} SSyncPeer;
@@ -141,6 +144,7 @@ typedef struct SSyncNode {
int8_t replica;
int8_t quorum;
uint32_t vgId;
+ int64_t rid;
void *ahandle;
int8_t selfIndex;
SSyncPeer *peerInfo[TAOS_SYNC_MAX_REPLICA+1]; // extra one for arbitrator
@@ -160,16 +164,16 @@ typedef struct SSyncNode {
} SSyncNode;
// sync module global
-extern int tsSyncNum;
-extern char tsNodeFqdn[TSDB_FQDN_LEN];
+extern int32_t tsSyncNum;
+extern char tsNodeFqdn[TSDB_FQDN_LEN];
void *syncRetrieveData(void *param);
void *syncRestoreData(void *param);
-int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead);
-void syncRestartConnection(SSyncPeer *pPeer);
-void syncBroadcastStatus(SSyncNode *pNode);
-void syncAddPeerRef(SSyncPeer *pPeer);
-int syncDecPeerRef(SSyncPeer *pPeer);
+int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead);
+void syncRestartConnection(SSyncPeer *pPeer);
+void syncBroadcastStatus(SSyncNode *pNode);
+void syncAddPeerRef(SSyncPeer *pPeer);
+int32_t syncDecPeerRef(SSyncPeer *pPeer);
#ifdef __cplusplus
}
diff --git a/src/sync/inc/taosTcpPool.h b/src/sync/inc/taosTcpPool.h
index 261d190ad3b7cbe4fbf46a061a1432c217262d24..41043b0cd4c886616d5cecd2739eae684052c395 100644
--- a/src/sync/inc/taosTcpPool.h
+++ b/src/sync/inc/taosTcpPool.h
@@ -20,23 +20,23 @@
extern "C" {
#endif
-typedef void* ttpool_h;
-typedef void* tthread_h;
+typedef void *ttpool_h;
+typedef void *tthread_h;
typedef struct {
- int numOfThreads;
+ int32_t numOfThreads;
uint32_t serverIp;
int16_t port;
- int bufferSize;
- void (*processBrokenLink)(void *ahandle);
- int (*processIncomingMsg)(void *ahandle, void *buffer);
- void (*processIncomingConn)(int fd, uint32_t ip);
+ int32_t bufferSize;
+ void (*processBrokenLink)(void *ahandle);
+ int32_t (*processIncomingMsg)(void *ahandle, void *buffer);
+ void (*processIncomingConn)(int32_t fd, uint32_t ip);
} SPoolInfo;
-ttpool_h taosOpenTcpThreadPool(SPoolInfo *pInfo);
-void taosCloseTcpThreadPool(ttpool_h);
-void *taosAllocateTcpConn(void *, void *ahandle, int connFd);
-void taosFreeTcpConn(void *);
+ttpool_h taosOpenTcpThreadPool(SPoolInfo *pInfo);
+void taosCloseTcpThreadPool(ttpool_h);
+void * taosAllocateTcpConn(void *, void *ahandle, int32_t connFd);
+void taosFreeTcpConn(void *);
#ifdef __cplusplus
}
diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c
index ef635e6efc1ca5f071c64dbe00920c3987837494..6ff04aad64b4a987aa9e8cd1370db1e1aff5f70c 100644
--- a/src/sync/src/syncMain.c
+++ b/src/sync/src/syncMain.c
@@ -20,6 +20,7 @@
#include "tlog.h"
#include "tutil.h"
#include "ttimer.h"
+#include "tref.h"
#include "tsocket.h"
#include "tglobal.h"
#include "taoserror.h"
@@ -30,37 +31,38 @@
#include "syncInt.h"
// global configurable
-int tsMaxSyncNum = 2;
-int tsSyncTcpThreads = 2;
-int tsMaxWatchFiles = 500;
-int tsMaxFwdInfo = 200;
-int tsSyncTimer = 1;
+int32_t tsMaxSyncNum = 2;
+int32_t tsSyncTcpThreads = 2;
+int32_t tsMaxWatchFiles = 500;
+int32_t tsMaxFwdInfo = 200;
+int32_t tsSyncTimer = 1;
// module global, not configurable
-int tsSyncNum; // number of sync in process in whole system
-char tsNodeFqdn[TSDB_FQDN_LEN];
+int32_t tsSyncNum; // number of sync in process in whole system
+char tsNodeFqdn[TSDB_FQDN_LEN];
static ttpool_h tsTcpPool;
-static void * syncTmrCtrl = NULL;
-static void * vgIdHash;
+static void * tsSyncTmrCtrl = NULL;
+static void * tsVgIdHash;
+static int32_t tsSyncRefId = -1;
// local functions
-static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
-static void syncRecoverFromMaster(SSyncPeer *pPeer);
-static void syncCheckPeerConnection(void *param, void *tmrId);
-static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack);
-static void syncProcessBrokenLink(void *param);
-static int syncProcessPeerMsg(void *param, void *buffer);
-static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp);
-static void syncRemovePeer(SSyncPeer *pPeer);
-static void syncAddArbitrator(SSyncNode *pNode);
-static void syncAddNodeRef(SSyncNode *pNode);
-static void syncDecNodeRef(SSyncNode *pNode);
-static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode);
-static void syncMonitorFwdInfos(void *param, void *tmrId);
-static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code);
-static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle);
-static void syncRestartPeer(SSyncPeer *pPeer);
+static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
+static void syncRecoverFromMaster(SSyncPeer *pPeer);
+static void syncCheckPeerConnection(void *param, void *tmrId);
+static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack);
+static void syncProcessBrokenLink(void *param);
+static int32_t syncProcessPeerMsg(void *param, void *buffer);
+static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp);
+static void syncRemovePeer(SSyncPeer *pPeer);
+static void syncAddArbitrator(SSyncNode *pNode);
+static void syncFreeNode(void *);
+static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode);
+static void syncMonitorFwdInfos(void *param, void *tmrId);
+static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code);
+static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle);
+static void syncRestartPeer(SSyncPeer *pPeer);
+static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtyp);
static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo);
char* syncRole[] = {
@@ -77,7 +79,7 @@ int32_t syncInit() {
info.numOfThreads = tsSyncTcpThreads;
info.serverIp = 0;
info.port = tsSyncPort;
- info.bufferSize = 640000;
+ info.bufferSize = SYNC_MAX_SIZE;
info.processBrokenLink = syncProcessBrokenLink;
info.processIncomingMsg = syncProcessPeerMsg;
info.processIncomingConn = syncProcessIncommingConnection;
@@ -88,21 +90,27 @@ int32_t syncInit() {
return -1;
}
- syncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC");
- if (syncTmrCtrl == NULL) {
+ tsSyncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC");
+ if (tsSyncTmrCtrl == NULL) {
sError("failed to init tmrCtrl");
taosCloseTcpThreadPool(tsTcpPool);
tsTcpPool = NULL;
return -1;
}
- vgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
- if (vgIdHash == NULL) {
- sError("failed to init vgIdHash");
- taosTmrCleanUp(syncTmrCtrl);
+ tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
+ if (tsVgIdHash == NULL) {
+ sError("failed to init tsVgIdHash");
+ taosTmrCleanUp(tsSyncTmrCtrl);
taosCloseTcpThreadPool(tsTcpPool);
tsTcpPool = NULL;
- syncTmrCtrl = NULL;
+ tsSyncTmrCtrl = NULL;
+ return -1;
+ }
+
+ tsSyncRefId = taosOpenRef(200, syncFreeNode);
+ if (tsSyncRefId < 0) {
+ syncCleanUp();
return -1;
}
@@ -118,27 +126,30 @@ void syncCleanUp() {
tsTcpPool = NULL;
}
- if (syncTmrCtrl) {
- taosTmrCleanUp(syncTmrCtrl);
- syncTmrCtrl = NULL;
+ if (tsSyncTmrCtrl) {
+ taosTmrCleanUp(tsSyncTmrCtrl);
+ tsSyncTmrCtrl = NULL;
}
- if (vgIdHash) {
- taosHashCleanup(vgIdHash);
- vgIdHash = NULL;
+ if (tsVgIdHash) {
+ taosHashCleanup(tsVgIdHash);
+ tsVgIdHash = NULL;
}
+ taosCloseRef(tsSyncRefId);
+ tsSyncRefId = -1;
+
sInfo("sync module is cleaned up");
}
-void *syncStart(const SSyncInfo *pInfo) {
+int64_t syncStart(const SSyncInfo *pInfo) {
const SSyncCfg *pCfg = &pInfo->syncCfg;
SSyncNode *pNode = (SSyncNode *)calloc(sizeof(SSyncNode), 1);
if (pNode == NULL) {
sError("no memory to allocate syncNode");
terrno = TAOS_SYSTEM_ERROR(errno);
- return NULL;
+ return -1;
}
tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path));
@@ -159,21 +170,32 @@ void *syncStart(const SSyncInfo *pInfo) {
pNode->quorum = pCfg->quorum;
if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
- for (int i = 0; i < pCfg->replica; ++i) {
+ pNode->rid = taosAddRef(tsSyncRefId, pNode);
+ if (pNode->rid < 0) {
+ syncFreeNode(pNode);
+ return -1;
+ }
+
+ for (int32_t i = 0; i < pCfg->replica; ++i) {
const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i;
pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo);
+ if (pNode->peerInfo[i] == NULL) {
+ sError("vgId:%d, node:%d fqdn:%s port:%u is not configured, stop taosd", pNode->vgId, pNodeInfo->nodeId, pNodeInfo->nodeFqdn,
+ pNodeInfo->nodePort);
+ syncStop(pNode->rid);
+ exit(1);
+ }
+
if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) {
pNode->selfIndex = i;
}
}
- syncAddNodeRef(pNode);
-
if (pNode->selfIndex < 0) {
sInfo("vgId:%d, this node is not configured", pNode->vgId);
terrno = TSDB_CODE_SYN_INVALID_CONFIG;
- syncStop(pNode);
- return NULL;
+ syncStop(pNode->rid);
+ return -1;
}
nodeVersion = pInfo->version; // set the initial version
@@ -185,40 +207,41 @@ void *syncStart(const SSyncInfo *pInfo) {
if (pNode->pSyncFwds == NULL) {
sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId);
terrno = TAOS_SYSTEM_ERROR(errno);
- syncStop(pNode);
- return NULL;
+ syncStop(pNode->rid);
+ return -1;
}
- pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
+ pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl);
if (pNode->pFwdTimer == NULL) {
sError("vgId:%d, failed to allocate timer", pNode->vgId);
- syncStop(pNode);
- return NULL;
+ syncStop(pNode->rid);
+ return -1;
}
syncAddArbitrator(pNode);
- taosHashPut(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *));
+ taosHashPut(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *));
if (pNode->notifyRole) {
(*pNode->notifyRole)(pNode->ahandle, nodeRole);
}
- return pNode;
+ return pNode->rid;
}
-void syncStop(void *param) {
- SSyncNode *pNode = param;
+void syncStop(int64_t rid) {
SSyncPeer *pPeer;
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return;
+
sInfo("vgId:%d, cleanup sync", pNode->vgId);
pthread_mutex_lock(&(pNode->mutex));
- if (vgIdHash) taosHashRemove(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t));
+ if (tsVgIdHash) taosHashRemove(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t));
if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer);
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
pPeer = pNode->peerInfo[i];
if (pPeer) syncRemovePeer(pPeer);
}
@@ -228,14 +251,16 @@ void syncStop(void *param) {
pthread_mutex_unlock(&(pNode->mutex));
- syncDecNodeRef(pNode);
+ taosReleaseRef(tsSyncRefId, rid);
+ taosRemoveRef(tsSyncRefId, rid);
}
-int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
- SSyncNode *pNode = param;
- int i, j;
+int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
+ int32_t i, j;
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG;
+
sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica,
pNode->replica);
@@ -298,105 +323,58 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
syncRole[nodeRole]);
syncBroadcastStatus(pNode);
+ taosReleaseRef(tsSyncRefId, rid);
+
return 0;
}
-int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) {
- SSyncNode *pNode = param;
- SSyncPeer *pPeer;
- SSyncHead *pSyncHead;
- SWalHead * pWalHead = data;
- int fwdLen;
- int code = 0;
-
- if (pNode == NULL) return 0;
-
- if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) {
- sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId,
- pWalHead->version, nodeVersion);
- for (int i = 0; i < pNode->replica; ++i) {
- pPeer = pNode->peerInfo[i];
- syncRestartConnection(pPeer);
- }
- return TSDB_CODE_SYN_INVALID_VERSION;
- }
-
- // always update version
- nodeVersion = pWalHead->version;
- sDebug("vgId:%d, replica:%d nodeRole:%s qtype:%d ver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole],
- qtype, pWalHead->version);
-
- if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0;
+int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype) {
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+ if (pNode == NULL) return 0;
- // only pkt from RPC or CQ can be forwarded
- if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0;
+ int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype);
- // a hacker way to improve the performance
- pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead));
- pSyncHead->type = TAOS_SMSG_FORWARD;
- pSyncHead->pversion = 0;
- pSyncHead->len = sizeof(SWalHead) + pWalHead->len;
- fwdLen = pSyncHead->len + sizeof(SSyncHead); // include the WAL and SYNC head
-
- pthread_mutex_lock(&(pNode->mutex));
-
- for (int i = 0; i < pNode->replica; ++i) {
- pPeer = pNode->peerInfo[i];
- if (pPeer == NULL || pPeer->peerFd < 0) continue;
- if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue;
-
- if (pNode->quorum > 1 && code == 0) {
- syncSaveFwdInfo(pNode, pWalHead->version, mhandle);
- code = 1;
- }
-
- int retLen = write(pPeer->peerFd, pSyncHead, fwdLen);
- if (retLen == fwdLen) {
- sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len);
- } else {
- sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen);
- syncRestartConnection(pPeer);
- }
- }
-
- pthread_mutex_unlock(&(pNode->mutex));
+ taosReleaseRef(tsSyncRefId, rid);
return code;
}
-void syncConfirmForward(void *param, uint64_t version, int32_t code) {
- SSyncNode *pNode = param;
+void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return;
- if (pNode->quorum <= 1) return;
SSyncPeer *pPeer = pNode->pMaster;
- if (pPeer == NULL) return;
-
- char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0};
+ if (pPeer && pNode->quorum > 1) {
+ char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0};
- SSyncHead *pHead = (SSyncHead *)msg;
- pHead->type = TAOS_SMSG_FORWARD_RSP;
- pHead->len = sizeof(SFwdRsp);
+ SSyncHead *pHead = (SSyncHead *)msg;
+ pHead->type = TAOS_SMSG_FORWARD_RSP;
+ pHead->len = sizeof(SFwdRsp);
- SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead));
- pFwdRsp->version = version;
- pFwdRsp->code = code;
+ SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead));
+ pFwdRsp->version = version;
+ pFwdRsp->code = code;
- int msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp);
- int retLen = write(pPeer->peerFd, msg, msgLen);
+ int32_t msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp);
+ int32_t retLen = write(pPeer->peerFd, msg, msgLen);
- if (retLen == msgLen) {
- sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version);
- } else {
- sDebug("%s, failed to send forward ack, restart", pPeer->id);
- syncRestartConnection(pPeer);
+ if (retLen == msgLen) {
+ sDebug("%s, forward-rsp is sent, code:%x hver:%" PRIu64, pPeer->id, code, version);
+ } else {
+ sDebug("%s, failed to send forward ack, restart", pPeer->id);
+ syncRestartConnection(pPeer);
+ }
}
+
+ taosReleaseRef(tsSyncRefId, rid);
}
-void syncRecover(void *param) {
- SSyncNode *pNode = param;
+void syncRecover(int64_t rid) {
SSyncPeer *pPeer;
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+ if (pNode == NULL) return;
+
// to do: add a few lines to check if recover is OK
// if take this node to unsync state, the whole system may not work
@@ -406,7 +384,7 @@ void syncRecover(void *param) {
pthread_mutex_lock(&(pNode->mutex));
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
pPeer = (SSyncPeer *)pNode->peerInfo[i];
if (pPeer->peerFd >= 0) {
syncRestartConnection(pPeer);
@@ -414,17 +392,22 @@ void syncRecover(void *param) {
}
pthread_mutex_unlock(&(pNode->mutex));
+
+ taosReleaseRef(tsSyncRefId, rid);
}
-int syncGetNodesRole(void *param, SNodesRole *pNodesRole) {
- SSyncNode *pNode = param;
+int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) {
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+ if (pNode == NULL) return -1;
pNodesRole->selfIndex = pNode->selfIndex;
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId;
pNodesRole->role[i] = pNode->peerInfo[i]->role;
}
+ taosReleaseRef(tsSyncRefId, rid);
+
return 0;
}
@@ -440,7 +423,7 @@ static void syncAddArbitrator(SSyncNode *pNode) {
SNodeInfo nodeInfo;
nodeInfo.nodeId = 0;
- int ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort);
+ int32_t ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort);
if (-1 == ret) {
nodeInfo.nodePort = tsArbitratorPort;
}
@@ -457,26 +440,24 @@ static void syncAddArbitrator(SSyncNode *pNode) {
pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo);
}
-static void syncAddNodeRef(SSyncNode *pNode) { atomic_add_fetch_8(&pNode->refCount, 1); }
+static void syncFreeNode(void *param) {
+ SSyncNode *pNode = param;
-static void syncDecNodeRef(SSyncNode *pNode) {
- if (atomic_sub_fetch_8(&pNode->refCount, 1) == 0) {
- pthread_mutex_destroy(&pNode->mutex);
- taosTFree(pNode->pRecv);
- taosTFree(pNode->pSyncFwds);
- taosTFree(pNode);
- }
+ pthread_mutex_destroy(&pNode->mutex);
+ tfree(pNode->pRecv);
+ tfree(pNode->pSyncFwds);
+ tfree(pNode);
}
void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); }
-int syncDecPeerRef(SSyncPeer *pPeer) {
+int32_t syncDecPeerRef(SSyncPeer *pPeer) {
if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) {
- syncDecNodeRef(pPeer->pSyncNode);
+ taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid);
sDebug("%s, resource is freed", pPeer->id);
- taosTFree(pPeer->watchFd);
- taosTFree(pPeer);
+ tfree(pPeer->watchFd);
+ tfree(pPeer);
return 0;
}
@@ -502,7 +483,11 @@ static void syncRemovePeer(SSyncPeer *pPeer) {
static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
uint32_t ip = taosGetIpFromFqdn(pInfo->nodeFqdn);
- if (ip == -1) return NULL;
+ if (ip == 0xFFFFFFFF) {
+ sError("failed to add peer, can resolve fqdn:%s since %s", pInfo->nodeFqdn, strerror(errno));
+ terrno = TSDB_CODE_RPC_FQDN_ERROR;
+ return NULL;
+ }
SSyncPeer *pPeer = calloc(1, sizeof(SSyncPeer));
if (pPeer == NULL) return NULL;
@@ -512,7 +497,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
pPeer->ip = ip;
pPeer->port = pInfo->nodePort;
pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0;
- snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d peer:%s:%u", pNode->vgId, pPeer->fqdn, pPeer->port);
+ snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, peer:%s:%u", pNode->vgId, pPeer->fqdn, pPeer->port);
pPeer->peerFd = -1;
pPeer->syncFd = -1;
@@ -521,22 +506,22 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
pPeer->refCount = 1;
sInfo("%s, it is configured", pPeer->id);
- int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
+ int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
int32_t checkMs = 100 + (pNode->vgId * 10) % 100;
- if (pNode->vgId > 1) checkMs = tsStatusInterval * 2000 + checkMs;
+ if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs;
sDebug("%s, start to check peer connection after %d ms", pPeer->id, checkMs);
- taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, syncTmrCtrl, &pPeer->timer);
+ taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer);
}
- syncAddNodeRef(pNode);
+ taosAcquireRef(tsSyncRefId, pNode->rid);
return pPeer;
}
void syncBroadcastStatus(SSyncNode *pNode) {
SSyncPeer *pPeer;
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
if (i == pNode->selfIndex) continue;
pPeer = pNode->peerInfo[i];
syncSendPeersStatusMsgToPeer(pPeer, 1);
@@ -544,7 +529,7 @@ void syncBroadcastStatus(SSyncNode *pNode) {
}
static void syncResetFlowCtrl(SSyncNode *pNode) {
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
pNode->peerInfo[i]->numOfRetrieves = 0;
}
@@ -555,13 +540,13 @@ static void syncResetFlowCtrl(SSyncNode *pNode) {
static void syncChooseMaster(SSyncNode *pNode) {
SSyncPeer *pPeer;
- int onlineNum = 0;
- int index = -1;
- int replica = pNode->replica;
+ int32_t onlineNum = 0;
+ int32_t index = -1;
+ int32_t replica = pNode->replica;
sDebug("vgId:%d, choose master", pNode->vgId);
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) {
onlineNum++;
}
@@ -570,7 +555,7 @@ static void syncChooseMaster(SSyncNode *pNode) {
if (onlineNum == pNode->replica) {
// if all peers are online, peer with highest version shall be master
index = 0;
- for (int i = 1; i < pNode->replica; ++i) {
+ for (int32_t i = 1; i < pNode->replica; ++i) {
if (pNode->peerInfo[i]->version > pNode->peerInfo[index]->version) {
index = i;
}
@@ -586,7 +571,7 @@ static void syncChooseMaster(SSyncNode *pNode) {
if (index < 0 && onlineNum > replica / 2.0) {
// over half of nodes are online
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
// slave with highest version shall be master
pPeer = pNode->peerInfo[i];
if (pPeer->role == TAOS_SYNC_ROLE_SLAVE || pPeer->role == TAOS_SYNC_ROLE_MASTER) {
@@ -601,6 +586,18 @@ static void syncChooseMaster(SSyncNode *pNode) {
if (index == pNode->selfIndex) {
sInfo("vgId:%d, start to work as master", pNode->vgId);
nodeRole = TAOS_SYNC_ROLE_MASTER;
+
+#if 0
+ for (int32_t i = 0; i < pNode->replica; ++i) {
+ if (i == index) continue;
+ pPeer = pNode->peerInfo[i];
+ if (pPeer->version == nodeVersion) {
+ pPeer->role = TAOS_SYNC_ROLE_SLAVE;
+ pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;
+ sInfo("%s, it shall work as slave", pPeer->id);
+ }
+ }
+#endif
syncResetFlowCtrl(pNode);
(*pNode->notifyRole)(pNode->ahandle, nodeRole);
} else {
@@ -613,11 +610,11 @@ static void syncChooseMaster(SSyncNode *pNode) {
}
static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
- int onlineNum = 0;
- int index = -1;
- int replica = pNode->replica;
+ int32_t onlineNum = 0;
+ int32_t index = -1;
+ int32_t replica = pNode->replica;
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) {
onlineNum++;
}
@@ -638,7 +635,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
}
} else {
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
SSyncPeer *pTemp = pNode->peerInfo[i];
if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue;
if (index < 0) {
@@ -657,9 +654,9 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
return pMaster;
}
-static int syncValidateMaster(SSyncPeer *pPeer) {
+static int32_t syncValidateMaster(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
- int code = 0;
+ int32_t code = 0;
if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) {
sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id);
@@ -667,7 +664,7 @@ static int syncValidateMaster(SSyncPeer *pPeer) {
(*pNode->notifyRole)(pNode->ahandle, nodeRole);
code = -1;
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
if (i == pNode->selfIndex) continue;
syncRestartPeer(pNode->peerInfo[i]);
}
@@ -709,7 +706,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne
}
} else {
// master not there, if all peer's state and version are consistent, choose the master
- int consistent = 0;
+ int32_t consistent = 0;
if (peersStatus) {
for (i = 0; i < pNode->replica; ++i) {
SSyncPeer *pTemp = pNode->peerInfo[i];
@@ -747,9 +744,9 @@ static void syncRestartPeer(SSyncPeer *pPeer) {
pPeer->sstatus = TAOS_SYNC_STATUS_INIT;
- int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
+ int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) {
- taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer);
+ taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
}
}
@@ -783,7 +780,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) {
pthread_t thread;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
- int ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer);
+ int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer);
pthread_attr_destroy(&thattr);
if (ret != 0) {
@@ -828,7 +825,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
// Ensure the sync of mnode not interrupted
if (pNode->vgId != 1 && tsSyncNum >= tsMaxSyncNum) {
sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum);
- taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, syncTmrCtrl, &pPeer->timer);
+ taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, tsSyncTmrCtrl, &pPeer->timer);
return;
}
@@ -841,7 +838,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
firstPkt.syncHead.len = sizeof(firstPkt) - sizeof(SSyncHead);
tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
firstPkt.port = tsSyncPort;
- taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer);
+ taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
if (write(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
sError("%s, failed to send sync-req to peer", pPeer->id);
@@ -857,12 +854,12 @@ static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) {
SSyncFwds *pSyncFwds = pNode->pSyncFwds;
SFwdInfo * pFwdInfo;
- sDebug("%s, forward-rsp is received, ver:%" PRIu64, pPeer->id, pFwdRsp->version);
+ sDebug("%s, forward-rsp is received, code:%x ver:%" PRIu64, pPeer->id, pFwdRsp->code, pFwdRsp->version);
SFwdInfo *pFirst = pSyncFwds->fwdInfo + pSyncFwds->first;
if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) {
// find the forwardInfo from first
- for (int i = 0; i < pSyncFwds->fwds; ++i) {
+ for (int32_t i = 0; i < pSyncFwds->fwds; ++i) {
pFwdInfo = pSyncFwds->fwdInfo + (i + pSyncFwds->first) % tsMaxFwdInfo;
if (pFwdRsp->version == pFwdInfo->version) break;
}
@@ -876,16 +873,16 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
SWalHead * pHead = (SWalHead *)cont;
- sDebug("%s, forward is received, ver:%" PRIu64, pPeer->id, pHead->version);
+ sDebug("%s, forward is received, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len);
if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
// nodeVersion = pHead->version;
- (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD);
+ (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL);
} else {
if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
syncSaveIntoBuffer(pPeer, pHead);
} else {
- sError("%s, forward discarded, ver:%" PRIu64, pPeer->id, pHead->version);
+ sError("%s, forward discarded, hver:%" PRIu64, pPeer->id, pHead->version);
}
}
}
@@ -905,10 +902,10 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) {
}
}
-static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
+static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
if (pPeer->peerFd < 0) return -1;
- int hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead));
+ int32_t hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead));
if (hlen != sizeof(SSyncHead)) {
sDebug("%s, failed to read msg, hlen:%d", pPeer->id, hlen);
return -1;
@@ -916,11 +913,12 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
// head.len = htonl(head.len);
if (pHead->len < 0) {
- sError("%s, invalid pkt length, len:%d", pPeer->id, pHead->len);
+ sError("%s, invalid pkt length, hlen:%d", pPeer->id, pHead->len);
return -1;
}
- int bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len);
+ assert(pHead->len <= TSDB_MAX_WAL_SIZE);
+ int32_t bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len);
if (bytes != pHead->len) {
sError("%s, failed to read, bytes:%d len:%d", pPeer->id, bytes, pHead->len);
return -1;
@@ -929,7 +927,7 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
return 0;
}
-static int syncProcessPeerMsg(void *param, void *buffer) {
+static int32_t syncProcessPeerMsg(void *param, void *buffer) {
SSyncPeer *pPeer = param;
SSyncHead head;
char * cont = buffer;
@@ -937,7 +935,7 @@ static int syncProcessPeerMsg(void *param, void *buffer) {
SSyncNode *pNode = pPeer->pSyncNode;
pthread_mutex_lock(&(pNode->mutex));
- int code = syncReadPeerMsg(pPeer, &head, cont);
+ int32_t code = syncReadPeerMsg(pPeer, &head, cont);
if (code == 0) {
if (head.type == TAOS_SMSG_FORWARD) {
@@ -974,12 +972,12 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) {
pPeersStatus->role = nodeRole;
pPeersStatus->ack = ack;
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
pPeersStatus->peersStatus[i].role = pNode->peerInfo[i]->role;
pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version;
}
- int retLen = write(pPeer->peerFd, msg, statusMsgLen);
+ int32_t retLen = write(pPeer->peerFd, msg, statusMsgLen);
if (retLen == statusMsgLen) {
sDebug("%s, status msg is sent, self:%s ver:%" PRIu64 ", ack:%d", pPeer->id, syncRole[pPeersStatus->role],
pPeersStatus->version, pPeersStatus->ack);
@@ -1001,10 +999,10 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
return;
}
- int connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
+ int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
if (connFd < 0) {
sDebug("%s, failed to open tcp socket(%s)", pPeer->id, strerror(errno));
- taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer);
+ taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
return;
}
@@ -1025,7 +1023,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
} else {
sDebug("try later");
close(connFd);
- taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer);
+ taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
}
}
@@ -1050,7 +1048,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
syncAddPeerRef(pPeer);
- int ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer);
+ int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer);
pthread_attr_destroy(&thattr);
if (ret < 0) {
@@ -1062,9 +1060,9 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
}
}
-static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) {
- char ipstr[24];
- int i;
+static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
+ char ipstr[24];
+ int32_t i;
tinet_ntoa(ipstr, sourceIp);
sDebug("peer TCP connection from ip:%s", ipstr);
@@ -1077,7 +1075,7 @@ static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) {
}
int32_t vgId = firstPkt.syncHead.vgId;
- SSyncNode **ppNode = (SSyncNode **)taosHashGet(vgIdHash, (const char *)&vgId, sizeof(int32_t));
+ SSyncNode **ppNode = (SSyncNode **)taosHashGet(tsVgIdHash, (const char *)&vgId, sizeof(int32_t));
if (ppNode == NULL || *ppNode == NULL) {
sError("vgId:%d, vgId could not be found", vgId);
taosCloseSocket(connFd);
@@ -1122,7 +1120,7 @@ static void syncProcessBrokenLink(void *param) {
SSyncPeer *pPeer = param;
SSyncNode *pNode = pPeer->pSyncNode;
- syncAddNodeRef(pNode);
+ if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return;
pthread_mutex_lock(&(pNode->mutex));
sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno));
@@ -1133,7 +1131,7 @@ static void syncProcessBrokenLink(void *param) {
}
pthread_mutex_unlock(&(pNode->mutex));
- syncDecNodeRef(pNode);
+ taosReleaseRef(tsSyncRefId, pNode->rid);
}
static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
@@ -1150,10 +1148,9 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
}
SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->last;
+ memset(pFwdInfo, 0, sizeof(SFwdInfo));
pFwdInfo->version = version;
pFwdInfo->mhandle = mhandle;
- pFwdInfo->acks = 0;
- pFwdInfo->confirmed = 0;
pFwdInfo->time = time;
pSyncFwds->fwds++;
@@ -1163,8 +1160,8 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) {
SSyncFwds *pSyncFwds = pNode->pSyncFwds;
- int fwds = pSyncFwds->fwds;
- for (int i = 0; i < fwds; ++i) {
+ int32_t fwds = pSyncFwds->fwds;
+ for (int32_t i = 0; i < fwds; ++i) {
SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->first;
if (pFwdInfo->confirmed == 0) break;
@@ -1178,7 +1175,7 @@ static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) {
}
static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) {
- int confirm = 0;
+ int32_t confirm = 0;
if (pFwdInfo->code == 0) pFwdInfo->code = code;
if (code == 0) {
@@ -1201,23 +1198,94 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code
}
static void syncMonitorFwdInfos(void *param, void *tmrId) {
- SSyncNode *pNode = param;
+ int64_t rid = (int64_t) param;
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+ if (pNode == NULL) return;
+
SSyncFwds *pSyncFwds = pNode->pSyncFwds;
- if (pSyncFwds == NULL) return;
- uint64_t time = taosGetTimestampMs();
+ if (pSyncFwds) {;
+ uint64_t time = taosGetTimestampMs();
- if (pSyncFwds->fwds > 0) {
- pthread_mutex_lock(&(pNode->mutex));
- for (int i = 0; i < pSyncFwds->fwds; ++i) {
- SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo;
- if (time - pFwdInfo->time < 2000) break;
- syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+ if (pSyncFwds->fwds > 0) {
+ pthread_mutex_lock(&(pNode->mutex));
+ for (int32_t i = 0; i < pSyncFwds->fwds; ++i) {
+ SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo;
+ if (time - pFwdInfo->time < 2000) break;
+ syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+ }
+
+ syncRemoveConfirmedFwdInfo(pNode);
+ pthread_mutex_unlock(&(pNode->mutex));
}
- syncRemoveConfirmedFwdInfo(pNode);
- pthread_mutex_unlock(&(pNode->mutex));
+ pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl);
+ }
+
+ taosReleaseRef(tsSyncRefId, rid);
+}
+
+static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype) {
+ SSyncPeer *pPeer;
+ SSyncHead *pSyncHead;
+ SWalHead * pWalHead = data;
+ int32_t fwdLen;
+ int32_t code = 0;
+
+
+ if (pWalHead->version > nodeVersion + 1) {
+ sError("vgId:%d, hver:%" PRIu64 ", inconsistent with ver:%" PRIu64, pNode->vgId, pWalHead->version, nodeVersion);
+ if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
+ sInfo("vgId:%d, restart connection", pNode->vgId);
+ for (int32_t i = 0; i < pNode->replica; ++i) {
+ pPeer = pNode->peerInfo[i];
+ syncRestartConnection(pPeer);
+ }
+ }
+
+ return TSDB_CODE_SYN_INVALID_VERSION;
+ }
+
+ // always update version
+ nodeVersion = pWalHead->version;
+ sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica,
+ syncRole[nodeRole], qtypeStr[qtype], pWalHead->version);
+
+ if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0;
+
+ // only pkt from RPC or CQ can be forwarded
+ if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0;
+
+ // a hacker way to improve the performance
+ pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead));
+ pSyncHead->type = TAOS_SMSG_FORWARD;
+ pSyncHead->pversion = 0;
+ pSyncHead->len = sizeof(SWalHead) + pWalHead->len;
+ fwdLen = pSyncHead->len + sizeof(SSyncHead); // include the WAL and SYNC head
+
+ pthread_mutex_lock(&(pNode->mutex));
+
+ for (int32_t i = 0; i < pNode->replica; ++i) {
+ pPeer = pNode->peerInfo[i];
+ if (pPeer == NULL || pPeer->peerFd < 0) continue;
+ if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue;
+
+ if (pNode->quorum > 1 && code == 0) {
+ syncSaveFwdInfo(pNode, pWalHead->version, mhandle);
+ code = 1;
+ }
+
+ int32_t retLen = write(pPeer->peerFd, pSyncHead, fwdLen);
+ if (retLen == fwdLen) {
+ sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len);
+ } else {
+ sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen);
+ syncRestartConnection(pPeer);
+ }
}
- pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
+ pthread_mutex_unlock(&(pNode->mutex));
+
+ return code;
}
+
diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c
index ebb6c3a0a9edff5acfc5f2ce7da8b58f03d8ab4a..5d7b9eac9b9386f6e3a48de0931bd4c4455f0328 100644
--- a/src/sync/src/syncRestore.c
+++ b/src/sync/src/syncRestore.c
@@ -48,20 +48,21 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex
}
}
-static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
+static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
SSyncNode *pNode = pPeer->pSyncNode;
SFileInfo minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */ // master file info
SFileInfo sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */ // slave file info
SFileAck fileAck;
- int code = -1;
+ int32_t code = -1;
char name[TSDB_FILENAME_LEN * 2] = {0};
uint32_t pindex = 0; // index in last restore
+ bool fileChanged = false;
*fversion = 0;
sinfo.index = 0;
while (1) {
// read file info
- int ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo));
+ int32_t ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo));
if (ret < 0) break;
// if no more file from master, break;
@@ -103,7 +104,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
minfo.name[sizeof(minfo.name) - 1] = 0;
snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name);
- int dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
+ int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
if (dfd < 0) {
sError("%s, failed to open file:%s", pPeer->id, name);
break;
@@ -114,10 +115,11 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
close(dfd);
if (ret < 0) break;
+ fileChanged = true;
sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
}
- if (code == 0 && (minfo.fversion != sinfo.fversion)) {
+ if (code == 0 && fileChanged) {
// data file is changed, code shall be set to 1
*fversion = minfo.fversion;
code = 1;
@@ -130,11 +132,11 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
return code;
}
-static int syncRestoreWal(SSyncPeer *pPeer) {
+static int32_t syncRestoreWal(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
- int ret, code = -1;
+ int32_t ret, code = -1;
- void *buffer = calloc(1024000, 1); // size for one record
+ void *buffer = calloc(SYNC_MAX_SIZE, 1); // size for one record
if (buffer == NULL) return -1;
SWalHead *pHead = (SWalHead *)buffer;
@@ -151,8 +153,8 @@ static int syncRestoreWal(SSyncPeer *pPeer) {
ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len);
if (ret < 0) break;
- sDebug("%s, restore a record, ver:%" PRIu64, pPeer->id, pHead->version);
- (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL);
+ sDebug("%s, restore a record, qtype:wal hver:%" PRIu64, pPeer->id, pHead->version);
+ (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL, NULL);
}
if (code < 0) {
@@ -167,16 +169,16 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) {
SSyncNode *pNode = pPeer->pSyncNode;
SWalHead * pHead = (SWalHead *)offset;
- (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD);
+ (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL);
offset += pHead->len + sizeof(SWalHead);
return offset;
}
-static int syncProcessBufferedFwd(SSyncPeer *pPeer) {
+static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) {
SSyncNode * pNode = pPeer->pSyncNode;
SRecvBuffer *pRecv = pNode->pRecv;
- int forwards = 0;
+ int32_t forwards = 0;
sDebug("%s, number of buffered forwards:%d", pPeer->id, pRecv->forwards);
@@ -201,12 +203,12 @@ static int syncProcessBufferedFwd(SSyncPeer *pPeer) {
return pRecv->code;
}
-int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
+int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
SSyncNode * pNode = pPeer->pSyncNode;
SRecvBuffer *pRecv = pNode->pRecv;
if (pRecv == NULL) return -1;
- int len = pHead->len + sizeof(SWalHead);
+ int32_t len = pHead->len + sizeof(SWalHead);
if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) {
memcpy(pRecv->offset, pHead, len);
@@ -223,19 +225,19 @@ int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
static void syncCloseRecvBuffer(SSyncNode *pNode) {
if (pNode->pRecv) {
- taosTFree(pNode->pRecv->buffer);
+ tfree(pNode->pRecv->buffer);
}
- taosTFree(pNode->pRecv);
+ tfree(pNode->pRecv);
}
-static int syncOpenRecvBuffer(SSyncNode *pNode) {
+static int32_t syncOpenRecvBuffer(SSyncNode *pNode) {
syncCloseRecvBuffer(pNode);
SRecvBuffer *pRecv = calloc(sizeof(SRecvBuffer), 1);
if (pRecv == NULL) return -1;
- pRecv->bufferSize = 5000000;
+ pRecv->bufferSize = SYNC_RECV_BUFFER_SIZE;
pRecv->buffer = malloc(pRecv->bufferSize);
if (pRecv->buffer == NULL) {
free(pRecv);
@@ -250,13 +252,13 @@ static int syncOpenRecvBuffer(SSyncNode *pNode) {
return 0;
}
-static int syncRestoreDataStepByStep(SSyncPeer *pPeer) {
+static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
nodeSStatus = TAOS_SYNC_STATUS_FILE;
uint64_t fversion = 0;
sDebug("%s, start to restore file", pPeer->id);
- int code = syncRestoreFile(pPeer, &fversion);
+ int32_t code = syncRestoreFile(pPeer, &fversion);
if (code < 0) {
sError("%s, failed to restore file", pPeer->id);
return -1;
diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c
index 60625d75eccdbe6bbb29f97b31ecd8e9855480a7..21151f119937519e477e73d0e5e7cb82c1155788 100644
--- a/src/sync/src/syncRetrieve.c
+++ b/src/sync/src/syncRetrieve.c
@@ -27,7 +27,7 @@
#include "tsync.h"
#include "syncInt.h"
-static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
+static int32_t syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
sDebug("%s, start to monitor:%s", pPeer->id, name);
if (pPeer->notifyFd <= 0) {
@@ -38,16 +38,16 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
return -1;
}
- if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles);
+ if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles);
if (pPeer->watchFd == NULL) {
sError("%s, failed to allocate watchFd", pPeer->id);
return -1;
}
- memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles);
+ memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles);
}
- int *wd = pPeer->watchFd + pPeer->watchNum;
+ int32_t *wd = pPeer->watchFd + pPeer->watchNum;
if (*wd >= 0) {
if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) {
@@ -69,17 +69,17 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
return 0;
}
-static int syncAreFilesModified(SSyncPeer *pPeer) {
+static int32_t syncAreFilesModified(SSyncPeer *pPeer) {
if (pPeer->notifyFd <= 0) return 0;
- char buf[2048];
- int len = read(pPeer->notifyFd, buf, sizeof(buf));
+ char buf[2048];
+ int32_t len = read(pPeer->notifyFd, buf, sizeof(buf));
if (len < 0 && errno != EAGAIN) {
sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));
return -1;
}
- int code = 0;
+ int32_t code = 0;
if (len > 0) {
const struct inotify_event *event;
char *ptr;
@@ -97,11 +97,11 @@ static int syncAreFilesModified(SSyncPeer *pPeer) {
return code;
}
-static int syncRetrieveFile(SSyncPeer *pPeer) {
+static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
SFileInfo fileInfo;
SFileAck fileAck;
- int code = -1;
+ int32_t code = -1;
char name[TSDB_FILENAME_LEN * 2] = {0};
memset(&fileInfo, 0, sizeof(fileInfo));
@@ -146,10 +146,10 @@ static int syncRetrieveFile(SSyncPeer *pPeer) {
}
// send the file to peer
- int sfd = open(name, O_RDONLY);
+ int32_t sfd = open(name, O_RDONLY);
if (sfd < 0) break;
- ret = taosTSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size);
+ ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size);
close(sfd);
if (ret < 0) break;
@@ -169,8 +169,8 @@ static int syncRetrieveFile(SSyncPeer *pPeer) {
/* if only a partial record is read out, set the IN_MODIFY flag in event,
so upper layer will reload the file to get a complete record */
-static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) {
- int ret;
+static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead, uint32_t *pEvent) {
+ int32_t ret;
ret = read(sfd, pHead, sizeof(SWalHead));
if (ret < 0) return -1;
@@ -182,6 +182,8 @@ static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) {
return 0;
}
+ assert(pHead->len <= TSDB_MAX_WAL_SIZE);
+
ret = read(sfd, pHead->cont, pHead->len);
if (ret < 0) return -1;
@@ -194,7 +196,7 @@ static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) {
return sizeof(SWalHead) + pHead->len;
}
-static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
+static int32_t syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
pPeer->watchNum = 0;
taosClose(pPeer->notifyFd);
pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
@@ -203,14 +205,14 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
return -1;
}
- if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles);
+ if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles);
if (pPeer->watchFd == NULL) {
sError("%s, failed to allocate watchFd", pPeer->id);
return -1;
}
- memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles);
- int *wd = pPeer->watchFd;
+ memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles);
+ int32_t *wd = pPeer->watchFd;
*wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE);
if (*wd == -1) {
@@ -222,8 +224,8 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
}
static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) {
- char buf[2048];
- int len = read(pPeer->notifyFd, buf, sizeof(buf));
+ char buf[2048];
+ int32_t len = read(pPeer->notifyFd, buf, sizeof(buf));
if (len < 0 && errno != EAGAIN) {
sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));
return -1;
@@ -243,11 +245,11 @@ static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) {
return 0;
}
-static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) {
- SWalHead *pHead = malloc(640000);
- int code = -1;
+static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) {
+ SWalHead *pHead = malloc(SYNC_MAX_SIZE);
+ int32_t code = -1;
int32_t bytes = 0;
- int sfd;
+ int32_t sfd;
sfd = open(name, O_RDONLY);
if (sfd < 0) {
@@ -256,10 +258,10 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion,
}
(void)lseek(sfd, offset, SEEK_SET);
- sDebug("%s, retrieve last wal, offset:%" PRId64 " fversion:%" PRIu64, pPeer->id, offset, fversion);
+ sDebug("%s, retrieve last wal, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, offset, fversion);
while (1) {
- int wsize = syncReadOneWalRecord(sfd, pHead, pEvent);
+ int32_t wsize = syncReadOneWalRecord(sfd, pHead, pEvent);
if (wsize < 0) break;
if (wsize == 0) {
code = 0;
@@ -267,7 +269,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion,
}
sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version);
- int ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
+ int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
if (ret != wsize) break;
pPeer->sversion = pHead->version;
@@ -287,9 +289,9 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion,
return -1;
}
-static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) {
+static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) {
SSyncNode *pNode = pPeer->pSyncNode;
- int code = -1;
+ int32_t code = -1;
char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file
if (syncAreFilesModified(pPeer) != 0) return -1;
@@ -325,7 +327,7 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) {
// if all data up to fversion is read out, it is over
if (pPeer->sversion >= fversion && fversion > 0) {
code = 0;
- sDebug("%s, data up to fversion:%" PRId64 " has been read out, bytes:%d", pPeer->id, fversion, bytes);
+ sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d", pPeer->id, fversion, bytes);
break;
}
@@ -370,14 +372,14 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) {
return code;
}
-static int syncRetrieveWal(SSyncPeer *pPeer) {
+static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
SSyncNode * pNode = pPeer->pSyncNode;
char fname[TSDB_FILENAME_LEN * 3];
char wname[TSDB_FILENAME_LEN * 2];
int32_t size;
struct stat fstat;
- int code = -1;
- uint32_t index = 0;
+ int32_t code = -1;
+ int64_t index = 0;
while (1) {
// retrieve wal info
@@ -403,10 +405,10 @@ static int syncRetrieveWal(SSyncPeer *pPeer) {
size = fstat.st_size;
sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size);
- int sfd = open(fname, O_RDONLY);
+ int32_t sfd = open(fname, O_RDONLY);
if (sfd < 0) break;
- code = taosTSendFile(pPeer->syncFd, sfd, NULL, size);
+ code = taosSendFile(pPeer->syncFd, sfd, NULL, size);
close(sfd);
if (code < 0) break;
@@ -428,7 +430,7 @@ static int syncRetrieveWal(SSyncPeer *pPeer) {
return code;
}
-static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
+static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
SFirstPkt firstPkt;
diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c
index 6a210a136ffe67b2e1394d26bac4cb5083452c8c..875528e66b188294707bf1cc67baf3780ece6490 100644
--- a/src/sync/src/taosTcpPool.c
+++ b/src/sync/src/taosTcpPool.c
@@ -13,18 +13,22 @@
* along with this program. If not, see .
*/
+#define _DEFAULT_SOURCE
#include "os.h"
#include "tulog.h"
#include "tutil.h"
#include "tsocket.h"
#include "taoserror.h"
#include "taosTcpPool.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
typedef struct SThreadObj {
pthread_t thread;
bool stop;
- int pollFd;
- int numOfFds;
+ int32_t pollFd;
+ int32_t numOfFds;
struct SPoolObj *pPool;
} SThreadObj;
@@ -32,15 +36,15 @@ typedef struct SPoolObj {
SPoolInfo info;
SThreadObj **pThread;
pthread_t thread;
- int nextId;
- int acceptFd; // FD for accept new connection
+ int32_t nextId;
+ int32_t acceptFd; // FD for accept new connection
} SPoolObj;
typedef struct {
SThreadObj *pThread;
- void *ahandle;
- int fd;
- int closedByApp;
+ void * ahandle;
+ int32_t fd;
+ int32_t closedByApp;
} SConnObj;
static void *taosAcceptPeerTcpConnection(void *argv);
@@ -53,66 +57,66 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) {
SPoolObj *pPool = calloc(sizeof(SPoolObj), 1);
if (pPool == NULL) {
- uError("TCP server, no enough memory");
+ sError("failed to alloc pool for TCP server since no enough memory");
return NULL;
}
pPool->info = *pInfo;
- pPool->pThread = (SThreadObj **)calloc(sizeof(SThreadObj *), pInfo->numOfThreads);
+ pPool->pThread = calloc(sizeof(SThreadObj *), pInfo->numOfThreads);
if (pPool->pThread == NULL) {
- uError("TCP server, no enough memory");
- free(pPool);
+ sError("failed to alloc pool thread for TCP server since no enough memory");
+ tfree(pPool);
return NULL;
}
pPool->acceptFd = taosOpenTcpServerSocket(pInfo->serverIp, pInfo->port);
if (pPool->acceptFd < 0) {
- free(pPool->pThread);
- free(pPool);
- uError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno));
+ tfree(pPool->pThread);
+ tfree(pPool);
+ sError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno));
return NULL;
}
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pPool->thread), &thattr, (void *)taosAcceptPeerTcpConnection, pPool) != 0) {
- uError("TCP server, failed to create accept thread, reason:%s", strerror(errno));
+ sError("failed to create accept thread for TCP server since %s", strerror(errno));
close(pPool->acceptFd);
- free(pPool->pThread);
- free(pPool);
+ tfree(pPool->pThread);
+ tfree(pPool);
return NULL;
}
pthread_attr_destroy(&thattr);
- uDebug("%p TCP pool is created", pPool);
+ sDebug("%p TCP pool is created", pPool);
return pPool;
}
void taosCloseTcpThreadPool(void *param) {
- SPoolObj * pPool = (SPoolObj *)param;
+ SPoolObj * pPool = param;
SThreadObj *pThread;
shutdown(pPool->acceptFd, SHUT_RD);
pthread_join(pPool->thread, NULL);
- for (int i = 0; i < pPool->info.numOfThreads; ++i) {
+ for (int32_t i = 0; i < pPool->info.numOfThreads; ++i) {
pThread = pPool->pThread[i];
if (pThread) taosStopPoolThread(pThread);
}
- uDebug("%p TCP pool is closed", pPool);
+ sDebug("%p TCP pool is closed", pPool);
- taosTFree(pPool->pThread);
- free(pPool);
+ tfree(pPool->pThread);
+ tfree(pPool);
}
-void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) {
+void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) {
struct epoll_event event;
- SPoolObj *pPool = (SPoolObj *)param;
+ SPoolObj *pPool = param;
- SConnObj *pConn = (SConnObj *)calloc(sizeof(SConnObj), 1);
+ SConnObj *pConn = calloc(sizeof(SConnObj), 1);
if (pConn == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
@@ -120,7 +124,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) {
SThreadObj *pThread = taosGetTcpThread(pPool);
if (pThread == NULL) {
- free(pConn);
+ tfree(pConn);
return NULL;
}
@@ -133,13 +137,13 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) {
event.data.ptr = pConn;
if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
- uError("failed to add fd:%d(%s)", connFd, strerror(errno));
+ sError("failed to add fd:%d since %s", connFd, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
- free(pConn);
+ tfree(pConn);
pConn = NULL;
} else {
pThread->numOfFds++;
- uDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds);
+ sDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds);
}
return pConn;
@@ -149,7 +153,7 @@ void taosFreeTcpConn(void *param) {
SConnObj * pConn = (SConnObj *)param;
SThreadObj *pThread = pConn->pThread;
- uDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd);
+ sDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd);
pConn->closedByApp = 1;
shutdown(pConn->fd, SHUT_WR);
}
@@ -164,9 +168,9 @@ static void taosProcessBrokenLink(SConnObj *pConn) {
pThread->numOfFds--;
epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL);
- uDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds);
+ sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds);
taosClose(pConn->fd);
- free(pConn);
+ tfree(pConn);
}
#define maxEvents 10
@@ -183,18 +187,18 @@ static void *taosProcessTcpData(void *param) {
while (1) {
if (pThread->stop) break;
- int fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME);
+ int32_t fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME);
if (pThread->stop) {
- uDebug("%p TCP epoll thread is exiting...", pThread);
+ sDebug("%p TCP epoll thread is exiting...", pThread);
break;
}
if (fdNum < 0) {
- uError("epoll_wait failed (%s)", strerror(errno));
+ sError("epoll_wait failed since %s", strerror(errno));
continue;
}
- for (int i = 0; i < fdNum; ++i) {
+ for (int32_t i = 0; i < fdNum; ++i) {
pConn = events[i].data.ptr;
assert(pConn);
@@ -219,17 +223,16 @@ static void *taosProcessTcpData(void *param) {
continue;
}
}
-
}
if (pThread->stop) break;
}
- uDebug("%p TCP epoll thread exits", pThread);
+ sDebug("%p TCP epoll thread exits", pThread);
close(pThread->pollFd);
- free(pThread);
- free(buffer);
+ tfree(pThread);
+ tfree(buffer);
return NULL;
}
@@ -242,18 +245,18 @@ static void *taosAcceptPeerTcpConnection(void *argv) {
while (1) {
struct sockaddr_in clientAddr;
socklen_t addrlen = sizeof(clientAddr);
- int connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen);
+ int32_t connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen);
if (connFd < 0) {
if (errno == EINVAL) {
- uDebug("%p TCP server accept is exiting...", pPool);
+ sDebug("%p TCP server accept is exiting...", pPool);
break;
} else {
- uError("TCP accept failure, reason:%s", strerror(errno));
+ sError("TCP accept failure since %s", strerror(errno));
continue;
}
}
- // uDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port);
+ // sDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port);
taosKeepTcpAlive(connFd);
(*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr);
}
@@ -273,23 +276,23 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) {
pThread->pPool = pPool;
pThread->pollFd = epoll_create(10); // size does not matter
if (pThread->pollFd < 0) {
- free(pThread);
+ tfree(pThread);
return NULL;
}
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
- int ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread);
+ int32_t ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread);
pthread_attr_destroy(&thattr);
if (ret != 0) {
close(pThread->pollFd);
- free(pThread);
+ tfree(pThread);
return NULL;
}
- uDebug("%p TCP epoll thread is created", pThread);
+ sDebug("%p TCP epoll thread is created", pThread);
pPool->pThread[pPool->nextId] = pThread;
pPool->nextId++;
pPool->nextId = pPool->nextId % pPool->info.numOfThreads;
@@ -298,31 +301,14 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) {
}
static void taosStopPoolThread(SThreadObj *pThread) {
+ pthread_t thread = pThread->thread;
+ if (!taosCheckPthreadValid(thread)) {
+ return;
+ }
pThread->stop = true;
-
- if (pThread->thread == pthread_self()) {
+ if (taosComparePthread(thread, pthread_self())) {
pthread_detach(pthread_self());
return;
}
-
- // save thread ID into a local variable, since pThread is freed when the thread exits
- pthread_t thread = pThread->thread;
-
- // signal the thread to stop, try graceful method first,
- // and use pthread_cancel when failed
- struct epoll_event event = {.events = EPOLLIN};
- eventfd_t fd = eventfd(1, 0);
- if (fd == -1) {
- // failed to create eventfd, call pthread_cancel instead, which may result in data corruption
- uError("failed to create eventfd(%s)", strerror(errno));
- pthread_cancel(pThread->thread);
- pThread->stop = true;
- } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
- // failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption
- uError("failed to call epoll_ctl(%s)", strerror(errno));
- pthread_cancel(pThread->thread);
- }
-
pthread_join(thread, NULL);
- if (fd >= 0) taosClose(fd);
}
diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c
index 360ea93f6c3fcd41e1df03702a0c963029deaacc..b7f819a3cd8f890d3580352322d8313f7cb2828e 100644
--- a/src/sync/src/tarbitrator.c
+++ b/src/sync/src/tarbitrator.c
@@ -28,22 +28,22 @@
#include "syncInt.h"
static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context);
-static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp);
+static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp);
static void arbProcessBrokenLink(void *param);
-static int arbProcessPeerMsg(void *param, void *buffer);
+static int32_t arbProcessPeerMsg(void *param, void *buffer);
static tsem_t tsArbSem;
static ttpool_h tsArbTcpPool;
typedef struct {
- char id[TSDB_EP_LEN + 24];
- int nodeFd;
- void *pConn;
+ char id[TSDB_EP_LEN + 24];
+ int32_t nodeFd;
+ void * pConn;
} SNodeConn;
-int main(int argc, char *argv[]) {
+int32_t main(int32_t argc, char *argv[]) {
char arbLogPath[TSDB_FILENAME_LEN + 16] = {0};
- for (int i = 1; i < argc; ++i) {
+ for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
tsArbitratorPort = atoi(argv[++i]);
} else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) {
@@ -86,7 +86,7 @@ int main(int argc, char *argv[]) {
info.numOfThreads = 1;
info.serverIp = 0;
info.port = tsArbitratorPort;
- info.bufferSize = 640000;
+ info.bufferSize = SYNC_MAX_SIZE;
info.processBrokenLink = arbProcessBrokenLink;
info.processIncomingMsg = arbProcessPeerMsg;
info.processIncomingConn = arbProcessIncommingConnection;
@@ -108,7 +108,7 @@ int main(int argc, char *argv[]) {
return 0;
}
-static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) {
+static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
char ipstr[24];
tinet_ntoa(ipstr, sourceIp);
sDebug("peer TCP connection from ip:%s", ipstr);
@@ -128,10 +128,10 @@ static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) {
}
firstPkt.fqdn[sizeof(firstPkt.fqdn) - 1] = 0;
- snprintf(pNode->id, sizeof(pNode->id), "vgId:%d peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port);
+ snprintf(pNode->id, sizeof(pNode->id), "vgId:%d, peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port);
if (firstPkt.syncHead.vgId) {
sDebug("%s, vgId in head is not zero, close the connection", pNode->id);
- taosTFree(pNode);
+ tfree(pNode);
taosCloseSocket(connFd);
return;
}
@@ -147,16 +147,16 @@ static void arbProcessBrokenLink(void *param) {
SNodeConn *pNode = param;
sDebug("%s, TCP link is broken(%s), close connection", pNode->id, strerror(errno));
- taosTFree(pNode);
+ tfree(pNode);
}
-static int arbProcessPeerMsg(void *param, void *buffer) {
+static int32_t arbProcessPeerMsg(void *param, void *buffer) {
SNodeConn *pNode = param;
SSyncHead head;
- int bytes = 0;
+ int32_t bytes = 0;
char * cont = (char *)buffer;
- int hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head));
+ int32_t hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head));
if (hlen != sizeof(head)) {
sDebug("%s, failed to read msg, hlen:%d", pNode->id, hlen);
return -1;
diff --git a/src/sync/test/syncClient.c b/src/sync/test/syncClient.c
index 23264dc8a0d969e238f35951b0a02e10261ab0c3..23ea54ee0c19b6ad2f93d7577d8d711874b10968 100644
--- a/src/sync/test/syncClient.c
+++ b/src/sync/test/syncClient.c
@@ -57,7 +57,7 @@ void *sendRequest(void *param) {
rpcMsg.ahandle = pInfo;
rpcMsg.msgType = 1;
uDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
- rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg);
+ rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL);
if (pInfo->num % 20000 == 0) {
uInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
}
diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c
index 380b971fa89bd1726e138c973a974bc995500693..9dd3feb4614105c13ba6a2d1069931345167b472 100644
--- a/src/sync/test/syncServer.c
+++ b/src/sync/test/syncServer.c
@@ -30,7 +30,7 @@ int dataFd = -1;
void * qhandle = NULL;
int walNum = 0;
uint64_t tversion = 0;
-void * syncHandle;
+int64_t syncHandle;
int role;
int nodeId;
char path[256];
@@ -254,7 +254,7 @@ uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex
return magic;
}
-int getWalInfo(void *ahandle, char *name, uint32_t *index) {
+int getWalInfo(void *ahandle, char *name, int64_t *index) {
struct stat fstat;
char aname[280];
diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h
index 1a73eae9082e219bb32d4e56813f1082171532b5..ef9222c59ea5024bb20bb7cb8ede7643166754f2 100644
--- a/src/tsdb/inc/tsdbMain.h
+++ b/src/tsdb/inc/tsdbMain.h
@@ -219,6 +219,18 @@ typedef struct {
} SFileGroupIter;
// ------------------ tsdbMain.c
+typedef struct {
+ int32_t totalLen;
+ int32_t len;
+ SDataRow row;
+} SSubmitBlkIter;
+
+typedef struct {
+ int32_t totalLen;
+ int32_t len;
+ void * pMsg;
+} SSubmitMsgIter;
+
typedef struct {
int8_t state;
@@ -231,10 +243,10 @@ typedef struct {
SMemTable* mem;
SMemTable* imem;
STsdbFileH* tsdbFileH;
- int commit;
- pthread_t commitThread;
+ sem_t readyToCommit;
pthread_mutex_t mutex;
bool repoLocked;
+ int32_t code; // Commit code
} STsdbRepo;
// ------------------ tsdbRWHelper.c
@@ -331,6 +343,15 @@ typedef struct {
void* compBuffer; // Buffer for temperary compress/decompress purpose
} SRWHelper;
+typedef struct {
+ int rowsInserted;
+ int rowsUpdated;
+ int rowsDeleteSucceed;
+ int rowsDeleteFailed;
+ int nOperations;
+ TSKEY keyFirst;
+ TSKEY keyLast;
+} SMergeInfo;
// ------------------ tsdbScan.c
typedef struct {
SFileGroup fGroup;
@@ -433,7 +454,6 @@ void tsdbCloseBufPool(STsdbRepo* pRepo);
SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
// ------------------ tsdbMemTable.c
-int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
@@ -441,7 +461,8 @@ void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem)
void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
int tsdbAsyncCommit(STsdbRepo* pRepo);
int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
- TSKEY* filterKeys, int nFilterKeys);
+ TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
+void* tsdbCommitData(STsdbRepo* pRepo);
static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
if (pIter == NULL) return NULL;
@@ -449,16 +470,23 @@ static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
SSkipListNode* node = tSkipListIterGet(pIter);
if (node == NULL) return NULL;
- return *(SDataRow *)SL_GET_NODE_DATA(node);
+ return (SDataRow)SL_GET_NODE_DATA(node);
}
static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
SDataRow row = tsdbNextIterRow(pIter);
- if (row == NULL) return -1;
+ if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
return dataRowKey(row);
}
+static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
+ SDataRow row = tsdbNextIterRow(pIter);
+ if (row == NULL) return TKEY_NULL;
+
+ return dataRowTKey(row);
+}
+
static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
ASSERT(pRepo != NULL);
if (pRepo->mem == NULL) return NULL;
@@ -586,6 +614,9 @@ int tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx);
int tsdbCloseScanFile(STsdbScanHandle* pScanHandle);
void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle);
+// ------------------ tsdbCommitQueue.c
+int tsdbScheduleCommit(STsdbRepo *pRepo);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c
index 2e097c6ff73543f2dd858f8424b1e942ebdd8c3e..7cea27658c80d689972e3cb0f5dda3269a34b720 100644
--- a/src/tsdb/src/tsdbBuffer.c
+++ b/src/tsdb/src/tsdbBuffer.c
@@ -110,7 +110,7 @@ void tsdbCloseBufPool(STsdbRepo *pRepo) {
}
}
- tsdbDebug("vgId:%d buffer pool is closed", REPO_ID(pRepo));
+ tsdbDebug("vgId:%d, buffer pool is closed", REPO_ID(pRepo));
}
SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) {
@@ -134,7 +134,7 @@ SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) {
pBufBlock->offset = 0;
pBufBlock->remain = pBufPool->bufBlockSize;
- tsdbDebug("vgId:%d buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId);
+ tsdbDebug("vgId:%d, buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId);
return pNode;
}
@@ -157,4 +157,4 @@ _err:
return NULL;
}
-static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { taosTFree(pBufBlock); }
\ No newline at end of file
+static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); }
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c
new file mode 100644
index 0000000000000000000000000000000000000000..637b02cd32ae8ad8e4077684609dcac23922d8a0
--- /dev/null
+++ b/src/tsdb/src/tsdbCommit.c
@@ -0,0 +1,340 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+#include "tsdbMain.h"
+
+static int tsdbCommitTSData(STsdbRepo *pRepo);
+static int tsdbCommitMeta(STsdbRepo *pRepo);
+static void tsdbEndCommit(STsdbRepo *pRepo, int eno);
+static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey);
+static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols);
+static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
+static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
+
+void *tsdbCommitData(STsdbRepo *pRepo) {
+ SMemTable * pMem = pRepo->imem;
+
+ tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64 " meta rows: %d",
+ REPO_ID(pRepo), pMem->keyFirst, pMem->keyLast, pMem->numOfRows, listNEles(pMem->actList));
+
+ pRepo->code = TSDB_CODE_SUCCESS;
+
+ // Commit to update meta file
+ if (tsdbCommitMeta(pRepo) < 0) {
+ tsdbError("vgId:%d error occurs while committing META data since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+
+ // Create the iterator to read from cache
+ if (tsdbCommitTSData(pRepo) < 0) {
+ tsdbError("vgId:%d error occurs while committing TS data since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+
+ tsdbFitRetention(pRepo);
+
+ tsdbInfo("vgId:%d commit over, succeed", REPO_ID(pRepo));
+ tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS);
+
+ return NULL;
+
+_err:
+ ASSERT(terrno != TSDB_CODE_SUCCESS);
+ pRepo->code = terrno;
+ tsdbInfo("vgId:%d commit over, failed", REPO_ID(pRepo));
+ tsdbEndCommit(pRepo, terrno);
+
+ return NULL;
+}
+
+static int tsdbCommitTSData(STsdbRepo *pRepo) {
+ SMemTable * pMem = pRepo->imem;
+ SDataCols * pDataCols = NULL;
+ STsdbMeta * pMeta = pRepo->tsdbMeta;
+ SCommitIter *iters = NULL;
+ SRWHelper whelper = {0};
+ STsdbCfg * pCfg = &(pRepo->config);
+
+ if (pMem->numOfRows <= 0) return 0;
+
+ iters = tsdbCreateCommitIters(pRepo);
+ if (iters == NULL) {
+ tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+
+ if (tsdbInitWriteHelper(&whelper, pRepo) < 0) {
+ tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+
+ if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s",
+ REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno));
+ goto _err;
+ }
+
+ int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision));
+ int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision));
+
+ // Loop to commit to each file
+ for (int fid = sfid; fid <= efid; fid++) {
+ if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
+ tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
+ goto _err;
+ }
+ }
+
+ tdFreeDataCols(pDataCols);
+ tsdbDestroyCommitIters(iters, pMem->maxTables);
+ tsdbDestroyHelper(&whelper);
+
+ return 0;
+
+_err:
+ tdFreeDataCols(pDataCols);
+ tsdbDestroyCommitIters(iters, pMem->maxTables);
+ tsdbDestroyHelper(&whelper);
+
+ return -1;
+}
+
+static int tsdbCommitMeta(STsdbRepo *pRepo) {
+ SMemTable *pMem = pRepo->imem;
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
+ SActObj * pAct = NULL;
+ SActCont * pCont = NULL;
+
+ if (listNEles(pMem->actList) <= 0) return 0;
+
+ if (tdKVStoreStartCommit(pMeta->pStore) < 0) {
+ tsdbError("vgId:%d failed to commit data while start commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+
+ SListNode *pNode = NULL;
+
+ while ((pNode = tdListPopHead(pMem->actList)) != NULL) {
+ pAct = (SActObj *)pNode->data;
+ if (pAct->act == TSDB_UPDATE_META) {
+ pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
+ if (tdUpdateKVStoreRecord(pMeta->pStore, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) {
+ tsdbError("vgId:%d failed to update meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
+ tstrerror(terrno));
+ tdKVStoreEndCommit(pMeta->pStore);
+ goto _err;
+ }
+ } else if (pAct->act == TSDB_DROP_META) {
+ if (tdDropKVStoreRecord(pMeta->pStore, pAct->uid) < 0) {
+ tsdbError("vgId:%d failed to drop meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
+ tstrerror(terrno));
+ tdKVStoreEndCommit(pMeta->pStore);
+ goto _err;
+ }
+ } else {
+ ASSERT(false);
+ }
+ }
+
+ if (tdKVStoreEndCommit(pMeta->pStore) < 0) {
+ tsdbError("vgId:%d failed to commit data while end commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+
+ return 0;
+
+_err:
+ return -1;
+}
+
+static void tsdbEndCommit(STsdbRepo *pRepo, int eno) {
+ if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno);
+ sem_post(&(pRepo->readyToCommit));
+}
+
+static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
+ for (int i = 0; i < nIters; i++) {
+ TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
+ if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1;
+ }
+ return 0;
+}
+
+static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) {
+ char * dataDir = NULL;
+ STsdbCfg * pCfg = &pRepo->config;
+ STsdbFileH *pFileH = pRepo->tsdbFileH;
+ SFileGroup *pGroup = NULL;
+ SMemTable * pMem = pRepo->imem;
+ bool newLast = false;
+
+ TSKEY minKey = 0, maxKey = 0;
+ tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey);
+
+ // Check if there are data to commit to this file
+ int hasDataToCommit = tsdbHasDataToCommit(iters, pMem->maxTables, minKey, maxKey);
+ if (!hasDataToCommit) {
+ tsdbDebug("vgId:%d no data to commit to file %d", REPO_ID(pRepo), fid);
+ return 0;
+ }
+
+ // Create and open files for commit
+ dataDir = tsdbGetDataDirName(pRepo->rootDir);
+ if (dataDir == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid)) == NULL) {
+ tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
+ goto _err;
+ }
+
+ // Open files for write/read
+ if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) {
+ tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+
+ newLast = TSDB_NLAST_FILE_OPENED(pHelper);
+
+ if (tsdbLoadCompIdx(pHelper, NULL) < 0) {
+ tsdbError("vgId:%d failed to load SCompIdx part since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+
+ // Loop to commit data in each table
+ for (int tid = 1; tid < pMem->maxTables; tid++) {
+ SCommitIter *pIter = iters + tid;
+ if (pIter->pTable == NULL) continue;
+
+ taosRLockLatch(&(pIter->pTable->latch));
+
+ if (tsdbSetHelperTable(pHelper, pIter->pTable, pRepo) < 0) goto _err;
+
+ if (pIter->pIter != NULL) {
+ if (tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1)) < 0) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) {
+ taosRUnLockLatch(&(pIter->pTable->latch));
+ tsdbError("vgId:%d failed to write data of table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo),
+ TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable),
+ tstrerror(terrno));
+ goto _err;
+ }
+ }
+
+ taosRUnLockLatch(&(pIter->pTable->latch));
+
+ // Move the last block to the new .l file if neccessary
+ if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) {
+ tsdbError("vgId:%d, failed to move last block, since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+
+ // Write the SCompBlock part
+ if (tsdbWriteCompInfo(pHelper) < 0) {
+ tsdbError("vgId:%d, failed to write compInfo part since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _err;
+ }
+ }
+
+ if (tsdbWriteCompIdx(pHelper) < 0) {
+ tsdbError("vgId:%d failed to write compIdx part to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
+ goto _err;
+ }
+
+ tfree(dataDir);
+ tsdbCloseHelperFile(pHelper, 0, pGroup);
+
+ pthread_rwlock_wrlock(&(pFileH->fhlock));
+
+ (void)rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname);
+ pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info;
+
+ if (newLast) {
+ (void)rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname);
+ pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info;
+ } else {
+ pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info;
+ }
+
+ pGroup->files[TSDB_FILE_TYPE_DATA].info = helperDataF(pHelper)->info;
+
+ pthread_rwlock_unlock(&(pFileH->fhlock));
+
+ return 0;
+
+_err:
+ tfree(dataDir);
+ tsdbCloseHelperFile(pHelper, 1, pGroup);
+ return -1;
+}
+
+static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) {
+ SMemTable *pMem = pRepo->imem;
+ STsdbMeta *pMeta = pRepo->tsdbMeta;
+
+ SCommitIter *iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter));
+ if (iters == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return NULL;
+ }
+
+ if (tsdbRLockRepoMeta(pRepo) < 0) goto _err;
+
+ // reference all tables
+ for (int i = 0; i < pMem->maxTables; i++) {
+ if (pMeta->tables[i] != NULL) {
+ tsdbRefTable(pMeta->tables[i]);
+ iters[i].pTable = pMeta->tables[i];
+ }
+ }
+
+ if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err;
+
+ for (int i = 0; i < pMem->maxTables; i++) {
+ if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TABLE_UID(iters[i].pTable) == pMem->tData[i]->uid)) {
+ if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ tSkipListIterNext(iters[i].pIter);
+ }
+ }
+
+ return iters;
+
+_err:
+ tsdbDestroyCommitIters(iters, pMem->maxTables);
+ return NULL;
+}
+
+static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables) {
+ if (iters == NULL) return;
+
+ for (int i = 1; i < maxTables; i++) {
+ if (iters[i].pTable != NULL) {
+ tsdbUnRefTable(iters[i].pTable);
+ tSkipListDestroyIter(iters[i].pIter);
+ }
+ }
+
+ free(iters);
+}
diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c
new file mode 100644
index 0000000000000000000000000000000000000000..c86b8f32b7ff6bfb30e7b734c7b38504200e44e6
--- /dev/null
+++ b/src/tsdb/src/tsdbCommitQueue.c
@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "os.h"
+#include "tlist.h"
+#include "tref.h"
+#include "tsdbMain.h"
+
+typedef struct {
+ bool stop;
+ pthread_mutex_t lock;
+ pthread_cond_t queueNotEmpty;
+ int nthreads;
+ int refCount;
+ SList * queue;
+ pthread_t * threads;
+} SCommitQueue;
+
+typedef struct {
+ STsdbRepo *pRepo;
+} SCommitReq;
+
+static void *tsdbLoopCommit(void *arg);
+
+SCommitQueue tsCommitQueue = {0};
+
+int tsdbInitCommitQueue(int nthreads) {
+ SCommitQueue *pQueue = &tsCommitQueue;
+
+ if (nthreads < 1) nthreads = 1;
+
+ pQueue->stop = false;
+ pQueue->nthreads = nthreads;
+
+ pQueue->queue = tdListNew(0);
+ if (pQueue->queue == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ pQueue->threads = (pthread_t *)calloc(nthreads, sizeof(pthread_t));
+ if (pQueue->threads == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ tdListFree(pQueue->queue);
+ return -1;
+ }
+
+ pthread_mutex_init(&(pQueue->lock), NULL);
+ pthread_cond_init(&(pQueue->queueNotEmpty), NULL);
+
+ for (int i = 0; i < nthreads; i++) {
+ pthread_create(pQueue->threads + i, NULL, tsdbLoopCommit, NULL);
+ }
+
+ return 0;
+}
+
+void tsdbDestroyCommitQueue() {
+ SCommitQueue *pQueue = &tsCommitQueue;
+
+ pthread_mutex_lock(&(pQueue->lock));
+
+ if (pQueue->stop) {
+ pthread_mutex_unlock(&(pQueue->lock));
+ return;
+ }
+
+ pQueue->stop = true;
+ pthread_cond_broadcast(&(pQueue->queueNotEmpty));
+
+ pthread_mutex_unlock(&(pQueue->lock));
+
+ for (size_t i = 0; i < pQueue->nthreads; i++) {
+ pthread_join(pQueue->threads[i], NULL);
+ }
+
+ free(pQueue->threads);
+ tdListFree(pQueue->queue);
+ pthread_cond_destroy(&(pQueue->queueNotEmpty));
+ pthread_mutex_destroy(&(pQueue->lock));
+}
+
+int tsdbScheduleCommit(STsdbRepo *pRepo) {
+ SCommitQueue *pQueue = &tsCommitQueue;
+
+ SListNode *pNode = (SListNode *)calloc(1, sizeof(SListNode) + sizeof(SCommitReq));
+ if (pNode == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ ((SCommitReq *)pNode->data)->pRepo = pRepo;
+
+ pthread_mutex_lock(&(pQueue->lock));
+
+ // ASSERT(pQueue->stop);
+
+ tdListAppendNode(pQueue->queue, pNode);
+ pthread_cond_signal(&(pQueue->queueNotEmpty));
+
+ pthread_mutex_unlock(&(pQueue->lock));
+ return 0;
+}
+
+static void *tsdbLoopCommit(void *arg) {
+ SCommitQueue *pQueue = &tsCommitQueue;
+ SListNode * pNode = NULL;
+ STsdbRepo * pRepo = NULL;
+
+ while (true) {
+ pthread_mutex_lock(&(pQueue->lock));
+
+ while (true) {
+ pNode = tdListPopHead(pQueue->queue);
+ if (pNode == NULL) {
+ if (pQueue->stop && pQueue->refCount <= 0) {
+ pthread_mutex_unlock(&(pQueue->lock));
+ goto _exit;
+ } else {
+ pthread_cond_wait(&(pQueue->queueNotEmpty), &(pQueue->lock));
+ }
+ } else {
+ break;
+ }
+ }
+
+ pthread_mutex_unlock(&(pQueue->lock));
+
+ pRepo = ((SCommitReq *)pNode->data)->pRepo;
+
+ tsdbCommitData(pRepo);
+ listNodeFree(pNode);
+ }
+
+_exit:
+ return NULL;
+}
+
+void tsdbIncCommitRef(int vgId) {
+ int refCount = atomic_add_fetch_32(&tsCommitQueue.refCount, 1);
+ tsdbDebug("vgId:%d, inc commit queue ref to %d", vgId, refCount);
+}
+
+void tsdbDecCommitRef(int vgId) {
+ int refCount = atomic_sub_fetch_32(&tsCommitQueue.refCount, 1);
+ pthread_cond_broadcast(&(tsCommitQueue.queueNotEmpty));
+ tsdbDebug("vgId:%d, dec commit queue ref to %d", vgId, refCount);
+}
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c
index 450ba879e8f96aef291678f917c2c3732f492ec7..c89c66d50b751032537742cd1000bc78fbead2eb 100644
--- a/src/tsdb/src/tsdbFile.c
+++ b/src/tsdb/src/tsdbFile.c
@@ -13,10 +13,8 @@
* along with this program. If not, see .
*/
#define _DEFAULT_SOURCE
-#include
-
#define TAOS_RANDOM_FILE_FAIL_TEST
-
+#include
#include "os.h"
#include "tglobal.h"
#include "talgo.h"
@@ -70,7 +68,7 @@ _err:
void tsdbFreeFileH(STsdbFileH *pFileH) {
if (pFileH) {
pthread_rwlock_destroy(&pFileH->fhlock);
- taosTFree(pFileH->pFGroup);
+ tfree(pFileH->pFGroup);
free(pFileH);
}
}
@@ -300,7 +298,7 @@ int tsdbUpdateFileHeader(SFile *pFile) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
- if (taosTWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+ if (taosWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@@ -372,7 +370,7 @@ int tsdbLoadFileHeader(SFile *pFile, uint32_t *version) {
return -1;
}
- if (taosTRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+ if (taosRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
tsdbError("failed to read file %s header part with %d bytes, reason:%s", pFile->fname, TSDB_FILE_HEAD_SIZE,
strerror(errno));
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
@@ -397,7 +395,7 @@ void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int64_t *size) {
SFile file;
SFile * pFile = &file;
- strncpy(pFile->fname, fname, TSDB_FILENAME_LEN);
+ strncpy(pFile->fname, fname, TSDB_FILENAME_LEN - 1);
pFile->fd = -1;
if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err;
diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c
index a1e6376304a54ea810f3662bd30e3f97fd53765c..3990c0c516d1768135934b098efbcd7746ab424c 100644
--- a/src/tsdb/src/tsdbMain.c
+++ b/src/tsdb/src/tsdbMain.c
@@ -32,18 +32,6 @@
#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP
#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP))
-typedef struct {
- int32_t totalLen;
- int32_t len;
- SDataRow row;
-} SSubmitBlkIter;
-
-typedef struct {
- int32_t totalLen;
- int32_t len;
- void * pMsg;
-} SSubmitMsgIter;
-
static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg);
static int32_t tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg);
static int32_t tsdbUnsetRepoEnv(char *rootDir);
@@ -52,20 +40,13 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg);
static char * tsdbGetCfgFname(char *rootDir);
static STsdbRepo * tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg);
static void tsdbFreeRepo(STsdbRepo *pRepo);
-static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
-static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows);
-static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock);
-static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
static int tsdbRestoreInfo(STsdbRepo *pRepo);
-static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression);
static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep);
static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks);
static int keyFGroupCompFunc(const void *key, const void *fgroup);
static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg);
static void * tsdbDecodeCfg(void *buf, STsdbCfg *pCfg);
-static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable);
-static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg);
static void tsdbStartStream(STsdbRepo *pRepo);
static void tsdbStopStream(STsdbRepo *pRepo);
@@ -153,17 +134,20 @@ _err:
}
// Note: all working thread and query thread must stopped when calling this function
-void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) {
- if (repo == NULL) return;
+int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) {
+ if (repo == NULL) return 0;
STsdbRepo *pRepo = (STsdbRepo *)repo;
int vgId = REPO_ID(pRepo);
+ terrno = TSDB_CODE_SUCCESS;
+
tsdbStopStream(pRepo);
if (toCommit) {
tsdbAsyncCommit(pRepo);
- if (pRepo->commit) pthread_join(pRepo->commitThread, NULL);
+ sem_wait(&(pRepo->readyToCommit));
+ terrno = pRepo->code;
}
tsdbUnRefMemTable(pRepo, pRepo->mem);
tsdbUnRefMemTable(pRepo, pRepo->imem);
@@ -175,40 +159,12 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) {
tsdbCloseMeta(pRepo);
tsdbFreeRepo(pRepo);
tsdbDebug("vgId:%d repository is closed", vgId);
-}
-int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) {
- STsdbRepo * pRepo = (STsdbRepo *)repo;
- SSubmitMsgIter msgIter = {0};
-
- if (tsdbScanAndConvertSubmitMsg(pRepo, pMsg) < 0) {
- if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
- tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pRepo), tstrerror(terrno));
- }
+ if (terrno != TSDB_CODE_SUCCESS) {
return -1;
+ } else {
+ return 0;
}
-
- if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) {
- tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pRepo), tstrerror(terrno));
- return -1;
- }
-
- SSubmitBlk *pBlock = NULL;
- int32_t affectedrows = 0;
-
- TSKEY now = taosGetTimestamp(pRepo->config.precision);
- while (true) {
- tsdbGetSubmitMsgNext(&msgIter, &pBlock);
- if (pBlock == NULL) break;
- if (tsdbInsertDataToTable(pRepo, pBlock, now, &affectedrows) < 0) {
- return -1;
- }
- }
-
- if (pRsp != NULL) pRsp->affectedRows = htonl(affectedrows);
-
- if (tsdbCheckCommit(pRepo) < 0) return -1;
- return 0;
}
uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) {
@@ -228,7 +184,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
int prefixLen = (int)strlen(prefix);
if (name[0] == 0) { // get the file from index or after, but not larger than eindex
- taosTFree(sdup);
+ tfree(sdup);
int fid = (*index) / TSDB_FILE_TYPE_MAX;
if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) {
@@ -260,8 +216,8 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
fname = malloc(prefixLen + strlen(name) + 2);
sprintf(fname, "%s/%s", prefix, name);
if (access(fname, F_OK) != 0) {
- taosFree(fname);
- taosFree(sdup);
+ tfree(fname);
+ tfree(sdup);
return 0;
}
if (*index == TSDB_META_FILE_INDEX) { // get meta file
@@ -269,20 +225,20 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
} else {
tsdbGetFileInfoImpl(fname, &magic, size);
}
- taosFree(fname);
- taosFree(sdup);
+ tfree(fname);
+ tfree(sdup);
return magic;
}
if (stat(fname, &fState) < 0) {
- taosTFree(fname);
+ tfree(fname);
return 0;
}
*size = fState.st_size;
// magic = *size;
- taosTFree(fname);
+ tfree(fname);
return magic;
}
@@ -294,6 +250,7 @@ STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) {
int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) {
// TODO: think about multithread cases
STsdbRepo *pRepo = (STsdbRepo *)repo;
+ STsdbCfg config = pRepo->config;
STsdbCfg * pRCfg = &pRepo->config;
if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1;
@@ -308,22 +265,25 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) {
bool configChanged = false;
if (pRCfg->compression != pCfg->compression) {
tsdbAlterCompression(pRepo, pCfg->compression);
+ config.compression = pCfg->compression;
configChanged = true;
}
if (pRCfg->keep != pCfg->keep) {
if (tsdbAlterKeep(pRepo, pCfg->keep) < 0) {
tsdbError("vgId:%d failed to configure repo when alter keep since %s", REPO_ID(pRepo), tstrerror(terrno));
+ config.keep = pCfg->keep;
return -1;
}
configChanged = true;
}
if (pRCfg->totalBlocks != pCfg->totalBlocks) {
tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks);
+ config.totalBlocks = pCfg->totalBlocks;
configChanged = true;
}
if (configChanged) {
- if (tsdbSaveConfig(pRepo->rootDir, &pRepo->config) < 0) {
+ if (tsdbSaveConfig(pRepo->rootDir, &config) < 0) {
tsdbError("vgId:%d failed to configure repository while save config since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
@@ -512,6 +472,9 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
}
}
+ // update check
+ if (pCfg->update != 0) pCfg->update = 1;
+
return 0;
_err:
@@ -579,7 +542,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) {
taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
- if (taosTWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+ if (taosWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, TSDB_FILE_HEAD_SIZE, fname,
strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -597,7 +560,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) {
return 0;
_err:
- taosTFree(fname);
+ tfree(fname);
if (fd >= 0) close(fd);
return -1;
}
@@ -620,7 +583,7 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) {
goto _err;
}
- if (taosTRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+ if (taosRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
tsdbError("failed to read %d bytes from file %s since %s", TSDB_FILE_HEAD_SIZE, fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
@@ -634,13 +597,13 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) {
tsdbDecodeCfg(buf, pCfg);
- taosTFree(fname);
+ tfree(fname);
close(fd);
return 0;
_err:
- taosTFree(fname);
+ tfree(fname);
if (fd >= 0) close(fd);
return -1;
}
@@ -665,6 +628,7 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) {
}
pRepo->state = TSDB_STATE_OK;
+ pRepo->code = TSDB_CODE_SUCCESS;
int code = pthread_mutex_init(&pRepo->mutex, NULL);
if (code != 0) {
@@ -672,6 +636,12 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) {
goto _err;
}
+ code = sem_init(&(pRepo->readyToCommit), 0, 1);
+ if (code != 0) {
+ terrno = TAOS_SYSTEM_ERROR(code);
+ goto _err;
+ }
+
pRepo->repoLocked = false;
pRepo->rootDir = strdup(rootDir);
@@ -715,99 +685,13 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) {
tsdbFreeMeta(pRepo->tsdbMeta);
// tsdbFreeMemTable(pRepo->mem);
// tsdbFreeMemTable(pRepo->imem);
- taosTFree(pRepo->rootDir);
+ tfree(pRepo->rootDir);
+ sem_destroy(&(pRepo->readyToCommit));
pthread_mutex_destroy(&pRepo->mutex);
free(pRepo);
}
}
-static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
- if (pMsg == NULL) {
- terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
- return -1;
- }
-
- pIter->totalLen = pMsg->length;
- pIter->len = 0;
- pIter->pMsg = pMsg;
- if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) {
- terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
- return -1;
- }
-
- return 0;
-}
-
-static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) {
- STsdbMeta *pMeta = pRepo->tsdbMeta;
- int64_t points = 0;
-
- ASSERT(pBlock->tid < pMeta->maxTables);
- STable *pTable = pMeta->tables[pBlock->tid];
- ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid);
-
- SSubmitBlkIter blkIter = {0};
- SDataRow row = NULL;
-
- TSKEY minKey = now - tsMsPerDay[pRepo->config.precision] * pRepo->config.keep;
- TSKEY maxKey = now + tsMsPerDay[pRepo->config.precision] * pRepo->config.daysPerFile;
-
- tsdbInitSubmitBlkIter(pBlock, &blkIter);
- while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
- if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) {
- tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64
- " maxKey %" PRId64,
- REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey);
- terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
- return -1;
- }
-
- if (tsdbInsertRowToMem(pRepo, row, pTable) < 0) return -1;
-
- (*affectedrows)++;
- points++;
- }
-
- STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion);
- pRepo->stat.pointsWritten += points * schemaNCols(pSchema);
- pRepo->stat.totalStorage += points * schemaVLen(pSchema);
-
- return 0;
-}
-
-static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
- if (pIter->len == 0) {
- pIter->len += TSDB_SUBMIT_MSG_HEAD_SIZE;
- } else {
- SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
- pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen);
- }
-
- if (pIter->len > pIter->totalLen) {
- terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
- *pPBlock = NULL;
- return -1;
- }
-
- *pPBlock = (pIter->len == pIter->totalLen) ? NULL : (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
-
- return 0;
-}
-
-static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) {
- SDataRow row = pIter->row;
- if (row == NULL) return NULL;
-
- pIter->len += dataRowLen(row);
- if (pIter->len >= pIter->totalLen) {
- pIter->row = NULL;
- } else {
- pIter->row = (char *)row + dataRowLen(row);
- }
-
- return row;
-}
-
static int tsdbRestoreInfo(STsdbRepo *pRepo) {
STsdbMeta * pMeta = pRepo->tsdbMeta;
STsdbFileH *pFileH = pRepo->tsdbFileH;
@@ -841,14 +725,6 @@ _err:
return -1;
}
-static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
- if (pBlock->dataLen <= 0) return -1;
- pIter->totalLen = pBlock->dataLen;
- pIter->len = 0;
- pIter->row = (SDataRow)(pBlock->data+pBlock->schemaLen);
- return 0;
-}
-
static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) {
int8_t ocompression = pRepo->config.compression;
pRepo->config.compression = compression;
@@ -923,6 +799,7 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) {
tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock);
tlen += taosEncodeFixedI8(buf, pCfg->precision);
tlen += taosEncodeFixedI8(buf, pCfg->compression);
+ tlen += taosEncodeFixedI8(buf, pCfg->update);
return tlen;
}
@@ -939,138 +816,11 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) {
buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock));
buf = taosDecodeFixedI8(buf, &(pCfg->precision));
buf = taosDecodeFixedI8(buf, &(pCfg->compression));
+ buf = taosDecodeFixedI8(buf, &(pCfg->update));
return buf;
}
-static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable) {
- ASSERT(pTable != NULL);
-
- STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
- int sversion = schemaVersion(pSchema);
-
- if (pBlock->sversion == sversion) {
- return 0;
- } else {
- if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { // stream table is not allowed to change schema
- terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
- return -1;
- }
- }
-
- if (pBlock->sversion > sversion) { // may need to update table schema
- if (pBlock->schemaLen > 0) {
- tsdbDebug(
- "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, update...",
- REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
- ASSERT(pBlock->schemaLen % sizeof(STColumn) == 0);
- int numOfCols = pBlock->schemaLen / sizeof(STColumn);
- STColumn *pTCol = (STColumn *)pBlock->data;
-
- STSchemaBuilder schemaBuilder = {0};
- if (tdInitTSchemaBuilder(&schemaBuilder, pBlock->sversion) < 0) {
- terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
- tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
- tstrerror(terrno));
- return -1;
- }
-
- for (int i = 0; i < numOfCols; i++) {
- if (tdAddColToSchema(&schemaBuilder, pTCol[i].type, htons(pTCol[i].colId), htons(pTCol[i].bytes)) < 0) {
- terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
- tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
- tstrerror(terrno));
- tdDestroyTSchemaBuilder(&schemaBuilder);
- return -1;
- }
- }
-
- STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder);
- if (pNSchema == NULL) {
- terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
- tdDestroyTSchemaBuilder(&schemaBuilder);
- return -1;
- }
-
- tdDestroyTSchemaBuilder(&schemaBuilder);
- tsdbUpdateTableSchema(pRepo, pTable, pNSchema, true);
- } else {
- tsdbDebug(
- "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, reconfigure...",
- REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
- terrno = TSDB_CODE_TDB_TABLE_RECONFIGURE;
- return -1;
- }
- } else {
- ASSERT(pBlock->sversion >= 0);
- if (tsdbGetTableSchemaImpl(pTable, false, false, pBlock->sversion) == NULL) {
- tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo),
- pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable));
- }
- terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
- return -1;
- }
-
- return 0;
-}
-
-static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) {
- ASSERT(pMsg != NULL);
- STsdbMeta * pMeta = pRepo->tsdbMeta;
- SSubmitMsgIter msgIter = {0};
- SSubmitBlk * pBlock = NULL;
-
- terrno = TSDB_CODE_SUCCESS;
- pMsg->length = htonl(pMsg->length);
- pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
-
- if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
- while (true) {
- if (tsdbGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
- if (pBlock == NULL) break;
-
- pBlock->uid = htobe64(pBlock->uid);
- pBlock->tid = htonl(pBlock->tid);
- pBlock->sversion = htonl(pBlock->sversion);
- pBlock->dataLen = htonl(pBlock->dataLen);
- pBlock->schemaLen = htonl(pBlock->schemaLen);
- pBlock->numOfRows = htons(pBlock->numOfRows);
-
- if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) {
- tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid,
- pBlock->tid);
- terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
- return -1;
- }
-
- STable *pTable = pMeta->tables[pBlock->tid];
- if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) {
- tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid,
- pBlock->tid);
- terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
- return -1;
- }
-
- if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
- tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable));
- terrno = TSDB_CODE_TDB_INVALID_ACTION;
- return -1;
- }
-
- // Check schema version and update schema if needed
- if (tsdbCheckTableSchema(pRepo, pBlock, pTable) < 0) {
- if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
- continue;
- } else {
- return -1;
- }
- }
- }
-
- if (terrno != TSDB_CODE_SUCCESS) return -1;
- return 0;
-}
-
static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) {
// TODO
// STsdbCache *pCache = pRepo->tsdbCache;
diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c
index b98ab71aa1cfa38ef278766787cc22d31d7122af..71944c87c6d68d530e07a13a05bf5ac89ee3754d 100644
--- a/src/tsdb/src/tsdbMemTable.c
+++ b/src/tsdb/src/tsdbMemTable.c
@@ -18,117 +18,56 @@
#define TSDB_DATA_SKIPLIST_LEVEL 5
-static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes);
static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo);
static void tsdbFreeMemTable(SMemTable *pMemTable);
static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable);
static void tsdbFreeTableData(STableData *pTableData);
static char * tsdbGetTsTupleKey(const void *data);
-static void * tsdbCommitData(void *arg);
-static int tsdbCommitMeta(STsdbRepo *pRepo);
-static void tsdbEndCommit(STsdbRepo *pRepo);
-static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey);
-static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols);
-static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
-static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables);
-static void tsdbSeekCommitIter(SCommitIter *pIters, int nIters, TSKEY minKey);
-
-// ---------------- INTERNAL FUNCTIONS ----------------
-int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
- STsdbCfg * pCfg = &pRepo->config;
- STsdbMeta * pMeta = pRepo->tsdbMeta;
- int32_t level = 0;
- int32_t headSize = 0;
- TSKEY key = dataRowKey(row);
- SMemTable * pMemTable = pRepo->mem;
- STableData *pTableData = NULL;
- SSkipList * pSList = NULL;
-
- if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL &&
- pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) {
- pTableData = pMemTable->tData[TABLE_TID(pTable)];
- pSList = pTableData->pData;
- }
-
- tSkipListNewNodeInfo(pSList, &level, &headSize);
-
- SSkipListNode *pNode = (SSkipListNode *)malloc(headSize + sizeof(SDataRow *));
- if (pNode == NULL) {
- terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
- return -1;
- }
-
- void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row));
- if (pRow == NULL) {
- tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s",
- REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno));
- free(pNode);
- return -1;
- }
-
- pNode->level = level;
- dataRowCpy(pRow, row);
- *(SDataRow *)SL_GET_NODE_DATA(pNode) = pRow;
-
- // Operations above may change pRepo->mem, retake those values
- ASSERT(pRepo->mem != NULL);
- pMemTable = pRepo->mem;
-
- if (TABLE_TID(pTable) >= pMemTable->maxTables) {
- if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) {
- tsdbFreeBytes(pRepo, pRow, dataRowLen(row));
- free(pNode);
- return -1;
+static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row);
+static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
+static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
+static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg);
+static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows);
+static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void **ppRow);
+static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
+static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock);
+static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable);
+static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter);
+static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter);
+
+static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey,
+ TSKEY now);
+
+int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) {
+ STsdbRepo * pRepo = (STsdbRepo *)repo;
+ SSubmitMsgIter msgIter = {0};
+ SSubmitBlk * pBlock = NULL;
+ int32_t affectedrows = 0;
+
+ if (tsdbScanAndConvertSubmitMsg(pRepo, pMsg) < 0) {
+ if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
+ tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pRepo), tstrerror(terrno));
}
+ return -1;
}
- pTableData = pMemTable->tData[TABLE_TID(pTable)];
-
- if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) {
- if (pTableData != NULL) {
- taosWLockLatch(&(pMemTable->latch));
- pMemTable->tData[TABLE_TID(pTable)] = NULL;
- tsdbFreeTableData(pTableData);
- taosWUnLockLatch(&(pMemTable->latch));
- }
- pTableData = tsdbNewTableData(pCfg, pTable);
- if (pTableData == NULL) {
- tsdbError("vgId:%d failed to insert row with key %" PRId64
- " to table %s while create new table data object since %s",
- REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno));
- tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
- free(pNode);
+ tsdbInitSubmitMsgIter(pMsg, &msgIter);
+ while (true) {
+ tsdbGetSubmitMsgNext(&msgIter, &pBlock);
+ if (pBlock == NULL) break;
+ if (tsdbInsertDataToTable(pRepo, pBlock, &affectedrows) < 0) {
return -1;
}
-
- pRepo->mem->tData[TABLE_TID(pTable)] = pTableData;
}
- ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable));
-
- if (tSkipListPut(pTableData->pData, pNode) == NULL) {
- tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
- free(pNode);
- } else {
- if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key;
- if (pMemTable->keyFirst > key) pMemTable->keyFirst = key;
- if (pMemTable->keyLast < key) pMemTable->keyLast = key;
- pMemTable->numOfRows++;
-
- if (pTableData->keyFirst > key) pTableData->keyFirst = key;
- if (pTableData->keyLast < key) pTableData->keyLast = key;
- pTableData->numOfRows++;
-
- ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData));
- }
-
- tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
- TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), key);
+ if (pRsp != NULL) pRsp->affectedRows = htonl(affectedrows);
+ if (tsdbCheckCommit(pRepo) < 0) return -1;
return 0;
}
+// ---------------- INTERNAL FUNCTIONS ----------------
int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) {
if (pMemTable == NULL) return 0;
int ref = T_REF_INC(pMemTable);
@@ -152,7 +91,7 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) {
}
int code = pthread_cond_signal(&pBufPool->poolNotEmpty);
if (code != 0) {
- tsdbUnlockRepo(pRepo);
+ if (tsdbUnlockRepo(pRepo) < 0) return -1;
tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code));
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
@@ -189,6 +128,8 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) {
}
void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem) {
+ tsdbDebug("vgId:%d untake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem);
+
if (pMem != NULL) {
taosRUnLockLatch(&(pMem->latch));
tsdbUnRefMemTable(pRepo, pMem);
@@ -228,6 +169,10 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
ASSERT(pRepo->mem->extraBuffList != NULL);
SListNode *pNode = (SListNode *)malloc(sizeof(SListNode) + bytes);
if (pNode == NULL) {
+ if (listNEles(pRepo->mem->extraBuffList) == 0) {
+ tdListFree(pRepo->mem->extraBuffList);
+ pRepo->mem->extraBuffList = NULL;
+ }
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return NULL;
}
@@ -258,123 +203,165 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
}
int tsdbAsyncCommit(STsdbRepo *pRepo) {
+ if (pRepo->mem == NULL) return 0;
+
SMemTable *pIMem = pRepo->imem;
- int code = 0;
- if (pIMem != NULL) {
- ASSERT(pRepo->commit);
- tsdbDebug("vgId:%d waiting for the commit thread", REPO_ID(pRepo));
- code = pthread_join(pRepo->commitThread, NULL);
- tsdbDebug("vgId:%d commit thread is finished", REPO_ID(pRepo));
- if (code != 0) {
- tsdbError("vgId:%d failed to thread join since %s", REPO_ID(pRepo), strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- return -1;
- }
- pRepo->commit = 0;
- }
+ sem_wait(&(pRepo->readyToCommit));
- ASSERT(pRepo->commit == 0);
- if (pRepo->mem != NULL) {
- if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
- if (tsdbLockRepo(pRepo) < 0) return -1;
- pRepo->imem = pRepo->mem;
- pRepo->mem = NULL;
- pRepo->commit = 1;
- code = pthread_create(&pRepo->commitThread, NULL, tsdbCommitData, (void *)pRepo);
- if (code != 0) {
- tsdbError("vgId:%d failed to create commit thread since %s", REPO_ID(pRepo), strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(code);
- tsdbUnlockRepo(pRepo);
- return -1;
- }
- if (tsdbUnlockRepo(pRepo) < 0) return -1;
+ if (pRepo->code != TSDB_CODE_SUCCESS) {
+ tsdbWarn("vgId:%d try to commit when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno));
}
- if (pIMem && tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1;
+ if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START, TSDB_CODE_SUCCESS);
+ if (tsdbLockRepo(pRepo) < 0) return -1;
+ pRepo->imem = pRepo->mem;
+ pRepo->mem = NULL;
+ tsdbScheduleCommit(pRepo);
+ if (tsdbUnlockRepo(pRepo) < 0) return -1;
+
+ if (tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1;
return 0;
}
+int tsdbSyncCommit(TSDB_REPO_T *repo) {
+ STsdbRepo *pRepo = (STsdbRepo *)repo;
+
+ tsdbAsyncCommit(pRepo);
+ sem_wait(&(pRepo->readyToCommit));
+ sem_post(&(pRepo->readyToCommit));
+
+ if (pRepo->code != TSDB_CODE_SUCCESS) {
+ terrno = pRepo->code;
+ return -1;
+ } else {
+ terrno = TSDB_CODE_SUCCESS;
+ return 0;
+ }
+}
+
+/**
+ * This is an important function to load data or try to load data from memory skiplist iterator.
+ *
+ * This function load memory data until:
+ * 1. iterator ends
+ * 2. data key exceeds maxKey
+ * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead
+ * 4. operations in pCols not exceeds its max capacity if pCols is given
+ *
+ * The function tries to procceed AS MUSH AS POSSIBLE.
+ */
int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
- TSKEY *filterKeys, int nFilterKeys) {
- ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
+ TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
+ ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0 && pMergeInfo != NULL);
if (pIter == NULL) return 0;
STSchema *pSchema = NULL;
- int numOfRows = 0;
- TSKEY keyNext = 0;
+ TSKEY rowKey = 0;
+ TSKEY fKey = 0;
+ bool isRowDel = false;
int filterIter = 0;
+ SDataRow row = NULL;
+
+ memset(pMergeInfo, 0, sizeof(*pMergeInfo));
+ pMergeInfo->keyFirst = INT64_MAX;
+ pMergeInfo->keyLast = INT64_MIN;
+ if (pCols) tdResetDataCols(pCols);
+
+ row = tsdbNextIterRow(pIter);
+ if (row == NULL || dataRowKey(row) > maxKey) {
+ rowKey = INT64_MAX;
+ isRowDel = false;
+ } else {
+ rowKey = dataRowKey(row);
+ isRowDel = dataRowDeleted(row);
+ }
+
+ if (filterIter >= nFilterKeys) {
+ fKey = INT64_MAX;
+ } else {
+ fKey = tdGetKey(filterKeys[filterIter]);
+ }
+
+ while (true) {
+ if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
+
+ if (fKey < rowKey) {
+ pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
+ pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
+
+ filterIter++;
+ if (filterIter >= nFilterKeys) {
+ fKey = INT64_MAX;
+ } else {
+ fKey = tdGetKey(filterKeys[filterIter]);
+ }
+ } else if (fKey > rowKey) {
+ if (isRowDel) {
+ pMergeInfo->rowsDeleteFailed++;
+ } else {
+ if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
+ if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+ pMergeInfo->rowsInserted++;
+ pMergeInfo->nOperations++;
+ pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
+ pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
+ tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
+ }
- if (nFilterKeys != 0) { // for filter purpose
- ASSERT(filterKeys != NULL);
- keyNext = tsdbNextIterKey(pIter);
- if (keyNext < 0 || keyNext > maxKey) return numOfRows;
- void *ptr = taosbsearch((void *)(&keyNext), (void *)filterKeys, nFilterKeys, sizeof(TSKEY), compTSKEY, TD_GE);
- filterIter = (ptr == NULL) ? nFilterKeys : (int)((POINTER_DISTANCE(ptr, filterKeys) / sizeof(TSKEY)));
- }
-
- do {
- SDataRow row = tsdbNextIterRow(pIter);
- if (row == NULL) break;
-
- keyNext = dataRowKey(row);
- if (keyNext > maxKey) break;
-
- bool keyFiltered = false;
- if (nFilterKeys != 0) {
- while (true) {
- if (filterIter >= nFilterKeys) break;
- if (keyNext == filterKeys[filterIter]) {
- keyFiltered = true;
- filterIter++;
- break;
- } else if (keyNext < filterKeys[filterIter]) {
- break;
+ tSkipListIterNext(pIter);
+ row = tsdbNextIterRow(pIter);
+ if (row == NULL || dataRowKey(row) > maxKey) {
+ rowKey = INT64_MAX;
+ isRowDel = false;
+ } else {
+ rowKey = dataRowKey(row);
+ isRowDel = dataRowDeleted(row);
+ }
+ } else {
+ if (isRowDel) {
+ ASSERT(!keepDup);
+ if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+ pMergeInfo->rowsDeleteSucceed++;
+ pMergeInfo->nOperations++;
+ tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
+ } else {
+ if (keepDup) {
+ if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+ pMergeInfo->rowsUpdated++;
+ pMergeInfo->nOperations++;
+ pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
+ pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
+ tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
} else {
- filterIter++;
+ pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
+ pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
}
}
- }
- if (!keyFiltered) {
- if (numOfRows >= maxRowsToRead) break;
- if (pCols) {
- if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
- pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
- if (pSchema == NULL) {
- ASSERT(0);
- }
- }
+ tSkipListIterNext(pIter);
+ row = tsdbNextIterRow(pIter);
+ if (row == NULL || dataRowKey(row) > maxKey) {
+ rowKey = INT64_MAX;
+ isRowDel = false;
+ } else {
+ rowKey = dataRowKey(row);
+ isRowDel = dataRowDeleted(row);
+ }
- tdAppendDataRowToDataCol(row, pSchema, pCols);
+ filterIter++;
+ if (filterIter >= nFilterKeys) {
+ fKey = INT64_MAX;
+ } else {
+ fKey = tdGetKey(filterKeys[filterIter]);
}
- numOfRows++;
}
- } while (tSkipListIterNext(pIter));
+ }
- return numOfRows;
+ return 0;
}
// ---------------- LOCAL FUNCTIONS ----------------
-static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes) {
- ASSERT(pRepo->mem != NULL);
- if (pRepo->mem->extraBuffList == NULL) {
- STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo);
- ASSERT(pBufBlock != NULL);
- pBufBlock->offset -= bytes;
- pBufBlock->remain += bytes;
- ASSERT(ptr == POINTER_SHIFT(pBufBlock->data, pBufBlock->offset));
- tsdbTrace("vgId:%d free %d bytes to TSDB buffer pool, nBlocks %d offset %d remain %d", REPO_ID(pRepo), bytes,
- listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain);
- } else {
- SListNode *pNode = (SListNode *)POINTER_SHIFT(ptr, -(int)(sizeof(SListNode)));
- ASSERT(listTail(pRepo->mem->extraBuffList) == pNode);
- tdListPopNode(pRepo->mem->extraBuffList, pNode);
- free(pNode);
- tsdbTrace("vgId:%d free %d bytes to SYSTEM buffer pool", REPO_ID(pRepo), bytes);
- }
-}
-
static SMemTable* tsdbNewMemTable(STsdbRepo *pRepo) {
STsdbMeta *pMeta = pRepo->tsdbMeta;
@@ -424,7 +411,7 @@ static void tsdbFreeMemTable(SMemTable* pMemTable) {
tdListFree(pMemTable->extraBuffList);
tdListFree(pMemTable->bufBlockList);
tdListFree(pMemTable->actList);
- taosTFree(pMemTable->tData);
+ tfree(pMemTable->tData);
free(pMemTable);
}
}
@@ -441,8 +428,9 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) {
pTableData->keyLast = 0;
pTableData->numOfRows = 0;
- pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP,
- TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 1, tsdbGetTsTupleKey);
+ pTableData->pData =
+ tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP],
+ tkeyComparFn, pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey);
if (pTableData->pData == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
@@ -462,336 +450,443 @@ static void tsdbFreeTableData(STableData *pTableData) {
}
}
-static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(*(SDataRow *)data); }
-
-static void *tsdbCommitData(void *arg) {
- STsdbRepo * pRepo = (STsdbRepo *)arg;
- SMemTable * pMem = pRepo->imem;
- STsdbCfg * pCfg = &pRepo->config;
- SDataCols * pDataCols = NULL;
- STsdbMeta * pMeta = pRepo->tsdbMeta;
- SCommitIter *iters = NULL;
- SFidGroup fidGroup = {0};
- SRWHelper whelper = {0};
- TSKEY minKey = 0, maxKey = 0;
- ASSERT(pRepo->commit == 1);
- ASSERT(pMem != NULL);
-
- tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64, REPO_ID(pRepo),
- pMem->keyFirst, pMem->keyLast, pMem->numOfRows);
-
- tsdbGetFidGroup(pCfg, &fidGroup);
- tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fidGroup.minFid, &minKey, &maxKey);
- tsdbRemoveFilesBeyondRetention(pRepo, &fidGroup);
-
- // Create the iterator to read from cache
- if (pMem->numOfRows > 0) {
- iters = tsdbCreateCommitIters(pRepo);
- if (iters == NULL) {
- tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno));
- goto _exit;
- }
+static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); }
- if (tsdbInitWriteHelper(&whelper, pRepo) < 0) {
- tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno));
- goto _exit;
- }
+void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) {
+ *minKey = fileId * daysPerFile * tsMsPerDay[precision];
+ *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1;
+}
- if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) {
- terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
- tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s",
- REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno));
- goto _exit;
- }
+static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
+ ASSERT(pMemTable->maxTables < maxTables);
- int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision));
- int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision));
+ STableData **pTableData = (STableData **)calloc(maxTables, sizeof(STableData *));
+ if (pTableData == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+ memcpy((void *)pTableData, (void *)pMemTable->tData, sizeof(STableData *) * pMemTable->maxTables);
- tsdbSeekCommitIter(iters, pMem->maxTables, minKey);
+ STableData **tData = pMemTable->tData;
- // Loop to commit to each file
- for (int fid = sfid; fid <= efid; fid++) {
- if (fid < fidGroup.minFid) continue;
+ taosWLockLatch(&(pMemTable->latch));
+ pMemTable->maxTables = maxTables;
+ pMemTable->tData = pTableData;
+ taosWUnLockLatch(&(pMemTable->latch));
- if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
- tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
- goto _exit;
+ tfree(tData);
+
+ return 0;
+}
+
+static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) {
+ if (pCols) {
+ if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) {
+ *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
+ if (*ppSchema == NULL) {
+ ASSERT(false);
+ return -1;
}
}
+
+ tdAppendDataRowToDataCol(row, *ppSchema, pCols);
}
- tsdbApplyRetention(pRepo, &fidGroup);
+ return 0;
+}
+
+static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
+ if (pBlock->dataLen <= 0) return -1;
+ pIter->totalLen = pBlock->dataLen;
+ pIter->len = 0;
+ pIter->row = (SDataRow)(pBlock->data+pBlock->schemaLen);
+ return 0;
+}
+
+static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) {
+ SDataRow row = pIter->row;
+ if (row == NULL) return NULL;
- // Commit to update meta file
- if (tsdbCommitMeta(pRepo) < 0) {
- tsdbError("vgId:%d failed to commit data while committing meta data since %s", REPO_ID(pRepo), tstrerror(terrno));
- goto _exit;
+ pIter->len += dataRowLen(row);
+ if (pIter->len >= pIter->totalLen) {
+ pIter->row = NULL;
+ } else {
+ pIter->row = (char *)row + dataRowLen(row);
}
-_exit:
- tdFreeDataCols(pDataCols);
- tsdbDestroyCommitIters(iters, pMem->maxTables);
- tsdbDestroyHelper(&whelper);
- tsdbEndCommit(pRepo);
- tsdbInfo("vgId:%d commit over", pRepo->config.tsdbId);
+ return row;
+}
- return NULL;
+static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey,
+ TSKEY now) {
+ if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) {
+ tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64
+ " maxKey %" PRId64 " row key %" PRId64,
+ REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey,
+ dataRowKey(row));
+ terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
+ return -1;
+ }
+
+ return 0;
}
-static int tsdbCommitMeta(STsdbRepo *pRepo) {
- SMemTable *pMem = pRepo->imem;
- STsdbMeta *pMeta = pRepo->tsdbMeta;
- SActObj * pAct = NULL;
- SActCont * pCont = NULL;
+static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) {
+ ASSERT(pMsg != NULL);
+ STsdbMeta * pMeta = pRepo->tsdbMeta;
+ SSubmitMsgIter msgIter = {0};
+ SSubmitBlk * pBlock = NULL;
+ SSubmitBlkIter blkIter = {0};
+ SDataRow row = NULL;
+ TSKEY now = taosGetTimestamp(pRepo->config.precision);
+ TSKEY minKey = now - tsMsPerDay[pRepo->config.precision] * pRepo->config.keep;
+ TSKEY maxKey = now + tsMsPerDay[pRepo->config.precision] * pRepo->config.daysPerFile;
+
+ terrno = TSDB_CODE_SUCCESS;
+ pMsg->length = htonl(pMsg->length);
+ pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
+
+ if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
+ while (true) {
+ if (tsdbGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
+ if (pBlock == NULL) break;
+
+ pBlock->uid = htobe64(pBlock->uid);
+ pBlock->tid = htonl(pBlock->tid);
+ pBlock->sversion = htonl(pBlock->sversion);
+ pBlock->dataLen = htonl(pBlock->dataLen);
+ pBlock->schemaLen = htonl(pBlock->schemaLen);
+ pBlock->numOfRows = htons(pBlock->numOfRows);
+
+ if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) {
+ tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid,
+ pBlock->tid);
+ terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+ return -1;
+ }
- if (listNEles(pMem->actList) > 0) {
- if (tdKVStoreStartCommit(pMeta->pStore) < 0) {
- tsdbError("vgId:%d failed to commit data while start commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
- goto _err;
+ STable *pTable = pMeta->tables[pBlock->tid];
+ if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) {
+ tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid,
+ pBlock->tid);
+ terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+ return -1;
}
- SListNode *pNode = NULL;
+ if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
+ tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable));
+ terrno = TSDB_CODE_TDB_INVALID_ACTION;
+ return -1;
+ }
- while ((pNode = tdListPopHead(pMem->actList)) != NULL) {
- pAct = (SActObj *)pNode->data;
- if (pAct->act == TSDB_UPDATE_META) {
- pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
- if (tdUpdateKVStoreRecord(pMeta->pStore, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) {
- tsdbError("vgId:%d failed to update meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
- tstrerror(terrno));
- tdKVStoreEndCommit(pMeta->pStore);
- goto _err;
- }
- } else if (pAct->act == TSDB_DROP_META) {
- if (tdDropKVStoreRecord(pMeta->pStore, pAct->uid) < 0) {
- tsdbError("vgId:%d failed to drop meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
- tstrerror(terrno));
- tdKVStoreEndCommit(pMeta->pStore);
- goto _err;
- }
+ // Check schema version and update schema if needed
+ if (tsdbCheckTableSchema(pRepo, pBlock, pTable) < 0) {
+ if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
+ continue;
} else {
- ASSERT(false);
+ return -1;
}
}
- if (tdKVStoreEndCommit(pMeta->pStore) < 0) {
- tsdbError("vgId:%d failed to commit data while end commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
- goto _err;
+ tsdbInitSubmitBlkIter(pBlock, &blkIter);
+ while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
+ if (tsdbCheckRowRange(pRepo, pTable, row, minKey, maxKey, now) < 0) {
+ return -1;
+ }
}
}
+ if (terrno != TSDB_CODE_SUCCESS) return -1;
return 0;
-
-_err:
- return -1;
}
-static void tsdbEndCommit(STsdbRepo *pRepo) {
- ASSERT(pRepo->commit == 1);
- if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER);
-}
-
-static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
- for (int i = 0; i < nIters; i++) {
- TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
- if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1;
+static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows) {
+ STsdbMeta * pMeta = pRepo->tsdbMeta;
+ int64_t points = 0;
+ STable * pTable = NULL;
+ SSubmitBlkIter blkIter = {0};
+ SDataRow row = NULL;
+ void ** rows = NULL;
+ int rowCounter = 0;
+
+ ASSERT(pBlock->tid < pMeta->maxTables);
+ pTable = pMeta->tables[pBlock->tid];
+ ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid);
+
+ rows = (void **)calloc(pBlock->numOfRows, sizeof(void *));
+ if (rows == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
}
- return 0;
-}
-void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) {
- *minKey = fileId * daysPerFile * tsMsPerDay[precision];
- *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1;
-}
+ tsdbInitSubmitBlkIter(pBlock, &blkIter);
+ while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
+ if (tsdbCopyRowToMem(pRepo, row, pTable, &(rows[rowCounter])) < 0) {
+ tsdbFreeRows(pRepo, rows, rowCounter);
+ goto _err;
+ }
-static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) {
- STsdbCfg * pCfg = &pRepo->config;
- STsdbFileH *pFileH = pRepo->tsdbFileH;
- SFileGroup *pGroup = NULL;
- SMemTable * pMem = pRepo->imem;
- bool newLast = false;
-
- TSKEY minKey = 0, maxKey = 0;
- tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey);
-
- // Check if there are data to commit to this file
- int hasDataToCommit = tsdbHasDataToCommit(iters, pMem->maxTables, minKey, maxKey);
- if (!hasDataToCommit) {
- tsdbDebug("vgId:%d no data to commit to file %d", REPO_ID(pRepo), fid);
- return 0;
- }
+ (*affectedrows)++;
+ points++;
- if ((pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ)) == NULL) {
- pGroup = tsdbCreateFGroup(pRepo, fid);
- if (pGroup == NULL) {
- tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
- return -1;
+ if (rows[rowCounter] != NULL) {
+ rowCounter++;
}
}
- // Open files for write/read
- if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) {
- tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno));
- goto _err;
- }
-
- newLast = TSDB_NLAST_FILE_OPENED(pHelper);
-
- if (tsdbLoadCompIdx(pHelper, NULL) < 0) {
- tsdbError("vgId:%d failed to load SCompIdx part since %s", REPO_ID(pRepo), tstrerror(terrno));
+ if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) {
goto _err;
}
- // Loop to commit data in each table
- for (int tid = 1; tid < pMem->maxTables; tid++) {
- SCommitIter *pIter = iters + tid;
- if (pIter->pTable == NULL) continue;
-
- taosRLockLatch(&(pIter->pTable->latch));
+ STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion);
+ pRepo->stat.pointsWritten += points * schemaNCols(pSchema);
+ pRepo->stat.totalStorage += points * schemaVLen(pSchema);
- if (tsdbSetHelperTable(pHelper, pIter->pTable, pRepo) < 0) goto _err;
-
- if (pIter->pIter != NULL) {
- if (tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1)) < 0) {
- terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
- goto _err;
- }
+ free(rows);
+ return 0;
- if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) {
- taosRUnLockLatch(&(pIter->pTable->latch));
- tsdbError("vgId:%d failed to write data of table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo),
- TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable),
- tstrerror(terrno));
- goto _err;
- }
- }
+_err:
+ free(rows);
+ return -1;
+}
- taosRUnLockLatch(&(pIter->pTable->latch));
+static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void **ppRow) {
+ STsdbCfg * pCfg = &pRepo->config;
+ TKEY tkey = dataRowTKey(row);
+ TSKEY key = dataRowKey(row);
+ bool isRowDelete = TKEY_IS_DELETED(tkey);
- // Move the last block to the new .l file if neccessary
- if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) {
- tsdbError("vgId:%d, failed to move last block, since %s", REPO_ID(pRepo), tstrerror(terrno));
- goto _err;
+ if (isRowDelete) {
+ if (!pCfg->update) {
+ tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo));
+ terrno = TSDB_CODE_TDB_INVALID_ACTION;
+ return -1;
}
- // Write the SCompBlock part
- if (tsdbWriteCompInfo(pHelper) < 0) {
- tsdbError("vgId:%d, failed to write compInfo part since %s", REPO_ID(pRepo), tstrerror(terrno));
- goto _err;
+ if (key > TABLE_LASTKEY(pTable)) {
+ tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64,
+ REPO_ID(pRepo), key, TABLE_LASTKEY(pTable));
+ return 0;
}
}
- if (tsdbWriteCompIdx(pHelper) < 0) {
- tsdbError("vgId:%d failed to write compIdx part to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
- goto _err;
+ void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row));
+ if (pRow == NULL) {
+ tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s",
+ REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno));
+ return -1;
}
- tsdbCloseHelperFile(pHelper, 0, pGroup);
+ dataRowCpy(pRow, row);
+ ppRow[0] = pRow;
- pthread_rwlock_wrlock(&(pFileH->fhlock));
+ tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
+ isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable),
+ key);
- (void)rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname);
- pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info;
+ return 0;
+}
- if (newLast) {
- (void)rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname);
- pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info;
- } else {
- pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info;
+static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
+ if (pMsg == NULL) {
+ terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
+ return -1;
}
- pGroup->files[TSDB_FILE_TYPE_DATA].info = helperDataF(pHelper)->info;
-
- pthread_rwlock_unlock(&(pFileH->fhlock));
+ pIter->totalLen = pMsg->length;
+ pIter->len = 0;
+ pIter->pMsg = pMsg;
+ if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) {
+ terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
+ return -1;
+ }
return 0;
-
-_err:
- tsdbCloseHelperFile(pHelper, 1, NULL);
- return -1;
}
-static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) {
- SMemTable *pMem = pRepo->imem;
- STsdbMeta *pMeta = pRepo->tsdbMeta;
+static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
+ if (pIter->len == 0) {
+ pIter->len += TSDB_SUBMIT_MSG_HEAD_SIZE;
+ } else {
+ SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
+ pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen);
+ }
- SCommitIter *iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter));
- if (iters == NULL) {
- terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
- return NULL;
+ if (pIter->len > pIter->totalLen) {
+ terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
+ *pPBlock = NULL;
+ return -1;
}
- if (tsdbRLockRepoMeta(pRepo) < 0) goto _err;
+ *pPBlock = (pIter->len == pIter->totalLen) ? NULL : (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
+
+ return 0;
+}
+
+static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable) {
+ ASSERT(pTable != NULL);
+
+ STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
+ int sversion = schemaVersion(pSchema);
- // reference all tables
- for (int i = 0; i < pMem->maxTables; i++) {
- if (pMeta->tables[i] != NULL) {
- tsdbRefTable(pMeta->tables[i]);
- iters[i].pTable = pMeta->tables[i];
+ if (pBlock->sversion == sversion) {
+ return 0;
+ } else {
+ if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { // stream table is not allowed to change schema
+ terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+ return -1;
}
}
- if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err;
+ if (pBlock->sversion > sversion) { // may need to update table schema
+ if (pBlock->schemaLen > 0) {
+ tsdbDebug(
+ "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, update...",
+ REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
+ ASSERT(pBlock->schemaLen % sizeof(STColumn) == 0);
+ int numOfCols = pBlock->schemaLen / sizeof(STColumn);
+ STColumn *pTCol = (STColumn *)pBlock->data;
+
+ STSchemaBuilder schemaBuilder = {0};
+ if (tdInitTSchemaBuilder(&schemaBuilder, pBlock->sversion) < 0) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+ tstrerror(terrno));
+ return -1;
+ }
+
+ for (int i = 0; i < numOfCols; i++) {
+ if (tdAddColToSchema(&schemaBuilder, pTCol[i].type, htons(pTCol[i].colId), htons(pTCol[i].bytes)) < 0) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+ tstrerror(terrno));
+ tdDestroyTSchemaBuilder(&schemaBuilder);
+ return -1;
+ }
+ }
- for (int i = 0; i < pMem->maxTables; i++) {
- if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TABLE_UID(iters[i].pTable) == pMem->tData[i]->uid)) {
- if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) {
+ STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder);
+ if (pNSchema == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
- goto _err;
+ tdDestroyTSchemaBuilder(&schemaBuilder);
+ return -1;
}
- tSkipListIterNext(iters[i].pIter);
+ tdDestroyTSchemaBuilder(&schemaBuilder);
+ tsdbUpdateTableSchema(pRepo, pTable, pNSchema, true);
+ } else {
+ tsdbDebug(
+ "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, reconfigure...",
+ REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
+ terrno = TSDB_CODE_TDB_TABLE_RECONFIGURE;
+ return -1;
+ }
+ } else {
+ ASSERT(pBlock->sversion >= 0);
+ if (tsdbGetTableSchemaImpl(pTable, false, false, pBlock->sversion) == NULL) {
+ tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo),
+ pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable));
}
+ terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+ return -1;
}
- return iters;
-
-_err:
- tsdbDestroyCommitIters(iters, pMem->maxTables);
- return NULL;
+ return 0;
}
-static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables) {
- if (iters == NULL) return;
+static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter) {
+ if (rowCounter < 1) return 0;
+
+ SMemTable * pMemTable = NULL;
+ STableData *pTableData = NULL;
+ STsdbMeta * pMeta = pRepo->tsdbMeta;
+ STsdbCfg * pCfg = &(pRepo->config);
+
+ ASSERT(pRepo->mem != NULL);
+ pMemTable = pRepo->mem;
- for (int i = 1; i < maxTables; i++) {
- if (iters[i].pTable != NULL) {
- tsdbUnRefTable(iters[i].pTable);
- tSkipListDestroyIter(iters[i].pIter);
+ if (TABLE_TID(pTable) >= pMemTable->maxTables) {
+ if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) {
+ tsdbFreeRows(pRepo, rows, rowCounter);
+ return -1;
}
}
+ pTableData = pMemTable->tData[TABLE_TID(pTable)];
- free(iters);
-}
+ if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) {
+ if (pTableData != NULL) {
+ taosWLockLatch(&(pMemTable->latch));
+ pMemTable->tData[TABLE_TID(pTable)] = NULL;
+ tsdbFreeTableData(pTableData);
+ taosWUnLockLatch(&(pMemTable->latch));
+ }
-static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
- ASSERT(pMemTable->maxTables < maxTables);
+ pTableData = tsdbNewTableData(pCfg, pTable);
+ if (pTableData == NULL) {
+ tsdbError("vgId:%d failed to insert data to table %s uid %" PRId64 " tid %d since %s", REPO_ID(pRepo),
+ TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), tstrerror(terrno));
+ tsdbFreeRows(pRepo, rows, rowCounter);
+ return -1;
+ }
- STableData **pTableData = (STableData **)calloc(maxTables, sizeof(STableData *));
- if (pTableData == NULL) {
- terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
- return -1;
+ pRepo->mem->tData[TABLE_TID(pTable)] = pTableData;
}
- memcpy((void *)pTableData, (void *)pMemTable->tData, sizeof(STableData *) * pMemTable->maxTables);
- STableData **tData = pMemTable->tData;
+ ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable));
- taosWLockLatch(&(pMemTable->latch));
- pMemTable->maxTables = maxTables;
- pMemTable->tData = pTableData;
- taosWUnLockLatch(&(pMemTable->latch));
+ int64_t osize = SL_SIZE(pTableData->pData);
+ tSkipListPutBatch(pTableData->pData, rows, rowCounter);
+ int64_t dsize = SL_SIZE(pTableData->pData) - osize;
+
+ if (pMemTable->keyFirst > dataRowKey(rows[0])) pMemTable->keyFirst = dataRowKey(rows[0]);
+ if (pMemTable->keyLast < dataRowKey(rows[rowCounter - 1])) pMemTable->keyLast = dataRowKey(rows[rowCounter - 1]);
+ pMemTable->numOfRows += dsize;
- taosTFree(tData);
+ if (pTableData->keyFirst > dataRowKey(rows[0])) pTableData->keyFirst = dataRowKey(rows[0]);
+ if (pTableData->keyLast < dataRowKey(rows[rowCounter - 1])) pTableData->keyLast = dataRowKey(rows[rowCounter - 1]);
+ pTableData->numOfRows += dsize;
+
+ // TODO: impl delete row thing
+ if (TABLE_LASTKEY(pTable) < dataRowKey(rows[rowCounter-1])) TABLE_LASTKEY(pTable) = dataRowKey(rows[rowCounter-1]);
return 0;
}
-static void tsdbSeekCommitIter(SCommitIter *pIters, int nIters, TSKEY key) {
- for (int i = 0; i < nIters; i++) {
- SCommitIter *pIter = pIters + i;
- if (pIter->pTable == NULL) continue;
- if (pIter->pIter == NULL) continue;
+static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) {
+ ASSERT(pRepo->mem != NULL);
+ STsdbBufPool *pBufPool = pRepo->pPool;
+
+ for (int i = rowCounter - 1; i >= 0; --i) {
+ SDataRow row = (SDataRow)rows[i];
+ int bytes = (int)dataRowLen(row);
- tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, key-1, INT32_MAX, NULL, NULL, 0);
+ if (pRepo->mem->extraBuffList == NULL) {
+ STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo);
+ ASSERT(pBufBlock != NULL && pBufBlock->offset >= bytes);
+
+ pBufBlock->offset -= bytes;
+ pBufBlock->remain += bytes;
+ ASSERT(row == POINTER_SHIFT(pBufBlock->data, pBufBlock->offset));
+ tsdbTrace("vgId:%d free %d bytes to TSDB buffer pool, nBlocks %d offset %d remain %d", REPO_ID(pRepo), bytes,
+ listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain);
+
+ if (pBufBlock->offset == 0) { // return the block to buffer pool
+ tsdbLockRepo(pRepo);
+ SListNode *pNode = tdListPopTail(pRepo->mem->bufBlockList);
+ tdListPrependNode(pBufPool->bufBlockList, pNode);
+ tsdbUnlockRepo(pRepo);
+ }
+ } else {
+ ASSERT(listNEles(pRepo->mem->extraBuffList) > 0);
+ SListNode *pNode = tdListPopTail(pRepo->mem->extraBuffList);
+ ASSERT(row == pNode->data);
+ free(pNode);
+ tsdbTrace("vgId:%d free %d bytes to SYSTEM buffer pool", REPO_ID(pRepo), bytes);
+
+ if (listNEles(pRepo->mem->extraBuffList) == 0) {
+ tdListFree(pRepo->mem->extraBuffList);
+ pRepo->mem->extraBuffList = NULL;
+ }
+ }
}
}
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c
index f3bd91f038cf209827b1c252160019a6b0aac27f..25c815b74e3bd5593bef9157cbd42ba869298cab 100644
--- a/src/tsdb/src/tsdbMeta.c
+++ b/src/tsdb/src/tsdbMeta.c
@@ -86,7 +86,8 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
if (pTable != NULL) {
tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
TABLE_TID(pTable), TABLE_UID(pTable));
- return TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
+ terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
+ goto _err;
}
if (pCfg->type == TSDB_CHILD_TABLE) {
@@ -191,7 +192,7 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) {
return 0;
_err:
- taosTFree(tbname);
+ tfree(tbname);
return -1;
}
@@ -461,7 +462,7 @@ void tsdbFreeMeta(STsdbMeta *pMeta) {
if (pMeta) {
taosHashCleanup(pMeta->uidMap);
tdListFree(pMeta->superList);
- taosTFree(pMeta->tables);
+ tfree(pMeta->tables);
pthread_rwlock_destroy(&pMeta->rwLock);
free(pMeta);
}
@@ -485,11 +486,11 @@ int tsdbOpenMeta(STsdbRepo *pRepo) {
}
tsdbDebug("vgId:%d open TSDB meta succeed", REPO_ID(pRepo));
- taosTFree(fname);
+ tfree(fname);
return 0;
_err:
- taosTFree(fname);
+ tfree(fname);
return -1;
}
@@ -562,12 +563,12 @@ int tsdbUnlockRepoMeta(STsdbRepo *pRepo) {
void tsdbRefTable(STable *pTable) {
int32_t ref = T_REF_INC(pTable);
UNUSED(ref);
- // tsdbDebug("ref table %"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref);
+ tsdbDebug("ref table %s uid %" PRIu64 " tid:%d, refCount:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), ref);
}
void tsdbUnRefTable(STable *pTable) {
int32_t ref = T_REF_DEC(pTable);
- tsdbDebug("unref table uid:%"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref);
+ tsdbDebug("unref table %s uid:%"PRIu64" tid:%d, refCount:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), ref);
if (ref == 0) {
// tsdbDebug("destory table name:%s uid:%"PRIu64", tid:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable));
@@ -643,7 +644,7 @@ static void tsdbOrgMeta(void *pHandle) {
}
static char *getTagIndexKey(const void *pData) {
- STable *pTable = *(STable **)pData;
+ STable *pTable = (STable *)pData;
STSchema *pSchema = tsdbGetTableTagSchema(pTable);
STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN);
@@ -700,7 +701,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) {
}
pTable->tagVal = NULL;
STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
- pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey);
+ pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, SL_ALLOW_DUP_KEY, getTagIndexKey);
if (pTable->pIndex == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
@@ -745,7 +746,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) {
T_REF_INC(pTable);
- tsdbTrace("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable),
+ tsdbDebug("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable),
TABLE_UID(pTable));
return pTable;
@@ -760,7 +761,7 @@ static void tsdbFreeTable(STable *pTable) {
if (pTable->name != NULL)
tsdbTrace("table %s tid %d uid %" PRIu64 " is freed", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable),
TABLE_UID(pTable));
- taosTFree(TABLE_NAME(pTable));
+ tfree(TABLE_NAME(pTable));
if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) {
for (int i = 0; i < TSDB_MAX_TABLE_SCHEMAS; i++) {
tdFreeSchema(pTable->schema[i]);
@@ -774,7 +775,7 @@ static void tsdbFreeTable(STable *pTable) {
kvRowFree(pTable->tagVal);
tSkipListDestroy(pTable->pIndex);
- taosTFree(pTable->sql);
+ tfree(pTable->sql);
free(pTable);
}
}
@@ -889,7 +890,7 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro
}
if (lock) tsdbUnlockRepoMeta(pRepo);
- tsdbDebug("vgId:%d table %s is removed from meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable));
+ tsdbDebug("vgId:%d table %s uid %" PRIu64 " is removed from meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_UID(pTable));
tsdbUnRefTable(pTable);
}
@@ -900,23 +901,8 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper
pTable->pSuper = pSTable;
- int32_t level = 0;
- int32_t headSize = 0;
-
- tSkipListNewNodeInfo(pSTable->pIndex, &level, &headSize);
-
- // NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the
- // actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function
- SSkipListNode *pNode = calloc(1, headSize + sizeof(STable *));
- if (pNode == NULL) {
- terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
- return -1;
- }
- pNode->level = level;
-
- memcpy(SL_GET_NODE_DATA(pNode), &pTable, sizeof(STable *));
+ tSkipListPut(pSTable->pIndex, (void *)pTable);
- tSkipListPut(pSTable->pIndex, pNode);
if (refSuper) T_REF_INC(pSTable);
return 0;
}
@@ -940,7 +926,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
SSkipListNode *pNode = taosArrayGetP(res, i);
// STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode);
- if (*(STable **)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need
+ if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need
tSkipListRemoveNode(pSTable->pIndex, pNode);
}
}
@@ -1080,9 +1066,9 @@ void tsdbClearTableCfg(STableCfg *config) {
if (config->schema) tdFreeSchema(config->schema);
if (config->tagSchema) tdFreeSchema(config->tagSchema);
if (config->tagValues) kvRowFree(config->tagValues);
- taosTFree(config->name);
- taosTFree(config->sname);
- taosTFree(config->sql);
+ tfree(config->name);
+ tfree(config->sname);
+ tfree(config->sql);
free(config);
}
}
@@ -1170,8 +1156,8 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) {
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
buf = tdDecodeSchema(buf, &(pTable->tagSchema));
STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
- pTable->pIndex =
- tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey);
+ pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL,
+ SL_ALLOW_DUP_KEY, getTagIndexKey);
if (pTable->pIndex == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbFreeTable(pTable);
@@ -1197,7 +1183,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) {
tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM);
} else {
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
- tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (tSkipListGetSize(pTable->pIndex) + 1));
+ tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_SIZE(pTable->pIndex) + 1));
} else {
tlen = sizeof(SListNode) + sizeof(SActObj);
}
@@ -1244,7 +1230,7 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) {
}
while (tSkipListIterNext(pIter)) {
- STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
+ STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE);
pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, tTable);
}
@@ -1269,7 +1255,7 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) {
tsdbWLockRepoMeta(pRepo);
while (tSkipListIterNext(pIter)) {
- STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
+ STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
tsdbRemoveTableFromMeta(pRepo, tTable, false, false);
}
@@ -1304,7 +1290,7 @@ static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) {
STable **tTables = pMeta->tables;
pMeta->tables = tables;
- taosTFree(tTables);
+ tfree(tTables);
tsdbDebug("vgId:%d tsdb meta maxTables is adjusted as %d", REPO_ID(pRepo), maxTables);
return 0;
diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c
index ec6f2a5f991927707cdd7a64cee4ff8067406ac9..3db64703f7b64326aca064b8fe67bdaf6464b920 100644
--- a/src/tsdb/src/tsdbRWHelper.c
+++ b/src/tsdb/src/tsdbRWHelper.c
@@ -14,9 +14,7 @@
*/
#define _DEFAULT_SOURCE
-
#define TAOS_RANDOM_FILE_FAIL_TEST
-
#include "os.h"
#include "talgo.h"
#include "tchecksum.h"
@@ -28,6 +26,7 @@
#define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM))
#define TSDB_KEY_COL_OFFSET 0
#define TSDB_GET_COMPBLOCK_IDX(h, b) (POINTER_DISTANCE(b, (h)->pCompInfo->blocks)/sizeof(SCompBlock))
+#define TSDB_IS_LAST_BLOCK(pb) ((pb)->last)
static bool tsdbShouldCreateNewLast(SRWHelper *pHelper);
static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock,
@@ -35,7 +34,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pD
static int compareKeyBlock(const void *arg1, const void *arg2);
static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize);
static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
-static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded);
+static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo);
static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
static void tsdbResetHelperFileImpl(SRWHelper *pHelper);
static int tsdbInitHelperFile(SRWHelper *pHelper);
@@ -62,8 +61,10 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pComp
static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock);
static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey,
int *blkIdx);
-static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
- TSKEY maxKey, int maxRows);
+static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
+ TSKEY maxKey, int maxRows, int8_t update);
+static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps);
+static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx);
// ---------------------- INTERNAL FUNCTIONS ----------------------
int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
@@ -284,7 +285,7 @@ int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols
while (true) {
ASSERT(blkIdx <= (int)pIdx->numOfBlocks);
TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
- if (keyFirst < 0 || keyFirst > maxKey) break; // iter over
+ if (keyFirst == TSDB_DATA_TIMESTAMP_NULL || keyFirst > maxKey) break; // iter over
if (pIdx->len <= 0 || keyFirst > pIdx->maxKey) {
if (tsdbProcessAppendCommit(pHelper, pCommitIter, pDataCols, maxKey) < 0) return -1;
@@ -340,7 +341,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
return -1;
}
- if (taosTSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) {
+ if (taosSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) {
tsdbError("vgId:%d failed to sendfile from file %s to file %s since %s", REPO_ID(pHelper->pRepo),
helperLastF(pHelper)->fname, helperNewLastF(pHelper)->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -385,7 +386,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
pIdx->tid = pHelper->tableInfo.tid;
ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
- if (taosTWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) {
+ if (taosWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) {
tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
pFile->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -437,7 +438,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) {
ASSERT(offset == pFile->info.size);
- if (taosTWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < (int)pFile->info.len) {
+ if (taosWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < (int)pFile->info.len) {
tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len,
pFile->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -459,7 +460,7 @@ int tsdbLoadCompIdxImpl(SFile *pFile, uint32_t offset, uint32_t len, void *buffe
return -1;
}
- if (taosTRead(pFile->fd, buffer, len) < len) {
+ if (taosRead(pFile->fd, buffer, len) < len) {
tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, offset, len,
strerror(errno));
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
@@ -556,7 +557,7 @@ int tsdbLoadCompInfoImpl(SFile *pFile, SCompIdx *pIdx, SCompInfo **ppCompInfo) {
return -1;
}
- if (taosTRead(pFile->fd, (void *)(*ppCompInfo), pIdx->len) < (int)pIdx->len) {
+ if (taosRead(pFile->fd, (void *)(*ppCompInfo), pIdx->len) < (int)pIdx->len) {
tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, pIdx->offset, pIdx->len,
strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -613,7 +614,7 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) {
return -1;
}
- if (taosTRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) {
+ if (taosRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) {
tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname,
strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -828,7 +829,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
sizeof(TSCKSUM));
// Write the whole block to file
- if (taosTWrite(pFile->fd, (void *)pCompData, lsize) < lsize) {
+ if (taosWrite(pFile->fd, (void *)pCompData, lsize) < lsize) {
tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, pFile->fname,
strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -930,7 +931,7 @@ _err:
return -1;
}
-static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) {
+static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo) {
ASSERT(pCompBlock->numOfSubBlocks == 0);
SCompIdx *pIdx = &(pHelper->curCompIdx);
@@ -963,9 +964,9 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
pSCompBlock->numOfSubBlocks++;
ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
pSCompBlock->len += sizeof(SCompBlock);
- pSCompBlock->numOfRows += rowsAdded;
- pSCompBlock->keyFirst = MIN(pSCompBlock->keyFirst, pCompBlock->keyFirst);
- pSCompBlock->keyLast = MAX(pSCompBlock->keyLast, pCompBlock->keyLast);
+ pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
+ pSCompBlock->keyFirst = pMergeInfo->keyFirst;
+ pSCompBlock->keyLast = pMergeInfo->keyLast;
pIdx->len += sizeof(SCompBlock);
} else { // Need to create two sub-blocks
void *ptr = NULL;
@@ -994,11 +995,11 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
((SCompBlock *)ptr)[1] = *pCompBlock;
pSCompBlock->numOfSubBlocks = 2;
- pSCompBlock->numOfRows += rowsAdded;
+ pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo);
pSCompBlock->len = sizeof(SCompBlock) * 2;
- pSCompBlock->keyFirst = MIN(((SCompBlock *)ptr)[0].keyFirst, ((SCompBlock *)ptr)[1].keyFirst);
- pSCompBlock->keyLast = MAX(((SCompBlock *)ptr)[0].keyLast, ((SCompBlock *)ptr)[1].keyLast);
+ pSCompBlock->keyFirst = pMergeInfo->keyFirst;
+ pSCompBlock->keyLast = pMergeInfo->keyLast;
pIdx->len += (sizeof(SCompBlock) * 2);
}
@@ -1052,6 +1053,45 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
return 0;
}
+static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) {
+ SCompIdx *pCompIdx = &(pHelper->curCompIdx);
+
+ ASSERT(pCompIdx->numOfBlocks > 0 && blkIdx < pCompIdx->numOfBlocks);
+
+ SCompBlock *pCompBlock= blockAtIdx(pHelper, blkIdx);
+ SCompBlock compBlock = *pCompBlock;
+ ASSERT(pCompBlock->numOfSubBlocks > 0 && pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
+
+ if (pCompIdx->numOfBlocks == 1) {
+ memset(pCompIdx, 0, sizeof(*pCompIdx));
+ } else {
+ int tsize = 0;
+
+ if (compBlock.numOfSubBlocks > 1) {
+ tsize = (int)(pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks));
+
+ ASSERT(tsize > 0);
+ memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset),
+ POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks),
+ tsize);
+
+ pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks;
+ }
+
+ tsize = (int)(pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo));
+ ASSERT(tsize > 0);
+ memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize);
+
+ pCompIdx->len -= sizeof(SCompBlock);
+
+ pCompIdx->numOfBlocks--;
+ pCompIdx->hasLast = (uint32_t)(blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last);
+ pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast;
+ }
+
+ return 0;
+}
+
static void tsdbResetHelperFileImpl(SRWHelper *pHelper) {
pHelper->idxH.numOfIdx = 0;
pHelper->idxH.curIdx = 0;
@@ -1215,7 +1255,7 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl
return -1;
}
- if (taosTRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) {
+ if (taosRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) {
tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompCol->len, pFile->fname,
strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -1330,7 +1370,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
- if (taosTRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) {
+ if (taosRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) {
tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompBlock->len,
pFile->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -1444,51 +1484,62 @@ static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) {
}
static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) {
- STsdbCfg * pCfg = &(pHelper->pRepo->config);
- STable * pTable = pCommitIter->pTable;
- SCompIdx * pIdx = &(pHelper->curCompIdx);
- TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
- int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
- SCompBlock compBlock = {0};
+ STsdbCfg * pCfg = &(pHelper->pRepo->config);
+ STable * pTable = pCommitIter->pTable;
+ SCompIdx * pIdx = &(pHelper->curCompIdx);
+ TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
+ int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
+ SCompBlock compBlock = {0};
+ SMergeInfo mergeInfo = {0};
+ SMergeInfo *pMergeInfo = &mergeInfo;
ASSERT(pIdx->len <= 0 || keyFirst > pIdx->maxKey);
if (pIdx->hasLast) { // append to with last block
ASSERT(pIdx->len > 0);
SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1);
ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock);
- tdResetDataCols(pDataCols);
- int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows,
- pDataCols, NULL, 0);
- ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows);
- if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock &&
- pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
- if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
- if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, rowsRead) < 0) return -1;
- } else {
- if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
- ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows);
+ tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols,
+ NULL, 0, pCfg->update, pMergeInfo);
- if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1;
- ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows);
+ ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
- if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1;
- if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1;
- }
+ if (pDataCols->numOfRows > 0) {
+ ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
- if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
+ if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock &&
+ pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
+ if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
+ pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, pCompBlock->keyFirst);
+ pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, pCompBlock->keyLast);
+ if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1;
+ } else {
+ if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
+ ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows);
+
+ if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1;
+ ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows);
+
+ if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1;
+ if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1;
+ }
+
+ if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
+ }
} else {
ASSERT(!pHelper->hasOldLastBlock);
- tdResetDataCols(pDataCols);
- int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0);
- ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows);
+ tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo);
+ ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
- if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
- if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1;
+ if (pDataCols->numOfRows > 0) {
+ ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
+ if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
+ if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1;
+ }
}
#ifndef NDEBUG
TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter);
- ASSERT(keyNext < 0 || keyNext > pIdx->maxKey);
+ ASSERT(keyNext == TSDB_DATA_TIMESTAMP_NULL || keyNext > pIdx->maxKey);
#endif
return 0;
@@ -1496,13 +1547,16 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey,
int *blkIdx) {
- STsdbCfg * pCfg = &(pHelper->pRepo->config);
- STable * pTable = pCommitIter->pTable;
- SCompIdx * pIdx = &(pHelper->curCompIdx);
- SCompBlock compBlock = {0};
- TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
- int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
- SDataCols *pDataCols0 = pHelper->pDataCols[0];
+ STsdbCfg * pCfg = &(pHelper->pRepo->config);
+ STable * pTable = pCommitIter->pTable;
+ SCompIdx * pIdx = &(pHelper->curCompIdx);
+ SCompBlock compBlock = {0};
+ TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
+ int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
+ SDataCols * pDataCols0 = pHelper->pDataCols[0];
+ SMergeInfo mergeInfo = {0};
+ SMergeInfo *pMergeInfo = &mergeInfo;
+ SCompBlock oBlock = {0};
SSkipListIterator slIter = {0};
@@ -1512,123 +1566,82 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE);
ASSERT(pCompBlock != NULL);
int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock));
+ oBlock = *pCompBlock;
+
+ ASSERT((!TSDB_IS_LAST_BLOCK(&oBlock)) || (tblkIdx == pIdx->numOfBlocks - 1));
- if (pCompBlock->last) {
- ASSERT(pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && tblkIdx == pIdx->numOfBlocks - 1);
+ if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) {
+ while (true) {
+ tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0,
+ pCfg->update, pMergeInfo);
+ ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
+ if (pDataCols->numOfRows == 0) break;
+
+ if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
+ if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
+ tblkIdx++;
+ }
+ ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) == TSDB_DATA_TIMESTAMP_NULL ||
+ tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
+ } else {
int16_t colId = 0;
+ if (tsdbLoadBlockDataCols(pHelper, &oBlock, NULL, &colId, 1) < 0) return -1;
+
+ TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1);
+
slIter = *(pCommitIter->pIter);
- if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1;
- ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows);
+ tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows,
+ pCfg->update, pMergeInfo);
- int rows1 = defaultRowsInBlock - pCompBlock->numOfRows;
- int rows2 =
- tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows);
- if (rows2 == 0) { // all data filtered out
+ if (pMergeInfo->nOperations == 0) {
+ // Do nothing
+ ASSERT(pMergeInfo->rowsDeleteFailed >= 0);
*(pCommitIter->pIter) = slIter;
+ tblkIdx++;
+ } else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) {
+ // Delete the block and do some stuff
+ // ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN);
+ if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1;
+ *pCommitIter->pIter = slIter;
+ if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
+ } else if (tsdbCheckAddSubBlockCond(pHelper, &oBlock, pMergeInfo, pDataCols->maxPoints)) {
+ // Append as a sub-block of the searched block
+ tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData,
+ pDataCols0->numOfRows, pCfg->update, pMergeInfo);
+ ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0);
+ if (tsdbWriteBlockToFile(pHelper, oBlock.last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols,
+ &compBlock, oBlock.last, false) < 0) {
+ return -1;
+ }
+ if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) {
+ return -1;
+ }
+ tblkIdx++;
} else {
- if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock &&
- pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
- tdResetDataCols(pDataCols);
- int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols,
- pDataCols0->cols[0].pData, pDataCols0->numOfRows);
- ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows);
- if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
- if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1;
- tblkIdx++;
- } else {
- if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
- int round = 0;
- int dIter = 0;
- while (true) {
- tdResetDataCols(pDataCols);
- int rowsRead =
- tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, defaultRowsInBlock);
- if (rowsRead == 0) break;
+ // load the block data, merge with the memory data
+ if (tsdbLoadBlockData(pHelper, &oBlock, NULL) < 0) return -1;
+ int round = 0;
+ int dIter = 0;
+ while (true) {
+ tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock,
+ pCfg->update);
+ if (pDataCols->numOfRows == 0) break;
+ if (tblkIdx == pIdx->numOfBlocks - 1) {
if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
- if (round == 0) {
- if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
- } else {
- if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
- }
-
- tblkIdx++;
- round++;
+ } else {
+ if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
}
- }
- if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
- }
- } else {
- TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (pCompBlock[1].keyFirst - 1);
- TSKEY blkKeyFirst = pCompBlock->keyFirst;
- TSKEY blkKeyLast = pCompBlock->keyLast;
- if (keyFirst < blkKeyFirst) {
- while (true) {
- tdResetDataCols(pDataCols);
- int rowsRead =
- tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, NULL, 0);
- if (rowsRead == 0) break;
-
- ASSERT(rowsRead == pDataCols->numOfRows);
- if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
- if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
- tblkIdx++;
- }
- ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
- tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
- } else {
- ASSERT(keyFirst <= blkKeyLast);
- int16_t colId = 0;
- if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1;
-
- slIter = *(pCommitIter->pIter);
- int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows);
- int rows2 = tsdbLoadDataFromCache(pTable, &slIter, blkKeyLast, INT_MAX, NULL, pDataCols0->cols[0].pData,
- pDataCols0->numOfRows);
-
- if (rows2 == 0) { // all filtered out
- *(pCommitIter->pIter) = slIter;
- ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
- tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
- } else {
- int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0) + rows2;
-
- if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) {
- int rows = (rows1 >= rows3) ? rows3 : rows2;
- tdResetDataCols(pDataCols);
- int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols,
- pDataCols0->cols[0].pData, pDataCols0->numOfRows);
- ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows);
- if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0)
- return -1;
- if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1;
- tblkIdx++;
- ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
- tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
+ if (round == 0) {
+ if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
+ if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
} else {
- if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
- int round = 0;
- int dIter = 0;
- while (true) {
- int rowsRead =
- tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock);
- if (rowsRead == 0) break;
-
- if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0)
- return -1;
- if (round == 0) {
- if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
- } else {
- if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
- }
-
- round++;
- tblkIdx++;
- }
- ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
- tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
+ if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
}
+
+ round++;
+ tblkIdx++;
}
}
}
@@ -1637,9 +1650,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
return 0;
}
-static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
- TSKEY maxKey, int maxRows) {
- int numOfRows = 0;
+static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
+ TSKEY maxKey, int maxRows, int8_t update) {
TSKEY key1 = INT64_MAX;
TSKEY key2 = INT64_MAX;
STSchema *pSchema = NULL;
@@ -1649,35 +1661,62 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte
while (true) {
key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
+ bool isRowDel = false;
SDataRow row = tsdbNextIterRow(pCommitIter->pIter);
- key2 = (row == NULL || dataRowKey(row) > maxKey) ? INT64_MAX : dataRowKey(row);
+ if (row == NULL || dataRowKey(row) > maxKey) {
+ key2 = INT64_MAX;
+ } else {
+ key2 = dataRowKey(row);
+ isRowDel = dataRowDeleted(row);
+ }
if (key1 == INT64_MAX && key2 == INT64_MAX) break;
- if (key1 <= key2) {
+ if (key1 < key2) {
for (int i = 0; i < pDataCols->numOfCols; i++) {
dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
pTarget->maxPoints);
}
+
pTarget->numOfRows++;
(*iter)++;
- if (key1 == key2) tSkipListIterNext(pCommitIter->pIter);
- } else {
- if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
- pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
- ASSERT(pSchema != NULL);
+ } else if (key1 > key2) {
+ if (!isRowDel) {
+ if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
+ pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
+ ASSERT(pSchema != NULL);
+ }
+
+ tdAppendDataRowToDataCol(row, pSchema, pTarget);
}
- tdAppendDataRowToDataCol(row, pSchema, pTarget);
+ tSkipListIterNext(pCommitIter->pIter);
+ } else {
+ if (update) {
+ if (!isRowDel) {
+ if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
+ pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
+ ASSERT(pSchema != NULL);
+ }
+
+ tdAppendDataRowToDataCol(row, pSchema, pTarget);
+ }
+ } else {
+ ASSERT(!isRowDel);
+
+ for (int i = 0; i < pDataCols->numOfCols; i++) {
+ dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
+ pTarget->maxPoints);
+ }
+
+ pTarget->numOfRows++;
+ }
+ (*iter)++;
tSkipListIterNext(pCommitIter->pIter);
}
- numOfRows++;
- if (numOfRows >= maxRows) break;
- ASSERT(numOfRows == pTarget->numOfRows && numOfRows <= pTarget->maxPoints);
+ if (pTarget->numOfRows >= maxRows) break;
}
-
- return numOfRows;
}
static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock) {
@@ -1700,3 +1739,20 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols,
return 0;
}
+
+static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps) {
+ STsdbCfg *pCfg = &(pHelper->pRepo->config);
+ int mergeRows = pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
+
+ ASSERT(mergeRows > 0);
+
+ if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pMergeInfo->nOperations <= maxOps) {
+ if (pCompBlock->last) {
+ if (!TSDB_NLAST_FILE_OPENED(pHelper) && mergeRows < pCfg->minRowsPerFileBlock) return true;
+ } else {
+ if (mergeRows < pCfg->maxRowsPerFileBlock) return true;
+ }
+ }
+
+ return false;
+}
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index ac6c2e0c5a9d5590e7dd57863dd18d8726912de5..d5cc566b5541a41e863d4c136746019b4b172ce0 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -20,7 +20,6 @@
#include "exception.h"
#include "../../query/inc/qAst.h" // todo move to common module
-#include "../../query/inc/qExecutor.h" // todo move to common module
#include "tlosertree.h"
#include "tsdb.h"
#include "tsdbMain.h"
@@ -72,8 +71,8 @@ typedef struct STableCheckInfo {
STable* pTableObj;
SCompInfo* pCompInfo;
int32_t compSize;
- int32_t numOfBlocks; // number of qualified data blocks not the original blocks
- int32_t chosen; // indicate which iterator should move forward
+ int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks
+ int8_t chosen:2; // indicate which iterator should move forward
bool initBuf; // whether to initialize the in-memory skip list iterator or not
SSkipListIterator* iter; // mem buffer skip list iterator
SSkipListIterator* iiter; // imem buffer skip list iterator
@@ -120,8 +119,7 @@ typedef struct STsdbQueryHandle {
SDataCols *pDataCols; // in order to hold current file data block
int32_t allocSize; // allocated data block size
- SMemTable *mem; // mem-table
- SMemTable *imem; // imem-table, acquired from snapshot
+ SMemRef *pMemRef;
SArray *defaultLoadColumn;// default load column
SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
@@ -184,27 +182,31 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS
return pLocalIdList;
}
-static void tsdbMayTakeMemSnapshot(TsdbQueryHandleT pHandle) {
- STsdbQueryHandle* pSecQueryHandle = (STsdbQueryHandle*) pHandle;
- SQInfo *pQInfo = (SQInfo *)(pSecQueryHandle->qinfo);
+static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
+ assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL);
- if (pQInfo->memRef.ref++ == 0) {
- tsdbTakeMemSnapshot(pSecQueryHandle->pTsdb, &pSecQueryHandle->mem, &pSecQueryHandle->imem);
- pQInfo->memRef.mem = pSecQueryHandle->mem;
- pQInfo->memRef.imem = pSecQueryHandle->imem;
- } else {
- pSecQueryHandle->mem = (SMemTable *)(pQInfo->memRef.mem);
- pSecQueryHandle->imem = (SMemTable *)(pQInfo->memRef.imem);
+ SMemRef* pMemRef = pQueryHandle->pMemRef;
+ if (pQueryHandle->pMemRef->ref++ == 0) {
+ tsdbTakeMemSnapshot(pQueryHandle->pTsdb, (SMemTable**)&(pMemRef->mem), (SMemTable**)&(pMemRef->imem));
}
}
-static void tsdbMayUnTakeMemSnapshot(TsdbQueryHandleT pHandle) {
- STsdbQueryHandle* pSecQueryHandle = (STsdbQueryHandle*) pHandle;
- SQInfo *pQInfo = (SQInfo *)(pSecQueryHandle->qinfo);
- if (--pQInfo->memRef.ref == 0) {
- tsdbUnTakeMemSnapShot(pSecQueryHandle->pTsdb, pSecQueryHandle->mem, pSecQueryHandle->imem);
+static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
+ assert(pQueryHandle != NULL);
+ SMemRef* pMemRef = pQueryHandle->pMemRef;
+ if (pMemRef == NULL) { // it has been freed
+ return;
+ }
+
+ if (--pMemRef->ref == 0) {
+ tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, pMemRef->mem, pMemRef->imem);
+ pMemRef->mem = NULL;
+ pMemRef->imem = NULL;
}
+
+ pQueryHandle->pMemRef = NULL;
}
+
static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta) {
size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList);
assert(sizeOfGroup >= 1 && pMeta != NULL);
@@ -270,7 +272,7 @@ static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey)
return pNew;
}
-static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo) {
+static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo, SMemRef* pMemRef) {
STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle));
if (pQueryHandle == NULL) {
goto out_of_memory;
@@ -288,13 +290,14 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond*
pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock;
pQueryHandle->allocSize = 0;
pQueryHandle->locateStart = false;
+ pQueryHandle->pMemRef = pMemRef;
if (tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb) != 0) {
goto out_of_memory;
}
tsdbMayTakeMemSnapshot(pQueryHandle);
- assert(pCond != NULL && pCond->numOfCols > 0);
+ assert(pCond != NULL && pCond->numOfCols > 0 && pMemRef != NULL);
if (ASCENDING_TRAVERSE(pCond->order)) {
assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey);
@@ -348,8 +351,8 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond*
return NULL;
}
-TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) {
- STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo);
+TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo, SMemRef* pRef) {
+ STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo, pRef);
STsdbMeta* pMeta = tsdbGetMeta(tsdb);
assert(pMeta != NULL);
@@ -366,7 +369,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
return (TsdbQueryHandleT) pQueryHandle;
}
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) {
+TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) {
pCond->twindow = changeTableGroupByLastrow(groupList);
// no qualified table
@@ -374,7 +377,7 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
return NULL;
}
- STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
+ STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pMemRef);
assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey);
return pQueryHandle;
@@ -396,8 +399,8 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) {
return res;
}
-TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo) {
- STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
+TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) {
+ STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pRef);
if (pQueryHandle != NULL) {
pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
changeQueryHandleForInterpQuery(pQueryHandle);
@@ -417,7 +420,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
int32_t order = pHandle->order;
// no data in buffer, abort
- if (pHandle->mem == NULL && pHandle->imem == NULL) {
+ if (pHandle->pMemRef->mem == NULL && pHandle->pMemRef->imem == NULL) {
return false;
}
@@ -426,16 +429,19 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
STableData* pMem = NULL;
STableData* pIMem = NULL;
- if (pHandle->mem && pCheckInfo->tableId.tid < pHandle->mem->maxTables) {
- pMem = pHandle->mem->tData[pCheckInfo->tableId.tid];
+ SMemTable* pMemT = pHandle->pMemRef->mem;
+ SMemTable* pIMemT = pHandle->pMemRef->imem;
+
+ if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) {
+ pMem = pMemT->tData[pCheckInfo->tableId.tid];
if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid
pCheckInfo->iter =
tSkipListCreateIterFromVal(pMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
}
}
- if (pHandle->imem && pCheckInfo->tableId.tid < pHandle->imem->maxTables) {
- pIMem = pHandle->imem->tData[pCheckInfo->tableId.tid];
+ if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) {
+ pIMem = pIMemT->tData[pCheckInfo->tableId.tid];
if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid
pCheckInfo->iiter =
tSkipListCreateIterFromVal(pIMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
@@ -457,7 +463,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
assert(node != NULL);
- SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
+ SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
@@ -479,7 +485,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
assert(node != NULL);
- SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
+ SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
@@ -504,19 +510,19 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
tSkipListDestroyIter(pCheckInfo->iiter);
}
-static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) {
+static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) {
SDataRow rmem = NULL, rimem = NULL;
if (pCheckInfo->iter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
if (node != NULL) {
- rmem = *(SDataRow *)SL_GET_NODE_DATA(node);
+ rmem = (SDataRow)SL_GET_NODE_DATA(node);
}
}
if (pCheckInfo->iiter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
if (node != NULL) {
- rimem = *(SDataRow *)SL_GET_NODE_DATA(node);
+ rimem = (SDataRow)SL_GET_NODE_DATA(node);
}
}
@@ -538,9 +544,15 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order
TSKEY r2 = dataRowKey(rimem);
if (r1 == r2) { // data ts are duplicated, ignore the data in mem
- tSkipListIterNext(pCheckInfo->iter);
- pCheckInfo->chosen = 1;
- return rimem;
+ if (!update) {
+ tSkipListIterNext(pCheckInfo->iter);
+ pCheckInfo->chosen = 1;
+ return rimem;
+ } else {
+ tSkipListIterNext(pCheckInfo->iiter);
+ pCheckInfo->chosen = 0;
+ return rmem;
+ }
} else {
if (ASCENDING_TRAVERSE(order)) {
if (r1 < r2) {
@@ -594,6 +606,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
}
static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
+ STsdbCfg *pCfg = &pHandle->pTsdb->config;
size_t size = taosArrayGetSize(pHandle->pTableCheckInfo);
assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1);
pHandle->cur.fid = -1;
@@ -607,7 +620,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
initTableMemIterator(pHandle, pCheckInfo);
}
- SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order);
+ SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update);
if (row == NULL) {
return false;
}
@@ -827,11 +840,12 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl
static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){
SQueryFilePos* cur = &pQueryHandle->cur;
+ STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
int32_t code = TSDB_CODE_SUCCESS;
/*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
- SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
+ SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
assert(cur->pos >= 0 && cur->pos <= binfo.rows);
@@ -1263,7 +1277,6 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl
int32_t end = endPos;
if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
- assert(start >= end);
SWAP(start, end, int32_t);
}
@@ -1317,6 +1330,7 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl
static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
SQueryFilePos* cur = &pQueryHandle->cur;
SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
+ STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
initTableMemIterator(pQueryHandle, pCheckInfo);
@@ -1354,7 +1368,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
} else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
SSkipListNode* node = NULL;
do {
- SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
+ SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
if (row == NULL) {
break;
}
@@ -1384,7 +1398,22 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
moveToNextRowInMem(pCheckInfo);
} else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it
- moveToNextRowInMem(pCheckInfo);
+ if (pCfg->update) {
+ copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable);
+ numOfRows += 1;
+ if (cur->win.skey == TSKEY_INITIAL_VAL) {
+ cur->win.skey = key;
+ }
+
+ cur->win.ekey = key;
+ cur->lastKey = key + step;
+ cur->mixBlock = true;
+
+ moveToNextRowInMem(pCheckInfo);
+ pos += step;
+ } else {
+ moveToNextRowInMem(pCheckInfo);
+ }
} else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
if (cur->win.skey == TSKEY_INITIAL_VAL) {
@@ -1395,7 +1424,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
assert(end != -1);
if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
- moveToNextRowInMem(pCheckInfo);
+ if (!pCfg->update) {
+ moveToNextRowInMem(pCheckInfo);
+ } else {
+ end -= step;
+ }
}
int32_t qstart = 0, qend = 0;
@@ -1415,8 +1448,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
* copy them all to result buffer, since it may be overlapped with file data block.
*/
if (node == NULL ||
- ((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
- ((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
+ ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+ ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
// no data in cache or data in cache is greater than the ekey of time window, load data from file block
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = tsArray[pos];
@@ -1513,15 +1546,15 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
}
static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) {
- taosTFree(pSupporter->numOfBlocksPerTable);
- taosTFree(pSupporter->blockIndexArray);
+ tfree(pSupporter->numOfBlocksPerTable);
+ tfree(pSupporter->blockIndexArray);
for (int32_t i = 0; i < numOfTables; ++i) {
STableBlockInfo* pBlockInfo = pSupporter->pDataBlockInfo[i];
- taosTFree(pBlockInfo);
+ tfree(pBlockInfo);
}
- taosTFree(pSupporter->pDataBlockInfo);
+ tfree(pSupporter->pDataBlockInfo);
}
static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) {
@@ -1818,6 +1851,8 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) {
pQueryHandle->activeIndex += 1;
}
+ // no data in memtable or imemtable, decrease the memory reference.
+ tsdbMayUnTakeMemSnapshot(pQueryHandle);
return false;
}
@@ -1863,13 +1898,14 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
STsdbQueryHandle* pQueryHandle) {
int numOfRows = 0;
int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns);
+ STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
win->skey = TSKEY_INITIAL_VAL;
int64_t st = taosGetTimestampUs();
STable* pTable = pCheckInfo->pTableObj;
do {
- SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
+ SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
if (row == NULL) {
break;
}
@@ -1920,9 +1956,9 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
while (tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter);
- STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode);
+ STable* pTable = (STable*) SL_GET_NODE_DATA((SSkipListNode*) pNode);
- STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = pTable, .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(list, &info);
}
@@ -1938,122 +1974,136 @@ static void destroyHelper(void* param) {
tQueryInfo* pInfo = (tQueryInfo*)param;
if (pInfo->optr != TSDB_RELATION_IN) {
- taosTFree(pInfo->q);
+ tfree(pInfo->q);
}
free(param);
}
-// handle data in cache situation
-bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
- STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
+static bool getNeighborRows(STsdbQueryHandle* pQueryHandle) {
+ assert (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL);
- int64_t stime = taosGetTimestampUs();
- int64_t elapsedTime = stime;
+ SDataBlockInfo blockInfo = {{0}, 0};
- size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
- assert(numOfTables > 0);
+ pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
+ pQueryHandle->order = TSDB_ORDER_DESC;
- SDataBlockInfo blockInfo = {{0}, 0};
- if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) {
- pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
- pQueryHandle->order = TSDB_ORDER_DESC;
+ if (!tsdbNextDataBlock((void*) pQueryHandle)) {
+ return false;
+ }
- if (!tsdbNextDataBlock(pHandle)) {
- return false;
+ tsdbRetrieveDataBlockInfo((void*) pQueryHandle, &blockInfo);
+ /*SArray *pDataBlock = */tsdbRetrieveDataBlock((void*) pQueryHandle, pQueryHandle->defaultLoadColumn);
+ if (terrno != TSDB_CODE_SUCCESS) {
+ return false;
+ }
+
+ if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) {
+ // data already retrieve, discard other data rows and return
+ int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle));
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
+ memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes);
}
- tsdbRetrieveDataBlockInfo(pHandle, &blockInfo);
- /*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, pQueryHandle->defaultLoadColumn);
- if (terrno != TSDB_CODE_SUCCESS) {
+ pQueryHandle->cur.win = (STimeWindow){pQueryHandle->window.skey, pQueryHandle->window.skey};
+ pQueryHandle->window = pQueryHandle->cur.win;
+ pQueryHandle->cur.rows = 1;
+ pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
+ return true;
+ } else {
+ STimeWindow win = (STimeWindow) {pQueryHandle->window.skey, INT64_MAX};
+ STsdbQueryCond cond = {
+ .order = TSDB_ORDER_ASC,
+ .numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle))
+ };
+ cond.twindow = win;
+
+ cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo));
+ if (cond.colList == NULL) {
+ terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return false;
}
- if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) {
- // data already retrieve, discard other data rows and return
- int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle));
- for (int32_t i = 0; i < numOfCols; ++i) {
- SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
- memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes);
- }
+ for(int32_t i = 0; i < cond.numOfCols; ++i) {
+ SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i);
+ memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo));
+ }
- pQueryHandle->cur.win = (STimeWindow){pQueryHandle->window.skey, pQueryHandle->window.skey};
- pQueryHandle->window = pQueryHandle->cur.win;
- pQueryHandle->cur.rows = 1;
- pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
- return true;
- } else {
- STimeWindow win = (STimeWindow) {pQueryHandle->window.skey, INT64_MAX};
- STsdbQueryCond cond = {
- .order = TSDB_ORDER_ASC,
- .numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle))
- };
- cond.twindow = win;
-
- cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo));
- if (cond.colList == NULL) {
- terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
- return false;
- }
+ STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pQueryHandle->pMemRef);
- for(int32_t i = 0; i < cond.numOfCols; ++i) {
- SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i);
- memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo));
- }
+ tfree(cond.colList);
- STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo);
+ pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey);
+ if (pSecQueryHandle->pTableCheckInfo == NULL) {
+ tsdbCleanupQueryHandle(pSecQueryHandle);
+ return false;
+ }
- taosTFree(cond.colList);
+ if (!tsdbNextDataBlock((void*) pSecQueryHandle)) {
+ tsdbCleanupQueryHandle(pSecQueryHandle);
+ return false;
+ }
- pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey);
- if (pSecQueryHandle->pTableCheckInfo == NULL) {
- tsdbCleanupQueryHandle(pSecQueryHandle);
- return false;
- }
+ tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo);
+ tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn);
- if (!tsdbNextDataBlock((void*) pSecQueryHandle)) {
- tsdbCleanupQueryHandle(pSecQueryHandle);
- return false;
- }
+ int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pSecQueryHandle));
+ size_t si = taosArrayGetSize(pSecQueryHandle->pTableCheckInfo);
- tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo);
- tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn);
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
+ memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes);
- int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pSecQueryHandle));
- size_t si = taosArrayGetSize(pSecQueryHandle->pTableCheckInfo);
+ SColumnInfoData* pCol1 = taosArrayGet(pSecQueryHandle->pColumns, i);
+ assert(pCol->info.colId == pCol1->info.colId);
- for (int32_t i = 0; i < numOfCols; ++i) {
- SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
- memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes);
+ memcpy((char*)pCol->pData + pCol->info.bytes, pCol1->pData, pCol1->info.bytes);
+ }
- SColumnInfoData* pCol1 = taosArrayGet(pSecQueryHandle->pColumns, i);
- assert(pCol->info.colId == pCol1->info.colId);
+ SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0);
- memcpy((char*)pCol->pData + pCol->info.bytes, pCol1->pData, pCol1->info.bytes);
- }
+ // it is ascending order
+ pQueryHandle->order = TSDB_ORDER_DESC;
+ pQueryHandle->window = pQueryHandle->cur.win;
+ pQueryHandle->cur.win = (STimeWindow){((TSKEY*)pTSCol->pData)[0], ((TSKEY*)pTSCol->pData)[1]};
+ pQueryHandle->cur.rows = 2;
+ pQueryHandle->cur.mixBlock = true;
- SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0);
+ int32_t step = -1;// one step for ascending order traverse
+ for (int32_t j = 0; j < si; ++j) {
+ STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j);
+ pCheckInfo->lastKey = pQueryHandle->cur.win.ekey + step;
+ }
- // it is ascending order
- pQueryHandle->order = TSDB_ORDER_DESC;
- pQueryHandle->window = pQueryHandle->cur.win;
- pQueryHandle->cur.win = (STimeWindow){((TSKEY*)pTSCol->pData)[0], ((TSKEY*)pTSCol->pData)[1]};
- pQueryHandle->cur.rows = 2;
- pQueryHandle->cur.mixBlock = true;
+ tsdbCleanupQueryHandle(pSecQueryHandle);
+ }
- int32_t step = -1;// one step for ascending order traverse
- for (int32_t j = 0; j < si; ++j) {
- STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j);
- pCheckInfo->lastKey = pQueryHandle->cur.win.ekey + step;
- }
+ //disable it after retrieve data
+ pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
+ pQueryHandle->checkFiles = false;
+ return true;
+}
- tsdbCleanupQueryHandle(pSecQueryHandle);
- }
+// handle data in cache situation
+bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
+ STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
- //disable it after retrieve data
- pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
- pQueryHandle->checkFiles = false;
- return true;
+ int64_t stime = taosGetTimestampUs();
+ int64_t elapsedTime = stime;
+
+ size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
+ assert(numOfTables > 0);
+
+ if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) {
+ SMemRef* pMemRef = pQueryHandle->pMemRef;
+ tsdbMayTakeMemSnapshot(pQueryHandle);
+ bool ret = getNeighborRows(pQueryHandle);
+ tsdbMayUnTakeMemSnapshot(pQueryHandle);
+
+ // restore the pMemRef
+ pQueryHandle->pMemRef = pMemRef;
+ return ret;
}
if (pQueryHandle->checkFiles) {
@@ -2122,7 +2172,16 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) {
}
}
- // clear current group
+ // clear current group, unref unused table
+ for (int32_t i = 0; i < numOfTables; ++i) {
+ STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pGroup, i);
+
+ // keyInfo.pTable may be NULL here.
+ if (pKeyInfo->pTable != keyInfo.pTable) {
+ tsdbUnRefTable(pKeyInfo->pTable);
+ }
+ }
+
taosArrayClear(pGroup);
// more than one table in each group, only one table left for each group
@@ -2302,8 +2361,13 @@ void filterPrepare(void* expr, void* param) {
if (pInfo->optr == TSDB_RELATION_IN) {
pInfo->q = (char*) pCond->arr;
- } else {
- pInfo->q = calloc(1, pSchema->bytes + TSDB_NCHAR_SIZE); // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space.
+ } else if (pCond != NULL) {
+ uint32_t size = pCond->nLen * TSDB_NCHAR_SIZE;
+ if (size < (uint32_t)pSchema->bytes) {
+ size = pSchema->bytes;
+ }
+ // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space.
+ pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
tVariantDump(pCond, pInfo->q, pSchema->type, true);
}
}
@@ -2452,7 +2516,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
static bool indexedNodeFilterFp(const void* pNode, void* param) {
tQueryInfo* pInfo = (tQueryInfo*) param;
- STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
+ STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
char* val = NULL;
@@ -2705,7 +2769,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
destroyTableMemIterator(pTableCheckInfo);
- taosTFree(pTableCheckInfo->pCompInfo);
+ tfree(pTableCheckInfo->pCompInfo);
}
taosArrayDestroy(pQueryHandle->pTableCheckInfo);
}
@@ -2714,14 +2778,14 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
size_t cols = taosArrayGetSize(pQueryHandle->pColumns);
for (int32_t i = 0; i < cols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
- taosTFree(pColInfo->pData);
+ tfree(pColInfo->pData);
}
taosArrayDestroy(pQueryHandle->pColumns);
}
taosArrayDestroy(pQueryHandle->defaultLoadColumn);
- taosTFree(pQueryHandle->pDataBlockInfo);
- taosTFree(pQueryHandle->statis);
+ tfree(pQueryHandle->pDataBlockInfo);
+ tfree(pQueryHandle->statis);
// todo check error
tsdbMayUnTakeMemSnapshot(pQueryHandle);
@@ -2735,7 +2799,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %p",
pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo);
- taosTFree(pQueryHandle);
+ tfree(pQueryHandle);
}
void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) {
diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp
index 605586515b7b1626751c01a550925c6e9ac4e183..ef5ed6f04459a4213e761f94ad00363ede9ecd26 100644
--- a/src/tsdb/tests/tsdbTests.cpp
+++ b/src/tsdb/tests/tsdbTests.cpp
@@ -80,7 +80,7 @@ static int insertData(SInsertInfo *pInfo) {
pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) {
- taosTFree(pMsg);
+ tfree(pMsg);
return -1;
}
}
@@ -88,7 +88,7 @@ static int insertData(SInsertInfo *pInfo) {
double etime = getCurTime();
printf("Spent %f seconds to write %d records\n", etime - stime, pInfo->totalRows);
- taosTFree(pMsg);
+ tfree(pMsg);
return 0;
}
diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt
index df4ef0d78d66001bcbc575060850ed975c55fdc4..3e212ba2d30fb6abb1177fb6883fedeb3626bcbb 100644
--- a/src/util/CMakeLists.txt
+++ b/src/util/CMakeLists.txt
@@ -1,13 +1,14 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z)
IF (TD_LINUX)
TARGET_LINK_LIBRARIES(tutil m rt)
- ADD_SUBDIRECTORY(tests)
+ # ADD_SUBDIRECTORY(tests)
FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
IF (ICONV_INCLUDE_EXIST)
diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h
index 2a58bec8830faab4cf9909bd417a44e2e8881570..42bc136584d618e62826c307b698746ca387ff41 100644
--- a/src/util/inc/hash.h
+++ b/src/util/inc/hash.h
@@ -31,14 +31,16 @@ extern "C" {
typedef void (*_hash_free_fn_t)(void *param);
typedef struct SHashNode {
- char *key;
-// struct SHashNode *prev;
+// char *key;
struct SHashNode *next;
- uint32_t hashVal; // the hash value of key, if hashVal == HASH_VALUE_IN_TRASH, this node is moved to trash
+ uint32_t hashVal; // the hash value of key
uint32_t keyLen; // length of the key
- char *data;
+// char *data;
} SHashNode;
+#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode))
+#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->keyLen)
+
typedef enum SHashLockTypeE {
HASH_NO_LOCK = 0,
HASH_ENTRY_LOCK = 1,
@@ -175,6 +177,8 @@ void* taosHashDestroyIter(SHashMutableIterator* iter);
*/
int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj);
+size_t taosHashGetMemSize(const SHashObj *pHashObj);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h
index 71838af15064ec3d84662dff031d40d2156924eb..bf922fe9c44b4f923d44f4a38c1c4c1a7fb20af3 100644
--- a/src/util/inc/tarray.h
+++ b/src/util/inc/tarray.h
@@ -70,6 +70,13 @@ void* taosArrayGet(const SArray* pArray, size_t index);
*/
void* taosArrayGetP(const SArray* pArray, size_t index);
+/**
+ * get the last element in the array list
+ * @param pArray
+ * @return
+ */
+void* taosArrayGetLast(const SArray* pArray);
+
/**
* return the size of array
* @param pArray
@@ -117,6 +124,13 @@ void taosArrayClear(SArray* pArray);
*/
void taosArrayDestroy(SArray* pArray);
+/**
+ *
+ * @param pArray
+ * @param fp
+ */
+void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*));
+
/**
* sort the array
* @param pArray
diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h
new file mode 100644
index 0000000000000000000000000000000000000000..10b7c1df35caaa675c9334106785b91af3fdeb05
--- /dev/null
+++ b/src/util/inc/tfile.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_TFILE_H
+#define TDENGINE_TFILE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include
+
+// init taos file module
+int32_t tfinit();
+
+// clean up taos file module
+void tfcleanup();
+
+// the same syntax as UNIX standard open/close/read/write
+// but FD is int64_t and will never be reused
+int64_t tfopen(const char *pathname, int32_t flags);
+int64_t tfclose(int64_t tfd);
+int64_t tfwrite(int64_t tfd, void *buf, int64_t count);
+int64_t tfread(int64_t tfd, void *buf, int64_t count);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TDENGINE_TREF_H
diff --git a/src/util/inc/tnettest.h b/src/util/inc/tnettest.h
index 3fe1dfa9204fbbf85f193078b17e0bb4f9643848..426df5cbb28b9c0fcada049c7242730359d2a3fc 100644
--- a/src/util/inc/tnettest.h
+++ b/src/util/inc/tnettest.h
@@ -20,7 +20,27 @@
extern "C" {
#endif
-void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole);
+typedef struct CmdArguments {
+ char* host;
+ char* password;
+ char* user;
+ char* auth;
+ char* database;
+ char* timezone;
+ bool is_raw_time;
+ bool is_use_passwd;
+ char file[TSDB_FILENAME_LEN];
+ char dir[TSDB_FILENAME_LEN];
+ int threadNum;
+ char* commands;
+ int abort;
+ int port;
+ int endPort;
+ int pktLen;
+ char* netTestRole;
+} CmdArguments;
+
+void taosNetTest(CmdArguments* args);
#ifdef __cplusplus
}
diff --git a/src/util/inc/tqueue.h b/src/util/inc/tqueue.h
index 8493a64315966aa32b58359652d0f429e8e0916a..c3051464e556860178be36f3473f5e4686f6082e 100644
--- a/src/util/inc/tqueue.h
+++ b/src/util/inc/tqueue.h
@@ -20,6 +20,23 @@
extern "C" {
#endif
+/*
+
+This set of API for queue is designed specially for vnode/mnode. The main purpose is to
+consume all the items instead of one item from a queue by one single read. Also, it can
+combine multiple queues into a queue set, a consumer thread can consume a queue set via
+a single API instead of looping every queue by itself.
+
+Notes:
+1: taosOpenQueue/taosCloseQueue, taosOpenQset/taosCloseQset is NOT multi-thread safe
+2: after taosCloseQueue/taosCloseQset is called, read/write operation APIs are not safe.
+3: read/write operation APIs are multi-thread safe
+
+To remove the limitation and make this set of queue APIs multi-thread safe, REF(tref.c)
+shall be used to set up the protection.
+
+*/
+
typedef void* taos_queue;
typedef void* taos_qset;
typedef void* taos_qall;
diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h
new file mode 100644
index 0000000000000000000000000000000000000000..cd5092f30a290c51de49e38b0226bbed637dc0e6
--- /dev/null
+++ b/src/util/inc/tref.h
@@ -0,0 +1,75 @@
+
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_TREF_H
+#define TDENGINE_TREF_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// open a reference set, max is the mod used by hash, fp is the pointer to free resource function
+// return rsetId which will be used by other APIs. On error, -1 is returned, and terrno is set appropriately
+int taosOpenRef(int max, void (*fp)(void *));
+
+// close the reference set, refId is the return value by taosOpenRef
+// return 0 if success. On error, -1 is returned, and terrno is set appropriately
+int taosCloseRef(int refId);
+
+// add ref, p is the pointer to resource or pointer ID
+// return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately
+int64_t taosAddRef(int refId, void *p);
+
+// remove ref, rid is the reference ID returned by taosAddRef
+// return 0 if success. On error, -1 is returned, and terrno is set appropriately
+int taosRemoveRef(int rsetId, int64_t rid);
+
+// acquire ref, rid is the reference ID returned by taosAddRef
+// return the resource p. On error, NULL is returned, and terrno is set appropriately
+void *taosAcquireRef(int rsetId, int64_t rid);
+
+// release ref, rid is the reference ID returned by taosAddRef
+// return 0 if success. On error, -1 is returned, and terrno is set appropriately
+int taosReleaseRef(int rsetId, int64_t rid);
+
+// return the first reference if rid is 0, otherwise return the next after current reference.
+// if return value is NULL, it means list is over(if terrno is set, it means error happens)
+void *taosIterateRef(int rsetId, int64_t rid);
+
+// return the number of references in system
+int taosListRef();
+
+/* sample code to iterate the refs
+
+void demoIterateRefs(int rsetId) {
+
+ void *p = taosIterateRef(refId, 0);
+ while (p) {
+ // process P
+
+ // get the rid from p
+
+ p = taosIterateRef(rsetId, rid);
+ }
+}
+
+*/
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TDENGINE_TREF_H
diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h
index a14a8565617bc08eefc989168a93f523f2e6caff..2c4d1a86ef7446f49a2c1a97e25db558f933bd57 100644
--- a/src/util/inc/tskiplist.h
+++ b/src/util/inc/tskiplist.h
@@ -27,33 +27,25 @@ extern "C" {
#define MAX_SKIP_LIST_LEVEL 15
#define SKIP_LIST_RECORD_PERFORMANCE 0
+// For key property setting
+#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage)
+#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case)
+#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update=1 case)
+// For thread safety setting
+#define SL_THREAD_SAFE (uint8_t)0x4
+
typedef char *SSkipListKey;
typedef char *(*__sl_key_fn_t)(const void *);
-/**
- * the skip list node is located in a consecutive memory area,
- * the format of skip list node is as follows:
- * +------------+-----------------------+------------------------+-----+------+
- * | node level | forward pointer array | backward pointer array | key | data |
- * +------------+-----------------------+------------------------+-----+------+
- */
typedef struct SSkipListNode {
- uint8_t level;
+ uint8_t level;
+ void * pData;
+ struct SSkipListNode *forwards[];
} SSkipListNode;
-#define SL_NODE_HEADER_SIZE(_l) (sizeof(SSkipListNode) + ((_l) << 1u) * POINTER_BYTES)
-
-#define SL_GET_FORWARD_POINTER(n, _l) ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode)))[(_l)]
-#define SL_GET_BACKWARD_POINTER(n, _l) \
- ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)]
-
-#define SL_GET_NODE_DATA(n) ((char *)(n) + SL_NODE_HEADER_SIZE((n)->level))
-#define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n)))
-
-#define SL_GET_SL_MIN_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_FORWARD_POINTER((s)->pHead, 0)))
-#define SL_GET_SL_MAX_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_BACKWARD_POINTER((s)->pTail, 0)))
-
-#define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n))
+#define SL_GET_NODE_DATA(n) (n)->pData
+#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)]
+#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)]
/*
* @version 0.3
@@ -103,34 +95,23 @@ typedef struct tSkipListState {
uint64_t nTotalElapsedTimeForInsert;
} tSkipListState;
-typedef struct SSkipListKeyInfo {
- uint8_t dupKey : 2; // if allow duplicated key in the skip list
- uint8_t type : 4; // key type
- uint8_t freeNode:2; // free node when destroy the skiplist
- uint8_t len; // maximum key length, used in case of string key
-} SSkipListKeyInfo;
-
typedef struct SSkipList {
__compar_fn_t comparFn;
__sl_key_fn_t keyFn;
- uint32_t size;
+ pthread_rwlock_t *lock;
+ uint16_t len;
uint8_t maxLevel;
+ uint8_t flags;
+ uint8_t type; // static info above
uint8_t level;
- SSkipListKeyInfo keyInfo;
- pthread_rwlock_t *lock;
- SSkipListNode * pHead; // point to the first element
- SSkipListNode * pTail; // point to the last element
+ uint32_t size;
+ SSkipListNode * pHead; // point to the first element
+ SSkipListNode * pTail; // point to the last element
#if SKIP_LIST_RECORD_PERFORMANCE
tSkipListState state; // skiplist state
#endif
} SSkipList;
-/*
- * iterate the skiplist
- * this will cause the multi-thread problem, when the skiplist is destroyed, the iterate may
- * continue iterating the skiplist, so add the reference count for skiplist
- * TODO add the ref for skip list when one iterator is created
- */
typedef struct SSkipListIterator {
SSkipList * pSkipList;
SSkipListNode *cur;
@@ -139,114 +120,27 @@ typedef struct SSkipListIterator {
SSkipListNode *next; // next points to the true qualified node in skip list
} SSkipListIterator;
-/**
- *
- * @param nMaxLevel maximum skip list level
- * @param keyType type of key
- * @param dupKey allow the duplicated key in the skip list
- * @return
- */
-SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe,
- uint8_t freeNode, __sl_key_fn_t fn);
-
-/**
- *
- * @param pSkipList
- * @return NULL will always be returned
- */
-void *tSkipListDestroy(SSkipList *pSkipList);
-
-/**
- *
- * @param pSkipList
- * @param level
- * @param headSize
- */
-void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize);
-
-/**
- * put the skip list node into the skip list.
- * If failed, NULL will be returned, otherwise, the pNode will be returned.
- *
- * @param pSkipList
- * @param pNode
- * @return
- */
-SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode);
-
-/**
- * get *all* nodes which key are equivalent to pKey
- *
- * @param pSkipList
- * @param pKey
- * @return
- */
-SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
-
-/**
- * get the size of skip list
- * @param pSkipList
- * @return
- */
-size_t tSkipListGetSize(const SSkipList *pSkipList);
-
-/**
- * display skip list of the given level, for debug purpose only
- * @param pSkipList
- * @param nlevel
- */
-void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
-
-/**
- * create skiplist iterator
- * @param pSkipList
- * @return
- */
+#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE)
+#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1))
+#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData))
+#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0))
+#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0))
+#define SL_SIZE(s) (s)->size
+
+SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags,
+ __sl_key_fn_t fn);
+void tSkipListDestroy(SSkipList *pSkipList);
+SSkipListNode * tSkipListPut(SSkipList *pSkipList, void *pData);
+void tSkipListPutBatch(SSkipList *pSkipList, void **ppData, int ndata);
+SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
+void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList);
-
-/**
- * create skip list iterator from the given node and specified the order
- * @param pSkipList
- * @param pNode start position, instead of the first node in skip list
- * @param order traverse order of the iterator
- * @return
- */
-SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order);
-
-/**
- * forward the skip list iterator
- * @param iter
- * @return
- */
-bool tSkipListIterNext(SSkipListIterator *iter);
-
-/**
- * get the element of skip list node
- * @param iter
- * @return
- */
-SSkipListNode *tSkipListIterGet(SSkipListIterator *iter);
-
-/**
- * destroy the skip list node
- * @param iter
- * @return
- */
-void *tSkipListDestroyIter(SSkipListIterator *iter);
-
-/*
- * remove nodes of the pKey value.
- * If more than one node has the same value, all will be removed
- *
- * @Return
- * the count of removed nodes
- */
-uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key);
-
-/*
- * remove the specified node in parameters
- */
-void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
+SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order);
+bool tSkipListIterNext(SSkipListIterator *iter);
+SSkipListNode * tSkipListIterGet(SSkipListIterator *iter);
+void * tSkipListDestroyIter(SSkipListIterator *iter);
+uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key);
+void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
#ifdef __cplusplus
}
diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h
index f14e8dbb356e131bbd580f8c5310bea944e71033..391cc44accadd785fb46e02aa79cde7617834c70 100644
--- a/src/util/inc/tsocket.h
+++ b/src/util/inc/tsocket.h
@@ -20,21 +20,21 @@
extern "C" {
#endif
-int taosReadn(SOCKET sock, char *buffer, int len);
-int taosWriteMsg(SOCKET fd, void *ptr, int nbytes);
-int taosReadMsg(SOCKET fd, void *ptr, int nbytes);
-int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes);
-int taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len);
-int taosSetNonblocking(SOCKET sock, int on);
+int32_t taosReadn(SOCKET sock, char *buffer, int32_t len);
+int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes);
+int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes);
+int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes);
+int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len);
+int32_t taosSetNonblocking(SOCKET sock, int32_t on);
-SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
-SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
-SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
-int taosKeepTcpAlive(SOCKET sockFd);
+SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
+SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
+SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
+int32_t taosKeepTcpAlive(SOCKET sockFd);
-int taosGetFqdn(char *);
+int32_t taosGetFqdn(char *);
uint32_t taosGetIpFromFqdn(const char *);
-void tinet_ntoa(char *ipstr, unsigned int ip);
+void tinet_ntoa(char *ipstr, uint32_t ip);
uint32_t ip2uint(const char *const ip_addr);
#ifdef __cplusplus
diff --git a/src/util/src/hash.c b/src/util/src/hash.c
index 625d4af1ac981c5d1f5f079f5b15533a4d63ef24..03a73424971ba7e4b77d508b12f71c729889a51a 100644
--- a/src/util/src/hash.c
+++ b/src/util/src/hash.c
@@ -22,14 +22,13 @@
#define DO_FREE_HASH_NODE(_n) \
do { \
- taosTFree((_n)->data); \
- taosTFree(_n); \
+ tfree(_n); \
} while (0)
#define FREE_HASH_NODE(_h, _n) \
do { \
if ((_h)->freeFp) { \
- (_h)->freeFp((_n)->data); \
+ (_h)->freeFp(GET_HASH_NODE_DATA(_n)); \
} \
\
DO_FREE_HASH_NODE(_n); \
@@ -77,7 +76,7 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) {
SHashNode *pNode = pe->next;
while (pNode) {
- if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) {
+ if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
assert(pNode->hashVal == hashVal);
break;
}
@@ -115,11 +114,15 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p
* @param dsize size of actual data
* @return hash node
*/
-static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode *pNode, SHashNode *pNewNode) {
+static FORCE_INLINE SHashNode *doUpdateHashNode(SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) {
assert(pNode->keyLen == pNewNode->keyLen);
- SWAP(pNode->key, pNewNode->key, void *);
- SWAP(pNode->data, pNewNode->data, void *);
+ if (prev != NULL) {
+ prev->next = pNewNode;
+ } else {
+ pe->next = pNewNode;
+ }
+ pNewNode->next = pNode->next;
return pNewNode;
}
@@ -208,12 +211,14 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
assert(pNode == NULL);
}
+ SHashNode* prev = NULL;
while (pNode) {
- if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) {
+ if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
assert(pNode->hashVal == hashVal);
break;
}
+ prev = pNode;
pNode = pNode->next;
}
@@ -239,7 +244,10 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
} else {
// not support the update operation, return error
if (pHashObj->enableUpdate) {
- doUpdateHashNode(pNode, pNewNode);
+ doUpdateHashNode(pe, prev, pNode, pNewNode);
+ DO_FREE_HASH_NODE(pNode);
+ } else {
+ DO_FREE_HASH_NODE(pNewNode);
}
if (pHashObj->type == HASH_ENTRY_LOCK) {
@@ -249,7 +257,6 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
// enable resize
__rd_unlock(&pHashObj->lock, pHashObj->type);
- DO_FREE_HASH_NODE(pNewNode);
return pHashObj->enableUpdate ? 0 : -1;
}
}
@@ -293,13 +300,13 @@ void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*f
SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal);
if (pNode != NULL) {
if (fp != NULL) {
- fp(pNode->data);
+ fp(GET_HASH_NODE_DATA(pNode));
}
if (d != NULL) {
- memcpy(d, pNode->data, dsize);
+ memcpy(d, GET_HASH_NODE_DATA(pNode), dsize);
} else {
- data = pNode->data;
+ data = GET_HASH_NODE_DATA(pNode);
}
}
@@ -357,13 +364,13 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
SHashNode *pRes = NULL;
// remove it
- if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) {
+ if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
pe->num -= 1;
pRes = pNode;
pe->next = pNode->next;
} else {
while (pNode->next != NULL) {
- if (((pNode->next)->keyLen == keyLen) && (memcmp((pNode->next)->key, key, keyLen) == 0)) {
+ if (((pNode->next)->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY((pNode->next)), key, keyLen) == 0)) {
assert((pNode->next)->hashVal == hashVal);
break;
}
@@ -392,7 +399,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
__rd_unlock(&pHashObj->lock, pHashObj->type);
if (data != NULL && pRes != NULL) {
- memcpy(data, pRes->data, dsize);
+ memcpy(data, GET_HASH_NODE_DATA(pRes), dsize);
}
if (pRes != NULL) {
@@ -426,7 +433,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi
// todo remove the first node
SHashNode *pNode = NULL;
while((pNode = pEntry->next) != NULL) {
- if (fp && (!fp(param, pNode->data))) {
+ if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) {
pEntry->num -= 1;
atomic_sub_fetch_64(&pHashObj->size, 1);
@@ -451,7 +458,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi
while ((pNext = pNode->next) != NULL) {
// not qualified, remove it
- if (fp && (!fp(param, pNext->data))) {
+ if (fp && (!fp(param, GET_HASH_NODE_DATA(pNext)))) {
pNode->next = pNext->next;
pEntry->num -= 1;
atomic_sub_fetch_64(&pHashObj->size, 1);
@@ -515,7 +522,7 @@ void taosHashCleanup(SHashObj *pHashObj) {
size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock);
for (int32_t i = 0; i < memBlock; ++i) {
void *p = taosArrayGetP(pHashObj->pMemBlock, i);
- taosTFree(p);
+ tfree(p);
}
taosArrayDestroy(pHashObj->pMemBlock);
@@ -605,7 +612,7 @@ bool taosHashIterNext(SHashMutableIterator *pIter) {
}
}
-void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : iter->pCur->data; }
+void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : GET_HASH_NODE_DATA(iter->pCur); }
void *taosHashDestroyIter(SHashMutableIterator *iter) {
if (iter == NULL) {
@@ -743,21 +750,19 @@ void taosHashTableResize(SHashObj *pHashObj) {
}
SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) {
- SHashNode *pNewNode = calloc(1, sizeof(SHashNode));
+ SHashNode *pNewNode = calloc(1, sizeof(SHashNode) + keyLen + dsize);
if (pNewNode == NULL) {
uError("failed to allocate memory, reason:%s", strerror(errno));
return NULL;
}
- pNewNode->data = malloc(dsize + keyLen);
- memcpy(pNewNode->data, pData, dsize);
-
- pNewNode->key = pNewNode->data + dsize;
- memcpy(pNewNode->key, key, keyLen);
-
pNewNode->keyLen = (uint32_t)keyLen;
pNewNode->hashVal = hashVal;
+
+ memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize);
+ memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen);
+
return pNewNode;
}
@@ -798,3 +803,11 @@ SHashNode *getNextHashNode(SHashMutableIterator *pIter) {
return NULL;
}
+
+size_t taosHashGetMemSize(const SHashObj *pHashObj) {
+ if (pHashObj == NULL) {
+ return 0;
+ }
+
+ return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj);
+}
diff --git a/src/util/src/talgo.c b/src/util/src/talgo.c
index 4b96e62e91ff440684328d7e7ea8c8c6cd783caa..278683539e3247b4b6dcd43687ac281368a7d31d 100644
--- a/src/util/src/talgo.c
+++ b/src/util/src/talgo.c
@@ -153,7 +153,7 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, size_t size, const
void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ext_compar_fn_t comparFn) {
char *buf = calloc(1, size); // prepare the swap buffer
tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf);
- taosTFree(buf);
+ tfree(buf);
}
void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t compar, int flags) {
diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c
index 65147b38def067bdca5de7c403835f8505a9b98e..bec2fac7dfd03ada681b9011df7e465cd0a49f0c 100644
--- a/src/util/src/tarray.c
+++ b/src/util/src/tarray.c
@@ -99,6 +99,10 @@ void* taosArrayGetP(const SArray* pArray, size_t index) {
return *(void**)d;
}
+void* taosArrayGetLast(const SArray* pArray) {
+ return TARRAY_GET_ELEM(pArray, pArray->size - 1);
+}
+
size_t taosArrayGetSize(const SArray* pArray) { return pArray->size; }
void* taosArrayInsert(SArray* pArray, size_t index, void* pData) {
@@ -189,6 +193,23 @@ void taosArrayDestroy(SArray* pArray) {
free(pArray);
}
+void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) {
+ if (pArray == NULL) {
+ return;
+ }
+
+ if (fp == NULL) {
+ taosArrayDestroy(pArray);
+ return;
+ }
+
+ for(int32_t i = 0; i < pArray->size; ++i) {
+ fp(TARRAY_GET_ELEM(pArray, i));
+ }
+
+ taosArrayDestroy(pArray);
+}
+
void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) {
assert(pArray != NULL);
assert(compar != NULL);
diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c
index 5be7253f6db80802feac5e51778fe89f5858733f..2571f11ba41b653e1fcc639cf1c22ef1f50b3448 100644
--- a/src/util/src/tcache.c
+++ b/src/util/src/tcache.c
@@ -228,7 +228,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v
pCacheObj->freeFp(p->data);
}
- taosTFree(p);
+ tfree(p);
} else {
taosAddToTrashcan(pCacheObj, p);
uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p->data);
@@ -614,7 +614,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) {
__cache_lock_destroy(pCacheObj);
- taosTFree(pCacheObj->name);
+ tfree(pCacheObj->name);
memset(pCacheObj, 0, sizeof(SCacheObj));
free(pCacheObj);
}
@@ -658,7 +658,11 @@ void* taosCacheTimedRefresh(void *handle) {
int64_t count = 0;
while(1) {
+#if defined LINUX
+ usleep(500*1000);
+#else
taosMsleep(500);
+#endif
// check if current cache object will be deleted every 500ms.
if (pCacheObj->deleting) {
@@ -677,6 +681,7 @@ void* taosCacheTimedRefresh(void *handle) {
continue;
}
+ uDebug("%s refresh thread timed scan", pCacheObj->name);
pCacheObj->statistics.refreshCount++;
// refresh data in hash table
diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c
index ba711ced8ff074e639a046c5dd9908865a3e9468..ff67b1f3ecf88c6da40d7b47cdac426a51636a48 100644
--- a/src/util/src/tcompare.c
+++ b/src/util/src/tcompare.c
@@ -1,7 +1,6 @@
#include "taosdef.h"
#include "tcompare.h"
#include "tarray.h"
-#include "tutil.h"
int32_t compareInt32Val(const void *pLeft, const void *pRight) {
int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);
@@ -367,7 +366,14 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) {
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_BOOL: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2));
case TSDB_DATA_TYPE_NCHAR: {
- int32_t ret = wcsncmp((wchar_t*) f1, (wchar_t*) f2, size/TSDB_NCHAR_SIZE);
+ tstr* t1 = (tstr*) f1;
+ tstr* t2 = (tstr*) f2;
+
+ if (t1->len != t2->len) {
+ return t1->len > t2->len? 1:-1;
+ }
+
+ int32_t ret = wcsncmp((wchar_t*) t1->data, (wchar_t*) t2->data, t2->len/TSDB_NCHAR_SIZE);
if (ret == 0) {
return ret;
}
diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c
index af54888cb4466dcd188f7a27146c78c9c5cc1d09..199f7d14f9996aee3d730a8ee3848f7e3b1ecd81 100644
--- a/src/util/src/tconfig.c
+++ b/src/util/src/tconfig.c
@@ -296,7 +296,7 @@ void taosReadGlobalLogCfg() {
option = value = NULL;
olen = vlen = 0;
- taosGetline(&line, &len, fp);
+ tgetline(&line, &len, fp);
line[len - 1] = 0;
paGetToken(line, &option, &olen);
@@ -310,7 +310,7 @@ void taosReadGlobalLogCfg() {
taosReadLogOption(option, value);
}
- taosTFree(line);
+ tfree(line);
fclose(fp);
}
@@ -342,7 +342,7 @@ bool taosReadGlobalCfg() {
option = value = value2 = value3 = NULL;
olen = vlen = vlen2 = vlen3 = 0;
- taosGetline(&line, &len, fp);
+ tgetline(&line, &len, fp);
line[len - 1] = 0;
paGetToken(line, &option, &olen);
@@ -365,8 +365,12 @@ bool taosReadGlobalCfg() {
fclose(fp);
- taosTFree(line);
-
+ tfree(line);
+
+ if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) {
+ taosSetAllDebugFlag();
+ }
+
return true;
}
diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c
new file mode 100644
index 0000000000000000000000000000000000000000..27ba30fe8179ecd5e213d4cf78862fd4982b3ba4
--- /dev/null
+++ b/src/util/src/tfile.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "os.h"
+#include "taoserror.h"
+#include "tulog.h"
+#include "tutil.h"
+#include "tref.h"
+
+static int32_t tsFileRsetId = -1;
+
+static void taosCloseFile(void *p) {
+ close((int32_t)(uintptr_t)p);
+}
+
+int32_t tfinit() {
+ tsFileRsetId = taosOpenRef(2000, taosCloseFile);
+ return tsFileRsetId;
+}
+
+void tfcleanup() {
+ if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
+ tsFileRsetId = -1;
+}
+
+int64_t tfopen(const char *pathname, int32_t flags) {
+ int32_t fd = open(pathname, flags);
+
+ if (fd < 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ void *p = (void *)(int64_t)fd;
+ int64_t rid = taosAddRef(tsFileRsetId, p);
+ if (rid < 0) close(fd);
+
+ return rid;
+}
+
+int64_t tfclose(int64_t tfd) {
+ return taosRemoveRef(tsFileRsetId, tfd);
+}
+
+int64_t tfwrite(int64_t tfd, void *buf, int64_t count) {
+ void *p = taosAcquireRef(tsFileRsetId, tfd);
+ if (p == NULL) return -1;
+
+ int32_t fd = (int32_t)(uintptr_t)p;
+
+ int64_t ret = taosWrite(fd, buf, count);
+ if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
+
+ taosReleaseRef(tsFileRsetId, tfd);
+ return ret;
+}
+
+int64_t tfread(int64_t tfd, void *buf, int64_t count) {
+ void *p = taosAcquireRef(tsFileRsetId, tfd);
+ if (p == NULL) return -1;
+
+ int32_t fd = (int32_t)(uintptr_t)p;
+
+ int64_t ret = taosRead(fd, buf, count);
+ if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
+
+ taosReleaseRef(tsFileRsetId, tfd);
+ return ret;
+}
diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c
index 6ba1d87d92ecf216bfde346f9d3ff1563515d34d..31641ac9a74d3fc914dcc05443da9384ba20c339 100644
--- a/src/util/src/tkvstore.c
+++ b/src/util/src/tkvstore.c
@@ -14,9 +14,7 @@
*/
#define _DEFAULT_SOURCE
-
#define TAOS_RANDOM_FILE_FAIL_TEST
-
#include "os.h"
#include "hash.h"
#include "taoserror.h"
@@ -188,7 +186,7 @@ int tdKVStoreStartCommit(SKVStore *pStore) {
goto _err;
}
- if (taosTSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+ if (taosSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
uError("failed to send file %d bytes since %s", TD_KVSTORE_HEADER_SIZE, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
@@ -238,6 +236,7 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe
rInfo.offset = lseek(pStore->fd, 0, SEEK_CUR);
if (rInfo.offset < 0) {
uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@@ -248,14 +247,15 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe
ASSERT(tlen == POINTER_DISTANCE(pBuf, buf));
ASSERT(tlen == sizeof(SKVRecord));
- if (taosTWrite(pStore->fd, buf, tlen) < tlen) {
+ if (taosWrite(pStore->fd, buf, tlen) < tlen) {
uError("failed to write %d bytes to file %s since %s", tlen, pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
- if (taosTWrite(pStore->fd, cont, contLen) < contLen) {
+ if (taosWrite(pStore->fd, cont, contLen) < contLen) {
uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@@ -292,7 +292,7 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) {
void *pBuf = buf;
tdEncodeKVRecord(&pBuf, &rInfo);
- if (taosTWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
+ if (taosWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
uError("failed to write %" PRId64 " bytes to file %s since %s", (int64_t)(POINTER_DISTANCE(pBuf, buf)), pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@@ -339,7 +339,7 @@ void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size) {
int fd = open(fname, O_RDONLY);
if (fd < 0) goto _err;
- if (taosTRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err;
+ if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err;
if (!taosCheckChecksumWhole((uint8_t *)buf, TD_KVSTORE_HEADER_SIZE)) goto _err;
void *pBuf = (void *)buf;
@@ -368,7 +368,7 @@ static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo, uint32_t
return -1;
}
- if (taosTRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+ if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
uError("failed to read %d bytes from file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@@ -402,7 +402,7 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE);
taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE);
- if (taosTWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+ if (taosWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@@ -477,9 +477,9 @@ _err:
static void tdFreeKVStore(SKVStore *pStore) {
if (pStore) {
- taosTFree(pStore->fname);
- taosTFree(pStore->fsnap);
- taosTFree(pStore->fnew);
+ tfree(pStore->fname);
+ tfree(pStore->fsnap);
+ tfree(pStore->fnew);
taosHashCleanup(pStore->map);
free(pStore);
}
@@ -535,7 +535,7 @@ static int tdRestoreKVStore(SKVStore *pStore) {
ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE);
while (true) {
- ssize_t tsize = taosTRead(pStore->fd, tbuf, sizeof(SKVRecord));
+ int64_t tsize = taosRead(pStore->fd, tbuf, sizeof(SKVRecord));
if (tsize == 0) break;
if (tsize < sizeof(SKVRecord)) {
uError("failed to read %" PRIzu " bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname,
@@ -598,7 +598,7 @@ static int tdRestoreKVStore(SKVStore *pStore) {
goto _err;
}
- if (taosTRead(pStore->fd, buf, (size_t)pRecord->size) < pRecord->size) {
+ if (taosRead(pStore->fd, buf, (size_t)pRecord->size) < pRecord->size) {
uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname,
strerror(errno), pRecord->offset);
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -618,11 +618,11 @@ static int tdRestoreKVStore(SKVStore *pStore) {
if (pStore->aFunc) (*pStore->aFunc)(pStore->appH);
taosHashDestroyIter(pIter);
- taosTFree(buf);
+ tfree(buf);
return 0;
_err:
taosHashDestroyIter(pIter);
- taosTFree(buf);
+ tfree(buf);
return -1;
}
diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c
index 09b0933fd6e32e9b65c8c7acbb81fbfe7d5c005b..ad3a92230482ff07b1533ceaf22b9ca30f72e10b 100644
--- a/src/util/src/tlog.c
+++ b/src/util/src/tlog.c
@@ -287,17 +287,17 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
tsLogObj.fileNum = maxFileNum;
taosGetLogFileName(fn);
+
if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
strcpy(name, fn);
strcat(name, ".0");
}
+ bool log0Exist = stat(name, &logstat0) >= 0;
if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
strcpy(name, fn);
strcat(name, ".1");
}
-
- bool log0Exist = stat(name, &logstat0) >= 0;
bool log1Exist = stat(name, &logstat1) >= 0;
// if none of the log files exist, open 0, if both exists, open the old one
@@ -336,11 +336,11 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
lseek(tsLogObj.logHandle->fd, 0, SEEK_END);
sprintf(name, "==================================================\n");
- taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+ taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
sprintf(name, " new log file \n");
- taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+ taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
sprintf(name, "==================================================\n");
- taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+ taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
return 0;
}
@@ -390,7 +390,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
if (tsAsyncLog) {
taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
} else {
- taosTWrite(tsLogObj.logHandle->fd, buffer, len);
+ taosWrite(tsLogObj.logHandle->fd, buffer, len);
}
if (tsLogObj.maxLines > 0) {
@@ -400,7 +400,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
}
}
- if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len);
+ if (dflag & DEBUG_SCREEN) taosWrite(1, buffer, (uint32_t)len);
}
void taosDumpData(unsigned char *msg, int32_t len) {
@@ -419,7 +419,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
pos += 3;
if (c >= 16) {
temp[pos++] = '\n';
- taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
+ taosWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
c = 0;
pos = 0;
}
@@ -427,9 +427,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
temp[pos++] = '\n';
- taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
-
- return;
+ taosWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
}
void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) {
@@ -467,7 +465,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
if (tsAsyncLog) {
taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
} else {
- taosTWrite(tsLogObj.logHandle->fd, buffer, len);
+ taosWrite(tsLogObj.logHandle->fd, buffer, len);
}
if (tsLogObj.maxLines > 0) {
@@ -477,7 +475,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
}
}
- if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len);
+ if (dflag & DEBUG_SCREEN) taosWrite(1, buffer, (uint32_t)len);
}
#if 0
@@ -514,8 +512,8 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) {
return tLogBuff;
_err:
- taosTFree(LOG_BUF_BUFFER(tLogBuff));
- taosTFree(tLogBuff);
+ tfree(LOG_BUF_BUFFER(tLogBuff));
+ tfree(tLogBuff);
return NULL;
}
@@ -524,7 +522,7 @@ static void taosLogBuffDestroy(SLogBuff *tLogBuff) {
tsem_destroy(&(tLogBuff->buffNotEmpty));
pthread_mutex_destroy(&(tLogBuff->buffMutex));
free(tLogBuff->buffer);
- taosTFree(tLogBuff);
+ tfree(tLogBuff);
}
#endif
@@ -606,7 +604,7 @@ static void *taosAsyncOutputLog(void *param) {
while (1) {
log_size = taosPollLogBuffer(tLogBuff, tempBuffer, TSDB_DEFAULT_LOG_BUF_UNIT);
if (log_size) {
- taosTWrite(tLogBuff->fd, tempBuffer, log_size);
+ taosWrite(tLogBuff->fd, tempBuffer, log_size);
LOG_BUF_START(tLogBuff) = (LOG_BUF_START(tLogBuff) + log_size) % LOG_BUF_SIZE(tLogBuff);
} else {
break;
diff --git a/src/util/src/tmempool.c b/src/util/src/tmempool.c
index a3d10355003c85d993b3a37190081436c0fb19ab..678c965eb1a7315977616778c0e4b39ceb4c7525 100644
--- a/src/util/src/tmempool.c
+++ b/src/util/src/tmempool.c
@@ -52,9 +52,9 @@ mpool_h taosMemPoolInit(int numOfBlock, int blockSize) {
if (pool_p->pool == NULL || pool_p->freeList == NULL) {
uError("failed to allocate memory\n");
- taosTFree(pool_p->freeList);
- taosTFree(pool_p->pool);
- taosTFree(pool_p);
+ tfree(pool_p->freeList);
+ tfree(pool_p->pool);
+ tfree(pool_p);
return NULL;
}
diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c
index 3793f3d3a944cc5c8d86c0dc0c0fa5bfe3cee764..6fd526598365f831addd1bacb0b7f748d9552fdb 100644
--- a/src/util/src/tnettest.c
+++ b/src/util/src/tnettest.c
@@ -15,11 +15,16 @@
#include "os.h"
#include "taosdef.h"
+#include "taosmsg.h"
#include "taoserror.h"
#include "tulog.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tsocket.h"
+#include "trpc.h"
+#include "rpcHead.h"
+#include "tutil.h"
+#include "tnettest.h"
#define MAX_PKG_LEN (64*1000)
#define BUFFER_SIZE (MAX_PKG_LEN + 1024)
@@ -30,9 +35,15 @@ typedef struct {
uint16_t pktLen;
} info_s;
-static char serverFqdn[TSDB_FQDN_LEN];
+extern int tsRpcMaxUdpSize;
+
+static char g_user[TSDB_USER_LEN+1] = {0};
+static char g_pass[TSDB_PASSWORD_LEN+1] = {0};
+static char g_serverFqdn[TSDB_FQDN_LEN] = {0};
static uint16_t g_startPort = 0;
static uint16_t g_endPort = 6042;
+static uint32_t g_pktLen = 0;
+
static void *bindUdpPort(void *sarg) {
info_s *pinfo = (info_s *)sarg;
@@ -321,19 +332,145 @@ static void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uin
return ;
}
-static void taosNetTestClient(const char* serverFqdn, uint16_t startPort, uint16_t endPort, int pktLen) {
- uint32_t serverIp = taosGetIpFromFqdn(serverFqdn);
- if (serverIp == 0xFFFFFFFF) {
- printf("Failed to resolve FQDN:%s", serverFqdn);
- exit(-1);
+void* tnetInitRpc(char* secretEncrypt, char spi) {
+ SRpcInit rpcInit;
+ void* pRpcConn = NULL;
+
+ taosEncryptPass((uint8_t *)g_pass, strlen(g_pass), secretEncrypt);
+
+ memset(&rpcInit, 0, sizeof(rpcInit));
+ rpcInit.localPort = 0;
+ rpcInit.label = "NET-TEST";
+ rpcInit.numOfThreads = 1; // every DB connection has only one thread
+ rpcInit.cfp = NULL;
+ rpcInit.sessions = 16;
+ rpcInit.connType = TAOS_CONN_CLIENT;
+ rpcInit.user = g_user;
+ rpcInit.idleTime = 2000;
+ rpcInit.ckey = "key";
+ rpcInit.spi = spi;
+ rpcInit.secret = secretEncrypt;
+
+ pRpcConn = rpcOpen(&rpcInit);
+ return pRpcConn;
+}
+
+static int rpcCheckPortImpl(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi) {
+ SRpcEpSet epSet;
+ SRpcMsg reqMsg;
+ SRpcMsg rspMsg;
+ void* pRpcConn;
+
+ char secretEncrypt[32] = {0};
+
+ pRpcConn = tnetInitRpc(secretEncrypt, spi);
+ if (NULL == pRpcConn) {
+ return -1;
}
- checkPort(serverIp, startPort, endPort, pktLen);
+ memset(&epSet, 0, sizeof(SRpcEpSet));
+ epSet.inUse = 0;
+ epSet.numOfEps = 1;
+ epSet.port[0] = port;
+ strcpy(epSet.fqdn[0], serverFqdn);
+
+ reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST;
+ reqMsg.pCont = rpcMallocCont(pktLen);
+ reqMsg.contLen = pktLen;
+ reqMsg.code = 0;
+ reqMsg.handle = NULL; // rpc handle returned to app
+ reqMsg.ahandle = NULL; // app handle set by client
+
+ rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg);
+
+ // handle response
+ if ((rspMsg.code != 0) || (rspMsg.msgType != TSDB_MSG_TYPE_NETWORK_TEST + 1)) {
+ //printf("code:%d[%s]\n", rspMsg.code, tstrerror(rspMsg.code));
+ return -1;
+ }
+
+ rpcFreeCont(rspMsg.pCont);
- return;
+ rpcClose(pRpcConn);
+
+ return 0;
+}
+
+static void rpcCheckPort(uint32_t hostIp) {
+ int ret;
+ char spi;
+
+ for (uint16_t port = g_startPort; port <= g_endPort; port++) {
+ //printf("test: %s:%d\n", info.host, port);
+ printf("\n");
+
+ //================ check tcp port ================
+ int32_t pktLen;
+ if (g_pktLen <= tsRpcMaxUdpSize) {
+ pktLen = tsRpcMaxUdpSize + 1000;
+ } else {
+ pktLen = g_pktLen;
+ }
+
+ spi = 1;
+ ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
+ if (ret != 0) {
+ spi = 0;
+ ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
+ if (ret != 0) {
+ printf("TCP port:%d test fail.\t\t", port);
+ } else {
+ //printf("tcp port:%d test ok.\t\t", port);
+ printf("TCP port:\033[32m%d test OK\033[0m\t\t", port);
+ }
+ } else {
+ //printf("tcp port:%d test ok.\t\t", port);
+ printf("TCP port:\033[32m%d test OK\033[0m\t\t", port);
+ }
+
+ //================ check udp port ================
+ if (g_pktLen >= tsRpcMaxUdpSize) {
+ pktLen = tsRpcMaxUdpSize - 1000;
+ } else {
+ pktLen = g_pktLen;
+ }
+
+ spi = 0;
+ ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
+ if (ret != 0) {
+ spi = 1;
+ ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
+ if (ret != 0) {
+ printf("udp port:%d test fail.\t\n", port);
+ } else {
+ //printf("udp port:%d test ok.\t\n", port);
+ printf("UDP port:\033[32m%d test OK\033[0m\t\n", port);
+ }
+ } else {
+ //printf("udp port:%d test ok.\t\n", port);
+ printf("UDP port:\033[32m%d test OK\033[0m\t\n", port);
+ }
+ }
+
+ printf("\n");
+ return ;
}
+static void taosNetTestClient(int flag) {
+ uint32_t serverIp = taosGetIpFromFqdn(g_serverFqdn);
+ if (serverIp == 0xFFFFFFFF) {
+ printf("Failed to resolve FQDN:%s", g_serverFqdn);
+ exit(-1);
+ }
+ if (0 == flag) {
+ checkPort(serverIp, g_startPort, g_endPort, g_pktLen);
+ } else {
+ rpcCheckPort(serverIp);
+ }
+
+ return;
+}
static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen) {
@@ -375,49 +512,66 @@ static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen)
}
-void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole) {
- if (pktLen > MAX_PKG_LEN) {
- printf("test packet len overflow: %d, max len not greater than %d bytes\n", pktLen, MAX_PKG_LEN);
- exit(-1);
+void taosNetTest(CmdArguments *args) {
+ if (0 == args->pktLen) {
+ g_pktLen = 1000;
+ } else {
+ g_pktLen = args->pktLen;
}
- if (port && endPort) {
- if (port > endPort) {
- printf("endPort[%d] must not lesss port[%d]\n", endPort, port);
+ if (args->port && args->endPort) {
+ if (args->port > args->endPort) {
+ printf("endPort[%d] must not lesss port[%d]\n", args->endPort, args->port);
exit(-1);
}
}
- if (host && host[0] != 0) {
- if (strlen(host) >= TSDB_EP_LEN) {
- printf("host invalid: %s\n", host);
+ if (args->host && args->host[0] != 0) {
+ if (strlen(args->host) >= TSDB_EP_LEN) {
+ printf("host invalid: %s\n", args->host);
exit(-1);
}
- taosGetFqdnPortFromEp(host, serverFqdn, &g_startPort);
+ taosGetFqdnPortFromEp(args->host, g_serverFqdn, &g_startPort);
} else {
- tstrncpy(serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN);
+ tstrncpy(g_serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN);
g_startPort = tsServerPort;
}
- if (port) {
- g_startPort = port;
+ if (args->port) {
+ g_startPort = args->port;
}
- if (endPort) {
- g_endPort = endPort;
+ if (args->endPort) {
+ g_endPort = args->endPort;
}
- if (port > endPort) {
+ if (g_startPort > g_endPort) {
printf("endPort[%d] must not lesss port[%d]\n", g_endPort, g_startPort);
exit(-1);
}
+
+
+ if (args->is_use_passwd) {
+ if (args->password == NULL) args->password = getpass("Enter password: ");
+ } else {
+ args->password = TSDB_DEFAULT_PASS;
+ }
+ tstrncpy(g_pass, args->password, TSDB_PASSWORD_LEN);
+
+ if (args->user == NULL) {
+ args->user = TSDB_DEFAULT_USER;
+ }
+ tstrncpy(g_user, args->user, TSDB_USER_LEN);
- if (0 == strcmp("client", netTestRole)) {
- printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", serverFqdn, g_startPort, g_endPort, pktLen);
- taosNetTestClient(serverFqdn, g_startPort, g_endPort, pktLen);
- } else if (0 == strcmp("server", netTestRole)) {
- taosNetTestServer(g_startPort, g_endPort, pktLen);
+ if (0 == strcmp("client", args->netTestRole)) {
+ printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen);
+ taosNetTestClient(0);
+ } else if (0 == strcmp("clients", args->netTestRole)) {
+ printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen);
+ taosNetTestClient(1);
+ } else if (0 == strcmp("server", args->netTestRole)) {
+ taosNetTestServer(g_startPort, g_endPort, g_pktLen);
}
}
diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c
index 4f05277a847a07831371278915011ebc4da400c6..9536f6fb70f9fe6d74c981274c849cab0c00ef1f 100644
--- a/src/util/src/tnote.c
+++ b/src/util/src/tnote.c
@@ -265,7 +265,7 @@ void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...)
buffer[len] = 0;
if (pNote->taosNoteFd >= 0) {
- taosTWrite(pNote->taosNoteFd, buffer, (unsigned int)len);
+ taosWrite(pNote->taosNoteFd, buffer, (unsigned int)len);
if (pNote->taosNoteMaxLines > 0) {
pNote->taosNoteLines++;
diff --git a/src/util/src/tref.c b/src/util/src/tref.c
new file mode 100644
index 0000000000000000000000000000000000000000..4c1a87c96070534e25149357b766a8539e362680
--- /dev/null
+++ b/src/util/src/tref.c
@@ -0,0 +1,496 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "os.h"
+#include "taoserror.h"
+#include "tulog.h"
+#include "tutil.h"
+
+#define TSDB_REF_OBJECTS 50
+#define TSDB_REF_STATE_EMPTY 0
+#define TSDB_REF_STATE_ACTIVE 1
+#define TSDB_REF_STATE_DELETED 2
+
+typedef struct SRefNode {
+ struct SRefNode *prev; // previous node
+ struct SRefNode *next; // next node
+ void *p; // pointer to resource protected,
+ int64_t rid; // reference ID
+ int32_t count; // number of references
+ int removed; // 1: removed
+} SRefNode;
+
+typedef struct {
+ SRefNode **nodeList; // array of SRefNode linked list
+ int state; // 0: empty, 1: active; 2: deleted
+ int rsetId; // refSet ID, global unique
+ int64_t rid; // increase by one for each new reference
+ int max; // mod
+ int32_t count; // total number of SRefNodes in this set
+ int64_t *lockedBy;
+ void (*fp)(void *);
+} SRefSet;
+
+static SRefSet tsRefSetList[TSDB_REF_OBJECTS];
+static pthread_once_t tsRefModuleInit = PTHREAD_ONCE_INIT;
+static pthread_mutex_t tsRefMutex;
+static int tsRefSetNum = 0;
+static int tsNextId = 0;
+
+static void taosInitRefModule(void);
+static void taosLockList(int64_t *lockedBy);
+static void taosUnlockList(int64_t *lockedBy);
+static void taosIncRsetCount(SRefSet *pSet);
+static void taosDecRsetCount(SRefSet *pSet);
+static int taosDecRefCount(int rsetId, int64_t rid, int remove);
+
+int taosOpenRef(int max, void (*fp)(void *))
+{
+ SRefNode **nodeList;
+ SRefSet *pSet;
+ int64_t *lockedBy;
+ int i, rsetId;
+
+ pthread_once(&tsRefModuleInit, taosInitRefModule);
+
+ nodeList = calloc(sizeof(SRefNode *), (size_t)max);
+ if (nodeList == NULL) {
+ terrno = TSDB_CODE_REF_NO_MEMORY;
+ return -1;
+ }
+
+ lockedBy = calloc(sizeof(int64_t), (size_t)max);
+ if (lockedBy == NULL) {
+ free(nodeList);
+ terrno = TSDB_CODE_REF_NO_MEMORY;
+ return -1;
+ }
+
+ pthread_mutex_lock(&tsRefMutex);
+
+ for (i = 0; i < TSDB_REF_OBJECTS; ++i) {
+ tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS;
+ if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId
+ if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break;
+ }
+
+ if (i < TSDB_REF_OBJECTS) {
+ rsetId = tsNextId;
+ pSet = tsRefSetList + rsetId;
+ pSet->max = max;
+ pSet->nodeList = nodeList;
+ pSet->lockedBy = lockedBy;
+ pSet->fp = fp;
+ pSet->rid = 1;
+ pSet->rsetId = rsetId;
+ pSet->state = TSDB_REF_STATE_ACTIVE;
+ taosIncRsetCount(pSet);
+
+ tsRefSetNum++;
+ uTrace("rsetId:%d is opened, max:%d, fp:%p refSetNum:%d", rsetId, max, fp, tsRefSetNum);
+ } else {
+ rsetId = TSDB_CODE_REF_FULL;
+ free (nodeList);
+ free (lockedBy);
+ uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum);
+ }
+
+ pthread_mutex_unlock(&tsRefMutex);
+
+ return rsetId;
+}
+
+int taosCloseRef(int rsetId)
+{
+ SRefSet *pSet;
+ int deleted = 0;
+
+ if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+ uTrace("rsetId:%d is invalid, out of range", rsetId);
+ terrno = TSDB_CODE_REF_INVALID_ID;
+ return -1;
+ }
+
+ pSet = tsRefSetList + rsetId;
+
+ pthread_mutex_lock(&tsRefMutex);
+
+ if (pSet->state == TSDB_REF_STATE_ACTIVE) {
+ pSet->state = TSDB_REF_STATE_DELETED;
+ deleted = 1;
+ uTrace("rsetId:%d is closed, count:%d", rsetId, pSet->count);
+ } else {
+ uTrace("rsetId:%d is already closed, count:%d", rsetId, pSet->count);
+ }
+
+ pthread_mutex_unlock(&tsRefMutex);
+
+ if (deleted) taosDecRsetCount(pSet);
+
+ return 0;
+}
+
+int64_t taosAddRef(int rsetId, void *p)
+{
+ int hash;
+ SRefNode *pNode;
+ SRefSet *pSet;
+ int64_t rid = 0;
+
+ if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+ uTrace("rsetId:%d p:%p failed to add, rsetId not valid", rsetId, p);
+ terrno = TSDB_CODE_REF_INVALID_ID;
+ return -1;
+ }
+
+ pSet = tsRefSetList + rsetId;
+ taosIncRsetCount(pSet);
+ if (pSet->state != TSDB_REF_STATE_ACTIVE) {
+ taosDecRsetCount(pSet);
+ uTrace("rsetId:%d p:%p failed to add, not active", rsetId, p);
+ terrno = TSDB_CODE_REF_ID_REMOVED;
+ return -1;
+ }
+
+ pNode = calloc(sizeof(SRefNode), 1);
+ if (pNode == NULL) {
+ terrno = TSDB_CODE_REF_NO_MEMORY;
+ return -1;
+ }
+
+ rid = atomic_add_fetch_64(&pSet->rid, 1);
+ hash = rid % pSet->max;
+ taosLockList(pSet->lockedBy+hash);
+
+ pNode->p = p;
+ pNode->rid = rid;
+ pNode->count = 1;
+
+ pNode->prev = NULL;
+ pNode->next = pSet->nodeList[hash];
+ if (pSet->nodeList[hash]) pSet->nodeList[hash]->prev = pNode;
+ pSet->nodeList[hash] = pNode;
+
+ uTrace("rsetId:%d p:%p rid:%" PRId64 " is added, count:%d", rsetId, p, rid, pSet->count);
+
+ taosUnlockList(pSet->lockedBy+hash);
+
+ return rid;
+}
+
+int taosRemoveRef(int rsetId, int64_t rid)
+{
+ return taosDecRefCount(rsetId, rid, 1);
+}
+
+// if rid is 0, return the first p in hash list, otherwise, return the next after current rid
+void *taosAcquireRef(int rsetId, int64_t rid)
+{
+ int hash;
+ SRefNode *pNode;
+ SRefSet *pSet;
+ void *p = NULL;
+
+ if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid);
+ terrno = TSDB_CODE_REF_INVALID_ID;
+ return NULL;
+ }
+
+ if (rid <= 0) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rid not valid", rsetId, rid);
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ return NULL;
+ }
+
+ pSet = tsRefSetList + rsetId;
+ taosIncRsetCount(pSet);
+ if (pSet->state != TSDB_REF_STATE_ACTIVE) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, not active", rsetId, rid);
+ taosDecRsetCount(pSet);
+ terrno = TSDB_CODE_REF_ID_REMOVED;
+ return NULL;
+ }
+
+ hash = rid % pSet->max;
+ taosLockList(pSet->lockedBy+hash);
+
+ pNode = pSet->nodeList[hash];
+
+ while (pNode) {
+ if (pNode->rid == rid) {
+ break;
+ }
+
+ pNode = pNode->next;
+ }
+
+ if (pNode) {
+ if (pNode->removed == 0) {
+ pNode->count++;
+ p = pNode->p;
+ uTrace("rsetId:%d p:%p rid:%" PRId64 " is acquired", rsetId, pNode->p, rid);
+ } else {
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ uTrace("rsetId:%d p:%p rid:%" PRId64 " is already removed, failed to acquire", rsetId, pNode->p, rid);
+ }
+ } else {
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to acquire", rsetId, rid);
+ }
+
+ taosUnlockList(pSet->lockedBy+hash);
+
+ taosDecRsetCount(pSet);
+
+ return p;
+}
+
+int taosReleaseRef(int rsetId, int64_t rid)
+{
+ return taosDecRefCount(rsetId, rid, 0);
+}
+
+// if rid is 0, return the first p in hash list, otherwise, return the next after current rid
+void *taosIterateRef(int rsetId, int64_t rid) {
+ SRefNode *pNode = NULL;
+ SRefSet *pSet;
+
+ if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rsetId not valid", rsetId, rid);
+ terrno = TSDB_CODE_REF_INVALID_ID;
+ return NULL;
+ }
+
+ if (rid <= 0) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rid not valid", rsetId, rid);
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ return NULL;
+ }
+
+ pSet = tsRefSetList + rsetId;
+ taosIncRsetCount(pSet);
+ if (pSet->state != TSDB_REF_STATE_ACTIVE) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rset not active", rsetId, rid);
+ terrno = TSDB_CODE_REF_ID_REMOVED;
+ taosDecRsetCount(pSet);
+ return NULL;
+ }
+
+ int hash = 0;
+ if (rid > 0) {
+ hash = rid % pSet->max;
+ taosLockList(pSet->lockedBy+hash);
+
+ pNode = pSet->nodeList[hash];
+ while (pNode) {
+ if (pNode->rid == rid) break;
+ pNode = pNode->next;
+ }
+
+ if (pNode == NULL) {
+ uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid);
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ taosUnlockList(pSet->lockedBy+hash);
+ return NULL;
+ }
+
+ // rid is there
+ pNode = pNode->next;
+ if (pNode == NULL) {
+ taosUnlockList(pSet->lockedBy+hash);
+ hash++;
+ }
+ }
+
+ if (pNode == NULL) {
+ for (; hash < pSet->max; ++hash) {
+ taosLockList(pSet->lockedBy+hash);
+ pNode = pSet->nodeList[hash];
+ if (pNode) break;
+ taosUnlockList(pSet->lockedBy+hash);
+ }
+ }
+
+ void *newP = NULL;
+ if (pNode) {
+ pNode->count++; // acquire it
+ newP = pNode->p;
+ taosUnlockList(pSet->lockedBy+hash);
+ uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid);
+ } else {
+ uTrace("rsetId:%d the list is over", rsetId);
+ }
+
+ if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one
+
+ taosDecRsetCount(pSet);
+
+ return newP;
+}
+
+int taosListRef() {
+ SRefSet *pSet;
+ SRefNode *pNode;
+ int num = 0;
+
+ pthread_mutex_lock(&tsRefMutex);
+
+ for (int i = 0; i < TSDB_REF_OBJECTS; ++i) {
+ pSet = tsRefSetList + i;
+
+ if (pSet->state == TSDB_REF_STATE_EMPTY)
+ continue;
+
+ uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count);
+
+ for (int j=0; j < pSet->max; ++j) {
+ pNode = pSet->nodeList[j];
+
+ while (pNode) {
+ uInfo("rsetId:%d p:%p rid:%" PRId64 "count:%d", i, pNode->p, pNode->rid, pNode->count);
+ pNode = pNode->next;
+ num++;
+ }
+ }
+ }
+
+ pthread_mutex_unlock(&tsRefMutex);
+
+ return num;
+}
+
+static int taosDecRefCount(int rsetId, int64_t rid, int remove) {
+ int hash;
+ SRefSet *pSet;
+ SRefNode *pNode;
+ int released = 0;
+ int code = 0;
+
+ if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rsetId not valid", rsetId, rid);
+ terrno = TSDB_CODE_REF_INVALID_ID;
+ return -1;
+ }
+
+ if (rid <= 0) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rid not valid", rsetId, rid);
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ return -1;
+ }
+
+ pSet = tsRefSetList + rsetId;
+ if (pSet->state == TSDB_REF_STATE_EMPTY) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to remove, cleaned", rsetId, rid);
+ terrno = TSDB_CODE_REF_ID_REMOVED;
+ return -1;
+ }
+
+ hash = rid % pSet->max;
+ taosLockList(pSet->lockedBy+hash);
+
+ pNode = pSet->nodeList[hash];
+ while (pNode) {
+ if (pNode->rid == rid)
+ break;
+
+ pNode = pNode->next;
+ }
+
+ if (pNode) {
+ pNode->count--;
+ if (remove) pNode->removed = 1;
+
+ if (pNode->count <= 0) {
+ if (pNode->prev) {
+ pNode->prev->next = pNode->next;
+ } else {
+ pSet->nodeList[hash] = pNode->next;
+ }
+
+ if (pNode->next) {
+ pNode->next->prev = pNode->prev;
+ }
+
+ (*pSet->fp)(pNode->p);
+
+ uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode);
+ free(pNode);
+ released = 1;
+ } else {
+ uTrace("rsetId:%d p:%p rid:%" PRId64 " is released, count:%d", rsetId, pNode->p, rid, pNode->count);
+ }
+ } else {
+ uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid);
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ code = -1;
+ }
+
+ taosUnlockList(pSet->lockedBy+hash);
+
+ if (released) taosDecRsetCount(pSet);
+
+ return code;
+}
+
+static void taosLockList(int64_t *lockedBy) {
+ int64_t tid = taosGetPthreadId();
+ int i = 0;
+ while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) {
+ if (++i % 100 == 0) {
+ sched_yield();
+ }
+ }
+}
+
+static void taosUnlockList(int64_t *lockedBy) {
+ int64_t tid = taosGetPthreadId();
+ if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) {
+ assert(false);
+ }
+}
+
+static void taosInitRefModule(void) {
+ pthread_mutex_init(&tsRefMutex, NULL);
+}
+
+static void taosIncRsetCount(SRefSet *pSet) {
+ atomic_add_fetch_32(&pSet->count, 1);
+ // uTrace("rsetId:%d inc count:%d", pSet->rsetId, count);
+}
+
+static void taosDecRsetCount(SRefSet *pSet) {
+ int32_t count = atomic_sub_fetch_32(&pSet->count, 1);
+ // uTrace("rsetId:%d dec count:%d", pSet->rsetId, count);
+
+ if (count > 0) return;
+
+ pthread_mutex_lock(&tsRefMutex);
+
+ if (pSet->state != TSDB_REF_STATE_EMPTY) {
+ pSet->state = TSDB_REF_STATE_EMPTY;
+ pSet->max = 0;
+ pSet->fp = NULL;
+
+ tfree(pSet->nodeList);
+ tfree(pSet->lockedBy);
+
+ tsRefSetNum--;
+ uTrace("rsetId:%d is cleaned, refSetNum:%d count:%d", pSet->rsetId, tsRefSetNum, pSet->count);
+ }
+
+ pthread_mutex_unlock(&tsRefMutex);
+}
+
diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c
index bacdaef6c8da1e42218331aaa34d25aa40ab5dc4..a36f7f0261ddb34cbfa82a5bc2485c6e9c45f03a 100644
--- a/src/util/src/tskiplist.c
+++ b/src/util/src/tskiplist.c
@@ -13,13 +13,508 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*/
+#include "tskiplist.h"
#include "os.h"
+#include "tcompare.h"
#include "tulog.h"
-#include "tskiplist.h"
#include "tutil.h"
-#include "tcompare.h"
-UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level) { // record link count in each level
+static int initForwardBackwardPtr(SSkipList *pSkipList);
+static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur);
+static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode);
+static void tSkipListCorrectLevel(SSkipList *pSkipList);
+static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order);
+static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward);
+static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData);
+static SSkipListNode *tSkipListNewNode(uint8_t level);
+#define tSkipListFreeNode(n) tfree((n))
+static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipListNode **direction, bool isForward,
+ bool hasDup);
+
+static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList);
+static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList);
+static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList);
+static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList);
+
+SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags,
+ __sl_key_fn_t fn) {
+ SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList));
+ if (pSkipList == NULL) return NULL;
+
+ if (maxLevel > MAX_SKIP_LIST_LEVEL) {
+ maxLevel = MAX_SKIP_LIST_LEVEL;
+ }
+
+ pSkipList->maxLevel = maxLevel;
+ pSkipList->type = keyType;
+ pSkipList->len = keyLen;
+ pSkipList->flags = flags;
+ pSkipList->keyFn = fn;
+ if (comparFn == NULL) {
+ pSkipList->comparFn = getKeyComparFunc(keyType);
+ } else {
+ pSkipList->comparFn = comparFn;
+ }
+
+ if (initForwardBackwardPtr(pSkipList) < 0) {
+ tSkipListDestroy(pSkipList);
+ return NULL;
+ }
+
+ if (SL_IS_THREAD_SAFE(pSkipList)) {
+ pSkipList->lock = (pthread_rwlock_t *)calloc(1, sizeof(pthread_rwlock_t));
+ if (pSkipList->lock == NULL) {
+ tSkipListDestroy(pSkipList);
+ return NULL;
+ }
+
+ if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) {
+ tSkipListDestroy(pSkipList);
+ return NULL;
+ }
+ }
+
+ srand((uint32_t)time(NULL));
+
+#if SKIP_LIST_RECORD_PERFORMANCE
+ pSkipList->state.nTotalMemSize += sizeof(SSkipList);
+#endif
+
+ return pSkipList;
+}
+
+void tSkipListDestroy(SSkipList *pSkipList) {
+ if (pSkipList == NULL) return;
+
+ tSkipListWLock(pSkipList);
+
+ SSkipListNode *pNode = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, 0);
+
+ while (pNode != pSkipList->pTail) {
+ SSkipListNode *pTemp = pNode;
+ pNode = SL_NODE_GET_FORWARD_POINTER(pNode, 0);
+ tSkipListFreeNode(pTemp);
+ }
+
+ tSkipListUnlock(pSkipList);
+ if (pSkipList->lock != NULL) {
+ pthread_rwlock_destroy(pSkipList->lock);
+ tfree(pSkipList->lock);
+ }
+
+ tSkipListFreeNode(pSkipList->pHead);
+ tSkipListFreeNode(pSkipList->pTail);
+ tfree(pSkipList);
+}
+
+SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) {
+ if (pSkipList == NULL || pData == NULL) return NULL;
+
+ SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0};
+ SSkipListNode *pNode = NULL;
+
+ tSkipListWLock(pSkipList);
+
+ bool hasDup = tSkipListGetPosToPut(pSkipList, backward, pData);
+ pNode = tSkipListPutImpl(pSkipList, pData, backward, false, hasDup);
+
+ tSkipListUnlock(pSkipList);
+
+ return pNode;
+}
+
+// Put a batch of data into skiplist. The batch of data must be in ascending order
+void tSkipListPutBatch(SSkipList *pSkipList, void **ppData, int ndata) {
+ SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0};
+ SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0};
+ bool hasDup = false;
+ char * pKey = NULL;
+ char * pDataKey = NULL;
+ int compare = 0;
+
+ tSkipListWLock(pSkipList);
+
+ // backward to put the first data
+ hasDup = tSkipListGetPosToPut(pSkipList, backward, ppData[0]);
+ tSkipListPutImpl(pSkipList, ppData[0], backward, false, hasDup);
+
+ for (int level = 0; level < pSkipList->maxLevel; level++) {
+ forward[level] = SL_NODE_GET_BACKWARD_POINTER(backward[level], level);
+ }
+
+ // forward to put the rest of data
+ for (int idata = 1; idata < ndata; idata++) {
+ pDataKey = pSkipList->keyFn(ppData[idata]);
+ hasDup = false;
+
+ // Compare max key
+ pKey = SL_GET_MAX_KEY(pSkipList);
+ compare = pSkipList->comparFn(pDataKey, pKey);
+ if (compare > 0) {
+ for (int i = 0; i < pSkipList->maxLevel; i++) {
+ forward[i] = SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i);
+ }
+ } else {
+ SSkipListNode *px = pSkipList->pHead;
+ for (int i = pSkipList->maxLevel - 1; i >= 0; --i) {
+ if (i < pSkipList->level) {
+ // set new px
+ if (forward[i] != pSkipList->pHead) {
+ if (px == pSkipList->pHead ||
+ pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, forward[i]), SL_GET_NODE_KEY(pSkipList, px)) > 0) {
+ px = forward[i];
+ }
+ }
+
+ SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(px, i);
+ while (p != pSkipList->pTail) {
+ pKey = SL_GET_NODE_KEY(pSkipList, p);
+
+ compare = pSkipList->comparFn(pKey, pDataKey);
+ if (compare >= 0) {
+ if (compare == 0 && !hasDup) hasDup = true;
+ break;
+ } else {
+ px = p;
+ p = SL_NODE_GET_FORWARD_POINTER(px, i);
+ }
+ }
+ }
+
+ forward[i] = px;
+ }
+ }
+
+ tSkipListPutImpl(pSkipList, ppData[idata], forward, true, hasDup);
+ }
+
+ tSkipListUnlock(pSkipList);
+}
+
+uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) {
+ uint32_t count = 0;
+
+ tSkipListWLock(pSkipList);
+
+ SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL);
+ while (1) {
+ SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0);
+ if (p == pSkipList->pTail) {
+ break;
+ }
+ if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) {
+ break;
+ }
+
+ tSkipListRemoveNodeImpl(pSkipList, p);
+
+ ++count;
+ }
+
+ tSkipListCorrectLevel(pSkipList);
+
+ tSkipListUnlock(pSkipList);
+
+ return count;
+}
+
+SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey key) {
+ SArray *sa = taosArrayInit(1, POINTER_BYTES);
+
+ tSkipListRLock(pSkipList);
+
+ SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL);
+ while (1) {
+ SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0);
+ if (p == pSkipList->pTail) {
+ break;
+ }
+ if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) {
+ break;
+ }
+ taosArrayPush(sa, &p);
+ pNode = p;
+ }
+
+ tSkipListUnlock(pSkipList);
+
+ return sa;
+}
+
+void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) {
+ tSkipListWLock(pSkipList);
+ tSkipListRemoveNodeImpl(pSkipList, pNode);
+ tSkipListCorrectLevel(pSkipList);
+ tSkipListUnlock(pSkipList);
+}
+
+SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList) {
+ if (pSkipList == NULL) return NULL;
+
+ return doCreateSkipListIterator(pSkipList, TSDB_ORDER_ASC);
+}
+
+SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order) {
+ ASSERT(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);
+ ASSERT(pSkipList != NULL);
+
+ SSkipListIterator *iter = doCreateSkipListIterator(pSkipList, order);
+ if (val == NULL) {
+ return iter;
+ }
+
+ tSkipListRLock(pSkipList);
+
+ iter->cur = getPriorNode(pSkipList, val, order, &(iter->next));
+
+ tSkipListUnlock(pSkipList);
+
+ return iter;
+}
+
+bool tSkipListIterNext(SSkipListIterator *iter) {
+ if (iter->pSkipList == NULL) return false;
+
+ SSkipList *pSkipList = iter->pSkipList;
+
+ tSkipListRLock(pSkipList);
+
+ if (iter->order == TSDB_ORDER_ASC) {
+ if (iter->cur == pSkipList->pTail) return false;
+ iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0);
+
+ // a new node is inserted into between iter->cur and iter->next, ignore it
+ if (iter->cur != iter->next && (iter->next != NULL)) {
+ iter->cur = iter->next;
+ }
+
+ iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0);
+ iter->step++;
+ } else {
+ if (iter->cur == pSkipList->pHead) return false;
+ iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0);
+
+ // a new node is inserted into between iter->cur and iter->next, ignore it
+ if (iter->cur != iter->next && (iter->next != NULL)) {
+ iter->cur = iter->next;
+ }
+
+ iter->next = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0);
+ iter->step++;
+ }
+
+ tSkipListUnlock(pSkipList);
+
+ return (iter->order == TSDB_ORDER_ASC) ? (iter->cur != pSkipList->pTail) : (iter->cur != pSkipList->pHead);
+}
+
+SSkipListNode *tSkipListIterGet(SSkipListIterator *iter) {
+ if (iter == NULL || iter->cur == iter->pSkipList->pTail || iter->cur == iter->pSkipList->pHead) {
+ return NULL;
+ } else {
+ return iter->cur;
+ }
+}
+
+void *tSkipListDestroyIter(SSkipListIterator *iter) {
+ if (iter == NULL) {
+ return NULL;
+ }
+
+ tfree(iter);
+ return NULL;
+}
+
+void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) {
+ if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) {
+ return;
+ }
+
+ SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1);
+
+ int32_t id = 1;
+ char * prev = NULL;
+
+ while (p != pSkipList->pTail) {
+ char *key = SL_GET_NODE_KEY(pSkipList, p);
+ if (prev != NULL) {
+ ASSERT(pSkipList->comparFn(prev, key) < 0);
+ }
+
+ switch (pSkipList->type) {
+ case TSDB_DATA_TYPE_INT:
+ fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key);
+ break;
+ case TSDB_DATA_TYPE_SMALLINT:
+ case TSDB_DATA_TYPE_TINYINT:
+ case TSDB_DATA_TYPE_BIGINT:
+ fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key);
+ break;
+ case TSDB_DATA_TYPE_BINARY:
+ fprintf(stdout, "%d: %s \n", id++, key);
+ break;
+ case TSDB_DATA_TYPE_DOUBLE:
+ fprintf(stdout, "%d: %lf \n", id++, *(double *)key);
+ break;
+ default:
+ fprintf(stdout, "\n");
+ }
+
+ prev = SL_GET_NODE_KEY(pSkipList, p);
+
+ p = SL_NODE_GET_FORWARD_POINTER(p, nlevel - 1);
+ }
+}
+
+static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward) {
+ for (int32_t i = 0; i < pNode->level; ++i) {
+ SSkipListNode *x = direction[i];
+ if (isForward) {
+ SL_NODE_GET_BACKWARD_POINTER(pNode, i) = x;
+
+ SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(x, i);
+ SL_NODE_GET_BACKWARD_POINTER(next, i) = pNode;
+
+ SL_NODE_GET_FORWARD_POINTER(pNode, i) = next;
+ SL_NODE_GET_FORWARD_POINTER(x, i) = pNode;
+ } else {
+ SL_NODE_GET_FORWARD_POINTER(pNode, i) = x;
+
+ SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(x, i);
+ SL_NODE_GET_FORWARD_POINTER(prev, i) = pNode;
+
+ SL_NODE_GET_BACKWARD_POINTER(pNode, i) = prev;
+ SL_NODE_GET_BACKWARD_POINTER(x, i) = pNode;
+ }
+ }
+
+ if (pSkipList->level < pNode->level) pSkipList->level = pNode->level;
+
+ pSkipList->size += 1;
+}
+
+static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) {
+ SSkipListIterator *iter = calloc(1, sizeof(SSkipListIterator));
+
+ iter->pSkipList = pSkipList;
+ iter->order = order;
+ if (order == TSDB_ORDER_ASC) {
+ iter->cur = pSkipList->pHead;
+ iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0);
+ } else {
+ iter->cur = pSkipList->pTail;
+ iter->next = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0);
+ }
+
+ return iter;
+}
+
+static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList) {
+ if (pSkipList->lock) {
+ return pthread_rwlock_wrlock(pSkipList->lock);
+ }
+ return 0;
+}
+
+static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList) {
+ if (pSkipList->lock) {
+ return pthread_rwlock_rdlock(pSkipList->lock);
+ }
+ return 0;
+}
+
+static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) {
+ if (pSkipList->lock) {
+ return pthread_rwlock_unlock(pSkipList->lock);
+ }
+ return 0;
+}
+
+static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData) {
+ int compare = 0;
+ bool hasDupKey = false;
+ char * pDataKey = pSkipList->keyFn(pData);
+
+ if (pSkipList->size == 0) {
+ for (int i = 0; i < pSkipList->maxLevel; i++) {
+ backward[i] = pSkipList->pTail;
+ }
+ } else {
+ char *pKey = NULL;
+
+ // Compare max key
+ pKey = SL_GET_MAX_KEY(pSkipList);
+ compare = pSkipList->comparFn(pDataKey, pKey);
+ if (compare >= 0) {
+ for (int i = 0; i < pSkipList->maxLevel; i++) {
+ backward[i] = pSkipList->pTail;
+ }
+
+ return (compare == 0);
+ }
+
+ // Compare min key
+ pKey = SL_GET_MIN_KEY(pSkipList);
+ compare = pSkipList->comparFn(pDataKey, pKey);
+ if (compare < 0) {
+ for (int i = 0; i < pSkipList->maxLevel; i++) {
+ backward[i] = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i);
+ }
+
+ return (compare == 0);
+ }
+
+ SSkipListNode *px = pSkipList->pTail;
+ for (int i = pSkipList->maxLevel - 1; i >= 0; --i) {
+ if (i < pSkipList->level) {
+ SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(px, i);
+ while (p != pSkipList->pHead) {
+ pKey = SL_GET_NODE_KEY(pSkipList, p);
+
+ compare = pSkipList->comparFn(pKey, pDataKey);
+ if (compare <= 0) {
+ if (compare == 0 && !hasDupKey) hasDupKey = true;
+ break;
+ } else {
+ px = p;
+ p = SL_NODE_GET_BACKWARD_POINTER(px, i);
+ }
+ }
+ }
+
+ backward[i] = px;
+ }
+ }
+
+ return hasDupKey;
+}
+
+static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) {
+ int32_t level = pNode->level;
+ uint8_t dupMode = SL_DUP_MODE(pSkipList);
+ ASSERT(dupMode != SL_DISCARD_DUP_KEY && dupMode != SL_UPDATE_DUP_KEY);
+
+ for (int32_t j = level - 1; j >= 0; --j) {
+ SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(pNode, j);
+ SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(pNode, j);
+
+ SL_NODE_GET_FORWARD_POINTER(prev, j) = next;
+ SL_NODE_GET_BACKWARD_POINTER(next, j) = prev;
+ }
+
+ tSkipListFreeNode(pNode);
+ pSkipList->size--;
+}
+
+// Function must be called after calling tSkipListRemoveNodeImpl() function
+static void tSkipListCorrectLevel(SSkipList *pSkipList) {
+ while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) {
+ pSkipList->level -= 1;
+ }
+}
+
+UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList,
+ int32_t level) { // record link count in each level
#if SKIP_LIST_RECORD_PERFORMANCE
for (int32_t i = 0; i < level; ++i) {
pSkipList->state.nLevelNodeCnt[i]++;
@@ -47,40 +542,28 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) {
}
static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) {
- int32_t level = getSkipListNodeRandomHeight(pSkipList);
+ int32_t level = 0;
if (pSkipList->size == 0) {
level = 1;
- pSkipList->level = 1;
} else {
+ level = getSkipListNodeRandomHeight(pSkipList);
if (level > pSkipList->level) {
if (pSkipList->level < pSkipList->maxLevel) {
- level = (++pSkipList->level);
+ level = pSkipList->level + 1;
} else {
level = pSkipList->level;
}
}
}
-
- assert(level <= pSkipList->maxLevel);
+
+ ASSERT(level <= pSkipList->maxLevel);
return level;
}
-#define DO_MEMSET_PTR_AREA(n) do {\
-int32_t _l = (n)->level;\
-memset(pNode, 0, SL_NODE_HEADER_SIZE(_l));\
-(n)->level = _l;\
-} while(0)
-
-static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode);
-static SSkipListNode* tSkipListPushBack(SSkipList *pSkipList, SSkipListNode *pNode);
-static SSkipListNode* tSkipListPushFront(SSkipList* pSkipList, SSkipListNode *pNode);
-static SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order);
-
-
// when order is TSDB_ORDER_ASC, return the last node with key less than val
// when order is TSDB_ORDER_DESC, return the first node with key large than val
-static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_t order, SSkipListNode** pCur) {
- __compar_fn_t comparFn = pSkipList->comparFn;
+static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur) {
+ __compar_fn_t comparFn = pSkipList->comparFn;
SSkipListNode *pNode = NULL;
if (pCur != NULL) {
*pCur = NULL;
@@ -89,12 +572,12 @@ static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_
if (order == TSDB_ORDER_ASC) {
pNode = pSkipList->pHead;
for (int32_t i = pSkipList->level - 1; i >= 0; --i) {
- SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, i);
+ SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, i);
while (p != pSkipList->pTail) {
char *key = SL_GET_NODE_KEY(pSkipList, p);
if (comparFn(key, val) < 0) {
pNode = p;
- p = SL_GET_FORWARD_POINTER(p, i);
+ p = SL_NODE_GET_FORWARD_POINTER(p, i);
} else {
if (pCur != NULL) {
*pCur = p;
@@ -106,12 +589,12 @@ static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_
} else {
pNode = pSkipList->pTail;
for (int32_t i = pSkipList->level - 1; i >= 0; --i) {
- SSkipListNode *p = SL_GET_BACKWARD_POINTER(pNode, i);
+ SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(pNode, i);
while (p != pSkipList->pHead) {
char *key = SL_GET_NODE_KEY(pSkipList, p);
if (comparFn(key, val) > 0) {
pNode = p;
- p = SL_GET_BACKWARD_POINTER(p, i);
+ p = SL_NODE_GET_BACKWARD_POINTER(p, i);
} else {
if (pCur != NULL) {
*pCur = p;
@@ -125,213 +608,62 @@ static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_
return pNode;
}
-
-static bool initForwardBackwardPtr(SSkipList* pSkipList) {
+static int initForwardBackwardPtr(SSkipList *pSkipList) {
uint32_t maxLevel = pSkipList->maxLevel;
-
- // head info
- pSkipList->pHead = (SSkipListNode *)calloc(1, SL_NODE_HEADER_SIZE(maxLevel) * 2);
- if (pSkipList->pHead == NULL) {
- return false;
- }
-
- pSkipList->pHead->level = pSkipList->maxLevel;
-
- // tail info
- pSkipList->pTail = (SSkipListNode*) ((char*) pSkipList->pHead + SL_NODE_HEADER_SIZE(maxLevel));
- pSkipList->pTail->level = pSkipList->maxLevel;
-
- for (uint32_t i = 0; i < maxLevel; ++i) {
- SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail;
- SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead;
- }
-
- return true;
-}
-
-SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t lock,
- uint8_t freeNode, __sl_key_fn_t fn) {
- SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList));
- if (pSkipList == NULL) {
- return NULL;
- }
- if (maxLevel > MAX_SKIP_LIST_LEVEL) {
- maxLevel = MAX_SKIP_LIST_LEVEL;
- }
+ // head info
+ pSkipList->pHead = tSkipListNewNode(maxLevel);
+ if (pSkipList->pHead == NULL) return -1;
- pSkipList->keyInfo.type = keyType;
- pSkipList->keyInfo.len = keyLen;
- pSkipList->keyInfo.dupKey = dupKey;
- pSkipList->keyInfo.freeNode = freeNode;
-
- pSkipList->keyFn = fn;
- pSkipList->comparFn = getKeyComparFunc(keyType);
- pSkipList->maxLevel = maxLevel;
- pSkipList->level = 1;
-
- if (!initForwardBackwardPtr(pSkipList)) {
- taosTFree(pSkipList);
- return NULL;
+ // tail info
+ pSkipList->pTail = tSkipListNewNode(maxLevel);
+ if (pSkipList->pTail == NULL) {
+ tSkipListFreeNode(pSkipList->pHead);
+ return -1;
}
-
- if (lock) {
- pSkipList->lock = calloc(1, sizeof(pthread_rwlock_t));
- if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) {
- taosTFree(pSkipList->pHead);
- taosTFree(pSkipList);
-
- return NULL;
- }
+ for (uint32_t i = 0; i < maxLevel; ++i) {
+ SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail;
+ SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead;
}
- srand((uint32_t)time(NULL));
-
-#if SKIP_LIST_RECORD_PERFORMANCE
- pSkipList->state.nTotalMemSize += sizeof(SSkipList);
-#endif
-
- return pSkipList;
+ return 0;
}
-void *tSkipListDestroy(SSkipList *pSkipList) {
- if (pSkipList == NULL) {
- return NULL;
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_wrlock(pSkipList->lock);
- }
-
- if (pSkipList->keyInfo.freeNode) {
- SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0);
-
- while (pNode != pSkipList->pTail) {
- SSkipListNode *pTemp = pNode;
- pNode = SL_GET_FORWARD_POINTER(pNode, 0);
- taosTFree(pTemp);
- }
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- pthread_rwlock_destroy(pSkipList->lock);
-
- taosTFree(pSkipList->lock);
- }
-
- taosTFree(pSkipList->pHead);
- taosTFree(pSkipList);
- return NULL;
-}
+static SSkipListNode *tSkipListNewNode(uint8_t level) {
+ int32_t tsize = sizeof(SSkipListNode) + sizeof(SSkipListNode *) * level * 2;
-void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize) {
- if (pSkipList == NULL) {
- *level = 1;
- *headSize = SL_NODE_HEADER_SIZE(*level);
- return;
- }
+ SSkipListNode *pNode = (SSkipListNode *)calloc(1, tsize);
+ if (pNode == NULL) return NULL;
- *level = getSkipListRandLevel(pSkipList);
- *headSize = SL_NODE_HEADER_SIZE(*level);
+ pNode->level = level;
+ return pNode;
}
-SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) {
- if (pSkipList == NULL || pNode == NULL) {
- return NULL;
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_wrlock(pSkipList->lock);
- }
-
- // if the new key is greater than the maximum key of skip list, push back this node at the end of skip list
- char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode);
- if (pSkipList->size == 0 || pSkipList->comparFn(SL_GET_SL_MAX_KEY(pSkipList), newDatakey) < 0) {
- return tSkipListPushBack(pSkipList, pNode);
- }
-
- // if the new key is less than the minimum key of skip list, push front this node at the front of skip list
- assert(pSkipList->size > 0);
- char* minKey = SL_GET_SL_MIN_KEY(pSkipList);
- if (pSkipList->comparFn(newDatakey, minKey) < 0) {
- return tSkipListPushFront(pSkipList, pNode);
- }
-
- // find the appropriated position to insert data
- SSkipListNode *px = pSkipList->pHead;
- SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0};
+static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipListNode **direction, bool isForward,
+ bool hasDup) {
+ uint8_t dupMode = SL_DUP_MODE(pSkipList);
+ SSkipListNode *pNode = NULL;
- int32_t ret = -1;
- for (int32_t i = pSkipList->level - 1; i >= 0; --i) {
- SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i);
- while (p != pSkipList->pTail) {
- char *key = SL_GET_NODE_KEY(pSkipList, p);
-
- // if the forward element is less than the specified key, forward one step
- ret = pSkipList->comparFn(key, newDatakey);
- if (ret < 0) {
- px = p;
- p = SL_GET_FORWARD_POINTER(px, i);
+ if (hasDup && (dupMode == SL_DISCARD_DUP_KEY || dupMode == SL_UPDATE_DUP_KEY)) {
+ if (dupMode == SL_UPDATE_DUP_KEY) {
+ if (isForward) {
+ pNode = SL_NODE_GET_FORWARD_POINTER(direction[0], 0);
} else {
- break;
+ pNode = SL_NODE_GET_BACKWARD_POINTER(direction[0], 0);
}
+ atomic_store_ptr(&(pNode->pData), pData);
}
-
- forward[i] = px;
- }
-
- // if the skip list does not allowed identical key inserted, the new data will be discarded.
- if (pSkipList->keyInfo.dupKey == 0 && ret == 0) {
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-
- return NULL;
- }
-
- tSkipListDoInsert(pSkipList, forward, pNode);
- return pNode;
-}
-
-
-
-SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey key) {
- SArray* sa = taosArrayInit(1, POINTER_BYTES);
-
- if (pSkipList->lock) {
- pthread_rwlock_wrlock(pSkipList->lock);
- }
+ } else {
+ pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList));
+ if (pNode != NULL) {
+ pNode->pData = pData;
- SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL);
- while (1) {
- SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0);
- if (p == pSkipList->pTail) {
- break;
+ tSkipListDoInsert(pSkipList, direction, pNode, isForward);
}
- if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) {
- break;
- }
- taosArrayPush(sa, &p);
- pNode = p;
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
}
- return sa;
-}
-
-
-
-size_t tSkipListGetSize(const SSkipList* pSkipList) {
- if (pSkipList == NULL) {
- return 0;
- }
-
- return pSkipList->size;
+ return pNode;
}
// static int32_t tSkipListEndParQuery(SSkipList *pSkipList, SSkipListNode *pStartNode, SSkipListKey *pEndKey,
@@ -447,178 +779,13 @@ size_t tSkipListGetSize(const SSkipList* pSkipList) {
// }
//
// // compress the minimum level of skip list
-// while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) {
+// while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) {
// pSkipList->level -= 1;
// }
//
// return true;
//}
-
-uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) {
- uint32_t count = 0;
-
- if (pSkipList->lock) {
- pthread_rwlock_wrlock(pSkipList->lock);
- }
-
- SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL);
- while (1) {
- SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0);
- if (p == pSkipList->pTail) {
- break;
- }
- if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) {
- break;
- }
-
- for (int32_t j = p->level - 1; j >= 0; --j) {
- SSkipListNode* prev = SL_GET_BACKWARD_POINTER(p, j);
- SSkipListNode* next = SL_GET_FORWARD_POINTER(p, j);
- SL_GET_FORWARD_POINTER(prev, j) = next;
- SL_GET_BACKWARD_POINTER(next, j) = prev;
- }
-
- if (pSkipList->keyInfo.freeNode) {
- taosTFree(p);
- }
-
- ++count;
- }
-
- // compress the minimum level of skip list
- while (pSkipList->level > 0) {
- if (SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) != NULL) {
- break;
- }
- pSkipList->level--;
- }
-
- pSkipList->size -= count;
-
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-
- return count;
-}
-
-void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) {
- int32_t level = pNode->level;
-
- if (pSkipList->lock) {
- pthread_rwlock_wrlock(pSkipList->lock);
- }
-
- for (int32_t j = level - 1; j >= 0; --j) {
- SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pNode, j);
- SSkipListNode* next = SL_GET_FORWARD_POINTER(pNode, j);
-
- SL_GET_FORWARD_POINTER(prev, j) = next;
- SL_GET_BACKWARD_POINTER(next, j) = prev;
- }
-
- if (pSkipList->keyInfo.freeNode) {
- taosTFree(pNode);
- }
-
- atomic_sub_fetch_32(&pSkipList->size, 1);
-
- // compress the minimum level of skip list
- while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) {
- pSkipList->level -= 1;
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-}
-
-SSkipListIterator* tSkipListCreateIter(SSkipList *pSkipList) {
- if (pSkipList == NULL) {
- return NULL;
- }
-
- return doCreateSkipListIterator(pSkipList, TSDB_ORDER_ASC);
-}
-
-SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order) {
- assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);
- assert(pSkipList != NULL);
-
- SSkipListIterator* iter = doCreateSkipListIterator(pSkipList, order);
- if (val == NULL) {
- return iter;
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_rdlock(pSkipList->lock);
- }
-
- iter->cur = getPriorNode(pSkipList, val, order, &iter->next);
-
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-
- return iter;
-}
-
-bool tSkipListIterNext(SSkipListIterator *iter) {
- if (iter->pSkipList == NULL) {
- return false;
- }
-
- SSkipList *pSkipList = iter->pSkipList;
-
- if (pSkipList->lock) {
- pthread_rwlock_rdlock(pSkipList->lock);
- }
-
- if (iter->order == TSDB_ORDER_ASC) { // ascending order iterate
- iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0);
-
- // a new node is inserted into between iter->cur and iter->next, ignore it
- if (iter->cur != iter->next && (iter->next != NULL)) {
- iter->cur = iter->next;
- }
-
- iter->next = SL_GET_FORWARD_POINTER(iter->cur, 0);
- } else { // descending order iterate
- iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0);
-
- // a new node is inserted into between iter->cur and iter->next, ignore it
- if (iter->cur != iter->next && (iter->next != NULL)) {
- iter->cur = iter->next;
- }
-
- iter->next = SL_GET_BACKWARD_POINTER(iter->cur, 0);
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-
- iter->step += 1;
- return (iter->order == TSDB_ORDER_ASC)? (iter->cur != pSkipList->pTail) : (iter->cur != pSkipList->pHead);
-}
-
-SSkipListNode *tSkipListIterGet(SSkipListIterator *iter) {
- if (iter == NULL || iter->cur == iter->pSkipList->pTail || iter->cur == iter->pSkipList->pHead) {
- return NULL;
- } else {
- return iter->cur;
- }
-}
-
-void* tSkipListDestroyIter(SSkipListIterator* iter) {
- if (iter == NULL) {
- return NULL;
- }
-
- taosTFree(iter);
- return NULL;
-}
-
+//
// bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey) {
// SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0};
// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType);
@@ -638,111 +805,3 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) {
//
// return ret;
//}
-
-void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) {
- if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) {
- return;
- }
-
- SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1);
-
- int32_t id = 1;
- char* prev = NULL;
-
- while (p != pSkipList->pTail) {
- char *key = SL_GET_NODE_KEY(pSkipList, p);
- if (prev != NULL) {
- assert(pSkipList->comparFn(prev, key) < 0);
- }
-
- switch (pSkipList->keyInfo.type) {
- case TSDB_DATA_TYPE_INT:
- fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key);
- break;
- case TSDB_DATA_TYPE_SMALLINT:
- case TSDB_DATA_TYPE_TINYINT:
- case TSDB_DATA_TYPE_BIGINT:
- fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key);
- break;
- case TSDB_DATA_TYPE_BINARY:
- fprintf(stdout, "%d: %s \n", id++, key);
- break;
- case TSDB_DATA_TYPE_DOUBLE:
- fprintf(stdout, "%d: %lf \n", id++, *(double *)key);
- break;
- default:
- fprintf(stdout, "\n");
- }
-
- prev = SL_GET_NODE_KEY(pSkipList, p);
-
- p = SL_GET_FORWARD_POINTER(p, nlevel - 1);
- }
-}
-
-void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) {
- DO_MEMSET_PTR_AREA(pNode);
-
- for (int32_t i = 0; i < pNode->level; ++i) {
- SSkipListNode *x = forward[i];
- SL_GET_BACKWARD_POINTER(pNode, i) = x;
-
- SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i);
- SL_GET_BACKWARD_POINTER(next, i) = pNode;
-
- SL_GET_FORWARD_POINTER(pNode, i) = next;
- SL_GET_FORWARD_POINTER(x, i) = pNode;
- }
-
- atomic_add_fetch_32(&pSkipList->size, 1);
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-}
-
-SSkipListNode* tSkipListPushFront(SSkipList* pSkipList, SSkipListNode *pNode) {
- SSkipListNode* forward[MAX_SKIP_LIST_LEVEL] = {0};
- for(int32_t i = 0; i < pSkipList->level; ++i) {
- forward[i] = pSkipList->pHead;
- }
-
- tSkipListDoInsert(pSkipList, forward, pNode);
- return pNode;
-}
-
-SSkipListNode* tSkipListPushBack(SSkipList *pSkipList, SSkipListNode *pNode) {
- // do clear pointer area
- DO_MEMSET_PTR_AREA(pNode);
-
- for(int32_t i = 0; i < pNode->level; ++i) {
- SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pSkipList->pTail, i);
- SL_GET_FORWARD_POINTER(prev, i) = pNode;
- SL_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail;
-
- SL_GET_BACKWARD_POINTER(pNode, i) = prev;
- SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode;
- }
-
- atomic_add_fetch_32(&pSkipList->size, 1);
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-
- return pNode;
-}
-
-SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) {
- SSkipListIterator* iter = calloc(1, sizeof(SSkipListIterator));
-
- iter->pSkipList = pSkipList;
- iter->order = order;
- if(order == TSDB_ORDER_ASC) {
- iter->cur = pSkipList->pHead;
- iter->next = SL_GET_FORWARD_POINTER(iter->cur, 0);
- } else {
- iter->cur = pSkipList->pTail;
- iter->next = SL_GET_BACKWARD_POINTER(iter->cur, 0);
- }
-
- return iter;
-}
\ No newline at end of file
diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c
index 4cf73e6dff8aa534e87145a3e1cb04e48d759c65..1be79b7bbd32a20c5f784ad10ca5aeff877eaac5 100644
--- a/src/util/src/tsocket.c
+++ b/src/util/src/tsocket.c
@@ -18,7 +18,7 @@
#include "tsocket.h"
#include "taoserror.h"
-int taosGetFqdn(char *fqdn) {
+int32_t taosGetFqdn(char *fqdn) {
char hostname[1024];
hostname[1023] = '\0';
if (gethostname(hostname, 1023) == -1) {
@@ -26,10 +26,10 @@ int taosGetFqdn(char *fqdn) {
return -1;
}
- struct addrinfo hints = {0};
+ struct addrinfo hints = {0};
struct addrinfo *result = NULL;
hints.ai_flags = AI_CANONNAME;
- int ret = getaddrinfo(hostname, NULL, &hints, &result);
+ int32_t ret = getaddrinfo(hostname, NULL, &hints, &result);
if (!result) {
uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
return -1;
@@ -49,28 +49,28 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) {
int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result);
if (result) {
- struct sockaddr *sa = result->ai_addr;
- struct sockaddr_in *si = (struct sockaddr_in*)sa;
- struct in_addr ia = si->sin_addr;
- uint32_t ip = ia.s_addr;
+ struct sockaddr * sa = result->ai_addr;
+ struct sockaddr_in *si = (struct sockaddr_in *)sa;
+ struct in_addr ia = si->sin_addr;
+ uint32_t ip = ia.s_addr;
freeaddrinfo(result);
return ip;
} else {
#ifdef EAI_SYSTEM
if (ret == EAI_SYSTEM) {
- uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, strerror(errno));
+ uError("failed to get the ip address, fqdn:%s, since:%s", fqdn, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
} else {
- uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret));
+ uError("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret));
}
#else
- uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret));
+ uError("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret));
#endif
return 0xFFFFFFFF;
}
}
-// Function converting an IP address string to an unsigned int.
+// Function converting an IP address string to an uint32_t.
uint32_t ip2uint(const char *const ip_addr) {
char ip_addr_cpy[20];
char ip[5];
@@ -81,7 +81,7 @@ uint32_t ip2uint(const char *const ip_addr) {
s_start = ip_addr_cpy;
s_end = ip_addr_cpy;
- int k;
+ int32_t k;
for (k = 0; *s_start != '\0'; s_start = s_end) {
for (s_end = s_start; *s_end != '.' && *s_end != '\0'; s_end++) {
@@ -95,19 +95,19 @@ uint32_t ip2uint(const char *const ip_addr) {
ip[k] = '\0';
- return *((unsigned int *)ip);
+ return *((uint32_t *)ip);
}
-int taosWriteMsg(SOCKET fd, void *buf, int nbytes) {
- int nleft, nwritten;
- char *ptr = (char *)buf;
+int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) {
+ int32_t nleft, nwritten;
+ char * ptr = (char *)buf;
nleft = nbytes;
while (nleft > 0) {
- nwritten = (int)taosWriteSocket(fd, (char *)ptr, (size_t)nleft);
+ nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft);
if (nwritten <= 0) {
- if (errno == EINTR)
+ if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)
continue;
else
return -1;
@@ -120,20 +120,20 @@ int taosWriteMsg(SOCKET fd, void *buf, int nbytes) {
return (nbytes - nleft);
}
-int taosReadMsg(SOCKET fd, void *buf, int nbytes) {
- int nleft, nread;
- char *ptr = (char *)buf;
+int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) {
+ int32_t nleft, nread;
+ char * ptr = (char *)buf;
nleft = nbytes;
if (fd < 0) return -1;
while (nleft > 0) {
- nread = (int)taosReadSocket(fd, ptr, (size_t)nleft);
+ nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft);
if (nread == 0) {
break;
} else if (nread < 0) {
- if (errno == EINTR) {
+ if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
continue;
} else {
return -1;
@@ -147,11 +147,11 @@ int taosReadMsg(SOCKET fd, void *buf, int nbytes) {
return (nbytes - nleft);
}
-int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) {
+int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) {
taosSetNonblocking(fd, 1);
- int nleft, nwritten, nready;
- fd_set fset;
+ int32_t nleft, nwritten, nready;
+ fd_set fset;
struct timeval tv;
nleft = nbytes;
@@ -160,7 +160,7 @@ int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) {
tv.tv_usec = 0;
FD_ZERO(&fset);
FD_SET(fd, &fset);
- if ((nready = select((int)(fd + 1), NULL, &fset, NULL, &tv)) == 0) {
+ if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) {
errno = ETIMEDOUT;
uError("fd %d timeout, no enough space to write", fd);
break;
@@ -172,7 +172,7 @@ int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) {
return -1;
}
- nwritten = (int)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL);
+ nwritten = (int32_t)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL);
if (nwritten <= 0) {
if (errno == EAGAIN || errno == EINTR) continue;
@@ -189,10 +189,10 @@ int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) {
return (nbytes - nleft);
}
-int taosReadn(SOCKET fd, char *ptr, int nbytes) {
- int nread, nready, nleft = nbytes;
+int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) {
+ int32_t nread, nready, nleft = nbytes;
- fd_set fset;
+ fd_set fset;
struct timeval tv;
while (nleft > 0) {
@@ -200,7 +200,7 @@ int taosReadn(SOCKET fd, char *ptr, int nbytes) {
tv.tv_usec = 0;
FD_ZERO(&fset);
FD_SET(fd, &fset);
- if ((nready = select((int)(fd + 1), NULL, &fset, NULL, &tv)) == 0) {
+ if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) {
errno = ETIMEDOUT;
uError("fd %d timeout\n", fd);
break;
@@ -210,7 +210,7 @@ int taosReadn(SOCKET fd, char *ptr, int nbytes) {
return -1;
}
- if ((nread = (int)taosReadSocket(fd, ptr, (size_t)nleft)) < 0) {
+ if ((nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft)) < 0) {
if (errno == EINTR) continue;
uError("read error, %d (%s)", errno, strerror(errno));
return -1;
@@ -229,8 +229,8 @@ int taosReadn(SOCKET fd, char *ptr, int nbytes) {
SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) {
struct sockaddr_in localAddr;
- SOCKET sockFd;
- int bufSize = 1024000;
+ SOCKET sockFd;
+ int32_t bufSize = 1024000;
uDebug("open udp socket:0x%x:%hu", ip, port);
@@ -239,7 +239,7 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) {
localAddr.sin_addr.s_addr = ip;
localAddr.sin_port = (uint16_t)htons(port);
- if ((sockFd = (int)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) {
+ if ((sockFd = (int32_t)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) {
uError("failed to open udp socket: %d (%s)", errno, strerror(errno));
taosCloseSocketNoCheck(sockFd);
return -1;
@@ -268,9 +268,9 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) {
}
SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) {
- SOCKET sockFd = 0;
+ SOCKET sockFd = 0;
+ int32_t ret;
struct sockaddr_in serverAddr, clientAddr;
- int ret;
sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
@@ -281,7 +281,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie
}
/* set REUSEADDR option, so the portnumber can be re-used */
- int reuse = 1;
+ int32_t reuse = 1;
if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) {
uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno));
taosCloseSocket(sockFd);
@@ -296,8 +296,8 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie
/* bind socket to client address */
if (bind(sockFd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) {
- uError("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)",
- clientIp, destIp, destPort, strerror(errno));
+ uError("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort,
+ strerror(errno));
taosCloseSocket(sockFd);
return -1;
}
@@ -311,7 +311,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie
ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr));
if (ret != 0) {
- //uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno));
+ // uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno));
taosCloseSocket(sockFd);
sockFd = -1;
} else {
@@ -321,36 +321,36 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie
return sockFd;
}
-int taosKeepTcpAlive(SOCKET sockFd) {
- int alive = 1;
+int32_t taosKeepTcpAlive(SOCKET sockFd) {
+ int32_t alive = 1;
if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) {
uError("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno));
taosCloseSocket(sockFd);
return -1;
}
- int probes = 3;
+ int32_t probes = 3;
if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) {
uError("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno));
taosCloseSocket(sockFd);
return -1;
}
- int alivetime = 10;
+ int32_t alivetime = 10;
if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) {
uError("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno));
taosCloseSocket(sockFd);
return -1;
}
- int interval = 3;
+ int32_t interval = 3;
if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) {
uError("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno));
taosCloseSocket(sockFd);
return -1;
}
- int nodelay = 1;
+ int32_t nodelay = 1;
if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) {
uError("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno));
taosCloseSocket(sockFd);
@@ -371,8 +371,8 @@ int taosKeepTcpAlive(SOCKET sockFd) {
SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
struct sockaddr_in serverAdd;
- SOCKET sockFd;
- int reuse;
+ SOCKET sockFd;
+ int32_t reuse;
uDebug("open tcp server socket:0x%x:%hu", ip, port);
@@ -381,7 +381,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
serverAdd.sin_addr.s_addr = ip;
serverAdd.sin_port = (uint16_t)htons(port);
- if ((sockFd = (int)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) {
+ if ((sockFd = (int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) {
uError("failed to open TCP socket: %d (%s)", errno, strerror(errno));
taosCloseSocketNoCheck(sockFd);
return -1;
@@ -417,38 +417,38 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
return sockFd;
}
-void tinet_ntoa(char *ipstr, unsigned int ip) {
+void tinet_ntoa(char *ipstr, uint32_t ip) {
sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24);
}
#define COPY_SIZE 32768
// sendfile shall be used
-int taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len) {
+int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len) {
int64_t leftLen;
- int readLen, writeLen;
+ int32_t readLen, writeLen;
char temp[COPY_SIZE];
leftLen = len;
while (leftLen > 0) {
if (leftLen < COPY_SIZE)
- readLen = (int)leftLen;
+ readLen = (int32_t)leftLen;
else
readLen = COPY_SIZE; // 4K
- int retLen = taosReadMsg(sfd, temp, (int)readLen);
+ int32_t retLen = taosReadMsg(sfd, temp, (int32_t)readLen);
if (readLen != retLen) {
- uError("read error, readLen:%d retLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, retLen, len, leftLen,
- strerror(errno));
+ uError("read error, readLen:%d retLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, retLen, len,
+ leftLen, strerror(errno));
return -1;
}
writeLen = taosWriteMsg(dfd, temp, readLen);
if (readLen != writeLen) {
- uError("copy error, readLen:%d writeLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, writeLen, len, leftLen,
- strerror(errno));
+ uError("copy error, readLen:%d writeLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, writeLen,
+ len, leftLen, strerror(errno));
return -1;
}
diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c
index 6c4af437b27b6623aeebb404e0dad1c70241cf47..451976f563740b4ac933766d5c0c8f4075ad42d8 100644
--- a/src/util/src/tutil.c
+++ b/src/util/src/tutil.c
@@ -326,6 +326,7 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) {
return 0;
}
+// TODO move to comm module
bool taosGetVersionNumber(char *versionStr, int *versionNubmer) {
if (versionStr == NULL || versionNubmer == NULL) {
return false;
@@ -376,7 +377,8 @@ int taosCheckVersion(char *input_client_version, char *input_server_version, int
for(int32_t i = 0; i < comparedSegments; ++i) {
if (clientVersionNumber[i] != serverVersionNumber[i]) {
- uError("the %d-th number of server version:%s not matched with client version:%s", i, server_version, version);
+ uError("the %d-th number of server version:%s not matched with client version:%s", i, server_version,
+ client_version);
return TSDB_CODE_TSC_INVALID_VERSION;
}
}
diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt
index 8687a8005ddeda7320c60c9ef90dd221f56b971f..0c96ed2a2f3dfb7f03268c9f8fbb1b0afa2397b9 100644
--- a/src/util/tests/CMakeLists.txt
+++ b/src/util/tests/CMakeLists.txt
@@ -9,7 +9,22 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
-
+
+ LIST(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c)
ADD_EXECUTABLE(utilTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(utilTest tutil common osdetail gtest pthread gcov)
+
+ LIST(APPEND BIN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c)
+ ADD_EXECUTABLE(trefTest ${BIN_SRC})
+ TARGET_LINK_LIBRARIES(trefTest common tutil)
+
ENDIF()
+
+#IF (TD_LINUX)
+# ADD_EXECUTABLE(trefTest ./trefTest.c)
+# TARGET_LINK_LIBRARIES(trefTest tutil common)
+#ENDIF ()
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+
+
diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp
index 77174f69fda1a49975c44717d8993996274ca4c9..2203ae8e4fc75c2aa0267055f79884f05e3a65e9 100644
--- a/src/util/tests/skiplistTest.cpp
+++ b/src/util/tests/skiplistTest.cpp
@@ -59,7 +59,7 @@ void doubleSkipListTest() {
}
if (size > 0) {
- taosTFree(pNodes);
+ tfree(pNodes);
}
}
@@ -196,7 +196,7 @@ void stringKeySkiplistTest() {
tSkipListRemoveNode(pSkipList, pres[0]);
if (num > 0) {
- taosTFree(pres);
+ tfree(pres);
}
}
@@ -247,7 +247,7 @@ void skiplistPerformanceTest() {
printf("total:%" PRIu64 " ms, avg:%f\n", e - s, (e - s) / (double)size);
printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level);
- assert(tSkipListGetSize(pSkipList) == size);
+ assert(SL_GET_SIZE(pSkipList) == size);
// printf("the level of skiplist is:\n");
//
@@ -273,10 +273,10 @@ void skiplistPerformanceTest() {
int64_t et = taosGetTimestampMs();
printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st);
- assert(tSkipListGetSize(pSkipList) == size);
+ assert(SL_GET_SIZE(pSkipList) == size);
tSkipListDestroy(pSkipList);
- taosTFree(total);
+ tfree(total);
}
// todo not support duplicated key yet
@@ -357,7 +357,7 @@ TEST(testCase, skiplist_test) {
printf("-----%lf\n", pNodes[i]->key.dKey);
}
printf("the range query result size is: %d\n", size);
- taosTFree(pNodes);
+ tfree(pNodes);
SSkipListKey *pKeys = malloc(sizeof(SSkipListKey) * 20);
for (int32_t i = 0; i < 8; i += 2) {
@@ -371,7 +371,7 @@ TEST(testCase, skiplist_test) {
for (int32_t i = 0; i < r; ++i) {
// printf("%lf ", pNodes[i]->key.dKey);
}
- taosTFree(pNodes);
+ tfree(pNodes);
free(pKeys);*/
}
diff --git a/src/util/tests/trefTest.c b/src/util/tests/trefTest.c
new file mode 100644
index 0000000000000000000000000000000000000000..454860410b4dd0c3fdf65c4b7fd7950a1a7d4446
--- /dev/null
+++ b/src/util/tests/trefTest.c
@@ -0,0 +1,195 @@
+#include
+#include
+#include
+#include
+#include
+#include "os.h"
+#include "tref.h"
+#include "tlog.h"
+#include "tglobal.h"
+#include "taoserror.h"
+#include "tulog.h"
+
+typedef struct {
+ int refNum;
+ int steps;
+ int rsetId;
+ int64_t rid;
+ void **p;
+} SRefSpace;
+
+void iterateRefs(int rsetId) {
+ int count = 0;
+
+ void *p = taosIterateRef(rsetId, NULL);
+ while (p) {
+ // process P
+ count++;
+ p = taosIterateRef(rsetId, p);
+ }
+
+ printf(" %d ", count);
+}
+
+void *addRef(void *param) {
+ SRefSpace *pSpace = (SRefSpace *)param;
+ int id;
+ int64_t rid;
+
+ for (int i=0; i < pSpace->steps; ++i) {
+ printf("a");
+ id = random() % pSpace->refNum;
+ if (pSpace->rid[id] <= 0) {
+ pSpace->p[id] = malloc(128);
+ pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]);
+ }
+ usleep(100);
+ }
+
+ return NULL;
+}
+
+void *removeRef(void *param) {
+ SRefSpace *pSpace = (SRefSpace *)param;
+ int id;
+ int64_t rid;
+
+ for (int i=0; i < pSpace->steps; ++i) {
+ printf("d");
+ id = random() % pSpace->refNum;
+ if (pSpace->rid[id] > 0) {
+ code = taosRemoveRef(pSpace->rsetId, pSpace->rid[id]);
+ if (code == 0) pSpace->rid[id] = 0;
+ }
+
+ usleep(100);
+ }
+
+ return NULL;
+}
+
+void *acquireRelease(void *param) {
+ SRefSpace *pSpace = (SRefSpace *)param;
+ int id;
+ int64_t rid;
+
+ for (int i=0; i < pSpace->steps; ++i) {
+ printf("a");
+
+ id = random() % pSpace->refNum;
+ void *p = taosAcquireRef(pSpace->rsetId, pSpace->p[id]);
+ if (p) {
+ usleep(id % 5 + 1);
+ taosReleaseRef(pSpace->rsetId, pSpace->p[id]);
+ }
+ }
+
+ return NULL;
+}
+
+void myfree(void *p) {
+ free(p);
+}
+
+void *openRefSpace(void *param) {
+ SRefSpace *pSpace = (SRefSpace *)param;
+
+ printf("c");
+ pSpace->rsetId = taosOpenRef(50, myfree);
+
+ if (pSpace->rsetId < 0) {
+ printf("failed to open ref, reson:%s\n", tstrerror(pSpace->rsetId));
+ return NULL;
+ }
+
+ pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum);
+
+ pthread_attr_t thattr;
+ pthread_attr_init(&thattr);
+ pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+
+ pthread_t thread1, thread2, thread3;
+ pthread_create(&(thread1), &thattr, addRef, (void *)(pSpace));
+ pthread_create(&(thread2), &thattr, removeRef, (void *)(pSpace));
+ pthread_create(&(thread3), &thattr, acquireRelease, (void *)(pSpace));
+
+ pthread_join(thread1, NULL);
+ pthread_join(thread2, NULL);
+ pthread_join(thread3, NULL);
+
+ for (int i=0; irefNum; ++i) {
+ taosRemoveRef(pSpace->rsetId, pSpace->rid[i]);
+ }
+
+ taosCloseRef(pSpace->rsetId);
+
+ uInfo("rsetId:%d main thread exit", pSpace->rsetId);
+ free(pSpace->p);
+ pSpace->p = NULL;
+
+ return NULL;
+}
+
+int main(int argc, char *argv[]) {
+ int refNum = 100;
+ int threads = 10;
+ int steps = 10000;
+ int loops = 1;
+
+ uDebugFlag = 143;
+
+ for (int i=1; i
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_VNODE_CFG_H
+#define TDENGINE_VNODE_CFG_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t vnodeReadCfg(SVnodeObj *pVnode);
+int32_t vnodeWriteCfg(SCreateVnodeMsg *pVnodeCfg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h
index 169334c6119a35f53fd0a69dd1f7cb6952fbb727..c9689ce8b266acd88b0b6212637768934e4833ed 100644
--- a/src/vnode/inc/vnodeInt.h
+++ b/src/vnode/inc/vnodeInt.h
@@ -37,17 +37,19 @@ extern int32_t vDebugFlag;
typedef struct {
int32_t vgId; // global vnode group ID
int32_t refCount; // reference count
+ int32_t queuedMsg;
int32_t delay;
int8_t status;
int8_t role;
int8_t accessState;
- int64_t version; // current version
- int64_t fversion; // version on saved data file
+ int8_t isFull;
+ uint64_t version; // current version
+ uint64_t fversion; // version on saved data file
void *wqueue;
void *rqueue;
void *wal;
void *tsdb;
- void *sync;
+ int64_t sync;
void *events;
void *cq; // continuous query
int32_t cfgVersion;
@@ -58,11 +60,9 @@ typedef struct {
char *rootDir;
tsem_t sem;
int8_t dropped;
- char db[TSDB_DB_NAME_LEN];
+ char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
} SVnodeObj;
-int vnodeWriteToQueue(void *param, void *pHead, int type);
-int vnodeWriteCqMsgToQueue(void *param, void *pHead, int type);
void vnodeInitWriteFp(void);
void vnodeInitReadFp(void);
diff --git a/src/vnode/inc/vnodeVersion.h b/src/vnode/inc/vnodeVersion.h
new file mode 100644
index 0000000000000000000000000000000000000000..1d086cb21fdab0247038c7f3d32d89b38c19d871
--- /dev/null
+++ b/src/vnode/inc/vnodeVersion.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_VNODE_VERSION_H
+#define TDENGINE_VNODE_VERSION_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t vnodeReadVersion(SVnodeObj *pVnode);
+int32_t vnodeSaveVersion(SVnodeObj *pVnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c
new file mode 100644
index 0000000000000000000000000000000000000000..2d56157328714a5ae4b8ac05abe3ac4468e361cb
--- /dev/null
+++ b/src/vnode/src/vnodeCfg.c
@@ -0,0 +1,324 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taosmsg.h"
+#include "taoserror.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "tsdb.h"
+#include "dnode.h"
+#include "vnodeInt.h"
+#include "vnodeCfg.h"
+
+static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
+ tstrncpy(pVnode->db, vnodeMsg->db, sizeof(pVnode->db));
+ pVnode->cfgVersion = vnodeMsg->cfg.cfgVersion;
+ pVnode->tsdbCfg.cacheBlockSize = vnodeMsg->cfg.cacheBlockSize;
+ pVnode->tsdbCfg.totalBlocks = vnodeMsg->cfg.totalBlocks;
+ pVnode->tsdbCfg.daysPerFile = vnodeMsg->cfg.daysPerFile;
+ pVnode->tsdbCfg.keep = vnodeMsg->cfg.daysToKeep;
+ pVnode->tsdbCfg.keep1 = vnodeMsg->cfg.daysToKeep1;
+ pVnode->tsdbCfg.keep2 = vnodeMsg->cfg.daysToKeep2;
+ pVnode->tsdbCfg.minRowsPerFileBlock = vnodeMsg->cfg.minRowsPerFileBlock;
+ pVnode->tsdbCfg.maxRowsPerFileBlock = vnodeMsg->cfg.maxRowsPerFileBlock;
+ pVnode->tsdbCfg.precision = vnodeMsg->cfg.precision;
+ pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression;
+ pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel;
+ pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod;
+ pVnode->walCfg.keep = TAOS_WAL_NOT_KEEP;
+ pVnode->syncCfg.replica = vnodeMsg->cfg.replications;
+ pVnode->syncCfg.quorum = vnodeMsg->cfg.quorum;
+
+ for (int i = 0; i < pVnode->syncCfg.replica; ++i) {
+ SVnodeDesc *node = &vnodeMsg->nodes[i];
+ pVnode->syncCfg.nodeInfo[i].nodeId = node->nodeId;
+ taosGetFqdnPortFromEp(node->nodeEp, pVnode->syncCfg.nodeInfo[i].nodeFqdn, &pVnode->syncCfg.nodeInfo[i].nodePort);
+ pVnode->syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC;
+ }
+
+ vInfo("vgId:%d, load vnode cfg successfully, replcia:%d", pVnode->vgId, pVnode->syncCfg.replica);
+ for (int32_t i = 0; i < pVnode->syncCfg.replica; i++) {
+ SNodeInfo *node = &pVnode->syncCfg.nodeInfo[i];
+ vInfo("vgId:%d, dnode:%d, %s:%u", pVnode->vgId, node->nodeId, node->nodeFqdn, node->nodePort);
+ }
+}
+
+int32_t vnodeReadCfg(SVnodeObj *pVnode) {
+ int32_t ret = TSDB_CODE_VND_APP_ERROR;
+ int32_t len = 0;
+ int maxLen = 1000;
+ char * content = calloc(1, maxLen + 1);
+ cJSON * root = NULL;
+ FILE * fp = NULL;
+ bool nodeChanged = false;
+ SCreateVnodeMsg vnodeMsg;
+
+ char file[TSDB_FILENAME_LEN + 30] = {0};
+ sprintf(file, "%s/vnode%d/config.json", tsVnodeDir, pVnode->vgId);
+
+ vnodeMsg.cfg.vgId = pVnode->vgId;
+
+ fp = fopen(file, "r");
+ if (!fp) {
+ vError("vgId:%d, failed to open vnode cfg file:%s to read, error:%s", pVnode->vgId, file, strerror(errno));
+ ret = TAOS_SYSTEM_ERROR(errno);
+ goto PARSE_VCFG_ERROR;
+ }
+
+ len = fread(content, 1, maxLen, fp);
+ if (len <= 0) {
+ vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+
+ content[len] = 0;
+ root = cJSON_Parse(content);
+ if (root == NULL) {
+ vError("vgId:%d, failed to read %s, invalid json format", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+
+ cJSON *db = cJSON_GetObjectItem(root, "db");
+ if (!db || db->type != cJSON_String || db->valuestring == NULL) {
+ vError("vgId:%d, failed to read %s, db not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ tstrncpy(vnodeMsg.db, db->valuestring, sizeof(vnodeMsg.db));
+
+ cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion");
+ if (!cfgVersion || cfgVersion->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, cfgVersion not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.cfgVersion = cfgVersion->valueint;
+
+ cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize");
+ if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, cacheBlockSize not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.cacheBlockSize = cacheBlockSize->valueint;
+
+ cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks");
+ if (!totalBlocks || totalBlocks->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, totalBlocks not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.totalBlocks = totalBlocks->valueint;
+
+ cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile");
+ if (!daysPerFile || daysPerFile->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, daysPerFile not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.daysPerFile = daysPerFile->valueint;
+
+ cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep");
+ if (!daysToKeep || daysToKeep->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, daysToKeep not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.daysToKeep = daysToKeep->valueint;
+
+ cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1");
+ if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, daysToKeep1 not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.daysToKeep1 = daysToKeep1->valueint;
+
+ cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2");
+ if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, daysToKeep2 not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.daysToKeep2 = daysToKeep2->valueint;
+
+ cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock");
+ if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, minRowsPerFileBlock not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.minRowsPerFileBlock = minRowsPerFileBlock->valueint;
+
+ cJSON *maxRowsPerFileBlock = cJSON_GetObjectItem(root, "maxRowsPerFileBlock");
+ if (!maxRowsPerFileBlock || maxRowsPerFileBlock->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, maxRowsPerFileBlock not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint;
+
+ cJSON *precision = cJSON_GetObjectItem(root, "precision");
+ if (!precision || precision->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, precision not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.precision = (int8_t)precision->valueint;
+
+ cJSON *compression = cJSON_GetObjectItem(root, "compression");
+ if (!compression || compression->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, compression not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.compression = (int8_t)compression->valueint;
+
+ cJSON *walLevel = cJSON_GetObjectItem(root, "walLevel");
+ if (!walLevel || walLevel->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, walLevel not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.walLevel = (int8_t)walLevel->valueint;
+
+ cJSON *fsyncPeriod = cJSON_GetObjectItem(root, "fsync");
+ if (!walLevel || walLevel->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, fsyncPeriod not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.fsyncPeriod = fsyncPeriod->valueint;
+
+ cJSON *wals = cJSON_GetObjectItem(root, "wals");
+ if (!wals || wals->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, wals not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.wals = (int8_t)wals->valueint;
+
+ cJSON *replica = cJSON_GetObjectItem(root, "replica");
+ if (!replica || replica->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, replica not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.replications = (int8_t)replica->valueint;
+
+ cJSON *quorum = cJSON_GetObjectItem(root, "quorum");
+ if (!quorum || quorum->type != cJSON_Number) {
+ vError("vgId: %d, failed to read %s, quorum not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.quorum = (int8_t)quorum->valueint;
+
+ cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
+ if (!nodeInfos || nodeInfos->type != cJSON_Array) {
+ vError("vgId:%d, failed to read %s, nodeInfos not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+
+ int size = cJSON_GetArraySize(nodeInfos);
+ if (size != vnodeMsg.cfg.replications) {
+ vError("vgId:%d, failed to read %s, nodeInfos size not matched", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+
+ for (int i = 0; i < size; ++i) {
+ cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
+ if (nodeInfo == NULL) continue;
+ SVnodeDesc *node = &vnodeMsg.nodes[i];
+
+ cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
+ if (!nodeId || nodeId->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, nodeId not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ node->nodeId = nodeId->valueint;
+
+ cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
+ if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
+ vError("vgId:%d, failed to read %s, nodeFqdn not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ tstrncpy(node->nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
+
+ if (!nodeChanged) {
+ nodeChanged = dnodeCheckEpChanged(node->nodeId, node->nodeEp);
+ }
+ }
+
+ ret = TSDB_CODE_SUCCESS;
+
+PARSE_VCFG_ERROR:
+ if (content != NULL) free(content);
+ if (root != NULL) cJSON_Delete(root);
+ if (fp != NULL) fclose(fp);
+
+ if (nodeChanged) {
+ vnodeWriteCfg(&vnodeMsg);
+ }
+
+ if (ret == TSDB_CODE_SUCCESS) {
+ vnodeLoadCfg(pVnode, &vnodeMsg);
+ }
+
+ terrno = 0;
+ return ret;
+}
+
+int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) {
+ char file[TSDB_FILENAME_LEN + 30] = {0};
+ sprintf(file, "%s/vnode%d/config.json", tsVnodeDir, pMsg->cfg.vgId);
+
+ FILE *fp = fopen(file, "w");
+ if (!fp) {
+ vError("vgId:%d, failed to write %s error:%s", pMsg->cfg.vgId, file, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return terrno;
+ }
+
+ int32_t len = 0;
+ int32_t maxLen = 1000;
+ char * content = calloc(1, maxLen + 1);
+
+ len += snprintf(content + len, maxLen - len, "{\n");
+ len += snprintf(content + len, maxLen - len, " \"db\": \"%s\",\n", pMsg->db);
+ len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pMsg->cfg.cfgVersion);
+ len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pMsg->cfg.cacheBlockSize);
+ len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pMsg->cfg.totalBlocks);
+ len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pMsg->cfg.daysPerFile);
+ len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pMsg->cfg.daysToKeep);
+ len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pMsg->cfg.daysToKeep1);
+ len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pMsg->cfg.daysToKeep2);
+ len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pMsg->cfg.minRowsPerFileBlock);
+ len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pMsg->cfg.maxRowsPerFileBlock);
+ len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pMsg->cfg.precision);
+ len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pMsg->cfg.compression);
+ len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pMsg->cfg.walLevel);
+ len += snprintf(content + len, maxLen - len, " \"fsync\": %d,\n", pMsg->cfg.fsyncPeriod);
+ len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pMsg->cfg.replications);
+ len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pMsg->cfg.wals);
+ len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pMsg->cfg.quorum);
+ len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
+ for (int32_t i = 0; i < pMsg->cfg.replications; i++) {
+ SVnodeDesc *node = &pMsg->nodes[i];
+ dnodeUpdateEp(node->nodeId, node->nodeEp, NULL, NULL);
+ len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", node->nodeId);
+ len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", node->nodeEp);
+ if (i < pMsg->cfg.replications - 1) {
+ len += snprintf(content + len, maxLen - len, " },{\n");
+ } else {
+ len += snprintf(content + len, maxLen - len, " }]\n");
+ }
+ }
+ len += snprintf(content + len, maxLen - len, "}\n");
+
+ fwrite(content, 1, len, fp);
+ fflush(fp);
+ fclose(fp);
+ free(content);
+ terrno = 0;
+
+ vInfo("vgId:%d, successed to write %s", pMsg->cfg.vgId, file);
+ return TSDB_CODE_SUCCESS;
+}
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index 7f1e222e06e4850b20bad348aaacb02afbaed049..dacc18c4114f485a98506a0aa8c970c050fb4dab 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -15,17 +15,11 @@
#define _DEFAULT_SOURCE
#include "os.h"
-
-#include "tcache.h"
-#include "cJSON.h"
-#include "dnode.h"
-#include "hash.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tglobal.h"
#include "trpc.h"
#include "tsdb.h"
-#include "ttimer.h"
#include "tutil.h"
#include "vnode.h"
#include "vnodeInt.h"
@@ -33,28 +27,22 @@
#include "tpath.h"
#include "tdisk.h"
-#define TSDB_VNODE_VERSION_CONTENT_LEN 31
-
-static SHashObj*tsDnodeVnodesHash;
+static SHashObj*tsVnodesHash;
static void vnodeCleanUp(SVnodeObj *pVnode);
-static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg);
-static int32_t vnodeReadCfg(SVnodeObj *pVnode);
-static int32_t vnodeSaveVersion(SVnodeObj *pVnode);
-static int32_t vnodeReadVersion(SVnodeObj *pVnode);
-static int vnodeProcessTsdbStatus(void *arg, int status);
+static int vnodeProcessTsdbStatus(void *arg, int status, int eno);
static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion);
-static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index);
+static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId);
static void vnodeNotifyRole(void *ahandle, int8_t role);
static void vnodeCtrlFlow(void *handle, int32_t mseconds);
static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion);
#ifndef _SYNC
-tsync_h syncStart(const SSyncInfo *info) { return NULL; }
-int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype) { return 0; }
-void syncStop(tsync_h shandle) {}
-int32_t syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; }
-int syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; }
-void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {}
+int64_t syncStart(const SSyncInfo *info) { return NULL; }
+int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype) { return 0; }
+void syncStop(int64_t rid) {}
+int32_t syncReconfig(int64_t rid, const SSyncCfg * cfg) { return 0; }
+int syncGetNodesRole(int64_t rid, SNodesRole * cfg) { return 0; }
+void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {}
#endif
char* vnodeStatus[] = {
@@ -72,26 +60,33 @@ int32_t vnodeInitResources() {
vnodeInitWriteFp();
vnodeInitReadFp();
- tsDnodeVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
- if (tsDnodeVnodesHash == NULL) {
+ tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
+ if (tsVnodesHash == NULL) {
vError("failed to init vnode list");
return TSDB_CODE_VND_OUT_OF_MEMORY;
}
+ if (tsdbInitCommitQueue(tsNumOfCommitThreads) < 0) {
+ vError("failed to init vnode commit queue");
+ return terrno;
+ }
+
return TSDB_CODE_SUCCESS;
}
void vnodeCleanupResources() {
- if (tsDnodeVnodesHash != NULL) {
+ tsdbDestroyCommitQueue();
+
+ if (tsVnodesHash != NULL) {
vDebug("vnode list is cleanup");
- taosHashCleanup(tsDnodeVnodesHash);
- tsDnodeVnodesHash = NULL;
+ taosHashCleanup(tsVnodesHash);
+ tsVnodesHash = NULL;
}
syncCleanUp();
}
-int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
+int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
int32_t code;
SVnodeObj *pVnode = vnodeAcquire(pVnodeCfg->cfg.vgId);
@@ -129,7 +124,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
}
}
- code = vnodeSaveCfg(pVnodeCfg);
+ code = vnodeWriteCfg(pVnodeCfg);
if (code != TSDB_CODE_SUCCESS) {
vError("vgId:%d, failed to save vnode cfg, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(code));
return code;
@@ -139,13 +134,13 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId;
tsdbCfg.cacheBlockSize = pVnodeCfg->cfg.cacheBlockSize;
tsdbCfg.totalBlocks = pVnodeCfg->cfg.totalBlocks;
- // tsdbCfg.maxTables = pVnodeCfg->cfg.maxTables;
tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile;
tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep;
tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock;
tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
tsdbCfg.precision = pVnodeCfg->cfg.precision;
tsdbCfg.compression = pVnodeCfg->cfg.compression;
+ tsdbCfg.update = pVnodeCfg->cfg.update;
char tsdbDir[TSDB_FILENAME_LEN] = {0};
tdGetTsdbRootDir(tsDataDir, pVnodeCfg->cfg.vgId, tsdbDir);
@@ -177,7 +172,7 @@ int32_t vnodeDrop(int32_t vgId) {
return TSDB_CODE_SUCCESS;
}
-int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) {
+int32_t vnodeAlter(void *param, SCreateVnodeMsg *pVnodeCfg) {
SVnodeObj *pVnode = param;
// vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS
@@ -187,7 +182,7 @@ int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) {
return TSDB_CODE_SUCCESS;
}
- int32_t code = vnodeSaveCfg(pVnodeCfg);
+ int32_t code = vnodeWriteCfg(pVnodeCfg);
if (code != TSDB_CODE_SUCCESS) {
pVnode->status = TAOS_VN_STATUS_READY;
return code;
@@ -238,6 +233,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
pVnode->vgId = vnode;
pVnode->status = TAOS_VN_STATUS_INIT;
+ pVnode->fversion = 0;
pVnode->version = 0;
pVnode->tsdbCfg.tsdbId = pVnode->vgId;
pVnode->rootDir = strdup(rootDir);
@@ -260,8 +256,8 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
pVnode->fversion = pVnode->version;
- pVnode->wqueue = dnodeAllocateVnodeWqueue(pVnode);
- pVnode->rqueue = dnodeAllocateVnodeRqueue(pVnode);
+ pVnode->wqueue = dnodeAllocVWriteQueue(pVnode);
+ pVnode->rqueue = dnodeAllocVReadQueue(pVnode);
if (pVnode->wqueue == NULL || pVnode->rqueue == NULL) {
vnodeCleanUp(pVnode);
return terrno;
@@ -272,7 +268,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
strcpy(cqCfg.pass, tsInternalPass);
strcpy(cqCfg.db, pVnode->db);
cqCfg.vgId = vnode;
- cqCfg.cqWrite = vnodeWriteCqMsgToQueue;
+ cqCfg.cqWrite = vnodeWriteToWQueue;
pVnode->cq = cqOpen(pVnode, &cqCfg);
if (pVnode->cq == NULL) {
vnodeCleanUp(pVnode);
@@ -292,25 +288,42 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
if (pVnode->tsdb == NULL) {
vnodeCleanUp(pVnode);
return terrno;
- } else if (terrno != TSDB_CODE_SUCCESS && pVnode->syncCfg.replica <= 1) {
+ } else if (terrno != TSDB_CODE_SUCCESS) {
vError("vgId:%d, failed to open tsdb, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
tstrerror(terrno));
- vnodeCleanUp(pVnode);
- return terrno;
+ if (pVnode->syncCfg.replica <= 1) {
+ vnodeCleanUp(pVnode);
+ return terrno;
+ } else {
+ pVnode->fversion = 0;
+ pVnode->version = 0;
+ }
}
sprintf(temp, "%s/wal", rootDir);
+ pVnode->walCfg.vgId = pVnode->vgId;
pVnode->wal = walOpen(temp, &pVnode->walCfg);
if (pVnode->wal == NULL) {
vnodeCleanUp(pVnode);
return terrno;
}
- walRestore(pVnode->wal, pVnode, vnodeWriteToQueue);
+ walRestore(pVnode->wal, pVnode, vnodeProcessWrite);
if (pVnode->version == 0) {
+ pVnode->fversion = 0;
pVnode->version = walGetVersion(pVnode->wal);
}
+ code = tsdbSyncCommit(pVnode->tsdb);
+ if (code != 0) {
+ vError("vgId:%d, failed to commit after restore from wal since %s", pVnode->vgId, tstrerror(code));
+ vnodeCleanUp(pVnode);
+ return code;
+ }
+
+ walRemoveAllOldFiles(pVnode->wal);
+ walRenew(pVnode->wal);
+
SSyncInfo syncInfo;
syncInfo.vgId = pVnode->vgId;
syncInfo.version = pVnode->version;
@@ -319,8 +332,8 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
syncInfo.ahandle = pVnode;
syncInfo.getWalInfo = vnodeGetWalInfo;
syncInfo.getFileInfo = vnodeGetFileInfo;
- syncInfo.writeToCache = vnodeWriteToQueue;
- syncInfo.confirmForward = dnodeSendRpcVnodeWriteRsp;
+ syncInfo.writeToCache = vnodeWriteToWQueue;
+ syncInfo.confirmForward = dnodeSendRpcVWriteRsp;
syncInfo.notifyRole = vnodeNotifyRole;
syncInfo.notifyFlowCtrl = vnodeCtrlFlow;
syncInfo.notifyFileSynced = vnodeNotifyFileSynced;
@@ -329,7 +342,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
#ifndef _SYNC
pVnode->role = TAOS_SYNC_ROLE_MASTER;
#else
- if (pVnode->sync == NULL) {
+ if (pVnode->sync <= 0) {
vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
tstrerror(terrno));
vnodeCleanUp(pVnode);
@@ -347,7 +360,8 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
pVnode->status = TAOS_VN_STATUS_READY;
vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode);
- taosHashPut(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *));
+ tsdbIncCommitRef(pVnode->vgId);
+ taosHashPut(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *));
return TSDB_CODE_SUCCESS;
}
@@ -364,7 +378,9 @@ int32_t vnodeClose(int32_t vgId) {
}
void vnodeRelease(void *pVnodeRaw) {
+ if (pVnodeRaw == NULL) return;
SVnodeObj *pVnode = pVnodeRaw;
+ int32_t code = 0;
int32_t vgId = pVnode->vgId;
int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
@@ -385,8 +401,12 @@ void vnodeRelease(void *pVnodeRaw) {
pVnode->qMgmt = NULL;
}
+ if (pVnode->wal) {
+ walStop(pVnode->wal);
+ }
+
if (pVnode->tsdb) {
- tsdbCloseRepo(pVnode->tsdb, 1);
+ code = tsdbCloseRepo(pVnode->tsdb, 1);
pVnode->tsdb = NULL;
}
@@ -398,21 +418,26 @@ void vnodeRelease(void *pVnodeRaw) {
}
if (pVnode->wal) {
+ if (code != 0) {
+ vError("vgId:%d, failed to commit while close tsdb repo, keep wal", pVnode->vgId);
+ } else {
+ walRemoveAllOldFiles(pVnode->wal);
+ }
walClose(pVnode->wal);
pVnode->wal = NULL;
}
if (pVnode->wqueue) {
- dnodeFreeVnodeWqueue(pVnode->wqueue);
+ dnodeFreeVWriteQueue(pVnode->wqueue);
pVnode->wqueue = NULL;
}
if (pVnode->rqueue) {
- dnodeFreeVnodeRqueue(pVnode->rqueue);
+ dnodeFreeVReadQueue(pVnode->rqueue);
pVnode->rqueue = NULL;
}
- taosTFree(pVnode->rootDir);
+ tfree(pVnode->rootDir);
if (pVnode->dropped) {
char rootDir[TSDB_FILENAME_LEN] = {0};
@@ -444,8 +469,9 @@ void vnodeRelease(void *pVnodeRaw) {
tsem_destroy(&pVnode->sem);
free(pVnode);
+ tsdbDecCommitRef(vgId);
- int32_t count = taosHashGetSize(tsDnodeVnodesHash);
+ int32_t count = taosHashGetSize(tsVnodesHash);
vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count);
}
@@ -462,7 +488,7 @@ static void vnodeIncRef(void *ptNode) {
}
void *vnodeAcquire(int32_t vgId) {
- SVnodeObj **ppVnode = taosHashGetCB(tsDnodeVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
+ SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
if (ppVnode == NULL || *ppVnode == NULL) {
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
@@ -473,41 +499,11 @@ void *vnodeAcquire(int32_t vgId) {
return *ppVnode;
}
-void *vnodeAcquireRqueue(int32_t vgId) {
- SVnodeObj *pVnode = vnodeAcquire(vgId);
- if (pVnode == NULL) return NULL;
-
- int32_t code = vnodeCheckRead(pVnode);
- if (code != TSDB_CODE_SUCCESS) {
- terrno = code;
- vInfo("vgId:%d, can not provide read service, status is %s", vgId, vnodeStatus[pVnode->status]);
- vnodeRelease(pVnode);
- return NULL;
- }
-
- return pVnode->rqueue;
-}
-
-void *vnodeAcquireWqueue(int32_t vgId) {
- SVnodeObj *pVnode = vnodeAcquire(vgId);
- if (pVnode == NULL) return NULL;
-
- int32_t code = vnodeCheckWrite(pVnode);
- if (code != TSDB_CODE_SUCCESS) {
- terrno = code;
- vInfo("vgId:%d, can not provide write service, status is %s", vgId, vnodeStatus[pVnode->status]);
- vnodeRelease(pVnode);
- return NULL;
- }
-
- return pVnode->wqueue;
-}
-
void *vnodeGetWal(void *pVnode) {
return ((SVnodeObj *)pVnode)->wal;
}
-static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) {
+static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) {
int64_t totalStorage = 0;
int64_t compStorage = 0;
int64_t pointsWritten = 0;
@@ -531,7 +527,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) {
}
int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
- SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash);
+ SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash);
while (taosHashIterNext(pIter)) {
SVnodeObj **pVnode = taosHashIterGet(pIter);
if (pVnode == NULL) continue;
@@ -551,8 +547,8 @@ int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
}
void vnodeBuildStatusMsg(void *param) {
- SDMStatusMsg *pStatus = param;
- SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash);
+ SStatusMsg *pStatus = param;
+ SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash);
while (taosHashIterNext(pIter)) {
SVnodeObj **pVnode = taosHashIterGet(pIter);
@@ -565,7 +561,7 @@ void vnodeBuildStatusMsg(void *param) {
taosHashDestroyIter(pIter);
}
-void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes) {
+void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) {
for (int32_t i = 0; i < numOfVnodes; ++i) {
pAccess[i].vgId = htonl(pAccess[i].vgId);
SVnodeObj *pVnode = vnodeAcquire(pAccess[i].vgId);
@@ -581,7 +577,7 @@ void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes) {
static void vnodeCleanUp(SVnodeObj *pVnode) {
// remove from hash, so new messages wont be consumed
- taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
+ taosHashRemove(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
if (pVnode->status != TAOS_VN_STATUS_INIT) {
// it may be in updateing or reset state, then it shall wait
@@ -595,9 +591,9 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
}
// stop replication module
- if (pVnode->sync) {
- void *sync = pVnode->sync;
- pVnode->sync = NULL;
+ if (pVnode->sync > 0) {
+ int64_t sync = pVnode->sync;
+ pVnode->sync = -1;
syncStop(sync);
}
@@ -609,29 +605,46 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
}
// TODO: this is a simple implement
-static int vnodeProcessTsdbStatus(void *arg, int status) {
+static int vnodeProcessTsdbStatus(void *arg, int status, int eno) {
SVnodeObj *pVnode = arg;
+ if (eno != TSDB_CODE_SUCCESS) {
+ vError("vgId:%d, failed to commit since %s, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, tstrerror(eno),
+ pVnode->fversion, pVnode->version);
+ pVnode->isFull = 1;
+ return 0;
+ }
+
if (status == TSDB_STATUS_COMMIT_START) {
- pVnode->fversion = pVnode->version;
- return walRenew(pVnode->wal);
+ pVnode->fversion = pVnode->version;
+ vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
+ if (pVnode->status == TAOS_VN_STATUS_INIT) {
+ return 0;
+ } else {
+ return walRenew(pVnode->wal);
+ }
}
- if (status == TSDB_STATUS_COMMIT_OVER)
+ if (status == TSDB_STATUS_COMMIT_OVER) {
+ vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
+ pVnode->isFull = 0;
+ walRemoveOneOldFile(pVnode->wal);
return vnodeSaveVersion(pVnode);
+ }
- return 0;
+ return 0;
}
-static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) {
+static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size,
+ uint64_t *fversion) {
SVnodeObj *pVnode = ahandle;
*fversion = pVnode->fversion;
return tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size);
}
-static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) {
+static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) {
SVnodeObj *pVnode = ahandle;
- return walGetWalFile(pVnode->wal, name, index);
+ return walGetWalFile(pVnode->wal, fileName, fileId);
}
static void vnodeNotifyRole(void *ahandle, int8_t role) {
@@ -648,18 +661,18 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) {
static void vnodeCtrlFlow(void *ahandle, int32_t mseconds) {
SVnodeObj *pVnode = ahandle;
- if (pVnode->delay != mseconds)
+ if (pVnode->delay != mseconds) {
vInfo("vgId:%d, sync flow control, mseconds:%d", pVnode->vgId, mseconds);
+ }
pVnode->delay = mseconds;
}
-static int vnodeResetTsdb(SVnodeObj *pVnode)
-{
+static int vnodeResetTsdb(SVnodeObj *pVnode) {
char rootDir[128] = "\0";
sprintf(rootDir, "%s/tsdb", pVnode->rootDir);
- if (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_RESET) != TAOS_VN_STATUS_READY) {
- return -1;
+ if (pVnode->status != TAOS_VN_STATUS_CLOSING && pVnode->status != TAOS_VN_STATUS_INIT) {
+ pVnode->status = TAOS_VN_STATUS_RESET;
}
void *tsdb = pVnode->tsdb;
@@ -683,375 +696,19 @@ static int vnodeResetTsdb(SVnodeObj *pVnode)
pVnode->tsdb = tsdbOpenRepo(rootDir, &appH);
pVnode->status = TAOS_VN_STATUS_READY;
- vnodeRelease(pVnode);
+ vnodeRelease(pVnode);
return 0;
}
static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion) {
SVnodeObj *pVnode = ahandle;
- vDebug("vgId:%d, data file is synced, fversion:%" PRId64, pVnode->vgId, fversion);
pVnode->fversion = fversion;
pVnode->version = fversion;
vnodeSaveVersion(pVnode);
+ vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion,
+ pVnode->version);
return vnodeResetTsdb(pVnode);
}
-
-static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) {
- char cfgFile[TSDB_FILENAME_LEN + 30] = {0};
- sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnodeCfg->cfg.vgId);
- FILE *fp = fopen(cfgFile, "w");
- if (!fp) {
- vError("vgId:%d, failed to open vnode cfg file for write, file:%s error:%s", pVnodeCfg->cfg.vgId, cfgFile,
- strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- return terrno;
- }
-
- int32_t len = 0;
- int32_t maxLen = 1000;
- char * content = calloc(1, maxLen + 1);
- if (content == NULL) {
- fclose(fp);
- return TSDB_CODE_VND_OUT_OF_MEMORY;
- }
-
- len += snprintf(content + len, maxLen - len, "{\n");
- len += snprintf(content + len, maxLen - len, " \"db\": \"%s\",\n", pVnodeCfg->db);
- len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnodeCfg->cfg.cfgVersion);
- len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize);
- len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks);
- // len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables);
- len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pVnodeCfg->cfg.daysPerFile);
- len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep);
- len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pVnodeCfg->cfg.daysToKeep1);
- len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pVnodeCfg->cfg.daysToKeep2);
- len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.minRowsPerFileBlock);
- len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.maxRowsPerFileBlock);
- // len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnodeCfg->cfg.commitTime);
- len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision);
- len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression);
- len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pVnodeCfg->cfg.walLevel);
- len += snprintf(content + len, maxLen - len, " \"fsync\": %d,\n", pVnodeCfg->cfg.fsyncPeriod);
- len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnodeCfg->cfg.replications);
- len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnodeCfg->cfg.wals);
- len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum);
-
- len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
-
- vInfo("vgId:%d, save vnode cfg, replica:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.replications);
- for (int32_t i = 0; i < pVnodeCfg->cfg.replications; i++) {
- len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", pVnodeCfg->nodes[i].nodeId);
- len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", pVnodeCfg->nodes[i].nodeEp);
- vInfo("vgId:%d, save vnode cfg, nodeId:%d nodeEp:%s", pVnodeCfg->cfg.vgId, pVnodeCfg->nodes[i].nodeId,
- pVnodeCfg->nodes[i].nodeEp);
-
- if (i < pVnodeCfg->cfg.replications - 1) {
- len += snprintf(content + len, maxLen - len, " },{\n");
- } else {
- len += snprintf(content + len, maxLen - len, " }]\n");
- }
- }
- len += snprintf(content + len, maxLen - len, "}\n");
-
- fwrite(content, 1, len, fp);
- fflush(fp);
- fclose(fp);
- free(content);
-
- vInfo("vgId:%d, save vnode cfg successed", pVnodeCfg->cfg.vgId);
-
- return TSDB_CODE_SUCCESS;
-}
-
-static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
- cJSON *root = NULL;
- char *content = NULL;
- char cfgFile[TSDB_FILENAME_LEN + 30] = {0};
- int maxLen = 1000;
-
- terrno = TSDB_CODE_VND_APP_ERROR;
- sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnode->vgId);
- FILE *fp = fopen(cfgFile, "r");
- if (!fp) {
- vError("vgId:%d, failed to open vnode cfg file:%s to read, error:%s", pVnode->vgId,
- cfgFile, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- goto PARSE_OVER;
- }
-
- content = calloc(1, maxLen + 1);
- if (content == NULL) goto PARSE_OVER;
- int len = fread(content, 1, maxLen, fp);
- if (len <= 0) {
- vError("vgId:%d, failed to read vnode cfg, content is null", pVnode->vgId);
- free(content);
- fclose(fp);
- return errno;
- }
-
- root = cJSON_Parse(content);
- if (root == NULL) {
- vError("vgId:%d, failed to read vnode cfg, invalid json format", pVnode->vgId);
- goto PARSE_OVER;
- }
-
- cJSON *db = cJSON_GetObjectItem(root, "db");
- if (!db || db->type != cJSON_String || db->valuestring == NULL) {
- vError("vgId:%d, failed to read vnode cfg, db not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- strcpy(pVnode->db, db->valuestring);
-
- cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion");
- if (!cfgVersion || cfgVersion->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, cfgVersion not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->cfgVersion = cfgVersion->valueint;
-
- cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize");
- if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, cacheBlockSize not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.cacheBlockSize = cacheBlockSize->valueint;
-
- cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks");
- if (!totalBlocks || totalBlocks->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, totalBlocks not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.totalBlocks = totalBlocks->valueint;
-
- // cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables");
- // if (!maxTables || maxTables->type != cJSON_Number) {
- // vError("vgId:%d, failed to read vnode cfg, maxTables not found", pVnode->vgId);
- // goto PARSE_OVER;
- // }
- // pVnode->tsdbCfg.maxTables = maxTables->valueint;
-
- cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile");
- if (!daysPerFile || daysPerFile->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, daysPerFile not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.daysPerFile = daysPerFile->valueint;
-
- cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep");
- if (!daysToKeep || daysToKeep->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.keep = daysToKeep->valueint;
-
- cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1");
- if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, daysToKeep1 not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.keep1 = daysToKeep1->valueint;
-
- cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2");
- if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, daysToKeep2 not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.keep2 = daysToKeep2->valueint;
-
- cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock");
- if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, minRowsPerFileBlock not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.minRowsPerFileBlock = minRowsPerFileBlock->valueint;
-
- cJSON *maxRowsPerFileBlock = cJSON_GetObjectItem(root, "maxRowsPerFileBlock");
- if (!maxRowsPerFileBlock || maxRowsPerFileBlock->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, maxRowsPerFileBlock not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint;
-
- // cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime");
- // if (!commitTime || commitTime->type != cJSON_Number) {
- // vError("vgId:%d, failed to read vnode cfg, commitTime not found", pVnode->vgId);
- // goto PARSE_OVER;
- // }
- // pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint;
-
- cJSON *precision = cJSON_GetObjectItem(root, "precision");
- if (!precision || precision->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, precision not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.precision = (int8_t)precision->valueint;
-
- cJSON *compression = cJSON_GetObjectItem(root, "compression");
- if (!compression || compression->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, compression not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.compression = (int8_t)compression->valueint;
-
- cJSON *walLevel = cJSON_GetObjectItem(root, "walLevel");
- if (!walLevel || walLevel->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, walLevel not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->walCfg.walLevel = (int8_t) walLevel->valueint;
-
- cJSON *fsyncPeriod = cJSON_GetObjectItem(root, "fsync");
- if (!walLevel || walLevel->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, fsyncPeriod not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->walCfg.fsyncPeriod = fsyncPeriod->valueint;
-
- cJSON *wals = cJSON_GetObjectItem(root, "wals");
- if (!wals || wals->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, wals not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->walCfg.wals = (int8_t)wals->valueint;
- pVnode->walCfg.keep = 0;
-
- cJSON *replica = cJSON_GetObjectItem(root, "replica");
- if (!replica || replica->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, replica not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->syncCfg.replica = (int8_t)replica->valueint;
-
- cJSON *quorum = cJSON_GetObjectItem(root, "quorum");
- if (!quorum || quorum->type != cJSON_Number) {
- vError("vgId: %d, failed to read vnode cfg, quorum not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->syncCfg.quorum = (int8_t)quorum->valueint;
-
- cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
- if (!nodeInfos || nodeInfos->type != cJSON_Array) {
- vError("vgId:%d, failed to read vnode cfg, nodeInfos not found", pVnode->vgId);
- goto PARSE_OVER;
- }
-
- int size = cJSON_GetArraySize(nodeInfos);
- if (size != pVnode->syncCfg.replica) {
- vError("vgId:%d, failed to read vnode cfg, nodeInfos size not matched", pVnode->vgId);
- goto PARSE_OVER;
- }
-
- for (int i = 0; i < size; ++i) {
- cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
- if (nodeInfo == NULL) continue;
-
- cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
- if (!nodeId || nodeId->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, nodeId not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->syncCfg.nodeInfo[i].nodeId = nodeId->valueint;
-
- cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
- if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
- vError("vgId:%d, failed to read vnode cfg, nodeFqdn not found", pVnode->vgId);
- goto PARSE_OVER;
- }
-
- taosGetFqdnPortFromEp(nodeEp->valuestring, pVnode->syncCfg.nodeInfo[i].nodeFqdn, &pVnode->syncCfg.nodeInfo[i].nodePort);
- pVnode->syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC;
- }
-
- terrno = TSDB_CODE_SUCCESS;
-
- vInfo("vgId:%d, read vnode cfg successfully, replcia:%d", pVnode->vgId, pVnode->syncCfg.replica);
- for (int32_t i = 0; i < pVnode->syncCfg.replica; i++) {
- vInfo("vgId:%d, dnode:%d, %s:%d", pVnode->vgId, pVnode->syncCfg.nodeInfo[i].nodeId,
- pVnode->syncCfg.nodeInfo[i].nodeFqdn, pVnode->syncCfg.nodeInfo[i].nodePort);
- }
-
-PARSE_OVER:
- taosTFree(content);
- cJSON_Delete(root);
- if (fp) fclose(fp);
- return terrno;
-}
-
-static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
- char versionFile[TSDB_FILENAME_LEN + 30] = {0};
- sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
- FILE *fp = fopen(versionFile, "w");
- if (!fp) {
- vError("vgId:%d, failed to open vnode version file for write, file:%s error:%s", pVnode->vgId,
- versionFile, strerror(errno));
- return TAOS_SYSTEM_ERROR(errno);
- }
-
- int32_t len = 0;
- int32_t maxLen = 30;
- char content[TSDB_VNODE_VERSION_CONTENT_LEN] = {0};
-
- len += snprintf(content + len, maxLen - len, "{\n");
- len += snprintf(content + len, maxLen - len, " \"version\": %" PRId64 "\n", pVnode->fversion);
- len += snprintf(content + len, maxLen - len, "}\n");
-
- fwrite(content, 1, len, fp);
- fflush(fp);
- fclose(fp);
-
- vInfo("vgId:%d, save vnode version:%" PRId64 " succeed", pVnode->vgId, pVnode->fversion);
-
- return TSDB_CODE_SUCCESS;
-}
-
-static int32_t vnodeReadVersion(SVnodeObj *pVnode) {
- char versionFile[TSDB_FILENAME_LEN + 30] = {0};
- char *content = NULL;
- cJSON *root = NULL;
- int maxLen = 100;
-
- terrno = TSDB_CODE_VND_INVALID_VRESION_FILE;
- sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
- FILE *fp = fopen(versionFile, "r");
- if (!fp) {
- if (errno != ENOENT) {
- vError("vgId:%d, failed to open version file:%s error:%s", pVnode->vgId, versionFile, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- } else {
- terrno = TSDB_CODE_SUCCESS;
- }
- goto PARSE_OVER;
- }
-
- content = calloc(1, maxLen + 1);
- int len = fread(content, 1, maxLen, fp);
- if (len <= 0) {
- vError("vgId:%d, failed to read vnode version, content is null", pVnode->vgId);
- goto PARSE_OVER;
- }
-
- root = cJSON_Parse(content);
- if (root == NULL) {
- vError("vgId:%d, failed to read vnode version, invalid json format", pVnode->vgId);
- goto PARSE_OVER;
- }
-
- cJSON *ver = cJSON_GetObjectItem(root, "version");
- if (!ver || ver->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode version, version not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->version = ver->valueint;
-
- terrno = TSDB_CODE_SUCCESS;
- vInfo("vgId:%d, read vnode version successfully, version:%" PRId64, pVnode->vgId, pVnode->version);
-
-PARSE_OVER:
- taosTFree(content);
- cJSON_Delete(root);
- if (fp) fclose(fp);
- return terrno;
-}
diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c
index 99aed03e54ccd069e5879104f62eb01ff7bb3d05..4cc8819bcba07847cb3fa1b09f0663b6368b148b 100644
--- a/src/vnode/src/vnodeRead.c
+++ b/src/vnode/src/vnodeRead.c
@@ -15,13 +15,10 @@
#define _DEFAULT_SOURCE
#define _NON_BLOCKING_RETRIEVE 0
-
#include "os.h"
-
#include "tglobal.h"
#include "taoserror.h"
#include "taosmsg.h"
-#include "tcache.h"
#include "query.h"
#include "trpc.h"
#include "tsdb.h"
@@ -29,9 +26,9 @@
#include "vnodeInt.h"
#include "tqueue.h"
-static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SReadMsg *pReadMsg);
-static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg);
-static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg);
+static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pRead);
+static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead);
+static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead);
static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId);
void vnodeInitReadFp(void) {
@@ -44,58 +41,92 @@ void vnodeInitReadFp(void) {
// still required, or there will be a deadlock, so we don’t do any check here, but put the check codes before the
// request enters the queue
//
-int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) {
+int32_t vnodeProcessRead(void *param, SVReadMsg *pRead) {
SVnodeObj *pVnode = (SVnodeObj *)param;
- int msgType = pReadMsg->rpcMsg.msgType;
+ int32_t msgType = pRead->msgType;
if (vnodeProcessReadMsgFp[msgType] == NULL) {
vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]);
return TSDB_CODE_VND_MSG_NOT_PROCESSED;
}
- return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg);
+ return (*vnodeProcessReadMsgFp[msgType])(pVnode, pRead);
}
-int32_t vnodeCheckRead(void *param) {
+static int32_t vnodeCheckRead(void *param) {
SVnodeObj *pVnode = param;
if (pVnode->status != TAOS_VN_STATUS_READY) {
- vDebug("vgId:%d, vnode status is %s, recCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status],
+ vDebug("vgId:%d, vnode status is %s, refCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status],
pVnode->refCount, pVnode);
return TSDB_CODE_APP_NOT_READY;
}
// tsdb may be in reset state
if (pVnode->tsdb == NULL) {
- vDebug("vgId:%d, tsdb is null, recCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
+ vDebug("vgId:%d, tsdb is null, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
return TSDB_CODE_APP_NOT_READY;
}
if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) {
- vDebug("vgId:%d, replica:%d role:%s, recCount:%d pVnode:%p", pVnode->vgId, pVnode->syncCfg.replica,
+ vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p", pVnode->vgId, pVnode->syncCfg.replica,
syncRole[pVnode->role], pVnode->refCount, pVnode);
return TSDB_CODE_APP_NOT_READY;
}
return TSDB_CODE_SUCCESS;
}
-static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) {
- int32_t code = vnodeCheckRead(pVnode);
- if (code != TSDB_CODE_SUCCESS) return code;
- SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg));
- pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
- pRead->pCont = qhandle;
- pRead->contLen = 0;
- pRead->rpcMsg.ahandle = ahandle;
+int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) {
+ SVnodeObj *pVnode = vparam;
- atomic_add_fetch_32(&pVnode->refCount, 1);
+ if (qtype == TAOS_QTYPE_RPC || qtype == TAOS_QTYPE_QUERY) {
+ int32_t code = vnodeCheckRead(pVnode);
+ if (code != TSDB_CODE_SUCCESS) return code;
+ }
+
+ int32_t size = sizeof(SVReadMsg) + contLen;
+ SVReadMsg *pRead = taosAllocateQitem(size);
+ if (pRead == NULL) {
+ return TSDB_CODE_VND_OUT_OF_MEMORY;
+ }
+
+ if (rparam != NULL) {
+ SRpcMsg *pRpcMsg = rparam;
+ pRead->rpcHandle = pRpcMsg->handle;
+ pRead->rpcAhandle = pRpcMsg->ahandle;
+ pRead->msgType = pRpcMsg->msgType;
+ pRead->code = pRpcMsg->code;
+ }
+
+ if (contLen != 0) {
+ pRead->contLen = contLen;
+ memcpy(pRead->pCont, pCont, contLen);
+ } else {
+ pRead->qhandle = pCont;
+ }
+
+ pRead->qtype = qtype;
- vDebug("QInfo:%p add to vread queue for exec query, msg:%p", *qhandle, pRead);
- taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead);
+ atomic_add_fetch_32(&pVnode->refCount, 1);
+ vTrace("vgId:%d, get vnode rqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
+ taosWriteQitem(pVnode->rqueue, qtype, pRead);
return TSDB_CODE_SUCCESS;
}
+static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) {
+ SRpcMsg rpcMsg = {0};
+ rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
+ rpcMsg.ahandle = ahandle;
+
+ int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg);
+ if (code == TSDB_CODE_SUCCESS) {
+ vDebug("QInfo:%p add to vread queue for exec query", *qhandle);
+ }
+
+ return code;
+}
+
/**
*
* @param pRet response message object
@@ -146,27 +177,27 @@ static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) {
pRsp->completed = true;
}
-static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
- void * pCont = pReadMsg->pCont;
- int32_t contLen = pReadMsg->contLen;
- SRspRet *pRet = &pReadMsg->rspRet;
+static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
+ void * pCont = pRead->pCont;
+ int32_t contLen = pRead->contLen;
+ SRspRet *pRet = &pRead->rspRet;
SQueryTableMsg *pQueryTableMsg = (SQueryTableMsg *)pCont;
memset(pRet, 0, sizeof(SRspRet));
// qHandle needs to be freed correctly
- if (pReadMsg->rpcMsg.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
- SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont;
+ if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
+ SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pRead->pCont;
killQueryMsg->free = htons(killQueryMsg->free);
killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle);
- vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
- assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1);
+ vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pRead->rpcHandle);
+ assert(pRead->contLen > 0 && killQueryMsg->free == 1);
void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle);
if (qhandle == NULL || *qhandle == NULL) {
vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle,
- pReadMsg->rpcMsg.handle);
+ pRead->rpcHandle);
} else {
assert(*qhandle == (void *)killQueryMsg->qhandle);
@@ -198,7 +229,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
if (handle == NULL) { // failed to register qhandle
pRsp->code = terrno;
terrno = 0;
- vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo,
+ vError("vgId:%d, QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo,
tstrerror(pRsp->code));
qDestroyQueryInfo(pQInfo); // destroy it directly
return pRsp->code;
@@ -208,9 +239,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
}
if (handle != NULL &&
- vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+ vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle,
- pReadMsg->rpcMsg.handle);
+ pRead->rpcHandle);
pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
return pRsp->code;
@@ -221,7 +252,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
if (handle != NULL) {
vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle);
- code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcMsg.ahandle);
+ code = vnodePutItemIntoReadQueue(pVnode, handle, pRead->rpcHandle);
if (code != TSDB_CODE_SUCCESS) {
pRsp->code = code;
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
@@ -230,7 +261,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
}
} else {
assert(pCont != NULL);
- void **qhandle = (void **)pCont;
+ void **qhandle = (void **)pRead->qhandle;
vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle);
@@ -242,14 +273,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
// build query rsp, the retrieve request has reached here already
if (buildRes) {
// update the connection info according to the retrieve connection
- pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*qhandle);
- assert(pReadMsg->rpcMsg.handle != NULL);
+ pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle);
+ assert(pRead->rpcHandle != NULL);
vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle,
- pReadMsg->rpcMsg.handle);
+ pRead->rpcHandle);
// set the real rsp error code
- pReadMsg->rpcMsg.code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcMsg.ahandle);
+ pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pRead->rpcHandle);
// NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client
code = TSDB_CODE_QRY_HAS_RSP;
@@ -274,16 +305,16 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
return code;
}
-static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
- void * pCont = pReadMsg->pCont;
- SRspRet *pRet = &pReadMsg->rspRet;
+static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
+ void * pCont = pRead->pCont;
+ SRspRet *pRet = &pRead->rspRet;
SRetrieveTableMsg *pRetrieve = pCont;
pRetrieve->free = htons(pRetrieve->free);
pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle,
- pRetrieve->free, pReadMsg->rpcMsg.handle);
+ pRetrieve->free, pRead->rpcHandle);
memset(pRet, 0, sizeof(SRspRet));
@@ -298,7 +329,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
}
if (code != TSDB_CODE_SUCCESS) {
- vDebug("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle);
+ vError("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle);
vnodeBuildNoResultQueryRsp(pRet);
return code;
}
@@ -314,9 +345,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
}
// register the qhandle to connect to quit query immediate if connection is broken
- if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
- vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle,
- pReadMsg->rpcMsg.handle);
+ if (vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+ vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pRead->rpcHandle);
code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
qKillQuery(*handle);
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
@@ -326,7 +356,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
bool freeHandle = true;
bool buildRes = false;
- code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle);
+ code = qRetrieveQueryResultInfo(*handle, &buildRes, pRead->rpcHandle);
if (code != TSDB_CODE_SUCCESS) {
// TODO handle malloc failure
pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
@@ -337,7 +367,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
assert(buildRes == true);
#if _NON_BLOCKING_RETRIEVE
if (!buildRes) {
- assert(pReadMsg->rpcMsg.handle != NULL);
+ assert(pRead->rpcHandle != NULL);
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false);
return TSDB_CODE_QRY_NOT_READY;
@@ -345,7 +375,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
#endif
// ahandle is the sqlObj pointer
- code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcMsg.ahandle);
+ code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pRead->rpcHandle);
}
// If qhandle is not added into vread queue, the query should be completed already or paused with error.
diff --git a/src/vnode/src/vnodeVersion.c b/src/vnode/src/vnodeVersion.c
new file mode 100644
index 0000000000000000000000000000000000000000..8f6360b4f98eac8f394f5078ed2b025cea4192b0
--- /dev/null
+++ b/src/vnode/src/vnodeVersion.c
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "tsdb.h"
+#include "vnodeInt.h"
+#include "vnodeVersion.h"
+
+int32_t vnodeReadVersion(SVnodeObj *pVnode) {
+ int32_t len = 0;
+ int32_t maxLen = 100;
+ char * content = calloc(1, maxLen + 1);
+ cJSON * root = NULL;
+ FILE * fp = NULL;
+
+ terrno = TSDB_CODE_VND_INVALID_VRESION_FILE;
+ char file[TSDB_FILENAME_LEN + 30] = {0};
+ sprintf(file, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
+
+ fp = fopen(file, "r");
+ if (!fp) {
+ if (errno != ENOENT) {
+ vError("vgId:%d, failed to read %s, error:%s", pVnode->vgId, file, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ } else {
+ terrno = TSDB_CODE_SUCCESS;
+ }
+ goto PARSE_VER_ERROR;
+ }
+
+ len = fread(content, 1, maxLen, fp);
+ if (len <= 0) {
+ vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file);
+ goto PARSE_VER_ERROR;
+ }
+
+ root = cJSON_Parse(content);
+ if (root == NULL) {
+ vError("vgId:%d, failed to read %s, invalid json format", pVnode->vgId, file);
+ goto PARSE_VER_ERROR;
+ }
+
+ cJSON *ver = cJSON_GetObjectItem(root, "version");
+ if (!ver || ver->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, version not found", pVnode->vgId, file);
+ goto PARSE_VER_ERROR;
+ }
+ pVnode->version = (uint64_t)ver->valueint;
+
+ terrno = TSDB_CODE_SUCCESS;
+ vInfo("vgId:%d, read %s successfully, fver:%" PRIu64, pVnode->vgId, file, pVnode->version);
+
+PARSE_VER_ERROR:
+ if (content != NULL) free(content);
+ if (root != NULL) cJSON_Delete(root);
+ if (fp != NULL) fclose(fp);
+
+ return terrno;
+}
+
+int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
+ char file[TSDB_FILENAME_LEN + 30] = {0};
+ sprintf(file, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
+
+ FILE *fp = fopen(file, "w");
+ if (!fp) {
+ vError("vgId:%d, failed to write %s, reason:%s", pVnode->vgId, file, strerror(errno));
+ return -1;
+ }
+
+ int32_t len = 0;
+ int32_t maxLen = 100;
+ char * content = calloc(1, maxLen + 1);
+
+ len += snprintf(content + len, maxLen - len, "{\n");
+ len += snprintf(content + len, maxLen - len, " \"version\": %" PRIu64 "\n", pVnode->fversion);
+ len += snprintf(content + len, maxLen - len, "}\n");
+
+ fwrite(content, 1, len, fp);
+ fflush(fp);
+ fclose(fp);
+ free(content);
+ terrno = 0;
+
+ vInfo("vgId:%d, successed to write %s, fver:%" PRIu64, pVnode->vgId, file, pVnode->fversion);
+ return TSDB_CODE_SUCCESS;
+}
\ No newline at end of file
diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c
index 855df81a1b9029d7ddd2f5e5ca6d42e8c680f7ac..2234b4f8eda7caa8622b60be8a9018d6aae40ad1 100644
--- a/src/vnode/src/vnodeWrite.c
+++ b/src/vnode/src/vnodeWrite.c
@@ -19,7 +19,6 @@
#include "taoserror.h"
#include "tqueue.h"
#include "trpc.h"
-#include "tutil.h"
#include "tsdb.h"
#include "twal.h"
#include "tsync.h"
@@ -29,13 +28,15 @@
#include "syncInt.h"
#include "tcq.h"
-static int32_t (*vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *, SRspRet *);
-static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pMsg, SRspRet *);
-static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pMsg, SRspRet *);
-static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pMsg, SRspRet *);
-static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pMsg, SRspRet *);
-static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pMsg, SRspRet *);
-static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet);
+#define MAX_QUEUED_MSG_NUM 10000
+
+static int32_t (*vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *pCont, SRspRet *);
+static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
+static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
+static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
+static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
+static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
+static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
void vnodeInitWriteFp(void) {
vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessSubmitMsg;
@@ -46,26 +47,31 @@ void vnodeInitWriteFp(void) {
vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessUpdateTagValMsg;
}
-int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
- int32_t code = 0;
- SVnodeObj *pVnode = (SVnodeObj *)param1;
- SWalHead * pHead = param2;
+int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) {
+ int32_t code = 0;
+ SVnodeObj * pVnode = vparam;
+ SWalHead * pHead = wparam;
+ SRspRet * pRspRet = rparam;
if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) {
- vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[pHead->msgType]);
+ vError("vgId:%d, msg:%s not processed since no handle, qtype:%s hver:%" PRIu64, pVnode->vgId,
+ taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version);
return TSDB_CODE_VND_MSG_NOT_PROCESSED;
}
+ vTrace("vgId:%d, msg:%s will be processed in vnode, qtype:%s hver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId,
+ taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version, pVnode->version);
+
if (pHead->version == 0) { // from client or CQ
if (pVnode->status != TAOS_VN_STATUS_READY) {
- vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[pHead->msgType],
- pVnode->status);
+ vDebug("vgId:%d, msg:%s not processed since vstatus:%d, qtype:%s hver:%" PRIu64, pVnode->vgId,
+ taosMsg[pHead->msgType], pVnode->status, qtypeStr[qtype], pHead->version);
return TSDB_CODE_APP_NOT_READY; // it may be in deleting or closing state
}
if (pVnode->role != TAOS_SYNC_ROLE_MASTER) {
- vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%s", pVnode->vgId, taosMsg[pHead->msgType],
- pVnode->syncCfg.replica, syncRole[pVnode->role]);
+ vDebug("vgId:%d, msg:%s not processed since replica:%d role:%s, qtype:%s hver:%" PRIu64, pVnode->vgId,
+ taosMsg[pHead->msgType], pVnode->syncCfg.replica, syncRole[pVnode->role], qtypeStr[qtype], pHead->version);
return TSDB_CODE_APP_NOT_READY;
}
@@ -80,7 +86,7 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
// forward to peers, even it is WAL/FWD, it shall be called to update version in sync
int32_t syncCode = 0;
- syncCode = syncForwardToPeer(pVnode->sync, pHead, item, qtype);
+ syncCode = syncForwardToPeer(pVnode->sync, pHead, pRspRet, qtype);
if (syncCode < 0) return syncCode;
// write into WAL
@@ -90,36 +96,41 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
pVnode->version = pHead->version;
// write data locally
- code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item);
+ code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, pRspRet);
if (code < 0) return code;
return syncCode;
}
-int32_t vnodeCheckWrite(void *param) {
- SVnodeObj *pVnode = param;
+static int32_t vnodeCheckWrite(void *vparam) {
+ SVnodeObj *pVnode = vparam;
if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) {
- vDebug("vgId:%d, no write auth, recCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
+ vDebug("vgId:%d, no write auth, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
return TSDB_CODE_VND_NO_WRITE_AUTH;
}
// tsdb may be in reset state
if (pVnode->tsdb == NULL) {
- vDebug("vgId:%d, tsdb is null, recCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
+ vDebug("vgId:%d, tsdb is null, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
return TSDB_CODE_APP_NOT_READY;
}
if (pVnode->status == TAOS_VN_STATUS_CLOSING) {
- vDebug("vgId:%d, vnode status is %s, recCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status],
+ vDebug("vgId:%d, vnode status is %s, refCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status],
pVnode->refCount, pVnode);
return TSDB_CODE_APP_NOT_READY;
}
+ if (pVnode->isFull) {
+ vDebug("vgId:%d, vnode is full, refCount:%d", pVnode->vgId, pVnode->refCount);
+ return TSDB_CODE_VND_IS_FULL;
+ }
+
return TSDB_CODE_SUCCESS;
}
-void vnodeConfirmForward(void *param, uint64_t version, int32_t code) {
- SVnodeObj *pVnode = (SVnodeObj *)param;
+void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code) {
+ SVnodeObj *pVnode = vparam;
syncConfirmForward(pVnode->sync, version, code);
}
@@ -183,8 +194,8 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
}
static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
- SMDDropSTableMsg *pTable = pCont;
- int32_t code = TSDB_CODE_SUCCESS;
+ SDropSTableMsg *pTable = pCont;
+ int32_t code = TSDB_CODE_SUCCESS;
vDebug("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableId);
@@ -204,37 +215,54 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR
return TSDB_CODE_SUCCESS;
}
+int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) {
+ SVnodeObj *pVnode = vparam;
+ SWalHead * pHead = wparam;
-int vnodeWriteCqMsgToQueue(void *param, void *data, int type) {
- SVnodeObj *pVnode = param;
- SWalHead * pHead = data;
+ if (qtype == TAOS_QTYPE_RPC) {
+ int32_t code = vnodeCheckWrite(pVnode);
+ if (code != TSDB_CODE_SUCCESS) return code;
+ }
- int size = sizeof(SWalHead) + pHead->len;
- SSyncHead *pSync = (SSyncHead*) taosAllocateQitem(size + sizeof(SSyncHead));
- SWalHead *pWal = (SWalHead *)(pSync + 1);
- memcpy(pWal, pHead, size);
+ if (pHead->len > TSDB_MAX_WAL_SIZE) {
+ vError("vgId:%d, wal len:%d exceeds limit, hver:%" PRIu64, pVnode->vgId, pHead->len, pHead->version);
+ return TSDB_CODE_WAL_SIZE_LIMIT;
+ }
- atomic_add_fetch_32(&pVnode->refCount, 1);
- vTrace("CQ: vgId:%d, get vnode wqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
+ int32_t size = sizeof(SVWriteMsg) + sizeof(SWalHead) + pHead->len;
+ SVWriteMsg *pWrite = taosAllocateQitem(size);
+ if (pWrite == NULL) {
+ return TSDB_CODE_VND_OUT_OF_MEMORY;
+ }
- taosWriteQitem(pVnode->wqueue, type, pSync);
+ if (rparam != NULL) {
+ SRpcMsg *pRpcMsg = rparam;
+ pWrite->rpcHandle = pRpcMsg->handle;
+ pWrite->rpcAhandle = pRpcMsg->ahandle;
+ }
- return 0;
-}
+ memcpy(pWrite->pHead, pHead, sizeof(SWalHead) + pHead->len);
+
+ atomic_add_fetch_32(&pVnode->refCount, 1);
+ int32_t queued = atomic_add_fetch_32(&pVnode->queuedMsg, 1);
+ if (queued > MAX_QUEUED_MSG_NUM) {
+ vDebug("vgId:%d, too many msg:%d in vwqueue, flow control", pVnode->vgId, queued);
+ taosMsleep(1);
+ }
-int vnodeWriteToQueue(void *param, void *data, int type) {
- SVnodeObj *pVnode = param;
- SWalHead * pHead = data;
+ vTrace("vgId:%d, write into vwqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedMsg);
- int size = sizeof(SWalHead) + pHead->len;
- SWalHead *pWal = (SWalHead *)taosAllocateQitem(size);
- memcpy(pWal, pHead, size);
+ taosWriteQitem(pVnode->wqueue, qtype, pWrite);
+ return TSDB_CODE_SUCCESS;
+}
- atomic_add_fetch_32(&pVnode->refCount, 1);
- vTrace("vgId:%d, get vnode wqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
+void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) {
+ SVnodeObj *pVnode = vparam;
- taosWriteQitem(pVnode->wqueue, type, pWal);
+ atomic_sub_fetch_32(&pVnode->queuedMsg, 1);
+ vTrace("vgId:%d, free from vwqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedMsg);
- return 0;
+ taosFreeQitem(pWrite);
+ vnodeRelease(pVnode);
}
diff --git a/src/wal/inc/walInt.h b/src/wal/inc/walInt.h
new file mode 100644
index 0000000000000000000000000000000000000000..b0edabfbd8b0f9e0c71d478994c796f4de755c0a
--- /dev/null
+++ b/src/wal/inc/walInt.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_WAL_INT_H
+#define TDENGINE_WAL_INT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tlog.h"
+
+extern int32_t wDebugFlag;
+
+#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }}
+#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }}
+#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }}
+#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }}
+#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
+#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
+
+#define WAL_PREFIX "wal"
+#define WAL_PREFIX_LEN 3
+#define WAL_REFRESH_MS 1000
+#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16)
+#define WAL_SIGNATURE ((uint32_t)(0xFAFBFDFE))
+#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12)
+#define WAL_FILE_LEN (WAL_PATH_LEN + 32)
+#define WAL_FILE_NUM 3
+
+typedef struct {
+ uint64_t version;
+ int64_t fileId;
+ int64_t rid;
+ int32_t vgId;
+ int32_t fd;
+ int32_t keep;
+ int32_t level;
+ int32_t fsyncPeriod;
+ int32_t fsyncSeq;
+ int8_t stop;
+ int8_t reserved[3];
+ char path[WAL_PATH_LEN];
+ char name[WAL_FILE_LEN];
+ pthread_mutex_t mutex;
+} SWal;
+
+int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId);
+int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId);
+int32_t walGetNewFile(SWal *pWal, int64_t *newFileId);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c
deleted file mode 100644
index 182600204259e703d171d6598f46a2a16cdcb27b..0000000000000000000000000000000000000000
--- a/src/wal/src/walMain.c
+++ /dev/null
@@ -1,588 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc.
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-
-#define _DEFAULT_SOURCE
-
-#define TAOS_RANDOM_FILE_FAIL_TEST
-
-#include "os.h"
-#include "tlog.h"
-#include "tchecksum.h"
-#include "tutil.h"
-#include "ttimer.h"
-#include "taoserror.h"
-#include "twal.h"
-#include "tqueue.h"
-
-#define walPrefix "wal"
-
-#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }}
-#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }}
-#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }}
-#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }}
-#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
-#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
-
-typedef struct {
- uint64_t version;
- int fd;
- int keep;
- int level;
- int32_t fsyncPeriod;
- void *timer;
- void *signature;
- int max; // maximum number of wal files
- uint32_t id; // increase continuously
- int num; // number of wal files
- char path[TSDB_FILENAME_LEN];
- char name[TSDB_FILENAME_LEN+16];
- pthread_mutex_t mutex;
-} SWal;
-
-static void *walTmrCtrl = NULL;
-static int tsWalNum = 0;
-static pthread_once_t walModuleInit = PTHREAD_ONCE_INIT;
-static uint32_t walSignature = 0xFAFBFDFE;
-static int walHandleExistingFiles(const char *path);
-static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp);
-static int walRemoveWalFiles(const char *path);
-static void walProcessFsyncTimer(void *param, void *tmrId);
-static void walRelease(SWal *pWal);
-static int walGetMaxOldFileId(char *odir);
-
-static void walModuleInitFunc() {
- walTmrCtrl = taosTmrInit(1000, 100, 300000, "WAL");
- if (walTmrCtrl == NULL)
- walModuleInit = PTHREAD_ONCE_INIT;
- else
- wDebug("WAL module is initialized");
-}
-
-static inline bool walNeedFsyncTimer(SWal *pWal) {
- if (pWal->fsyncPeriod > 0 && pWal->level == TAOS_WAL_FSYNC) {
- return true;
- }
- return false;
-}
-
-void *walOpen(const char *path, const SWalCfg *pCfg) {
- SWal *pWal = calloc(sizeof(SWal), 1);
- if (pWal == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- return NULL;
- }
-
- pthread_once(&walModuleInit, walModuleInitFunc);
- if (walTmrCtrl == NULL) {
- free(pWal);
- terrno = TAOS_SYSTEM_ERROR(errno);
- return NULL;
- }
-
- atomic_add_fetch_32(&tsWalNum, 1);
- pWal->fd = -1;
- pWal->max = pCfg->wals;
- pWal->id = 0;
- pWal->num = 0;
- pWal->level = pCfg->walLevel;
- pWal->keep = pCfg->keep;
- pWal->fsyncPeriod = pCfg->fsyncPeriod;
- pWal->signature = pWal;
- tstrncpy(pWal->path, path, sizeof(pWal->path));
- pthread_mutex_init(&pWal->mutex, NULL);
-
- if (walNeedFsyncTimer(pWal)) {
- pWal->timer = taosTmrStart(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, walTmrCtrl);
- if (pWal->timer == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- walRelease(pWal);
- return NULL;
- }
- }
-
- if (taosMkDir(path, 0755) != 0) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- wError("wal:%s, failed to create directory(%s)", path, strerror(errno));
- walRelease(pWal);
- pWal = NULL;
- }
-
- if (pCfg->keep == 1) return pWal;
-
- if (walHandleExistingFiles(path) == 0) walRenew(pWal);
-
- if (pWal && pWal->fd < 0) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- wError("wal:%s, failed to open(%s)", path, strerror(errno));
- walRelease(pWal);
- pWal = NULL;
- }
-
- if (pWal) wDebug("wal:%s, it is open, level:%d fsyncPeriod:%d", path, pWal->level, pWal->fsyncPeriod);
- return pWal;
-}
-
-int walAlter(twalh wal, const SWalCfg *pCfg) {
- SWal *pWal = wal;
- if (pWal == NULL) {
- return TSDB_CODE_WAL_APP_ERROR;
- }
-
- if (pWal->level == pCfg->walLevel && pWal->fsyncPeriod == pCfg->fsyncPeriod) {
- wDebug("wal:%s, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->name, pWal->level,
- pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod);
- return TSDB_CODE_SUCCESS;
- }
-
- wInfo("wal:%s, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->name, pWal->level, pWal->fsyncPeriod,
- pCfg->walLevel, pCfg->fsyncPeriod);
-
- pthread_mutex_lock(&pWal->mutex);
- pWal->level = pCfg->walLevel;
- pWal->fsyncPeriod = pCfg->fsyncPeriod;
- if (walNeedFsyncTimer(pWal)) {
- wInfo("wal:%s, reset fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod);
- taosTmrReset(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, &pWal->timer, walTmrCtrl);
- } else {
- wInfo("wal:%s, stop fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod);
- taosTmrStop(pWal->timer);
- pWal->timer = NULL;
- }
- pthread_mutex_unlock(&pWal->mutex);
-
- return TSDB_CODE_SUCCESS;
-}
-
-void walClose(void *handle) {
- if (handle == NULL) return;
-
- SWal *pWal = handle;
- taosClose(pWal->fd);
- if (pWal->timer) taosTmrStopA(&pWal->timer);
-
- if (pWal->keep == 0) {
- // remove all files in the directory
- for (int i = 0; i < pWal->num; ++i) {
- snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id - i);
- if (remove(pWal->name) < 0) {
- wError("wal:%s, failed to remove", pWal->name);
- } else {
- wDebug("wal:%s, it is removed", pWal->name);
- }
- }
- } else {
- wDebug("wal:%s, it is closed and kept", pWal->name);
- }
-
- walRelease(pWal);
-}
-
-int walRenew(void *handle) {
- if (handle == NULL) return 0;
- SWal *pWal = handle;
-
- terrno = 0;
-
- pthread_mutex_lock(&pWal->mutex);
-
- if (pWal->fd >= 0) {
- close(pWal->fd);
- pWal->id++;
- wDebug("wal:%s, it is closed", pWal->name);
- }
-
- pWal->num++;
-
- snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id);
- pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
-
- if (pWal->fd < 0) {
- wError("wal:%s, failed to open(%s)", pWal->name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- } else {
- wDebug("wal:%s, it is created", pWal->name);
-
- if (pWal->num > pWal->max) {
- // remove the oldest wal file
- char name[TSDB_FILENAME_LEN * 3];
- snprintf(name, sizeof(name), "%s/%s%d", pWal->path, walPrefix, pWal->id - pWal->max);
- if (remove(name) < 0) {
- wError("wal:%s, failed to remove(%s)", name, strerror(errno));
- } else {
- wDebug("wal:%s, it is removed", name);
- }
-
- pWal->num--;
- }
- }
-
- pthread_mutex_unlock(&pWal->mutex);
-
- return terrno;
-}
-
-int walWrite(void *handle, SWalHead *pHead) {
- SWal *pWal = handle;
- if (pWal == NULL) return -1;
-
- terrno = 0;
-
- // no wal
- if (pWal->level == TAOS_WAL_NOLOG) return 0;
- if (pHead->version <= pWal->version) return 0;
-
- pHead->signature = walSignature;
- taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead));
- int contLen = pHead->len + sizeof(SWalHead);
-
- if (taosTWrite(pWal->fd, pHead, contLen) != contLen) {
- wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- } else {
- pWal->version = pHead->version;
- }
-
- return terrno;
-}
-
-void walFsync(void *handle) {
- SWal *pWal = handle;
- if (pWal == NULL || pWal->level != TAOS_WAL_FSYNC || pWal->fd < 0) return;
-
- if (pWal->fsyncPeriod == 0) {
- if (fsync(pWal->fd) < 0) {
- wError("wal:%s, fsync failed(%s)", pWal->name, strerror(errno));
- }
- }
-}
-
-int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) {
- SWal *pWal = handle;
- struct dirent *ent;
- int count = 0;
- uint32_t maxId = 0, minId = -1, index =0;
-
- terrno = 0;
- int plen = strlen(walPrefix);
- char opath[TSDB_FILENAME_LEN + 5];
-
- int slen = snprintf(opath, sizeof(opath), "%s", pWal->path);
- if (pWal->keep == 0) strcpy(opath + slen, "/old");
-
- DIR *dir = opendir(opath);
- if (dir == NULL && errno == ENOENT) return 0;
- if (dir == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- return terrno;
- }
-
- while ((ent = readdir(dir)) != NULL) {
- if (strncmp(ent->d_name, walPrefix, plen) == 0) {
- index = atol(ent->d_name + plen);
- if (index > maxId) maxId = index;
- if (index < minId) minId = index;
- count++;
- }
- }
-
- closedir(dir);
-
- if (count == 0) {
- if (pWal->keep) terrno = walRenew(pWal);
- return terrno;
- }
-
- if (count != (maxId - minId + 1)) {
- wError("wal:%s, messed up, count:%d max:%d min:%d", opath, count, maxId, minId);
- terrno = TSDB_CODE_WAL_APP_ERROR;
- } else {
- wDebug("wal:%s, %d files will be restored", opath, count);
-
- for (index = minId; index <= maxId; ++index) {
- snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, index);
- terrno = walRestoreWalFile(pWal, pVnode, writeFp);
- if (terrno < 0) continue;
- }
- }
-
- if (terrno == 0) {
- if (pWal->keep == 0) {
- terrno = walRemoveWalFiles(opath);
- if (terrno == 0) {
- if (remove(opath) < 0) {
- wError("wal:%s, failed to remove directory(%s)", opath, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- }
- }
- } else {
- // open the existing WAL file in append mode
- pWal->num = count;
- pWal->id = maxId;
- snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, maxId);
- pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
- if (pWal->fd < 0) {
- wError("wal:%s, failed to open file(%s)", pWal->name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- }
- }
- }
-
- return terrno;
-}
-
-int walGetWalFile(void *handle, char *name, uint32_t *index) {
- SWal * pWal = handle;
- int code = 1;
- int32_t first = 0;
-
- name[0] = 0;
- if (pWal == NULL || pWal->num == 0) return 0;
-
- pthread_mutex_lock(&(pWal->mutex));
-
- first = pWal->id + 1 - pWal->num;
- if (*index == 0) *index = first; // set to first one
-
- if (*index < first && *index > pWal->id) {
- code = -1; // index out of range
- } else {
- sprintf(name, "wal/%s%d", walPrefix, *index);
- code = (*index == pWal->id) ? 0 : 1;
- }
-
- pthread_mutex_unlock(&(pWal->mutex));
-
- return code;
-}
-
-static void walRelease(SWal *pWal) {
- pthread_mutex_destroy(&pWal->mutex);
- pWal->signature = NULL;
- free(pWal);
-
- if (atomic_sub_fetch_32(&tsWalNum, 1) == 0) {
- if (walTmrCtrl) taosTmrCleanUp(walTmrCtrl);
- walTmrCtrl = NULL;
- walModuleInit = PTHREAD_ONCE_INIT;
- wDebug("WAL module is cleaned up");
- }
-}
-
-static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
- char *name = pWal->name;
- int size = 1024 * 1024; // default 1M buffer size
-
- terrno = 0;
- char *buffer = malloc(size);
- if (buffer == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- return terrno;
- }
-
- SWalHead *pHead = (SWalHead *)buffer;
-
- int fd = open(name, O_RDWR);
- if (fd < 0) {
- wError("wal:%s, failed to open for restore(%s)", name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- free(buffer);
- return terrno;
- }
-
- wDebug("wal:%s, start to restore", name);
-
- size_t offset = 0;
- while (1) {
- int ret = taosTRead(fd, pHead, sizeof(SWalHead));
- if (ret == 0) break;
-
- if (ret < 0) {
- wError("wal:%s, failed to read wal head part since %s", name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- break;
- }
-
- if (ret < sizeof(SWalHead)) {
- wError("wal:%s, failed to read head, ret:%d, skip the rest of file", name, ret);
- taosFtruncate(fd, offset);
- fsync(fd);
- break;
- }
-
- if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) {
- wWarn("wal:%s, cksum is messed up, skip the rest of file", name);
- terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
- // ASSERT(false);
- break;
- }
-
- if (pHead->len > size - sizeof(SWalHead)) {
- size = sizeof(SWalHead) + pHead->len;
- buffer = realloc(buffer, size);
- if (buffer == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- break;
- }
-
- pHead = (SWalHead *)buffer;
- }
-
- ret = taosTRead(fd, pHead->cont, pHead->len);
- if (ret < 0) {
- wError("wal:%s failed to read wal body part since %s", name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- break;
- }
-
- if (ret < pHead->len) {
- wError("wal:%s, failed to read body, len:%d ret:%d, skip the rest of file", name, pHead->len, ret);
- taosFtruncate(fd, offset);
- fsync(fd);
- break;
- }
-
- offset = offset + sizeof(SWalHead) + pHead->len;
-
- if (pWal->keep) pWal->version = pHead->version;
- (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL);
- }
-
- close(fd);
- free(buffer);
-
- return terrno;
-}
-
-int walHandleExistingFiles(const char *path) {
- char oname[TSDB_FILENAME_LEN * 3];
- char nname[TSDB_FILENAME_LEN * 3];
- char opath[TSDB_FILENAME_LEN];
-
- snprintf(opath, sizeof(opath), "%s/old", path);
-
- struct dirent *ent;
- DIR *dir = opendir(path);
- int plen = strlen(walPrefix);
- terrno = 0;
-
- int midx = walGetMaxOldFileId(opath);
- int count = 0;
- while ((ent = readdir(dir)) != NULL) {
- if (strncmp(ent->d_name, walPrefix, plen) == 0) {
- midx++;
- snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name);
- snprintf(nname, sizeof(nname), "%s/old/wal%d", path, midx);
- if (taosMkDir(opath, 0755) != 0) {
- wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- break;
- }
-
- if (rename(oname, nname) < 0) {
- wError("wal:%s, failed to move to new:%s", oname, nname);
- terrno = TAOS_SYSTEM_ERROR(errno);
- break;
- }
-
- count++;
- }
-
- wDebug("wal:%s, %d files are moved for restoration", path, count);
- }
-
- closedir(dir);
- return terrno;
-}
-
-static int walRemoveWalFiles(const char *path) {
- int plen = strlen(walPrefix);
- char name[TSDB_FILENAME_LEN * 3];
-
- terrno = 0;
-
- struct dirent *ent;
- DIR *dir = opendir(path);
- if (dir == NULL && errno == ENOENT) return 0;
- if (dir == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- return terrno;
- }
-
- while ((ent = readdir(dir)) != NULL) {
- if (strncmp(ent->d_name, walPrefix, plen) == 0) {
- snprintf(name, sizeof(name), "%s/%s", path, ent->d_name);
- if (remove(name) < 0) {
- wError("wal:%s, failed to remove(%s)", name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- }
- }
- }
-
- closedir(dir);
-
- return terrno;
-}
-
-static void walProcessFsyncTimer(void *param, void *tmrId) {
- SWal *pWal = param;
-
- if (pWal->signature != pWal) return;
- if (pWal->fd < 0) return;
-
- if (fsync(pWal->fd) < 0) {
- wError("wal:%s, fsync failed(%s)", pWal->name, strerror(errno));
- }
-
- if (walNeedFsyncTimer(pWal)) {
- pWal->timer = taosTmrStart(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, walTmrCtrl);
- } else {
- wInfo("wal:%s, stop fsync timer for walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod);
- taosTmrStop(pWal->timer);
- pWal->timer = NULL;
- }
-}
-
-int64_t walGetVersion(twalh param) {
- SWal *pWal = param;
- if (pWal == 0) return 0;
-
- return pWal->version;
-}
-
-static int walGetMaxOldFileId(char *odir) {
- int midx = 0;
- DIR * dir = NULL;
- struct dirent *dp = NULL;
- int plen = strlen(walPrefix);
-
- if (access(odir, F_OK) != 0) return midx;
-
- dir = opendir(odir);
- if (dir == NULL) {
- wError("failed to open directory %s since %s", odir, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- return -1;
- }
-
- while ((dp = readdir(dir)) != NULL) {
- if (strncmp(dp->d_name, walPrefix, plen) == 0) {
- int idx = atol(dp->d_name + plen);
- if (midx < idx) midx = idx;
- }
- }
-
- closedir(dir);
- return midx;
-}
\ No newline at end of file
diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c
new file mode 100644
index 0000000000000000000000000000000000000000..36c190be3e0f72f96b9075b7fb90db39ee3c5412
--- /dev/null
+++ b/src/wal/src/walMgmt.c
@@ -0,0 +1,217 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "tref.h"
+#include "twal.h"
+#include "walInt.h"
+
+typedef struct {
+ int32_t refId;
+ int32_t seq;
+ int8_t stop;
+ pthread_t thread;
+ pthread_mutex_t mutex;
+} SWalMgmt;
+
+static SWalMgmt tsWal = {0};
+static int32_t walCreateThread();
+static void walStopThread();
+static int32_t walInitObj(SWal *pWal);
+static void walFreeObj(void *pWal);
+
+int32_t walInit() {
+ tsWal.refId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj);
+
+ int32_t code = walCreateThread();
+ if (code != TSDB_CODE_SUCCESS) {
+ wError("failed to init wal module since %s", tstrerror(code));
+ return code;
+ }
+
+ wInfo("wal module is initialized, refId:%d", tsWal.refId);
+ return code;
+}
+
+void walCleanUp() {
+ walStopThread();
+ taosCloseRef(tsWal.refId);
+ wInfo("wal module is cleaned up");
+}
+
+void *walOpen(char *path, SWalCfg *pCfg) {
+ SWal *pWal = tcalloc(1, sizeof(SWal));
+ if (pWal == NULL) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return NULL;
+ }
+
+ pWal->vgId = pCfg->vgId;
+ pWal->fd = -1;
+ pWal->fileId = -1;
+ pWal->level = pCfg->walLevel;
+ pWal->keep = pCfg->keep;
+ pWal->fsyncPeriod = pCfg->fsyncPeriod;
+ tstrncpy(pWal->path, path, sizeof(pWal->path));
+ pthread_mutex_init(&pWal->mutex, NULL);
+
+ pWal->fsyncSeq = pCfg->fsyncPeriod / 1000;
+ if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1;
+
+ if (walInitObj(pWal) != TSDB_CODE_SUCCESS) {
+ walFreeObj(pWal);
+ return NULL;
+ }
+
+ pWal->rid = taosAddRef(tsWal.refId, pWal);
+ if (pWal->rid < 0) {
+ walFreeObj(pWal);
+ return NULL;
+ }
+
+ wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->vgId, pWal, pWal->level, pWal->fsyncPeriod);
+
+ return pWal;
+}
+
+int32_t walAlter(void *handle, SWalCfg *pCfg) {
+ if (handle == NULL) return TSDB_CODE_WAL_APP_ERROR;
+ SWal *pWal = handle;
+
+ if (pWal->level == pCfg->walLevel && pWal->fsyncPeriod == pCfg->fsyncPeriod) {
+ wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->vgId, pWal->level,
+ pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod);
+ return TSDB_CODE_SUCCESS;
+ }
+
+ wInfo("vgId:%d, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->vgId, pWal->level,
+ pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod);
+
+ pWal->level = pCfg->walLevel;
+ pWal->fsyncPeriod = pCfg->fsyncPeriod;
+ pWal->fsyncSeq = pCfg->fsyncPeriod % 1000;
+ if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1;
+
+ return TSDB_CODE_SUCCESS;
+}
+
+void walStop(void *handle) {
+ if (handle == NULL) return;
+ SWal *pWal = handle;
+
+ pthread_mutex_lock(&pWal->mutex);
+ pWal->stop = 1;
+ pthread_mutex_unlock(&pWal->mutex);
+ wDebug("vgId:%d, stop write wal", pWal->vgId);
+}
+
+void walClose(void *handle) {
+ if (handle == NULL) return;
+
+ SWal *pWal = handle;
+ pthread_mutex_lock(&pWal->mutex);
+ taosClose(pWal->fd);
+ pthread_mutex_unlock(&pWal->mutex);
+ taosRemoveRef(tsWal.refId, pWal->rid);
+}
+
+static int32_t walInitObj(SWal *pWal) {
+ if (taosMkDir(pWal->path, 0755) != 0) {
+ wError("vgId:%d, path:%s, failed to create directory since %s", pWal->vgId, pWal->path, strerror(errno));
+ return TAOS_SYSTEM_ERROR(errno);
+ }
+
+ wDebug("vgId:%d, object is initialized", pWal->vgId);
+ return TSDB_CODE_SUCCESS;
+}
+
+static void walFreeObj(void *wal) {
+ SWal *pWal = wal;
+ wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal);
+
+ taosClose(pWal->fd);
+ pthread_mutex_destroy(&pWal->mutex);
+ tfree(pWal);
+}
+
+static bool walNeedFsync(SWal *pWal) {
+ if (pWal->fsyncPeriod <= 0 || pWal->level != TAOS_WAL_FSYNC) {
+ return false;
+ }
+
+ if (tsWal.seq % pWal->fsyncSeq == 0) {
+ return true;
+ }
+
+ return false;
+}
+
+static void walUpdateSeq() {
+ taosMsleep(WAL_REFRESH_MS);
+ if (++tsWal.seq <= 0) {
+ tsWal.seq = 1;
+ }
+}
+
+static void walFsyncAll() {
+ SWal *pWal = taosIterateRef(tsWal.refId, 0);
+ while (pWal) {
+ if (walNeedFsync(pWal)) {
+ wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq);
+ int32_t code = fsync(pWal->fd);
+ if (code != 0) {
+ wError("vgId:%d, file:%s, failed to fsync since %s", pWal->vgId, pWal->name, strerror(code));
+ }
+ }
+ pWal = taosIterateRef(tsWal.refId, pWal->rid);
+ }
+}
+
+static void *walThreadFunc(void *param) {
+ while (1) {
+ walUpdateSeq();
+ walFsyncAll();
+ if (tsWal.stop) break;
+ }
+
+ return NULL;
+}
+
+static int32_t walCreateThread() {
+ pthread_attr_t thAttr;
+ pthread_attr_init(&thAttr);
+ pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+
+ if (pthread_create(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) {
+ wError("failed to create wal thread since %s", strerror(errno));
+ return TAOS_SYSTEM_ERROR(errno);
+ }
+
+ pthread_attr_destroy(&thAttr);
+ wDebug("wal thread is launched");
+
+ return TSDB_CODE_SUCCESS;
+}
+
+static void walStopThread() {
+ tsWal.stop = 1;
+ if (tsWal.thread) {
+ pthread_join(tsWal.thread, NULL);
+ }
+
+ wDebug("wal thread is stopped");
+}
diff --git a/src/wal/src/walUtil.c b/src/wal/src/walUtil.c
new file mode 100644
index 0000000000000000000000000000000000000000..e4d9a555b3a60cb6be1e6584652ec4a309b1c301
--- /dev/null
+++ b/src/wal/src/walUtil.c
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "walInt.h"
+
+int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId) {
+ int64_t curFileId = *nextFileId;
+ int64_t minFileId = INT64_MAX;
+
+ DIR *dir = opendir(pWal->path);
+ if (dir == NULL) {
+ wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno));
+ return -1;
+ }
+
+ struct dirent *ent;
+ while ((ent = readdir(dir)) != NULL) {
+ char *name = ent->d_name;
+
+ if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) {
+ int64_t id = atoll(name + WAL_PREFIX_LEN);
+ if (id <= curFileId) continue;
+
+ if (id < minFileId) {
+ minFileId = id;
+ }
+ }
+ }
+ closedir(dir);
+
+ if (minFileId == INT64_MAX) return -1;
+
+ *nextFileId = minFileId;
+ wTrace("vgId:%d, path:%s, curFileId:%" PRId64 " nextFileId:%" PRId64, pWal->vgId, pWal->path, curFileId, *nextFileId);
+
+ return 0;
+}
+
+int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId) {
+ int64_t minFileId = INT64_MAX;
+
+ DIR *dir = opendir(pWal->path);
+ if (dir == NULL) {
+ wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno));
+ return -1;
+ }
+
+ struct dirent *ent;
+ while ((ent = readdir(dir)) != NULL) {
+ char *name = ent->d_name;
+
+ if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) {
+ int64_t id = atoll(name + WAL_PREFIX_LEN);
+ if (id >= curFileId) continue;
+
+ minDiff--;
+ if (id < minFileId) {
+ minFileId = id;
+ }
+ }
+ }
+ closedir(dir);
+
+ if (minFileId == INT64_MAX) return -1;
+ if (minDiff > 0) return -1;
+
+ *oldFileId = minFileId;
+ wTrace("vgId:%d, path:%s, curFileId:%" PRId64 " oldFildId:%" PRId64, pWal->vgId, pWal->path, curFileId, *oldFileId);
+
+ return 0;
+}
+
+int32_t walGetNewFile(SWal *pWal, int64_t *newFileId) {
+ int64_t maxFileId = INT64_MIN;
+
+ DIR *dir = opendir(pWal->path);
+ if (dir == NULL) {
+ wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno));
+ return -1;
+ }
+
+ struct dirent *ent;
+ while ((ent = readdir(dir)) != NULL) {
+ char *name = ent->d_name;
+
+ if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) {
+ int64_t id = atoll(name + WAL_PREFIX_LEN);
+ if (id > maxFileId) {
+ maxFileId = id;
+ }
+ }
+ }
+ closedir(dir);
+
+ if (maxFileId == INT64_MIN) {
+ *newFileId = 0;
+ } else {
+ *newFileId = maxFileId;
+ }
+
+ wTrace("vgId:%d, path:%s, newFileId:%" PRId64, pWal->vgId, pWal->path, *newFileId);
+
+ return 0;
+}
\ No newline at end of file
diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c
new file mode 100644
index 0000000000000000000000000000000000000000..72464d4309b5fe2b6e40f5e197bee59482843526
--- /dev/null
+++ b/src/wal/src/walWrite.c
@@ -0,0 +1,346 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#define TAOS_RANDOM_FILE_FAIL_TEST
+#include "os.h"
+#include "taoserror.h"
+#include "tchecksum.h"
+#include "twal.h"
+#include "walInt.h"
+
+static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId);
+
+int32_t walRenew(void *handle) {
+ if (handle == NULL) return 0;
+
+ SWal * pWal = handle;
+ int32_t code = 0;
+
+ if (pWal->stop) {
+ wDebug("vgId:%d, do not create a new wal file", pWal->vgId);
+ return 0;
+ }
+
+ pthread_mutex_lock(&pWal->mutex);
+
+ if (pWal->fd >= 0) {
+ tclose(pWal->fd);
+ wDebug("vgId:%d, file:%s, it is closed", pWal->vgId, pWal->name);
+ }
+
+ if (pWal->keep == TAOS_WAL_KEEP) {
+ pWal->fileId = 0;
+ } else {
+ if (walGetNewFile(pWal, &pWal->fileId) != 0) pWal->fileId = 0;
+ pWal->fileId++;
+ }
+
+ snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId);
+ pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
+
+ if (pWal->fd < 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno));
+ } else {
+ wDebug("vgId:%d, file:%s, it is created", pWal->vgId, pWal->name);
+ }
+
+ pthread_mutex_unlock(&pWal->mutex);
+
+ return code;
+}
+
+void walRemoveOneOldFile(void *handle) {
+ SWal *pWal = handle;
+ if (pWal == NULL) return;
+ if (pWal->keep == TAOS_WAL_KEEP) return;
+ if (pWal->fd <= 0) return;
+
+ pthread_mutex_lock(&pWal->mutex);
+
+ // remove the oldest wal file
+ int64_t oldFileId = -1;
+ if (walGetOldFile(pWal, pWal->fileId, WAL_FILE_NUM, &oldFileId) == 0) {
+ char walName[WAL_FILE_LEN] = {0};
+ snprintf(walName, sizeof(walName), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, oldFileId);
+
+ if (remove(walName) < 0) {
+ wError("vgId:%d, file:%s, failed to remove since %s", pWal->vgId, walName, strerror(errno));
+ } else {
+ wInfo("vgId:%d, file:%s, it is removed", pWal->vgId, walName);
+ }
+ }
+
+ pthread_mutex_unlock(&pWal->mutex);
+}
+
+void walRemoveAllOldFiles(void *handle) {
+ if (handle == NULL) return;
+
+ SWal * pWal = handle;
+ int64_t fileId = -1;
+
+ pthread_mutex_lock(&pWal->mutex);
+ while (walGetNextFile(pWal, &fileId) >= 0) {
+ snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId);
+
+ if (remove(pWal->name) < 0) {
+ wError("vgId:%d, wal:%p file:%s, failed to remove", pWal->vgId, pWal, pWal->name);
+ } else {
+ wInfo("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->name);
+ }
+ }
+ pthread_mutex_unlock(&pWal->mutex);
+}
+
+int32_t walWrite(void *handle, SWalHead *pHead) {
+ if (handle == NULL) return -1;
+
+ SWal * pWal = handle;
+ int32_t code = 0;
+
+ // no wal
+ if (pWal->fd <= 0) return 0;
+ if (pWal->level == TAOS_WAL_NOLOG) return 0;
+ if (pHead->version <= pWal->version) return 0;
+
+ pHead->signature = WAL_SIGNATURE;
+ taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead));
+ int32_t contLen = pHead->len + sizeof(SWalHead);
+
+ pthread_mutex_lock(&pWal->mutex);
+
+ if (taosWrite(pWal->fd, pHead, contLen) != contLen) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ wError("vgId:%d, file:%s, failed to write since %s", pWal->vgId, pWal->name, strerror(errno));
+ } else {
+ wTrace("vgId:%d, write wal, fileId:%" PRId64 " fd:%d hver:%" PRId64 " wver:%" PRIu64 " len:%d", pWal->vgId,
+ pWal->fileId, pWal->fd, pHead->version, pWal->version, pHead->len);
+ pWal->version = pHead->version;
+ }
+
+ pthread_mutex_unlock(&pWal->mutex);
+
+ ASSERT(contLen == pHead->len + sizeof(SWalHead));
+
+ return code;
+}
+
+void walFsync(void *handle, bool forceFsync) {
+ SWal *pWal = handle;
+ if (pWal == NULL || pWal->fd < 0) return;
+
+ if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) {
+ wTrace("vgId:%d, file:%s, do fsync", pWal->vgId, pWal->name);
+ if (fsync(pWal->fd) < 0) {
+ wError("vgId:%d, file:%s, fsync failed since %s", pWal->vgId, pWal->name, strerror(errno));
+ }
+ }
+}
+
+int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) {
+ if (handle == NULL) return -1;
+
+ SWal * pWal = handle;
+ int32_t count = 0;
+ int32_t code = 0;
+ int64_t fileId = -1;
+
+ while ((code = walGetNextFile(pWal, &fileId)) >= 0) {
+ if (fileId == pWal->fileId) continue;
+
+ char walName[WAL_FILE_LEN];
+ snprintf(walName, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId);
+
+ wDebug("vgId:%d, file:%s, will be restored", pWal->vgId, walName);
+ int32_t code = walRestoreWalFile(pWal, pVnode, writeFp, walName, fileId);
+ if (code != TSDB_CODE_SUCCESS) {
+ wError("vgId:%d, file:%s, failed to restore since %s", pWal->vgId, walName, tstrerror(code));
+ continue;
+ }
+
+ wDebug("vgId:%d, file:%s, restore success", pWal->vgId, walName);
+
+ count++;
+ }
+
+ if (pWal->keep != TAOS_WAL_KEEP) return TSDB_CODE_SUCCESS;
+
+ if (count == 0) {
+ wDebug("vgId:%d, wal file not exist, renew it", pWal->vgId);
+ return walRenew(pWal);
+ } else {
+ // open the existing WAL file in append mode
+ pWal->fileId = 0;
+ snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId);
+ pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
+ if (pWal->fd < 0) {
+ wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno));
+ return TAOS_SYSTEM_ERROR(errno);
+ }
+ wDebug("vgId:%d, file:%s open success", pWal->vgId, pWal->name);
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) {
+ if (handle == NULL) return -1;
+ SWal *pWal = handle;
+
+ if (*fileId == 0) *fileId = -1;
+
+ pthread_mutex_lock(&(pWal->mutex));
+
+ int32_t code = walGetNextFile(pWal, fileId);
+ if (code >= 0) {
+ sprintf(fileName, "wal/%s%" PRId64, WAL_PREFIX, *fileId);
+ code = (*fileId == pWal->fileId) ? 0 : 1;
+ }
+
+ wTrace("vgId:%d, get wal file, code:%d curId:%" PRId64 " outId:%" PRId64, pWal->vgId, code, pWal->fileId, *fileId);
+ pthread_mutex_unlock(&(pWal->mutex));
+
+ return code;
+}
+
+static void walFtruncate(SWal *pWal, int32_t fd, int64_t offset) {
+ taosFtruncate(fd, offset);
+ fsync(fd);
+}
+
+static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int32_t fd, int64_t *offset) {
+ int64_t pos = *offset;
+ while (1) {
+ pos++;
+
+ if (lseek(fd, pos, SEEK_SET) < 0) {
+ wError("vgId:%d, failed to seek from corrupted wal file since %s", pWal->vgId, strerror(errno));
+ return TSDB_CODE_WAL_FILE_CORRUPTED;
+ }
+
+ if (taosRead(fd, pHead, sizeof(SWalHead)) <= 0) {
+ wError("vgId:%d, read to end of corrupted wal file, offset:%" PRId64, pWal->vgId, pos);
+ return TSDB_CODE_WAL_FILE_CORRUPTED;
+ }
+
+ if (pHead->signature != WAL_SIGNATURE) {
+ continue;
+ }
+
+ if (taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) {
+ wInfo("vgId:%d, wal head cksum check passed, offset:%" PRId64, pWal->vgId, pos);
+ *offset = pos;
+ return TSDB_CODE_SUCCESS;
+ }
+ }
+
+ return TSDB_CODE_WAL_FILE_CORRUPTED;
+}
+
+static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId) {
+ int32_t size = WAL_MAX_SIZE;
+ void * buffer = tmalloc(size);
+ if (buffer == NULL) {
+ wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno));
+ return TAOS_SYSTEM_ERROR(errno);
+ }
+
+ int32_t fd = open(name, O_RDWR);
+ if (fd < 0) {
+ wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno));
+ tfree(buffer);
+ return TAOS_SYSTEM_ERROR(errno);
+ }
+
+ wDebug("vgId:%d, file:%s, start to restore", pWal->vgId, name);
+
+ int32_t code = TSDB_CODE_SUCCESS;
+ int64_t offset = 0;
+ SWalHead *pHead = buffer;
+
+ while (1) {
+ int32_t ret = taosRead(fd, pHead, sizeof(SWalHead));
+ if (ret == 0) break;
+
+ if (ret < 0) {
+ wError("vgId:%d, file:%s, failed to read wal head since %s", pWal->vgId, name, strerror(errno));
+ code = TAOS_SYSTEM_ERROR(errno);
+ break;
+ }
+
+ if (ret < sizeof(SWalHead)) {
+ wError("vgId:%d, file:%s, failed to read wal head, ret is %d", pWal->vgId, name, ret);
+ walFtruncate(pWal, fd, offset);
+ break;
+ }
+
+ if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) {
+ wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name,
+ pHead->version, pHead->len, offset);
+ code = walSkipCorruptedRecord(pWal, pHead, fd, &offset);
+ if (code != TSDB_CODE_SUCCESS) {
+ walFtruncate(pWal, fd, offset);
+ break;
+ }
+ }
+
+ if (pHead->len > size - sizeof(SWalHead)) {
+ size = sizeof(SWalHead) + pHead->len;
+ buffer = realloc(buffer, size);
+ if (buffer == NULL) {
+ wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno));
+ code = TAOS_SYSTEM_ERROR(errno);
+ break;
+ }
+
+ pHead = buffer;
+ }
+
+ ret = taosRead(fd, pHead->cont, pHead->len);
+ if (ret < 0) {
+ wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno));
+ code = TAOS_SYSTEM_ERROR(errno);
+ break;
+ }
+
+ if (ret < pHead->len) {
+ wError("vgId:%d, file:%s, failed to read wal body, ret:%d len:%d", pWal->vgId, name, ret, pHead->len);
+ offset += sizeof(SWalHead);
+ continue;
+ }
+
+ offset = offset + sizeof(SWalHead) + pHead->len;
+
+ wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId,
+ fileId, pHead->version, pWal->version, pHead->len);
+
+ pWal->version = pHead->version;
+ (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL);
+ }
+
+ tclose(fd);
+ tfree(buffer);
+
+ return code;
+}
+
+uint64_t walGetVersion(twalh param) {
+ SWal *pWal = param;
+ if (pWal == 0) return 0;
+
+ return pWal->version;
+}
diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c
index bbee1347b8f92aa6cfad448fdfb369de8f5a6301..7a473ed18c958afa8be3c5b94b04d2fd548a56fd 100644
--- a/src/wal/test/waltest.c
+++ b/src/wal/test/waltest.c
@@ -23,7 +23,7 @@
int64_t ver = 0;
void *pWal = NULL;
-int writeToQueue(void *pVnode, void *data, int type) {
+int writeToQueue(void *pVnode, void *data, int type, void *pMsg) {
// do nothing
SWalHead *pHead = data;
@@ -37,7 +37,6 @@ int writeToQueue(void *pVnode, void *data, int type) {
int main(int argc, char *argv[]) {
char path[128] = "/home/jhtao/test/wal";
- int max = 3;
int level = 2;
int total = 5;
int rows = 10000;
@@ -47,8 +46,6 @@ int main(int argc, char *argv[]) {
for (int i=1; ijava -version
+java version "1.8.0_131"
+Java(TM) SE Runtime Environment (build 1.8.0_131-b11)
+Java HotSpot(TM) 64-Bit Server VM (build 25.131-b11, mixed mode)
+```
+
+
+(2)安装配置maven
+
+官网下载maven,下载地址:http://maven.apache.org/download.cgi
+
+配置环境变量MAVEN_HOME,将MAVEN_HOME/bin添加到PATH
+
+命令行里查看maven的版本
+
+```shell
+>mvn --version
+Apache Maven 3.5.0 (ff8f5e7444045639af65f6095c62210b5713f426; 2017-04-04T03:39:06+08:00)
+Maven home: D:\apache-maven-3.5.0\bin\..
+Java version: 1.8.0_131, vendor: Oracle Corporation
+Java home: C:\Program Files\Java\jdk1.8.0_131\jre
+Default locale: zh_CN, platform encoding: GBK
+OS name: "windows 10", version: "10.0", arch: "amd64", family: "windows"
+```
+
+为了加快maven下载依赖的速度,可以为maven配置mirror,修改MAVEN_HOME\config\settings.xml文件
+
+```xml
+
+
+ D:\apache-maven-localRepository
+
+
+
+
+ alimaven
+ aliyun maven
+ http://maven.aliyun.com/nexus/content/groups/public/
+ central
+
+
+
+
+
+
+ jdk-1.8
+
+ true
+ 1.8
+
+
+ 1.8
+ 1.8
+ 1.8
+
+
+
+
+```
+
+
+
+(3)在linux服务器上安装TDengine-server
+
+在taosdata官网下载TDengine-server,下载地址:https://www.taosdata.com/cn/all-downloads/
+
+在linux服务器上安装TDengine-server
+
+```shell
+# tar -zxvf package/TDengine-server-2.0.1.1-Linux-x64.tar.gz
+# cd TDengine-server/
+# ./install.sh
+```
+
+启动taosd
+
+```shell
+# systemctl start taosd
+```
+
+在server上用taos连接taosd
+
+```shell
+# taos
+taos> show dnodes;
+ id | end_point | vnodes | cores | status | role | create_time |
+==================================================================================================================
+ 1 | td01:6030 | 2 | 4 | ready | any | 2020-08-19 18:40:25.045 |
+Query OK, 1 row(s) in set (0.005765s)
+```
+
+如果可以正确连接到taosd实例,并打印出databases的信息,说明TDengine的server已经正确启动。这里查看server的hostname
+
+```shell
+# hostname -f
+td01
+```
+
+注意,如果安装TDengine后,使用默认的taos.cfg配置文件,taosd会使用当前server的hostname创建dnode实例。之后,在client也需要使用这个hostname来连接taosd。
+
+
+
+(4)在windows上安装TDengine-client
+
+在taosdata官网下载taos客户端,下载地址:
+https://www.taosdata.com/cn/all-downloads/
+下载后,双击exe安装。
+
+修改client的hosts文件(C:\Windows\System32\drivers\etc\hosts),将server的hostname和ip配置到client的hosts文件中
+
+```
+192.168.236.136 td01
+```
+
+配置完成后,在命令行内使用taos shell连接server端
+
+```shell
+C:\TDengine>taos -h td01
+Welcome to the TDengine shell from Linux, Client Version:2.0.1.1
+Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.
+
+taos> show databases;
+ name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | precision | status |
+===================================================================================================================================================================================================================================================================
+ test | 2020-08-19 18:43:50.731 | 1 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready |
+ log | 2020-08-19 18:40:28.064 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready |
+Query OK, 2 row(s) in set (0.068000s)
+```
+
+如果windows上的client能够正常连接,并打印database信息,说明client可以正常连接server了。
+
+
+
+## 应用开发
+
+(1)新建maven工程,在pom.xml中引入taos-jdbcdriver依赖。
+
+```xml
+
+
+ 4.0.0
+
+ com.taosdata.demo
+ JdbcDemo
+ 1.0-SNAPSHOT
+
+
+
+ com.taosdata.jdbc
+ taos-jdbcdriver
+ 2.0.8
+
+
+
+```
+
+(2)使用jdbc查询TDengine数据库
+
+下面是示例代码:
+
+```java
+public class JdbcDemo {
+
+ public static void main(String[] args) throws Exception {
+ Connection conn = getConn();
+ Statement stmt = conn.createStatement();
+ // create database
+ stmt.executeUpdate("create database if not exists db");
+ // use database
+ stmt.executeUpdate("use db");
+ // create table
+ stmt.executeUpdate("create table if not exists tb (ts timestamp, temperature int, humidity float)");
+ // insert data
+ int affectedRows = stmt.executeUpdate("insert into tb values(now, 23, 10.3) (now + 1s, 20, 9.3)");
+ System.out.println("insert " + affectedRows + " rows.");
+ // query data
+ ResultSet resultSet = stmt.executeQuery("select * from tb");
+ Timestamp ts = null;
+ int temperature = 0;
+ float humidity = 0;
+ while(resultSet.next()){
+ ts = resultSet.getTimestamp(1);
+ temperature = resultSet.getInt(2);
+ humidity = resultSet.getFloat("humidity");
+ System.out.printf("%s, %d, %s\n", ts, temperature, humidity);
+ }
+ }
+
+ public static Connection getConn() throws Exception{
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ String jdbcUrl = "jdbc:TAOS://td01:0/log?user=root&password=taosdata";
+ Properties connProps = new Properties();
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ Connection conn = DriverManager.getConnection(jdbcUrl, connProps);
+ return conn;
+ }
+
+}
+```
+
+(3)测试jdbc访问tdengine的sever实例
+
+console输出:
+
+```
+insert 2 rows.
+2020-08-26 00:06:34.575, 23, 10.3
+2020-08-26 00:06:35.575, 20, 9.3
+```
+
+
+
+## 指南
+
+(1)如何设置主机名和hosts
+
+在server上查看hostname和fqdn
+```shell
+查看hostname
+# hostname
+taos-server
+
+查看fqdn
+# hostname -f
+taos-server
+```
+
+windows下hosts文件位于:
+C:\\Windows\System32\drivers\etc\hosts
+修改hosts文件,添加server的ip和hostname
+
+```s
+192.168.56.101 node5
+```
+
+(2)什么是fqdn?
+
+
+> 什么是FQDN?
+>
+> FQDN(Full qualified domain name)全限定域名,fqdn由2部分组成:hostname+domainname。
+>
+> 例如,一个邮件服务器的fqdn可能是:mymail.somecollege.edu,其中mymail是hostname(主机名),somcollege.edu是domainname(域名)。本例中,.edu是顶级域名,.somecollege是二级域名。
+>
+> 当连接服务器时,必须指定fqdn,然后,dns服务器通过查看dns表,将hostname解析为相应的ip地址。如果只指定hostname(不指定domainname),应用程序可能服务解析主机名。因为如果你试图访问不在本地的远程服务器时,本地的dns服务器和可能没有远程服务器的hostname列表。
+>
+> 参考:https://kb.iu.edu/d/aiuv
diff --git a/tests/examples/JDBC/connectionPools/README-cn.md b/tests/examples/JDBC/connectionPools/README-cn.md
new file mode 100644
index 0000000000000000000000000000000000000000..761596dfc55a3e2c9f449ed34fd72ac96c277512
--- /dev/null
+++ b/tests/examples/JDBC/connectionPools/README-cn.md
@@ -0,0 +1,33 @@
+这个example中,我们适配了java常见的连接池:
+* c3p0
+* dbcp
+* druid
+* HikariCP
+
+### 说明
+ConnectionPoolDemo的程序逻辑:
+1. 创建到host的connection连接池
+2. 创建名称为pool_test的database,创建表超级weather,创建tableSize个子表
+3. 不断向所有子表进行插入。
+
+### 如何运行这个例子:
+```shell script
+# mvn exec:java -Dexec.mainClass="com.taosdata.demo.ConnectionPoolDemo" -Dexec.args="-host localhost"
+```
+使用mvn运行ConnectionPoolDemo的main方法,可以指定参数
+```shell script
+Usage:
+mvn exec:java -Dexec.mainClass="com.taosdata.demo.ConnectionPoolDemo" -Dexec.args=""
+-host : hostname
+-poolType
+-poolSize
+-tableSize
+-batchSize : 每条Insert SQL中values的数量
+-sleep : 每次插入任务提交后的
+```
+
+### 如何停止程序:
+ConnectionPoolDemo不会自己停止,会一直执行插入,需要手动Ctrl+C运行。
+
+### 日志
+使用log4j,将日志和错误分别输出到了debug.log和error.log中
\ No newline at end of file
diff --git a/tests/examples/JDBC/connectionPools/pom.xml b/tests/examples/JDBC/connectionPools/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..2793f0a83ddc88711796c133802c82979ae14be5
--- /dev/null
+++ b/tests/examples/JDBC/connectionPools/pom.xml
@@ -0,0 +1,56 @@
+
+
+ 4.0.0
+
+ com.taosdata.demo
+ connectionPools
+ 1.0-SNAPSHOT
+
+
+
+ com.taosdata.jdbc
+ taos-jdbcdriver
+ 2.0.11
+
+
+
+
+ com.alibaba
+ druid
+ 1.1.17
+
+
+
+ com.zaxxer
+ HikariCP
+ 3.2.0
+
+
+
+ commons-pool
+ commons-pool
+ 1.5.4
+
+
+ commons-dbcp
+ commons-dbcp
+ 1.4
+
+
+
+ com.mchange
+ c3p0
+ 0.9.5.4
+
+
+
+
+ log4j
+ log4j
+ 1.2.17
+
+
+
+
\ No newline at end of file
diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java
new file mode 100644
index 0000000000000000000000000000000000000000..79c0aacea740dcb6fca9780c7f64872c537c3225
--- /dev/null
+++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java
@@ -0,0 +1,117 @@
+package com.taosdata.demo;
+
+import com.taosdata.demo.common.InsertTask;
+import com.taosdata.demo.pool.C3p0Builder;
+import com.taosdata.demo.pool.DbcpBuilder;
+import com.taosdata.demo.pool.DruidPoolBuilder;
+import com.taosdata.demo.pool.HikariCpBuilder;
+import org.apache.log4j.Logger;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+public class ConnectionPoolDemo {
+
+ private static Logger logger = Logger.getLogger(DruidPoolBuilder.class);
+ private static final String dbName = "pool_test";
+
+ private static int batchSize = 10;
+ private static int sleep = 1000;
+ private static int poolSize = 50;
+ private static int tableSize = 1000;
+ private static int threadCount = 50;
+ private static String poolType = "hikari";
+
+
+ public static void main(String[] args) throws InterruptedException {
+ String host = null;
+ for (int i = 0; i < args.length; i++) {
+ if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+ host = args[++i];
+ }
+ if ("-batchSize".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+ batchSize = Integer.parseInt(args[++i]);
+ }
+ if ("-sleep".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+ sleep = Integer.parseInt(args[++i]);
+ }
+ if ("-poolSize".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+ poolSize = Integer.parseInt(args[++i]);
+ }
+ if ("-tableSize".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+ tableSize = Integer.parseInt(args[++i]);
+ }
+ if ("-poolType".equalsIgnoreCase(args[i]) && i < args.length - 1) {
+ poolType = args[++i];
+ }
+ }
+ if (host == null) {
+ System.out.println("Usage: java -jar XXX.jar " +
+ "-host " +
+ "-batchSize " +
+ "-sleep " +
+ "-poolSize " +
+ "-tableSize " +
+ "-poolType ");
+ return;
+ }
+
+ DataSource dataSource;
+ switch (poolType) {
+ case "c3p0":
+ dataSource = C3p0Builder.getDataSource(host, poolSize);
+ break;
+ case "dbcp":
+ dataSource = DbcpBuilder.getDataSource(host, poolSize);
+ break;
+ case "druid":
+ dataSource = DruidPoolBuilder.getDataSource(host, poolSize);
+ break;
+ case "hikari":
+ default:
+ dataSource = HikariCpBuilder.getDataSource(host, poolSize);
+ poolType = "hikari";
+ }
+
+ logger.info(">>>>>>>>>>>>>> connection pool Type: " + poolType);
+
+ init(dataSource);
+
+ ExecutorService executor = Executors.newFixedThreadPool(threadCount);
+ while (true) {
+ executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize));
+ if (sleep > 0)
+ TimeUnit.MILLISECONDS.sleep(sleep);
+ }
+ }
+
+ private static void init(DataSource dataSource) {
+ try (Connection conn = dataSource.getConnection()) {
+ execute(conn, "drop database if exists " + dbName + "");
+ execute(conn, "create database if not exists " + dbName + "");
+ execute(conn, "use " + dbName + "");
+ execute(conn, "create table weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)");
+ for (int tb_ind = 1; tb_ind <= tableSize; tb_ind++) {
+ execute(conn, "create table t_" + tb_ind + " using weather tags('beijing'," + (tb_ind + 1) + ")");
+ }
+ logger.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>> init finished.");
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private static void execute(Connection con, String sql) {
+ try (Statement stmt = con.createStatement()) {
+ stmt.executeUpdate(sql);
+ logger.info("SQL >>> " + sql);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
+}
diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java
new file mode 100644
index 0000000000000000000000000000000000000000..ed86acd6e9f8bfb8c862c1764e39f541d3f054eb
--- /dev/null
+++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java
@@ -0,0 +1,77 @@
+package com.taosdata.demo.common;
+
+import org.apache.log4j.Logger;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Random;
+
+public class InsertTask implements Runnable {
+ private final Random random = new Random(System.currentTimeMillis());
+ private static final Logger logger = Logger.getLogger(InsertTask.class);
+
+ private final DataSource ds;
+ private final int batchSize;
+ private final String dbName;
+ private final int tableSize;
+
+ public InsertTask(DataSource ds, String dbName, int tableSize, int batchSize) {
+ this.ds = ds;
+ this.dbName = dbName;
+ this.tableSize = tableSize;
+ this.batchSize = batchSize;
+ }
+
+ @Override
+ public void run() {
+ Connection conn = null;
+ Statement stmt = null;
+ int affectedRows = 0;
+
+ long start = System.currentTimeMillis();
+ try {
+ conn = ds.getConnection();
+ stmt = conn.createStatement();
+
+ for (int tb_index = 1; tb_index <= tableSize; tb_index++) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("insert into ");
+ sb.append(dbName);
+ sb.append(".t_");
+ sb.append(tb_index);
+ sb.append("(ts, temperature, humidity) values ");
+ for (int i = 0; i < batchSize; i++) {
+ sb.append("(");
+ sb.append(start + i);
+ sb.append(", ");
+ sb.append(random.nextFloat() * 30);
+ sb.append(", ");
+ sb.append(random.nextInt(70));
+ sb.append(") ");
+ }
+ logger.info("SQL >>> " + sb.toString());
+ affectedRows += stmt.executeUpdate(sb.toString());
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ } finally {
+ if (stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ logger.info(">>> affectedRows:" + affectedRows + " TimeCost:" + (System.currentTimeMillis() - start) + " ms");
+ }
+ }
+}
diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/C3p0Builder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/C3p0Builder.java
new file mode 100644
index 0000000000000000000000000000000000000000..587f417410f96f43be2ced5a4820cd49cdb99a17
--- /dev/null
+++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/C3p0Builder.java
@@ -0,0 +1,28 @@
+package com.taosdata.demo.pool;
+
+import com.mchange.v2.c3p0.ComboPooledDataSource;
+import org.apache.commons.dbcp.BasicDataSource;
+
+import javax.sql.DataSource;
+import java.beans.PropertyVetoException;
+
+public class C3p0Builder {
+
+ public static DataSource getDataSource(String host, int poolSize) {
+ ComboPooledDataSource ds = new ComboPooledDataSource();
+
+ try {
+ ds.setDriverClass("com.taosdata.jdbc.TSDBDriver");
+ } catch (PropertyVetoException e) {
+ e.printStackTrace();
+ }
+ ds.setJdbcUrl("jdbc:TAOS://" + host + ":6030");
+ ds.setUser("root");
+ ds.setPassword("taosdata");
+
+ ds.setMinPoolSize(poolSize);
+ ds.setMaxPoolSize(poolSize);
+ ds.setAcquireIncrement(5);
+ return ds;
+ }
+}
diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DbcpBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DbcpBuilder.java
new file mode 100644
index 0000000000000000000000000000000000000000..3c34a32532f595bf3134942094e96e952bd09dbb
--- /dev/null
+++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DbcpBuilder.java
@@ -0,0 +1,21 @@
+package com.taosdata.demo.pool;
+
+import org.apache.commons.dbcp.BasicDataSource;
+
+import javax.sql.DataSource;
+
+public class DbcpBuilder {
+
+ public static DataSource getDataSource(String host, int poolSize) {
+ BasicDataSource ds = new BasicDataSource();
+ ds.setDriverClassName("com.taosdata.jdbc.TSDBDriver");
+ ds.setUrl("jdbc:TAOS://" + host + ":6030");
+ ds.setUsername("root");
+ ds.setPassword("taosdata");
+
+ ds.setMaxActive(poolSize);
+ ds.setMinIdle(poolSize);
+ ds.setInitialSize(poolSize);
+ return ds;
+ }
+}
diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java
new file mode 100644
index 0000000000000000000000000000000000000000..e5dc14c6a5ef69c2a7059d5d78b621e25ff3d799
--- /dev/null
+++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java
@@ -0,0 +1,31 @@
+package com.taosdata.demo.pool;
+
+import com.alibaba.druid.pool.DruidDataSource;
+
+import javax.sql.DataSource;
+
+public class DruidPoolBuilder {
+
+ public static DataSource getDataSource(String host, int poolSize) {
+ final String url = "jdbc:TAOS://" + host + ":6030";
+
+ DruidDataSource dataSource = new DruidDataSource();
+ dataSource.setUrl(url);
+ dataSource.setDriverClassName("com.taosdata.jdbc.TSDBDriver");
+ dataSource.setUsername("root");
+ dataSource.setPassword("taosdata");
+
+ //初始连接数,默认0
+ dataSource.setInitialSize(poolSize);
+ //最大连接数,默认8
+ dataSource.setMaxActive(poolSize);
+ //最小闲置数
+ dataSource.setMinIdle(poolSize);
+ //获取连接的最大等待时间,单位毫秒
+ dataSource.setMaxWait(2000);
+
+ return dataSource;
+ }
+
+
+}
diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java
new file mode 100644
index 0000000000000000000000000000000000000000..87f1f4ad2cbba41a779f0247f2214ef2bf04a8ca
--- /dev/null
+++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java
@@ -0,0 +1,22 @@
+package com.taosdata.demo.pool;
+
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+
+import javax.sql.DataSource;
+
+public class HikariCpBuilder {
+
+ public static DataSource getDataSource(String host, int poolSize) {
+ HikariConfig config = new HikariConfig();
+ config.setDriverClassName("com.taosdata.jdbc.TSDBDriver");
+ config.setJdbcUrl("jdbc:TAOS://" + host + ":6030");
+ config.setUsername("root");
+ config.setPassword("taosdata");
+
+ config.setMaximumPoolSize(poolSize);
+ config.setMinimumIdle(poolSize);
+ HikariDataSource ds = new HikariDataSource(config);
+ return ds;
+ }
+}
diff --git a/tests/examples/JDBC/connectionPools/src/main/resources/log4j.properties b/tests/examples/JDBC/connectionPools/src/main/resources/log4j.properties
new file mode 100644
index 0000000000000000000000000000000000000000..1299357be3d2e99ca6b79227f14ca7a587718914
--- /dev/null
+++ b/tests/examples/JDBC/connectionPools/src/main/resources/log4j.properties
@@ -0,0 +1,21 @@
+### 设置###
+log4j.rootLogger=debug,stdout,DebugLog,ErrorLog
+### 输出信息到控制抬 ###
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n
+### 输出DEBUG 级别以上的日志到=logs/debug.log
+log4j.appender.DebugLog=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DebugLog.File=logs/debug.log
+log4j.appender.DebugLog.Append=true
+log4j.appender.DebugLog.Threshold=DEBUG
+log4j.appender.DebugLog.layout=org.apache.log4j.PatternLayout
+log4j.appender.DebugLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
+### 输出ERROR 级别以上的日志到=logs/error.log
+log4j.appender.ErrorLog=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.ErrorLog.File=logs/error.log
+log4j.appender.ErrorLog.Append=true
+log4j.appender.ErrorLog.Threshold=ERROR
+log4j.appender.ErrorLog.layout=org.apache.log4j.PatternLayout
+log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
\ No newline at end of file
diff --git a/tests/examples/JDBC/mybatisplus-demo/.gitignore b/tests/examples/JDBC/mybatisplus-demo/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..b56f1dd0d04da4e03f710af3917e4fbdd9be4aa8
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/.gitignore
@@ -0,0 +1,33 @@
+README.md
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/
diff --git a/tests/examples/JDBC/mybatisplus-demo/.mvn/wrapper/MavenWrapperDownloader.java b/tests/examples/JDBC/mybatisplus-demo/.mvn/wrapper/MavenWrapperDownloader.java
new file mode 100644
index 0000000000000000000000000000000000000000..a45eb6ba269cd38f8965cef786729790945d9537
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/.mvn/wrapper/MavenWrapperDownloader.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright 2007-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.net.*;
+import java.io.*;
+import java.nio.channels.*;
+import java.util.Properties;
+
+public class MavenWrapperDownloader {
+
+ private static final String WRAPPER_VERSION = "0.5.6";
+ /**
+ * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
+ */
+ private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+ + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
+
+ /**
+ * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
+ * use instead of the default one.
+ */
+ private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
+ ".mvn/wrapper/maven-wrapper.properties";
+
+ /**
+ * Path where the maven-wrapper.jar will be saved to.
+ */
+ private static final String MAVEN_WRAPPER_JAR_PATH =
+ ".mvn/wrapper/maven-wrapper.jar";
+
+ /**
+ * Name of the property which should be used to override the default download url for the wrapper.
+ */
+ private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
+
+ public static void main(String args[]) {
+ System.out.println("- Downloader started");
+ File baseDirectory = new File(args[0]);
+ System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
+
+ // If the maven-wrapper.properties exists, read it and check if it contains a custom
+ // wrapperUrl parameter.
+ File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
+ String url = DEFAULT_DOWNLOAD_URL;
+ if (mavenWrapperPropertyFile.exists()) {
+ FileInputStream mavenWrapperPropertyFileInputStream = null;
+ try {
+ mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
+ Properties mavenWrapperProperties = new Properties();
+ mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
+ url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
+ } catch (IOException e) {
+ System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
+ } finally {
+ try {
+ if (mavenWrapperPropertyFileInputStream != null) {
+ mavenWrapperPropertyFileInputStream.close();
+ }
+ } catch (IOException e) {
+ // Ignore ...
+ }
+ }
+ }
+ System.out.println("- Downloading from: " + url);
+
+ File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
+ if (!outputFile.getParentFile().exists()) {
+ if (!outputFile.getParentFile().mkdirs()) {
+ System.out.println(
+ "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
+ }
+ }
+ System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
+ try {
+ downloadFileFromURL(url, outputFile);
+ System.out.println("Done");
+ System.exit(0);
+ } catch (Throwable e) {
+ System.out.println("- Error downloading");
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ private static void downloadFileFromURL(String urlString, File destination) throws Exception {
+ if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
+ String username = System.getenv("MVNW_USERNAME");
+ char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
+ Authenticator.setDefault(new Authenticator() {
+ @Override
+ protected PasswordAuthentication getPasswordAuthentication() {
+ return new PasswordAuthentication(username, password);
+ }
+ });
+ }
+ URL website = new URL(urlString);
+ ReadableByteChannel rbc;
+ rbc = Channels.newChannel(website.openStream());
+ FileOutputStream fos = new FileOutputStream(destination);
+ fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+ fos.close();
+ rbc.close();
+ }
+
+}
diff --git a/tests/examples/JDBC/mybatisplus-demo/.mvn/wrapper/maven-wrapper.jar b/tests/examples/JDBC/mybatisplus-demo/.mvn/wrapper/maven-wrapper.jar
new file mode 100644
index 0000000000000000000000000000000000000000..2cc7d4a55c0cd0092912bf49ae38b3a9e3fd0054
Binary files /dev/null and b/tests/examples/JDBC/mybatisplus-demo/.mvn/wrapper/maven-wrapper.jar differ
diff --git a/tests/examples/JDBC/mybatisplus-demo/.mvn/wrapper/maven-wrapper.properties b/tests/examples/JDBC/mybatisplus-demo/.mvn/wrapper/maven-wrapper.properties
new file mode 100644
index 0000000000000000000000000000000000000000..642d572ce90e5085986bdd9c9204b9404f028084
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/.mvn/wrapper/maven-wrapper.properties
@@ -0,0 +1,2 @@
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar
diff --git a/tests/examples/JDBC/mybatisplus-demo/mvnw b/tests/examples/JDBC/mybatisplus-demo/mvnw
new file mode 100755
index 0000000000000000000000000000000000000000..3c8a5537314954d53ec2fb774b34fe5d5a5f253a
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/mvnw
@@ -0,0 +1,322 @@
+#!/bin/sh
+# ----------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Maven Start Up Batch script
+#
+# Required ENV vars:
+# ------------------
+# JAVA_HOME - location of a JDK home dir
+#
+# Optional ENV vars
+# -----------------
+# M2_HOME - location of maven2's installed home dir
+# MAVEN_OPTS - parameters passed to the Java VM when running Maven
+# e.g. to debug Maven itself, use
+# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+# ----------------------------------------------------------------------------
+
+if [ -z "$MAVEN_SKIP_RC" ]; then
+
+ if [ -f /etc/mavenrc ]; then
+ . /etc/mavenrc
+ fi
+
+ if [ -f "$HOME/.mavenrc" ]; then
+ . "$HOME/.mavenrc"
+ fi
+
+fi
+
+# OS specific support. $var _must_ be set to either true or false.
+cygwin=false
+darwin=false
+mingw=false
+case "$(uname)" in
+CYGWIN*) cygwin=true ;;
+MINGW*) mingw=true ;;
+Darwin*)
+ darwin=true
+ # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
+ # See https://developer.apple.com/library/mac/qa/qa1170/_index.html
+ if [ -z "$JAVA_HOME" ]; then
+ if [ -x "/usr/libexec/java_home" ]; then
+ export JAVA_HOME="$(/usr/libexec/java_home)"
+ else
+ export JAVA_HOME="/Library/Java/Home"
+ fi
+ fi
+ ;;
+esac
+
+if [ -z "$JAVA_HOME" ]; then
+ if [ -r /etc/gentoo-release ]; then
+ JAVA_HOME=$(java-config --jre-home)
+ fi
+fi
+
+if [ -z "$M2_HOME" ]; then
+ ## resolve links - $0 may be a link to maven's home
+ PRG="$0"
+
+ # need this for relative symlinks
+ while [ -h "$PRG" ]; do
+ ls=$(ls -ld "$PRG")
+ link=$(expr "$ls" : '.*-> \(.*\)$')
+ if expr "$link" : '/.*' >/dev/null; then
+ PRG="$link"
+ else
+ PRG="$(dirname "$PRG")/$link"
+ fi
+ done
+
+ saveddir=$(pwd)
+
+ M2_HOME=$(dirname "$PRG")/..
+
+ # make it fully qualified
+ M2_HOME=$(cd "$M2_HOME" && pwd)
+
+ cd "$saveddir"
+ # echo Using m2 at $M2_HOME
+fi
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin; then
+ [ -n "$M2_HOME" ] &&
+ M2_HOME=$(cygpath --unix "$M2_HOME")
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=$(cygpath --unix "$JAVA_HOME")
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=$(cygpath --path --unix "$CLASSPATH")
+fi
+
+# For Mingw, ensure paths are in UNIX format before anything is touched
+if $mingw; then
+ [ -n "$M2_HOME" ] &&
+ M2_HOME="$( (
+ cd "$M2_HOME"
+ pwd
+ ))"
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME="$( (
+ cd "$JAVA_HOME"
+ pwd
+ ))"
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+ javaExecutable="$(which javac)"
+ if [ -n "$javaExecutable" ] && ! [ "$(expr \"$javaExecutable\" : '\([^ ]*\)')" = "no" ]; then
+ # readlink(1) is not available as standard on Solaris 10.
+ readLink=$(which readlink)
+ if [ ! $(expr "$readLink" : '\([^ ]*\)') = "no" ]; then
+ if $darwin; then
+ javaHome="$(dirname \"$javaExecutable\")"
+ javaExecutable="$(cd \"$javaHome\" && pwd -P)/javac"
+ else
+ javaExecutable="$(readlink -f \"$javaExecutable\")"
+ fi
+ javaHome="$(dirname \"$javaExecutable\")"
+ javaHome=$(expr "$javaHome" : '\(.*\)/bin')
+ JAVA_HOME="$javaHome"
+ export JAVA_HOME
+ fi
+ fi
+fi
+
+if [ -z "$JAVACMD" ]; then
+ if [ -n "$JAVA_HOME" ]; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ]; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD="$JAVA_HOME/jre/sh/java"
+ else
+ JAVACMD="$JAVA_HOME/bin/java"
+ fi
+ else
+ JAVACMD="$(which java)"
+ fi
+fi
+
+if [ ! -x "$JAVACMD" ]; then
+ echo "Error: JAVA_HOME is not defined correctly." >&2
+ echo " We cannot execute $JAVACMD" >&2
+ exit 1
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+ echo "Warning: JAVA_HOME environment variable is not set."
+fi
+
+CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
+
+# traverses directory structure from process work directory to filesystem root
+# first directory with .mvn subdirectory is considered project base directory
+find_maven_basedir() {
+
+ if [ -z "$1" ]; then
+ echo "Path not specified to find_maven_basedir"
+ return 1
+ fi
+
+ basedir="$1"
+ wdir="$1"
+ while [ "$wdir" != '/' ]; do
+ if [ -d "$wdir"/.mvn ]; then
+ basedir=$wdir
+ break
+ fi
+ # workaround for JBEAP-8937 (on Solaris 10/Sparc)
+ if [ -d "${wdir}" ]; then
+ wdir=$(
+ cd "$wdir/.."
+ pwd
+ )
+ fi
+ # end of workaround
+ done
+ echo "${basedir}"
+}
+
+# concatenates all lines of a file
+concat_lines() {
+ if [ -f "$1" ]; then
+ echo "$(tr -s '\n' ' ' <"$1")"
+ fi
+}
+
+BASE_DIR=$(find_maven_basedir "$(pwd)")
+if [ -z "$BASE_DIR" ]; then
+ exit 1
+fi
+
+##########################################################################################
+# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+# This allows using the maven wrapper in projects that prohibit checking in binary data.
+##########################################################################################
+if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Found .mvn/wrapper/maven-wrapper.jar"
+ fi
+else
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
+ fi
+ if [ -n "$MVNW_REPOURL" ]; then
+ jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
+ else
+ jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
+ fi
+ while IFS="=" read key value; do
+ case "$key" in wrapperUrl)
+ jarUrl="$value"
+ break
+ ;;
+ esac
+ done <"$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Downloading from: $jarUrl"
+ fi
+ wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
+ if $cygwin; then
+ wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath")
+ fi
+
+ if command -v wget >/dev/null; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Found wget ... using wget"
+ fi
+ if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+ wget "$jarUrl" -O "$wrapperJarPath"
+ else
+ wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath"
+ fi
+ elif command -v curl >/dev/null; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Found curl ... using curl"
+ fi
+ if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+ curl -o "$wrapperJarPath" "$jarUrl" -f
+ else
+ curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f
+ fi
+
+ else
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo "Falling back to using Java to download"
+ fi
+ javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
+ # For Cygwin, switch paths to Windows format before running javac
+ if $cygwin; then
+ javaClass=$(cygpath --path --windows "$javaClass")
+ fi
+ if [ -e "$javaClass" ]; then
+ if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo " - Compiling MavenWrapperDownloader.java ..."
+ fi
+ # Compiling the Java class
+ ("$JAVA_HOME/bin/javac" "$javaClass")
+ fi
+ if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
+ # Running the downloader
+ if [ "$MVNW_VERBOSE" = true ]; then
+ echo " - Running MavenWrapperDownloader.java ..."
+ fi
+ ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
+ fi
+ fi
+ fi
+fi
+##########################################################################################
+# End of extension
+##########################################################################################
+
+export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
+if [ "$MVNW_VERBOSE" = true ]; then
+ echo $MAVEN_PROJECTBASEDIR
+fi
+MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin; then
+ [ -n "$M2_HOME" ] &&
+ M2_HOME=$(cygpath --path --windows "$M2_HOME")
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME")
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=$(cygpath --path --windows "$CLASSPATH")
+ [ -n "$MAVEN_PROJECTBASEDIR" ] &&
+ MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR")
+fi
+
+# Provide a "standardized" way to retrieve the CLI args that will
+# work with both Windows and non-Windows executions.
+MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@"
+export MAVEN_CMD_LINE_ARGS
+
+WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+exec "$JAVACMD" \
+ $MAVEN_OPTS \
+ -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
+ "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
+ ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"
diff --git a/tests/examples/JDBC/mybatisplus-demo/mvnw.cmd b/tests/examples/JDBC/mybatisplus-demo/mvnw.cmd
new file mode 100644
index 0000000000000000000000000000000000000000..c8d43372c986d97911cdc21bd87e0cbe3d83bdda
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/mvnw.cmd
@@ -0,0 +1,182 @@
+@REM ----------------------------------------------------------------------------
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM https://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM ----------------------------------------------------------------------------
+
+@REM ----------------------------------------------------------------------------
+@REM Maven Start Up Batch script
+@REM
+@REM Required ENV vars:
+@REM JAVA_HOME - location of a JDK home dir
+@REM
+@REM Optional ENV vars
+@REM M2_HOME - location of maven2's installed home dir
+@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
+@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
+@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
+@REM e.g. to debug Maven itself, use
+@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+@REM ----------------------------------------------------------------------------
+
+@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
+@echo off
+@REM set title of command window
+title %0
+@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
+@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
+
+@REM set %HOME% to equivalent of $HOME
+if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
+
+@REM Execute a user defined script before this one
+if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
+@REM check for pre script, once with legacy .bat ending and once with .cmd ending
+if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
+if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
+:skipRcPre
+
+@setlocal
+
+set ERROR_CODE=0
+
+@REM To isolate internal variables from possible post scripts, we use another setlocal
+@setlocal
+
+@REM ==== START VALIDATION ====
+if not "%JAVA_HOME%" == "" goto OkJHome
+
+echo.
+echo Error: JAVA_HOME not found in your environment. >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+:OkJHome
+if exist "%JAVA_HOME%\bin\java.exe" goto init
+
+echo.
+echo Error: JAVA_HOME is set to an invalid directory. >&2
+echo JAVA_HOME = "%JAVA_HOME%" >&2
+echo Please set the JAVA_HOME variable in your environment to match the >&2
+echo location of your Java installation. >&2
+echo.
+goto error
+
+@REM ==== END VALIDATION ====
+
+:init
+
+@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
+@REM Fallback to current working directory if not found.
+
+set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
+IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
+
+set EXEC_DIR=%CD%
+set WDIR=%EXEC_DIR%
+:findBaseDir
+IF EXIST "%WDIR%"\.mvn goto baseDirFound
+cd ..
+IF "%WDIR%"=="%CD%" goto baseDirNotFound
+set WDIR=%CD%
+goto findBaseDir
+
+:baseDirFound
+set MAVEN_PROJECTBASEDIR=%WDIR%
+cd "%EXEC_DIR%"
+goto endDetectBaseDir
+
+:baseDirNotFound
+set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
+cd "%EXEC_DIR%"
+
+:endDetectBaseDir
+
+IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
+
+@setlocal EnableExtensions EnableDelayedExpansion
+for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
+@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
+
+:endReadAdditionalConfig
+
+SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
+set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
+set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
+
+FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
+ IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B
+)
+
+@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
+if exist %WRAPPER_JAR% (
+ if "%MVNW_VERBOSE%" == "true" (
+ echo Found %WRAPPER_JAR%
+ )
+) else (
+ if not "%MVNW_REPOURL%" == "" (
+ SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
+ )
+ if "%MVNW_VERBOSE%" == "true" (
+ echo Couldn't find %WRAPPER_JAR%, downloading it ...
+ echo Downloading from: %DOWNLOAD_URL%
+ )
+
+ powershell -Command "&{"^
+ "$webclient = new-object System.Net.WebClient;"^
+ "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
+ "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
+ "}"^
+ "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^
+ "}"
+ if "%MVNW_VERBOSE%" == "true" (
+ echo Finished downloading %WRAPPER_JAR%
+ )
+)
+@REM End of extension
+
+@REM Provide a "standardized" way to retrieve the CLI args that will
+@REM work with both Windows and non-Windows executions.
+set MAVEN_CMD_LINE_ARGS=%*
+
+%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
+if ERRORLEVEL 1 goto error
+goto end
+
+:error
+set ERROR_CODE=1
+
+:end
+@endlocal & set ERROR_CODE=%ERROR_CODE%
+
+if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
+@REM check for post script, once with legacy .bat ending and once with .cmd ending
+if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
+if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
+:skipRcPost
+
+@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
+if "%MAVEN_BATCH_PAUSE%" == "on" pause
+
+if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
+
+exit /B %ERROR_CODE%
diff --git a/tests/examples/JDBC/mybatisplus-demo/pom.xml b/tests/examples/JDBC/mybatisplus-demo/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..8535f3b797dcf13bd47d968f735ba5e7873fad51
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/pom.xml
@@ -0,0 +1,101 @@
+
+
+ 4.0.0
+
+ org.springframework.boot
+ spring-boot-starter-parent
+ 2.4.0
+
+
+ com.taosdata.example
+ mybatisplus-demo
+ 0.0.1-SNAPSHOT
+ mybatisplus-demo
+ Demo project for tdengine
+
+
+ 1.8
+
+
+
+
+ org.springframework.boot
+ spring-boot-starter
+
+
+ org.projectlombok
+ lombok
+ true
+
+
+ com.baomidou
+ mybatis-plus-boot-starter
+ 3.1.2
+
+
+ com.h2database
+ h2
+ runtime
+
+
+
+ com.taosdata.jdbc
+ taos-jdbcdriver
+ 2.0.11
+
+
+
+
+ mysql
+ mysql-connector-java
+ 5.1.47
+
+
+ org.springframework.boot
+ spring-boot-starter-web
+
+
+ org.springframework.boot
+ spring-boot-devtools
+ runtime
+ true
+
+
+ org.springframework.boot
+ spring-boot-starter-test
+ test
+
+
+ junit
+ junit
+ 4.12
+ test
+
+
+
+
+
+
+ org.springframework.boot
+ spring-boot-maven-plugin
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+ 2.17
+
+
+ **/*Test.java
+
+
+ **/Abstract*.java
+
+
+
+
+
+
+
+
diff --git a/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/MybatisplusDemoApplication.java b/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/MybatisplusDemoApplication.java
new file mode 100644
index 0000000000000000000000000000000000000000..7aaebca0846c15c2055596c95ae76d0cee773e41
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/MybatisplusDemoApplication.java
@@ -0,0 +1,15 @@
+package com.taosdata.example.mybatisplusdemo;
+
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+@MapperScan("com.taosdata.example.mybatisplusdemo.mapper")
+public class MybatisplusDemoApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(MybatisplusDemoApplication.class, args);
+ }
+
+}
\ No newline at end of file
diff --git a/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/config/MybatisPlusConfig.java b/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/config/MybatisPlusConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..a6ac7f7fc247a361286333de4b3c03ffba306336
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/config/MybatisPlusConfig.java
@@ -0,0 +1,34 @@
+package com.taosdata.example.mybatisplusdemo.config;
+
+import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class MybatisPlusConfig {
+
+
+ /** mybatis 3.4.1 pagination config start ***/
+// @Bean
+// public MybatisPlusInterceptor mybatisPlusInterceptor() {
+// MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor();
+// interceptor.addInnerInterceptor(new PaginationInnerInterceptor());
+// return interceptor;
+// }
+
+// @Bean
+// public ConfigurationCustomizer configurationCustomizer() {
+// return configuration -> configuration.setUseDeprecatedExecutor(false);
+// }
+
+ @Bean
+ public PaginationInterceptor paginationInterceptor() {
+// return new PaginationInterceptor();
+ PaginationInterceptor paginationInterceptor = new PaginationInterceptor();
+ //TODO: mybatis-plus do not support TDengine, use postgresql Dialect
+ paginationInterceptor.setDialectType("postgresql");
+
+ return paginationInterceptor;
+ }
+
+}
diff --git a/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/domain/Temperature.java b/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/domain/Temperature.java
new file mode 100644
index 0000000000000000000000000000000000000000..97e50b06f6b71c26d1edd65c5ae9e7ff29a03e4d
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/domain/Temperature.java
@@ -0,0 +1,15 @@
+package com.taosdata.example.mybatisplusdemo.domain;
+
+import lombok.Data;
+
+import java.sql.Timestamp;
+
+@Data
+public class Temperature {
+
+ private Timestamp ts;
+ private float temperature;
+ private String location;
+ private int tbIndex;
+
+}
diff --git a/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/domain/Weather.java b/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/domain/Weather.java
new file mode 100644
index 0000000000000000000000000000000000000000..361757411a15d29e742a07a92060e20190921223
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/domain/Weather.java
@@ -0,0 +1,15 @@
+package com.taosdata.example.mybatisplusdemo.domain;
+
+import lombok.Data;
+
+import java.sql.Timestamp;
+
+@Data
+public class Weather {
+
+ private Timestamp ts;
+ private float temperature;
+ private int humidity;
+ private String location;
+
+}
diff --git a/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapper.java b/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapper.java
new file mode 100644
index 0000000000000000000000000000000000000000..3e122524d57b5a54e08ff1cfc54101d517f32c32
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapper.java
@@ -0,0 +1,23 @@
+package com.taosdata.example.mybatisplusdemo.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.taosdata.example.mybatisplusdemo.domain.Temperature;
+import org.apache.ibatis.annotations.Insert;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Update;
+
+public interface TemperatureMapper extends BaseMapper {
+
+ @Update("CREATE TABLE if not exists temperature(ts timestamp, temperature float) tags(location nchar(64), tbIndex int)")
+ int createSuperTable();
+
+ @Update("create table #{tbName} using temperature tags( #{location}, #{tbindex})")
+ int createTable(@Param("tbName") String tbName, @Param("location") String location, @Param("tbindex") int tbindex);
+
+ @Update("drop table if exists temperature")
+ void dropSuperTable();
+
+ @Insert("insert into t${tbIndex}(ts, temperature) values(#{ts}, #{temperature})")
+ int insertOne(Temperature one);
+
+}
diff --git a/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapper.java b/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapper.java
new file mode 100644
index 0000000000000000000000000000000000000000..6733cbded9d1d180408eccaad9e8badad7d39a3d
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/src/main/java/com/taosdata/example/mybatisplusdemo/mapper/WeatherMapper.java
@@ -0,0 +1,8 @@
+package com.taosdata.example.mybatisplusdemo.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.taosdata.example.mybatisplusdemo.domain.Weather;
+
+public interface WeatherMapper extends BaseMapper {
+
+}
diff --git a/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml b/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml
new file mode 100644
index 0000000000000000000000000000000000000000..96667f28b8d45d74541609f3d44176534c609f23
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml
@@ -0,0 +1,34 @@
+spring:
+ datasource:
+ # driver-class-name: org.h2.Driver
+ # schema: classpath:db/schema-mysql.sql
+ # data: classpath:db/data-mysql.sql
+ # url: jdbc:h2:mem:test
+ # username: root
+ # password: test
+
+ # driver-class-name: com.mysql.jdbc.Driver
+ # url: jdbc:mysql://master:3306/test?useSSL=false
+ # username: root
+ # password: 123456
+
+ driver-class-name: com.taosdata.jdbc.TSDBDriver
+ url: jdbc:TAOS://localhost:6030/mp_test
+ user: root
+ password: taosdata
+ charset: UTF-8
+ locale: en_US.UTF-8
+ timezone: UTC-8
+
+mybatis-plus:
+ configuration:
+ map-underscore-to-camel-case: false
+
+logging:
+ level:
+ com:
+ taosdata:
+ example:
+ mybatisplusdemo:
+ mapper: debug
+
diff --git a/tests/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java b/tests/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..4331d15d3476d3428e72a186664ed77cc59aad3e
--- /dev/null
+++ b/tests/examples/JDBC/mybatisplus-demo/src/test/java/com/taosdata/example/mybatisplusdemo/mapper/TemperatureMapperTest.java
@@ -0,0 +1,140 @@
+package com.taosdata.example.mybatisplusdemo.mapper;
+
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.taosdata.example.mybatisplusdemo.domain.Temperature;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import java.sql.Timestamp;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@SpringBootTest
+public class TemperatureMapperTest {
+
+ private static Random random = new Random(System.currentTimeMillis());
+ private static String[] locations = {"北京", "上海", "深圳", "广州", "杭州"};
+
+ @Before
+ public void before() {
+ mapper.dropSuperTable();
+ // create table temperature
+ mapper.createSuperTable();
+ // create table t_X using temperature
+ for (int i = 0; i < 10; i++) {
+ mapper.createTable("t" + i, locations[random.nextInt(locations.length)], i);
+ }
+ // insert into table
+ int affectRows = 0;
+ // insert 10 tables
+ for (int i = 0; i < 10; i++) {
+ // each table insert 5 rows
+ for (int j = 0; j < 5; j++) {
+ Temperature one = new Temperature();
+ one.setTs(new Timestamp(1605024000000l));
+ one.setTemperature(random.nextFloat() * 50);
+ one.setLocation("望京");
+ one.setTbIndex(i);
+ affectRows += mapper.insertOne(one);
+ }
+ }
+ Assert.assertEquals(50, affectRows);
+ }
+
+ @After
+ public void after() {
+ mapper.dropSuperTable();
+ }
+
+ @Autowired
+ private TemperatureMapper mapper;
+
+ /***
+ * test SelectList
+ * **/
+ @Test
+ public void testSelectList() {
+ List temperatureList = mapper.selectList(null);
+ temperatureList.forEach(System.out::println);
+ }
+
+ /***
+ * test InsertOne which is a custom metheod
+ * ***/
+ @Test
+ public void testInsert() {
+ Temperature one = new Temperature();
+ one.setTs(new Timestamp(1605024000000l));
+ one.setTemperature(random.nextFloat() * 50);
+ one.setLocation("望京");
+ int affectRows = mapper.insertOne(one);
+ Assert.assertEquals(1, affectRows);
+ }
+
+ /***
+ * test SelectOne
+ * **/
+ @Test
+ public void testSelectOne() {
+ QueryWrapper wrapper = new QueryWrapper<>();
+ wrapper.eq("location", "beijing");
+ Temperature one = mapper.selectOne(wrapper);
+ System.out.println(one);
+ Assert.assertNotNull(one);
+ }
+
+ /***
+ * test select By map
+ * ***/
+ @Test
+ public void testSelectByMap() {
+ Map map = new HashMap<>();
+ map.put("location", "beijing");
+ List temperatures = mapper.selectByMap(map);
+ Assert.assertEquals(1, temperatures.size());
+ }
+
+ /***
+ * test selectObjs
+ * **/
+ @Test
+ public void testSelectObjs() {
+ List