diff --git a/.gitmodules b/.gitmodules
index 7c84eac8a4ee7529005855bc836387561c49ae2d..049b39abfb2cf5f31abe10f194e7a09c4dc932f0 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -4,3 +4,6 @@
[submodule "src/connector/grafanaplugin"]
path = src/connector/grafanaplugin
url = https://github.com/taosdata/grafanaplugin
+[submodule "src/connector/hivemq-tdengine-extension"]
+ path = src/connector/hivemq-tdengine-extension
+ url = https://github.com/huskar-t/hivemq-tdengine-extension.git
diff --git a/Jenkinsfile b/Jenkinsfile
index ea50d6ef5a13fed2114868f8a36cd0acace78dd5..dc7836c3daacaa457f721f9278687b99770fc394 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -27,6 +27,7 @@ pipeline {
cd debug
cmake .. > /dev/null
make > /dev/null
+ make install > /dev/null
cd ${WKC}/tests
#./test-all.sh smoke
./test-all.sh pytest
@@ -79,7 +80,20 @@ pipeline {
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests/pytest
- ./crash_gen.sh -a -p -t 4 -s 2000
+ '''
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WKC}/tests/pytest
+ ./crash_gen.sh -a -p -t 4 -s 2000
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WKC}/tests/pytest
+ ./handle_crash_gen_val_log.sh
+ '''
+ }
+ sh '''
date
cd ${WKC}/tests
./test-all.sh b2
@@ -124,14 +138,33 @@ pipeline {
sh'''
cd ${WORKSPACE}
git checkout develop
- cd tests/gotest
- bash batchtest.sh
- cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
- mvn clean package assembly:single >/dev/null
- java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
- cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
- python3 PythonChecker.py
'''
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WORKSPACE}/tests/gotest
+ bash batchtest.sh
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
+ python3 PythonChecker.py
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
+ mvn clean package assembly:single >/dev/null
+ java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
+ '''
+ }
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ sh '''
+ cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
+ dotnet run
+ '''
+ }
+
}
}
@@ -139,5 +172,82 @@ pipeline {
}
}
-
+ post {
+ success {
+ emailext (
+ subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
+ body: '''
+
+
+
+
+
+
+
+
+ 构建信息
+
|
+
+
+
+
+
+ - 构建名称>>分支:${PROJECT_NAME}
+ - 构建结果: Successful
+ - 构建编号:${BUILD_NUMBER}
+ - 触发用户:${CAUSE}
+ - 变更概要:${CHANGES}
+ - 构建地址:${BUILD_URL}
+ - 构建日志:${BUILD_URL}console
+ - 变更集:${JELLY_SCRIPT}
+
+
+ |
+
+
+
+ ''',
+ to: "yqliu@taosdata.com,pxiao@taosdata.com",
+ from: "support@taosdata.com"
+ )
+ }
+ failure {
+ emailext (
+ subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
+ body: '''
+
+
+
+
+
+
+
+
+ 构建信息
+
|
+
+
+
+
+
+ - 构建名称>>分支:${PROJECT_NAME}
+ - 构建结果: Successful
+ - 构建编号:${BUILD_NUMBER}
+ - 触发用户:${CAUSE}
+ - 变更概要:${CHANGES}
+ - 构建地址:${BUILD_URL}
+ - 构建日志:${BUILD_URL}console
+ - 变更集:${JELLY_SCRIPT}
+
+
+ |
+
+
+
+ ''',
+ to: "yqliu@taosdata.com,pxiao@taosdata.com",
+ from: "support@taosdata.com"
+ )
+ }
+ }
}
\ No newline at end of file
diff --git a/cmake/install.inc b/cmake/install.inc
index dfca758b9362c96bec0ce45aa385d54a4e75a9e5..9bbcc2cf40520a9ab5e7aa4d19f46b47b6b42192 100755
--- a/cmake/install.inc
+++ b/cmake/install.inc
@@ -13,6 +13,7 @@ ELSEIF (TD_WINDOWS)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/nodejs DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector)
+ INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/C\# DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)
diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md
index 44b3ad46712019870be6fefb5234611f55f6e03e..79388a2edb9404a0f7b31b9182eb5ce2cb0d52be 100644
--- a/documentation/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation/webdocs/markdowndocs/administrator-ch.md
@@ -87,6 +87,7 @@ TDengine系统后台服务由taosd提供,可以在配置文件taos.cfg里修
- httpPort: RESTful服务使用的端口号,所有的HTTP请求(TCP)都需要向该接口发起查询/写入请求。
- dataDir: 数据文件目录,所有的数据文件都将写入该目录。默认值:/var/lib/taos。
- logDir:日志文件目录,客户端和服务器的运行日志文件将写入该目录。默认值:/var/log/taos。
+- tempDir:临时文件目录,客户端和服务器的临时文件(主要是查询时用于保存中间结果的问题)将写入该目录。 默认值:Linux下为 /tmp/,Windows下为环境变量 tmp 或 temp 指向的目录。
- arbitrator:系统中裁决器的end point, 缺省值为空。
- role:dnode的可选角色。0-any; 既可作为mnode,也可分配vnode;1-mgmt;只能作为mnode,不能分配vnode;2-dnode;不能作为mnode,只能分配vnode
- debugFlag:运行日志开关。131(输出错误和警告日志),135( 输出错误、警告和调试日志),143( 输出错误、警告、调试和跟踪日志)。默认值:131或135(不同模块有不同的默认值)。
diff --git a/documentation20/webdocs/markdowndocs/Documentation-ch.md b/documentation20/webdocs/markdowndocs/Documentation-ch.md
index 077a0431383da8067b6bd33c4b7218627b7adbd0..f1f2d58f0562b831395e5177eb34ec3534aaa715 100644
--- a/documentation20/webdocs/markdowndocs/Documentation-ch.md
+++ b/documentation20/webdocs/markdowndocs/Documentation-ch.md
@@ -34,7 +34,8 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台,专
- [SQL写入](https://www.taosdata.com/cn/documentation20/insert/#SQL写入):使用SQL insert命令向一张或多张表写入单条或多条记录
- [Telegraf写入](https://www.taosdata.com/cn/documentation20/insert/#Telegraf直接写入):配置Telegraf, 不用任何代码,将采集数据直接写入
- [Prometheus写入](https://www.taosdata.com/cn/documentation20/insert/#Prometheus直接写入):配置Prometheus, 不用任何代码,将数据直接写入
-- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入):配置EMQ X,不用任何代码,就可将MQTT数据直接写入
+- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入):配置EMQ X,不用任何代码,就可将 MQTT 数据直接写入
+- [HiveMQ Broker](https://www.taosdata.com/cn/documentation20/insert/#HiveMQ-Broker直接写入):通过 HiveMQ Extension,不用任何代码,就可将 MQTT 数据直接写入
## [高效查询数据](https://www.taosdata.com/cn/documentation20/queries)
diff --git a/documentation20/webdocs/markdowndocs/Evaluation-ch.md b/documentation20/webdocs/markdowndocs/Evaluation-ch.md
index 9e7e0ec6aa8b60bfa9dbe603a45a265a1d1eba00..a92f97a8d9dfc9a47d5554daa076f4d2f0774c92 100644
--- a/documentation20/webdocs/markdowndocs/Evaluation-ch.md
+++ b/documentation20/webdocs/markdowndocs/Evaluation-ch.md
@@ -10,7 +10,7 @@ TDengine的模块之一是时序数据库。但除此之外,为减少研发的
* __硬件或云服务成本降至1/5__:由于超强性能,计算资源不到通用大数据方案的1/5;通过列式存储和先进的压缩算法,存储空间不到通用数据库的1/10。
* __全栈时序数据处理引擎__:将数据库、消息队列、缓存、流式计算等功能融为一体,应用无需再集成Kafka/Redis/HBase/Spark/HDFS等软件,大幅降低应用开发和维护的复杂度成本。
* __强大的分析功能__:无论是十年前还是一秒钟前的数据,指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell, Python, R, Matlab随时进行。
-* __与第三方工具无缝连接__:不用一行代码,即可与Telegraf, Grafana, EMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。
+* __与第三方工具无缝连接__:不用一行代码,即可与Telegraf, Grafana, EMQ, HiveMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。
* __零运维成本、零学习成本__:安装集群简单快捷,无需分库分表,实时备份。类似标准SQL,支持RESTful, 支持Python/Java/C/C++/C#/Go/Node.js, 与MySQL相似,零学习成本。
采用TDengine,可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是,因充分利用了物联网时序数据的特点,它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。
diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md
index 4082d72f112e8ff62d2e8b2c8b15391dd6f39d8a..905d3b2cd72044e621c125641ca053d1cda809d9 100644
--- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md
+++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md
@@ -844,7 +844,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
- **PERCENTILE**
```mysql
- SELECT PERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause];
+ SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause];
```
功能说明:统计表中某列的值百分比分位数。
返回结果数据类型: 双精度浮点数Double。
@@ -1016,9 +1016,9 @@ SELECT AVG(current),MAX(current),LEASTSQUARES(current, start_val, step_val), PER
```
## TAOS SQL 边界限制
-- 数据库名最大长度为33
-- 表名最大长度为193,每行数据最大长度16k个字符
-- 列名最大长度为65,最多允许1024列,最少需要2列,第一列必须是时间戳
+- 数据库名最大长度为32
+- 表名最大长度为192,每行数据最大长度16k个字符
+- 列名最大长度为64,最多允许1024列,最少需要2列,第一列必须是时间戳
- 标签最多允许128个,可以0个,标签总长度不超过16k个字符
- SQL语句最大长度65480个字符,但可通过系统配置参数maxSQLLength修改,最长可配置为1M
- 库的数目,超级表的数目、表的数目,系统不做限制,仅受系统资源限制
diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md
index 4b274e05e6e392879fc887ec1137968270a8e4b8..36466d2b7ea29307b5c17ad21b0db47a98598fa4 100644
--- a/documentation20/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation20/webdocs/markdowndocs/administrator-ch.md
@@ -35,7 +35,7 @@ TDengine相对于通用数据库,有超高的压缩比,在绝大多数场景
Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable
```
-示例:1000万台智能电表,每台电表每15分钟采集一次数据,每次采集的数据128字节,那么一年的原始数据量是:10000000\*128\*24\*60/15*365 = 44851T。TDengine大概需要消耗44851/5=8970T, 8.9P空间。
+示例:1000万台智能电表,每台电表每15分钟采集一次数据,每次采集的数据128字节,那么一年的原始数据量是:10000000\*128\*24\*60/15*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。
用户可以通过参数keep,设置数据在磁盘中的最大保存时长。为进一步减少存储成本,TDengine还提供多级存储,最冷的数据可以存放在最廉价的存储介质上,应用的访问不用做任何调整,只是读取速度降低了。
@@ -253,7 +253,7 @@ ALTER USER PASS <'password'>;
修改用户密码, 为避免被转换为小写,密码需要用单引号引用,单引号为英文半角
```
-ALTER USER PRIVILEDGE ;
+ALTER USER PRIVILEGE ;
```
修改用户权限为:super/write/read,不需要添加单引号
diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md
index d4705ccb05c092d8da38072368a167466bd78968..3721d3c2cd7d28327887c8fe90b53b8c2f63e564 100644
--- a/documentation20/webdocs/markdowndocs/architecture-ch.md
+++ b/documentation20/webdocs/markdowndocs/architecture-ch.md
@@ -221,7 +221,7 @@ TDengine采用时间驱动缓存管理策略(First-In-First-Out,FIFO),
TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中,可以更加快速地响应用户针对最近一条或一批数据的查询分析,整体上提供更快的数据库查询响应能力。从这个意义上来说,**可通过设置合适的配置参数将TDengine作为数据缓存来使用,而不需要再部署Redis或其他额外的缓存系统**,可有效地简化系统架构,降低运维的成本。需要注意的是,TDengine重启以后系统的缓存将被清空,之前缓存的数据均会被批量写入磁盘,缓存的数据将不会像专门的Key-value缓存系统再将之前缓存的数据重新加载到缓存中。
-每个vnode有自己独立的内存,而且由多个固定大小的内存块组成,不同vnode之间完全隔离。数据写入时,类似于日志的写法,数据被顺序追加写入内存,但每个vnode维护有自己的skip list,便于迅速查找。当一半以上的内存块写满时,启动落盘操作,而且后续写的操作在新的内存块进行。这样,一个vnode里有一半内存块是保留有最近的数据的,以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定,内存块的大小由配置参数cache决定。
+每个vnode有自己独立的内存,而且由多个固定大小的内存块组成,不同vnode之间完全隔离。数据写入时,类似于日志的写法,数据被顺序追加写入内存,但每个vnode维护有自己的skip list,便于迅速查找。当三分之一以上的内存块写满时,启动落盘操作,而且后续写的操作在新的内存块进行。这样,一个vnode里有三分之一内存块是保留有最近的数据的,以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定,内存块的大小由配置参数cache决定。
### 持久化存储
TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久化存储。当vnode中缓存的数据达到一定规模时,为了不阻塞后续数据的写入,TDengine也会拉起落盘线程将缓存的数据写入持久化存储。TDengine在数据落盘时会打开新的数据库日志文件,在落盘成功后则会删除老的数据库日志文件,避免日志文件无限制的增长。
diff --git a/documentation20/webdocs/markdowndocs/connector-ch.md b/documentation20/webdocs/markdowndocs/connector-ch.md
index 0e29b324871e61d184fde17710c863f06b60bdcc..c5a955f43fcc30e70d8aac3919433c2e767f76ba 100644
--- a/documentation20/webdocs/markdowndocs/connector-ch.md
+++ b/documentation20/webdocs/markdowndocs/connector-ch.md
@@ -616,6 +616,43 @@ HTTP请求URL采用`sqlutc`时,返回结果集的时间戳将采用UTC时间
- httpEnableCompress: 是否支持压缩,默认不支持,目前TDengine仅支持gzip压缩格式
- httpDebugFlag: 日志开关,131:仅错误和报警信息,135:调试信息,143:非常详细的调试信息,默认131
+## CSharp Connector
+
+在Windows系统上,C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作,后续版本将提供ORM(dapper)框架驱动。
+
+#### 安装TDengine客户端
+
+C#连接器需要使用`libtaos.so`和`taos.h`。因此,在使用C#连接器之前,需在程序运行的Windows环境安装TDengine的Windows客户端,以便获得相关驱动文件。
+
+安装完成后,在文件夹`C:/TDengine/examples/C#`中,将会看到两个文件
+
+- TDengineDriver.cs 调用taos.dll文件的Native C方法
+- TDengineTest.cs 参考程序示例
+
+在文件夹`C:\Windows\System32`,将会看到`taos.dll`文件
+
+#### 使用方法
+
+- 将C#接口文件TDengineDriver.cs加入到应用程序所在.NET项目中
+- 参考TDengineTest.cs来定义数据库连接参数,及执行数据插入、查询等操作的方法
+- 因为C#接口需要用到`taos.dll`文件,用户可以将`taos.dll`文件加入.NET解决方案中
+
+#### 注意事项
+
+- `taos.dll`文件使用x64平台编译,所以.NET项目在生成.exe文件时,“解决方案”/“项目”的“平台”请均选择“x64”。
+- 此.NET接口目前已经在Visual Studio 2013/2015/2017中验证过,其它VS版本尚待验证。
+
+#### 第三方驱动
+
+Maikebing.Data.Taos是一个TDengine的ADO.Net提供器,支持linux,windows。该开发包由热心贡献者`麦壳饼@@maikebing`提供,具体请参考
+
+```
+//接口下载
+https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos
+//用法说明
+https://www.taosdata.com/blog/2020/11/02/1901.html
+```
+
## Go Connector
diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md
index 80deb889ef0eee8e9b47f86b5e58a76c6c070d5b..757b6d9929b68a90e7bc558b553233f24b09ba17 100644
--- a/documentation20/webdocs/markdowndocs/faq-ch.md
+++ b/documentation20/webdocs/markdowndocs/faq-ch.md
@@ -38,9 +38,9 @@
6. 检查防火墙设置,确认TCP/UDP 端口6030-6042 是打开的
-7. 对于Linux上的JDBC(ODBC, Python, Go等接口类似)连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里
+7. 对于Linux上的JDBC(ODBC, Python, Go等接口类似)连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里
-8. 对于windows上的JDBC, ODBC, Python, Go等连接,确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
+8. 对于windows上的JDBC, ODBC, Python, Go等连接,确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
9. 如果仍不能排除连接故障,请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
检查UDP端口连接是否工作:`nc -vuz {hostIP} {port} `
diff --git a/documentation20/webdocs/markdowndocs/insert-ch.md b/documentation20/webdocs/markdowndocs/insert-ch.md
index fa53cbd62b17169c0f54877a62da8c48ac21edcf..77ba596d4ec6de76a39b878e57ea255b081fea45 100644
--- a/documentation20/webdocs/markdowndocs/insert-ch.md
+++ b/documentation20/webdocs/markdowndocs/insert-ch.md
@@ -1,6 +1,6 @@
# 高效写入数据
-TDengine支持多种接口写入数据,包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, CSV文件等,后续还将提供Kafka, OPC等接口。数据可以单条插入,也可以批量插入,可以插入一个数据采集点的数据,也可以同时插入多个数据采集点的数据。支持多线程插入,支持时间乱序数据插入,也支持历史数据插入。
+TDengine支持多种接口写入数据,包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, HiveMQ Broker, CSV文件等,后续还将提供Kafka, OPC等接口。数据可以单条插入,也可以批量插入,可以插入一个数据采集点的数据,也可以同时插入多个数据采集点的数据。支持多线程插入,支持时间乱序数据插入,也支持历史数据插入。
## SQL写入
@@ -218,7 +218,15 @@ use telegraf;
select * from cpu;
```
-## EMQ X Broker直接写入
-MQTT是一流行的物联网数据传输协议,[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件,无需任何代码,只需要在EMQ Dashboard里使用“规则”做简单配置,即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDEngine,也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。
+
+MQTT是一流行的物联网数据传输协议,TDengine 可以很方便的接入 MQTT Broker 接受的数据并写入到 TDengine。
+
+## EMQ Broker 直接写入
+
+[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件,无需任何代码,只需要在EMQ Dashboard里使用“规则”做简单配置,即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine,也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。
+
+## HiveMQ Broker 直接写入
+
+[HiveMQ](https://www.hivemq.com/) 是一个提供免费个人版和企业版的 MQTT 代理,主要用于企业和新兴的机器到机器M2M通讯和内部传输,满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 [HiveMQ extension - TDengine 说明文档](https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md)。
diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg
index 974b2b05c12f9fd5a482ef0072a9dacc2d1f09dc..ca88bca3c863dcfd4a95497fc572499dc93f31f9 100644
--- a/packaging/cfg/taos.cfg
+++ b/packaging/cfg/taos.cfg
@@ -20,6 +20,9 @@
# data file's directory
# dataDir /var/lib/taos
+# temporary file's directory
+# tempDir /tmp/
+
# the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only
# arbitrator arbitrator_hostname:6042
@@ -256,3 +259,5 @@
# maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden
# maxBinaryDisplayWidth 30
+# enable/disable telemetry reporting
+# telemetryReporting 1
\ No newline at end of file
diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh
index 450c6a8f551fe7dd8dff5e53f5dad639aec4656e..516d91258a759545842a6086a2966e02e32826a0 100755
--- a/packaging/deb/makedeb.sh
+++ b/packaging/deb/makedeb.sh
@@ -48,6 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_pat
cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin
+#cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver
@@ -58,7 +59,7 @@ cp -r ${top_dir}/src/connector/grafanaplugin ${pkg_dir}${install_home_pat
cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector
cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector
cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector
-cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.* ${pkg_dir}${install_home_path}/connector
+cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.* ${pkg_dir}${install_home_path}/connector ||:
cp -r ${compile_dir}/../packaging/deb/DEBIAN ${pkg_dir}/
chmod 755 ${pkg_dir}/DEBIAN/*
diff --git a/packaging/release.sh b/packaging/release.sh
index 7542a5b4cafb69d5cee16bddfc9a5651eb717b92..68f947ccab3ef18a1b351b91a58db64a8f465c8e 100755
--- a/packaging/release.sh
+++ b/packaging/release.sh
@@ -156,9 +156,15 @@ build_time=$(date +"%F %R")
# get commint id from git
gitinfo=$(git rev-parse --verify HEAD)
-enterprise_dir="${top_dir}/../enterprise"
-cd ${enterprise_dir}
-gitinfoOfInternal=$(git rev-parse --verify HEAD)
+
+if [[ "$verMode" == "cluster" ]]; then
+ enterprise_dir="${top_dir}/../enterprise"
+ cd ${enterprise_dir}
+ gitinfoOfInternal=$(git rev-parse --verify HEAD)
+else
+ gitinfoOfInternal=NULL
+fi
+
cd ${curr_dir}
# 2. cmake executable file
@@ -193,23 +199,35 @@ cd ${curr_dir}
# 3. Call the corresponding script for packaging
if [ "$osType" != "Darwin" ]; then
if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]] && [[ "$dbName" == "taos" ]]; then
- echo "====do deb package for the ubuntu system===="
- output_dir="${top_dir}/debs"
- if [ -d ${output_dir} ]; then
- ${csudo} rm -rf ${output_dir}
+ ret='0'
+ command -v dpkg >/dev/null 2>&1 || { ret='1'; }
+ if [ "$ret" -eq 0 ]; then
+ echo "====do deb package for the ubuntu system===="
+ output_dir="${top_dir}/debs"
+ if [ -d ${output_dir} ]; then
+ ${csudo} rm -rf ${output_dir}
+ fi
+ ${csudo} mkdir -p ${output_dir}
+ cd ${script_dir}/deb
+ ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
+ else
+ echo "==========dpkg command not exist, so not release deb package!!!"
fi
- ${csudo} mkdir -p ${output_dir}
- cd ${script_dir}/deb
- ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
-
- echo "====do rpm package for the centos system===="
- output_dir="${top_dir}/rpms"
- if [ -d ${output_dir} ]; then
- ${csudo} rm -rf ${output_dir}
+
+ ret='0'
+ command -v rpmbuild >/dev/null 2>&1 || { ret='1'; }
+ if [ "$ret" -eq 0 ]; then
+ echo "====do rpm package for the centos system===="
+ output_dir="${top_dir}/rpms"
+ if [ -d ${output_dir} ]; then
+ ${csudo} rm -rf ${output_dir}
+ fi
+ ${csudo} mkdir -p ${output_dir}
+ cd ${script_dir}/rpm
+ ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
+ else
+ echo "==========rpmbuild command not exist, so not release rpm package!!!"
fi
- ${csudo} mkdir -p ${output_dir}
- cd ${script_dir}/rpm
- ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
fi
echo "====do tar.gz package for all systems===="
diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec
index 4e40263dc4ebaf6b566d20890ecd97f64e160340..19d932e0745e4f3106f10bc360fb110a0db12c3b 100644
--- a/packaging/rpm/tdengine.spec
+++ b/packaging/rpm/tdengine.spec
@@ -2,7 +2,7 @@
%define cfg_install_dir /etc/taos
%define __strip /bin/true
-Name: TDengine
+Name: tdengine
Version: %{_version}
Release: 3%{?dist}
Summary: tdengine from taosdata
@@ -58,6 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/scri
cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin
+#cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver
cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include
cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include
@@ -65,7 +66,7 @@ cp -r %{_compiledir}/../src/connector/grafanaplugin %{buildroot}%{homepath}/conn
cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector
cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector
cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector
-cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.* %{buildroot}%{homepath}/connector
+cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.* %{buildroot}%{homepath}/connector ||:
cp -r %{_compiledir}/../tests/examples/* %{buildroot}%{homepath}/examples
#Scripts executed before installation
@@ -134,6 +135,7 @@ if [ $1 -eq 0 ];then
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
+ #${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${cfg_link_dir}/* || :
${csudo} rm -f ${inc_link_dir}/taos.h || :
${csudo} rm -f ${inc_link_dir}/taoserror.h || :
diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh
index eff70d8035af0291f6dc7040ec13632fec4fa3be..831012851ad70d05080bfae161c0b925d5215ae9 100755
--- a/packaging/tools/make_install.sh
+++ b/packaging/tools/make_install.sh
@@ -278,11 +278,11 @@ function install_service_on_sysvinit() {
# Install taosd service
if ((${os_type}==1)); then
- ${csudo} cp -f ${script_dir}/../deb/init.d/taosd ${install_main_dir}/init.d
- ${csudo} cp ${script_dir}/../deb/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
+ ${csudo} cp -f ${script_dir}/../deb/taosd ${install_main_dir}/init.d
+ ${csudo} cp ${script_dir}/../deb/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
elif ((${os_type}==2)); then
- ${csudo} cp -f ${script_dir}/../rpm/init.d/taosd ${install_main_dir}/init.d
- ${csudo} cp ${script_dir}/../rpm/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
+ ${csudo} cp -f ${script_dir}/../rpm/taosd ${install_main_dir}/init.d
+ ${csudo} cp ${script_dir}/../rpm/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd
fi
#restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start"
diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh
index e17c678f263cb6b7a0ccbc32250265b9bc5cbd0e..4f4c6dee1dfc82dfaa486efb6c0447274a9eb4bd 100755
--- a/packaging/tools/makeclient.sh
+++ b/packaging/tools/makeclient.sh
@@ -45,6 +45,7 @@ if [ "$osType" != "Darwin" ]; then
strip ${build_dir}/bin/taos
bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
else
+ #bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
fi
lib_files="${build_dir}/lib/libtaos.so.${version}"
@@ -110,7 +111,7 @@ mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
if [ "$osType" != "Darwin" ]; then
- cp ${build_dir}/lib/*.jar ${install_dir}/connector
+ cp ${build_dir}/lib/*.jar ${install_dir}/connector ||:
fi
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
cp -r ${connector_dir}/python ${install_dir}/connector/
diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh
index faa5a03f52b4c9b56981a6b1c0918e543262b3bb..7e8bef0dffffa3028c82ed00ab5695970222770d 100755
--- a/packaging/tools/makeclient_power.sh
+++ b/packaging/tools/makeclient_power.sh
@@ -135,7 +135,7 @@ mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
if [ "$osType" != "Darwin" ]; then
- cp ${build_dir}/lib/*.jar ${install_dir}/connector
+ cp ${build_dir}/lib/*.jar ${install_dir}/connector ||:
fi
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
cp -r ${connector_dir}/python ${install_dir}/connector/
diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh
index 75b45b544e0a4abbf709cc4c5b3a3b55dc315f0f..2be60709aa2ca3191542fd24d03599cd7cbb2895 100755
--- a/packaging/tools/makepkg.sh
+++ b/packaging/tools/makepkg.sh
@@ -36,6 +36,7 @@ if [ "$pagMode" == "lite" ]; then
strip ${build_dir}/bin/taos
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh"
else
+ #bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
fi
@@ -124,7 +125,7 @@ cp ${lib_files} ${install_dir}/driver
connector_dir="${code_dir}/connector"
mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
- cp ${build_dir}/lib/*.jar ${install_dir}/connector
+ cp ${build_dir}/lib/*.jar ${install_dir}/connector ||:
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
cp -r ${connector_dir}/python ${install_dir}/connector/
cp -r ${connector_dir}/go ${install_dir}/connector
diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh
index 2c02b99787c6d5ad6234de2319bf78b0b09d7e8a..0ffcb63e3f93c300ce41d627ccf43b7f089d6c2e 100755
--- a/packaging/tools/makepkg_power.sh
+++ b/packaging/tools/makepkg_power.sh
@@ -156,7 +156,7 @@ cp ${lib_files} ${install_dir}/driver
connector_dir="${code_dir}/connector"
mkdir -p ${install_dir}/connector
if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
- cp ${build_dir}/lib/*.jar ${install_dir}/connector
+ cp ${build_dir}/lib/*.jar ${install_dir}/connector ||:
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
cp -r ${connector_dir}/python ${install_dir}/connector/
cp -r ${connector_dir}/go ${install_dir}/connector
diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh
index d91daaa5c44488e34dea7ec2ddec0863699446f2..00705fad778c065eddbd7cb65a2f9c5583a9997b 100755
--- a/packaging/tools/post.sh
+++ b/packaging/tools/post.sh
@@ -81,8 +81,10 @@ function install_lib() {
${csudo} ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1
${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
- ${csudo} ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
- ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
+ if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
+ ${csudo} ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
+ ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
+ fi
}
function install_bin() {
@@ -121,8 +123,11 @@ function install_config() {
echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join${NC}"
echo
echo -e -n "${GREEN}OR leave it blank to build one${NC}:"
- read firstEp
- while true; do
+ #read firstEp
+ if exec < /dev/tty; then
+ read firstEp;
+ fi
+ while true; do
if [ ! -z "$firstEp" ]; then
# check the format of the firstEp
#if [[ $firstEp == $FQDN_PATTERN ]]; then
diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c
index 0e9bb85b25defd169fea8711d3e0b40304500de4..df78f4fe270d34d17e436e80322f6c81b68d0d2c 100644
--- a/src/balance/src/balance.c
+++ b/src/balance/src/balance.c
@@ -490,7 +490,7 @@ static bool balanceMontiorDropping() {
if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
if (pDnode->lastAccess + tsOfflineThreshold > tsAccessSquence) continue;
- if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) continue;
+ if (dnodeIsMasterEp(pDnode->dnodeEp)) continue;
if (mnodeGetDnodesNum() <= 1) continue;
mLInfo("dnode:%d, set to removing state for it offline:%d seconds", pDnode->dnodeId,
@@ -571,8 +571,8 @@ static void balanceCheckDnodeAccess() {
if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) {
pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
- mInfo("dnode:%d, set to offline state, access seq:%d, last seq:%d", pDnode->dnodeId, tsAccessSquence,
- pDnode->lastAccess);
+ mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence,
+ pDnode->lastAccess, pDnode->status);
balanceSetVgroupOffline(pDnode);
}
}
diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h
index 5baa66a9e0229f35c431cea7a0d2dbb9e2ffb0e2..0af8c8b5768c844954a072521e67a3cb6e914ac3 100644
--- a/src/client/inc/tscLocalMerge.h
+++ b/src/client/inc/tscLocalMerge.h
@@ -64,9 +64,8 @@ typedef struct SLocalReducer {
SColumnModel * resColModel;
tExtMemBuffer ** pExtMemBuffer; // disk-based buffer
SFillInfo* pFillInfo; // interpolation support structure
- char * pFinalRes; // result data after interpo
- tFilePage * discardData;
- SResultInfo * pResInfo;
+ char* pFinalRes; // result data after interpo
+ tFilePage* discardData;
bool discard;
int32_t offset; // limit offset value
bool orderPrjOnSTable; // projection query on stable
diff --git a/src/client/inc/tscSubquery.h b/src/client/inc/tscSubquery.h
index bc01de110345e4c90cf5c15d3d7f6b010cb7308d..3406dcd8589e149f2a62040d1271fbfd40caaac7 100644
--- a/src/client/inc/tscSubquery.h
+++ b/src/client/inc/tscSubquery.h
@@ -23,7 +23,7 @@ extern "C" {
#include "tscUtil.h"
#include "tsclient.h"
-void tscFetchDatablockFromSubquery(SSqlObj* pSql);
+void tscFetchDatablockForSubquery(SSqlObj* pSql);
void tscSetupOutputColumnIndex(SSqlObj* pSql);
void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code);
diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h
index 3df493349e387c1dd33a945886389218f7f7cf0f..6628e3087456c52b2daf1b9ef84efa9577ad4f36 100644
--- a/src/client/inc/tscUtil.h
+++ b/src/client/inc/tscUtil.h
@@ -75,6 +75,7 @@ typedef struct SJoinSupporter {
SArray* exprList;
SFieldInfo fieldsInfo;
STagCond tagCond;
+ SSqlGroupbyExpr groupInfo; // group by info
struct STSBuf* pTSBuf; // the TSBuf struct that holds the compressed timestamp array
FILE* f; // temporary file in order to create TSBuf
char path[PATH_MAX]; // temporary file path, todo dynamic allocate memory
@@ -82,11 +83,12 @@ typedef struct SJoinSupporter {
char* pIdTagList; // result of first stage tags
int32_t totalLen;
int32_t num;
+ SArray* pVgroupTables;
} SJoinSupporter;
typedef struct SVgroupTableInfo {
- SCMVgroupInfo vgInfo;
- SArray* itemList; //SArray
+ SVgroupInfo vgInfo;
+ SArray* itemList; //SArray
} SVgroupTableInfo;
static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) {
@@ -215,7 +217,7 @@ SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex);
void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache);
STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta,
- SVgroupsInfo* vgroupList, SArray* pTagCols);
+ SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables);
STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo);
int32_t tscAddSubqueryInfo(SSqlCmd *pCmd);
@@ -224,6 +226,9 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo);
void tscClearSubqueryInfo(SSqlCmd* pCmd);
void tscFreeVgroupTableInfo(SArray* pVgroupTables);
+SArray* tscVgroupTableInfoClone(SArray* pVgroupTables);
+void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index);
+void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo);
int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex);
int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo);
@@ -234,7 +239,7 @@ void tscDoQuery(SSqlObj* pSql);
SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *pInfo);
void* tscVgroupInfoClear(SVgroupsInfo *pInfo);
-void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src);
+void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src);
/**
* The create object function must be successful expect for the out of memory issue.
*
@@ -262,6 +267,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t sub
void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex);
int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid);
+int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId);
void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex);
diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h
index 67942ad42a0942756efe18e44eff711df59ba1d9..f6dc45398f35c38598f3f3132b2f6f5601a4ed68 100644
--- a/src/client/inc/tschemautil.h
+++ b/src/client/inc/tschemautil.h
@@ -77,7 +77,7 @@ SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex);
* @param colId
* @return
*/
-SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId);
+SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId);
/**
* check if the schema is valid or not, including following aspects:
@@ -107,9 +107,6 @@ SSchema tscGetTbnameColumnSchema();
*/
STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size);
-//todo tags value as well as the table id structure needs refactor
-char *tsGetTagsValue(STableMeta *pMeta);
-
#ifdef __cplusplus
}
#endif
diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h
index fa215db2702f63aff4c704f8393255a94987d681..4c85af4919c74de5f41272272bd1bdbf4f6c6da3 100644
--- a/src/client/inc/tsclient.h
+++ b/src/client/inc/tsclient.h
@@ -30,6 +30,7 @@ extern "C" {
#include "tsqlfunction.h"
#include "tutil.h"
#include "tcache.h"
+#include "tref.h"
#include "qExecutor.h"
#include "qSqlparser.h"
@@ -89,12 +90,12 @@ typedef struct STableComInfo {
int32_t rowSize;
} STableComInfo;
-typedef struct SCMCorVgroupInfo {
- int32_t version;
- int8_t inUse;
- int8_t numOfEps;
- SEpAddr1 epAddr[TSDB_MAX_REPLICA];
-} SCMCorVgroupInfo;
+typedef struct SCorVgroupInfo {
+ int32_t version;
+ int8_t inUse;
+ int8_t numOfEps;
+ SEpAddr1 epAddr[TSDB_MAX_REPLICA];
+} SCorVgroupInfo;
typedef struct STableMeta {
STableComInfo tableInfo;
@@ -102,8 +103,8 @@ typedef struct STableMeta {
int16_t sversion;
int16_t tversion;
char sTableId[TSDB_TABLE_FNAME_LEN];
- SCMVgroupInfo vgroupInfo;
- SCMCorVgroupInfo corVgroupInfo;
+ SVgroupInfo vgroupInfo;
+ SCorVgroupInfo corVgroupInfo;
STableId id;
SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
} STableMeta;
@@ -127,7 +128,7 @@ typedef struct STableMetaInfo {
typedef struct SSqlExpr {
char aliasName[TSDB_COL_NAME_LEN]; // as aliasName
SColIndex colInfo;
- int64_t uid; // refactor use the pointer
+ uint64_t uid; // refactor use the pointer
int16_t functionId; // function id in aAgg array
int16_t resType; // return value type
int16_t resBytes; // length of return value
@@ -329,6 +330,7 @@ typedef struct STscObj {
char writeAuth : 1;
char superAuth : 1;
uint32_t connId;
+ uint64_t rid; // ref ID returned by taosAddRef
struct SSqlObj * pHb;
struct SSqlObj * sqlList;
struct SSqlStream *streamList;
@@ -338,16 +340,16 @@ typedef struct STscObj {
} STscObj;
typedef struct SSubqueryState {
- int32_t numOfRemain; // the number of remain unfinished subquery
- int32_t numOfSub; // the number of total sub-queries
- uint64_t numOfRetrievedRows; // total number of points in this query
+ int32_t numOfRemain; // the number of remain unfinished subquery
+ int32_t numOfSub; // the number of total sub-queries
+ uint64_t numOfRetrievedRows; // total number of points in this query
} SSubqueryState;
typedef struct SSqlObj {
void *signature;
pthread_t owner; // owner of sql object, by which it is executed
STscObj *pTscObj;
- void *pRpcCtx;
+ int64_t rpcRid;
void (*fp)();
void (*fetchFp)();
void *param;
@@ -430,14 +432,6 @@ void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache);
*/
void tscFreeSqlResult(SSqlObj *pSql);
-/**
- * only free part of resources allocated during query.
- * TODO remove it later
- * Note: this function is multi-thread safe.
- * @param pObj
- */
-void tscPartiallyFreeSqlObj(SSqlObj *pSql);
-
/**
* free sql object, release allocated resource
* @param pObj
@@ -446,7 +440,7 @@ void tscFreeSqlObj(SSqlObj *pSql);
void tscFreeRegisteredSqlObj(void *pSql);
void tscFreeTableMetaHelper(void *pTableMeta);
-void tscCloseTscObj(STscObj *pObj);
+void tscCloseTscObj(void *pObj);
// todo move to taos? or create a new file: taos_internal.h
TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
@@ -516,12 +510,12 @@ extern void * tscQhandle;
extern int tscKeepConn[];
extern int tsInsertHeadSize;
extern int tscNumOfThreads;
+extern int tscRefId;
extern SRpcCorEpSet tscMgmtEpSet;
extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
-int32_t tscCompareTidTags(const void* p1, const void* p2);
void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables);
#ifdef __cplusplus
diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c
index c996bb2a76ff505fca4e09cd8763b324e5c4cb8d..99c03c65801a64eb6b913373603cfa9c48d36911 100644
--- a/src/client/src/tscAsync.c
+++ b/src/client/src/tscAsync.c
@@ -176,7 +176,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
}
if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
- tscFetchDatablockFromSubquery(pSql);
+ tscFetchDatablockForSubquery(pSql);
} else {
tscProcessSql(pSql);
}
@@ -226,7 +226,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) {
// handle the sub queries of join query
if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
- tscFetchDatablockFromSubquery(pSql);
+ tscFetchDatablockForSubquery(pSql);
} else if (pRes->completed) {
if(pCmd->command == TSDB_SQL_FETCH || (pCmd->command >= TSDB_SQL_SERV_STATUS && pCmd->command <= TSDB_SQL_CURRENT_USER)) {
if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes.
@@ -405,7 +405,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
SSqlRes *pRes = &pSql->res;
pRes->code = code;
- const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta";
+ SSqlObj *sub = (SSqlObj*) res;
+ const char* msg = (sub->cmd.command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta";
if (code != TSDB_CODE_SUCCESS) {
tscError("%p get %s failed, code:%s", pSql, msg, tstrerror(code));
goto _error;
@@ -427,8 +428,12 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
} else {
assert(code == TSDB_CODE_SUCCESS);
}
-
- assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pSql->param != NULL);
+
+ // param already freed by other routine and pSql in tscCache when ctrl + c
+ if (atomic_load_ptr(&pSql->param) == NULL) {
+ return;
+ }
+ assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0));
SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param;
SSqlObj * pParObj = trs->pParentSql;
@@ -437,6 +442,20 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex &&
pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL);
+ // tscProcessSql can add error into async res
+ tscProcessSql(pSql);
+ return;
+ } else if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) {
+ tscDebug("%p update table meta in local cache, continue to process sql and send corresponding tid_tag query", pSql);
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+ code = tscGetTableMeta(pSql, pTableMetaInfo);
+ if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+ return;
+ } else {
+ assert(code == TSDB_CODE_SUCCESS);
+ }
+
+ assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0));
// tscProcessSql can add error into async res
tscProcessSql(pSql);
return;
@@ -461,7 +480,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
tscResetSqlCmdObj(pCmd, false);
code = tsParseSql(pSql, true);
-
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return;
} else if (code != TSDB_CODE_SUCCESS) {
diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c
index 1b4f92d3fc9fc78951e23eaec4c438d7522bf7f1..39cc753d404925fdf74efc26e48d54df61af664e 100644
--- a/src/client/src/tscFunctionImpl.c
+++ b/src/client/src/tscFunctionImpl.c
@@ -99,7 +99,7 @@ typedef struct SSumInfo {
// the attribute of hasResult is not needed since the num attribute would server as this purpose
typedef struct SAvgInfo {
double sum;
- int64_t num; // num servers as the hasResult attribute in other struct
+ int64_t num;
} SAvgInfo;
typedef struct SStddevInfo {
@@ -167,7 +167,13 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) {
*type = (int16_t)dataType;
*bytes = (int16_t)dataBytes;
- *interBytes = *bytes + sizeof(SResultInfo);
+
+ if (functionId == TSDB_FUNC_INTERP) {
+ *interBytes = sizeof(SInterpInfoDetail);
+ } else {
+ *interBytes = 0;
+ }
+
return TSDB_CODE_SUCCESS;
}
@@ -175,21 +181,21 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
if (functionId == TSDB_FUNC_TID_TAG) { // todo use struct
*type = TSDB_DATA_TYPE_BINARY;
*bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE);
- *interBytes = *bytes;
+ *interBytes = 0;
return TSDB_CODE_SUCCESS;
}
if (functionId == TSDB_FUNC_COUNT) {
*type = TSDB_DATA_TYPE_BIGINT;
*bytes = sizeof(int64_t);
- *interBytes = *bytes;
+ *interBytes = 0;
return TSDB_CODE_SUCCESS;
}
if (functionId == TSDB_FUNC_ARITHM) {
*type = TSDB_DATA_TYPE_DOUBLE;
*bytes = sizeof(double);
- *interBytes = *bytes;
+ *interBytes = 0;
return TSDB_CODE_SUCCESS;
}
@@ -298,7 +304,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
} else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) {
*type = (int16_t)dataType;
*bytes = (int16_t)dataBytes;
- *interBytes = dataBytes + sizeof(SResultInfo);
+ *interBytes = dataBytes;
} else if (functionId == TSDB_FUNC_SPREAD) {
*type = (int16_t)TSDB_DATA_TYPE_DOUBLE;
*bytes = sizeof(double);
@@ -310,7 +316,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
} else if (functionId == TSDB_FUNC_LEASTSQR) {
*type = TSDB_DATA_TYPE_BINARY;
*bytes = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE; // string
- *interBytes = *bytes + sizeof(SResultInfo);
+ *interBytes = *bytes;
} else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) {
*type = TSDB_DATA_TYPE_BINARY;
*bytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo));
@@ -334,28 +340,20 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
return TSDB_CODE_SUCCESS;
}
-void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf) {
- assert(pResInfo->interResultBuf == NULL);
-
- pResInfo->bufLen = size;
- pResInfo->superTableQ = superTable;
- pResInfo->interResultBuf = buf;
-}
-
// set the query flag to denote that query is completed
static void no_next_step(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->complete = true;
}
static bool function_setup(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pResInfo->initialized) {
return false;
}
memset(pCtx->aOutputBuf, 0, (size_t)pCtx->outputBytes);
- initResultInfo(pResInfo);
+ initResultInfo(pResInfo, pCtx->interBufBytes);
return true;
}
@@ -367,7 +365,7 @@ static bool function_setup(SQLFunctionCtx *pCtx) {
* @param pCtx
*/
static void function_finalizer(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pResInfo->hasResult != DATA_SET_FLAG) {
if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
@@ -431,7 +429,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) {
*((int64_t *)pCtx->aOutputBuf) += 1;
// do not need it actually
- SResultInfo *pInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
pInfo->hasResult = DATA_SET_FLAG;
}
@@ -592,8 +590,8 @@ static void sum_function(SQLFunctionCtx *pCtx) {
do_sum(pCtx);
// keep the result data in output buffer, not in the intermediate buffer
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) {
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) {
// set the flag for super table query
SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf;
pSum->hasResult = DATA_SET_FLAG;
@@ -604,8 +602,8 @@ static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) {
do_sum_f(pCtx, index);
// keep the result data in output buffer, not in the intermediate buffer
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) {
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) {
SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf;
pSum->hasResult = DATA_SET_FLAG;
}
@@ -615,8 +613,7 @@ static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
GET_TRUE_DATA_TYPE();
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ assert(pCtx->stableQuery);
for (int32_t i = 0; i < pCtx->size; ++i) {
char * input = GET_INPUT_CHAR_INDEX(pCtx, i);
@@ -661,7 +658,7 @@ static void sum_func_second_merge(SQLFunctionCtx *pCtx) {
int32_t notNullElems = sum_merge_impl(pCtx);
SET_VAL(pCtx, notNullElems, 1);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (notNullElems > 0) {
pResInfo->hasResult = DATA_SET_FLAG;
@@ -755,9 +752,9 @@ static void avg_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
// NOTE: keep the intermediate result into the interResultBuf
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
- SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf;
+ SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo);
double * pVal = &pAvgInfo->sum;
if (pCtx->preAggVals.isSet) {
@@ -800,8 +797,8 @@ static void avg_function(SQLFunctionCtx *pCtx) {
}
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo));
}
}
@@ -814,9 +811,9 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) {
SET_VAL(pCtx, 1, 1);
// NOTE: keep the intermediate result into the interResultBuf
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
- SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf;
+ SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo);
if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) {
pAvgInfo->sum += GET_INT8_VAL(pData);
@@ -839,16 +836,16 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) {
pResInfo->hasResult = DATA_SET_FLAG;
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo));
}
}
static void avg_func_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ assert(pCtx->stableQuery);
- SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf;
+ SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo);
char * input = GET_INPUT_CHAR(pCtx);
for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) {
@@ -864,12 +861,12 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) {
// if the data set hasResult is not set, the result is null
if (pAvgInfo->num > 0) {
pResInfo->hasResult = DATA_SET_FLAG;
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo));
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo));
}
}
static void avg_func_second_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
double *sum = (double*) pCtx->aOutputBuf;
char * input = GET_INPUT_CHAR(pCtx);
@@ -883,7 +880,7 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) {
*sum += pInput->sum;
// keep the number of data into the temp buffer
- *(int64_t *)pResInfo->interResultBuf += pInput->num;
+ *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo) += pInput->num;
}
}
@@ -891,21 +888,21 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) {
* the average value is calculated in finalize routine, since current routine does not know the exact number of points
*/
static void avg_finalizer(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pCtx->currentStage == SECONDARY_STAGE_MERGE) {
assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY);
- if (GET_INT64_VAL(pResInfo->interResultBuf) <= 0) {
+ if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) {
setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
return; // empty table
}
- *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)pResInfo->interResultBuf;
+ *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo);
} else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY
assert(pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE);
- SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf;
+ SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo);
if (pAvgInfo->num == 0) { // all data are NULL or empty table
setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
@@ -1116,11 +1113,11 @@ static void min_function(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, notNullElems, 1);
if (notNullElems > 0) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
// set the flag for super table query
- if (pResInfo->superTableQ) {
+ if (pCtx->stableQuery) {
*(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG;
}
}
@@ -1133,11 +1130,11 @@ static void max_function(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, notNullElems, 1);
if (notNullElems > 0) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
// set the flag for super table query
- if (pResInfo->superTableQ) {
+ if (pCtx->stableQuery) {
*(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG;
}
}
@@ -1148,8 +1145,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp
GET_TRUE_DATA_TYPE();
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ assert(pCtx->stableQuery);
for (int32_t i = 0; i < pCtx->size; ++i) {
char *input = GET_INPUT_CHAR_INDEX(pCtx, i);
@@ -1210,7 +1206,7 @@ static void min_func_merge(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, notNullElems, 1);
- if (notNullElems > 0) { // for super table query, SResultInfo is not used
+ if (notNullElems > 0) { // for super table query, SResultRowCellInfo is not used
char *flag = pCtx->aOutputBuf + pCtx->inputBytes;
*flag = DATA_SET_FLAG;
}
@@ -1221,7 +1217,7 @@ static void min_func_second_merge(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, notNullElems, 1);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (notNullElems > 0) {
pResInfo->hasResult = DATA_SET_FLAG;
}
@@ -1242,7 +1238,7 @@ static void max_func_second_merge(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, numOfElem, 1);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (numOfElem > 0) {
pResInfo->hasResult = DATA_SET_FLAG;
}
@@ -1297,8 +1293,8 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) {
SET_VAL(pCtx, 1, 1);
minMax_function_f(pCtx, index, 0);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) {
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) {
char *flag = pCtx->aOutputBuf + pCtx->inputBytes;
*flag = DATA_SET_FLAG;
}
@@ -1313,8 +1309,8 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) {
SET_VAL(pCtx, 1, 1);
minMax_function_f(pCtx, index, 1);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) {
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) {
char *flag = pCtx->aOutputBuf + pCtx->inputBytes;
*flag = DATA_SET_FLAG;
}
@@ -1330,7 +1326,7 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) {
static void stddev_function(SQLFunctionCtx *pCtx) {
// the second stage to calculate standard deviation
- SStddevInfo *pStd = GET_RES_INFO(pCtx)->interResultBuf;
+ SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
if (pStd->stage == 0) { // the first stage is to calculate average value
avg_function(pCtx);
@@ -1381,8 +1377,8 @@ static void stddev_function(SQLFunctionCtx *pCtx) {
static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) {
// the second stage to calculate standard deviation
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SStddevInfo *pStd = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo);
/* the first stage is to calculate average value */
if (pStd->stage == 0) {
@@ -1433,8 +1429,8 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) {
* the stddevInfo and the average info struct share the same buffer area
* And the position of each element in their struct is exactly the same matched
*/
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SStddevInfo *pStd = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo);
if (pStd->stage == 0) {
/*
@@ -1449,7 +1445,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) {
pResInfo->initialized = true; // set it initialized to avoid re-initialization
// save average value into tmpBuf, for second stage scan
- SAvgInfo *pAvg = pResInfo->interResultBuf;
+ SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo);
pStd->avg = GET_DOUBLE_VAL(pCtx->aOutputBuf);
assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum));
@@ -1459,7 +1455,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) {
}
static void stddev_finalizer(SQLFunctionCtx *pCtx) {
- SStddevInfo *pStd = (SStddevInfo *)GET_RES_INFO(pCtx)->interResultBuf;
+ SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
if (pStd->num <= 0) {
setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
@@ -1505,7 +1501,7 @@ static void first_function(SQLFunctionCtx *pCtx) {
TSKEY k = pCtx->ptsList[i];
DO_UPDATE_TAG_COLUMNS(pCtx, k);
- SResultInfo *pInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
pInfo->hasResult = DATA_SET_FLAG;
pInfo->complete = true;
@@ -1532,7 +1528,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) {
TSKEY ts = pCtx->ptsList[index];
DO_UPDATE_TAG_COLUMNS(pCtx, ts);
- SResultInfo *pInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
pInfo->hasResult = DATA_SET_FLAG;
pInfo->complete = true; // get the first not-null data, completed
}
@@ -1576,7 +1572,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) {
first_data_assign_impl(pCtx, data, i);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
notNullElems++;
@@ -1604,8 +1600,7 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) {
static void first_dist_func_merge(SQLFunctionCtx *pCtx) {
char *pData = GET_INPUT_CHAR(pCtx);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pCtx->size == 1 && pResInfo->superTableQ);
+ assert(pCtx->size == 1 && pCtx->stableQuery);
SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes);
if (pInput->hasResult != DATA_SET_FLAG) {
@@ -1620,8 +1615,8 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) {
}
static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) {
- assert(pCtx->resultInfo->superTableQ);
-
+ assert(pCtx->stableQuery);
+
char * pData = GET_INPUT_CHAR(pCtx);
SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes);
if (pInput->hasResult != DATA_SET_FLAG) {
@@ -1668,7 +1663,7 @@ static void last_function(SQLFunctionCtx *pCtx) {
TSKEY ts = pCtx->ptsList[i];
DO_UPDATE_TAG_COLUMNS(pCtx, ts);
- SResultInfo *pInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
pInfo->hasResult = DATA_SET_FLAG;
pInfo->complete = true; // set query completed on this column
@@ -1691,7 +1686,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) {
TSKEY ts = pCtx->ptsList[index];
DO_UPDATE_TAG_COLUMNS(pCtx, ts);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
pResInfo->complete = true; // set query completed
}
@@ -1740,7 +1735,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) {
last_data_assign_impl(pCtx, data, i);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
notNullElems++;
@@ -1776,8 +1771,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) {
static void last_dist_func_merge(SQLFunctionCtx *pCtx) {
char *pData = GET_INPUT_CHAR(pCtx);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pCtx->size == 1 && pResInfo->superTableQ);
+ assert(pCtx->size == 1 && pCtx->stableQuery);
// the input data is null
SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes);
@@ -1833,11 +1827,11 @@ static void last_row_function(SQLFunctionCtx *pCtx) {
// assign the last element in current data block
assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
// set the result to final result buffer in case of super table query
- if (pResInfo->superTableQ) {
+ if (pCtx->stableQuery) {
SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes);
pInfo1->ts = pCtx->ptsList[pCtx->size - 1];
pInfo1->hasResult = DATA_SET_FLAG;
@@ -1852,7 +1846,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) {
static void last_row_finalizer(SQLFunctionCtx *pCtx) {
// do nothing at the first stage
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pResInfo->hasResult != DATA_SET_FLAG) {
if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pCtx->aOutputBuf, pCtx->outputType);
@@ -2044,8 +2038,8 @@ static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) {
static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); }
static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- STopBotInfo *pRes = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo);
tValuePair **tvp = pRes->res;
@@ -2123,7 +2117,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) {
}
}
- taosTFree(pData);
+ tfree(pData);
}
/*
@@ -2135,18 +2129,18 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) {
* top/bottom use the intermediate result buffer to keep the intermediate result
*/
static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
// only the first_stage_merge is directly written data into final output buffer
- if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) {
+ if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) {
return (STopBotInfo*) pCtx->aOutputBuf;
} else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer
- return pResInfo->interResultBuf;
+ return GET_ROWCELL_INTERBUF(pResInfo);
}
}
bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pResInfo == NULL) {
return true;
}
@@ -2252,7 +2246,7 @@ static void top_function(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, notNullElems, 1);
if (notNullElems > 0) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
}
}
@@ -2270,7 +2264,7 @@ static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) {
do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL,
0);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
}
@@ -2285,8 +2279,7 @@ static void top_func_merge(SQLFunctionCtx *pCtx) {
// remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary
buildTopBotStruct(pInput, pCtx);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1);
+ assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1);
STopBotInfo *pOutput = getTopBotOutputInfo(pCtx);
@@ -2314,7 +2307,7 @@ static void top_func_second_merge(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, pInput->num, pOutput->num);
if (pOutput->num > 0) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
}
}
@@ -2343,7 +2336,7 @@ static void bottom_function(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, notNullElems, 1);
if (notNullElems > 0) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
}
}
@@ -2359,7 +2352,7 @@ static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) {
do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo,
NULL, 0);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
}
@@ -2374,8 +2367,7 @@ static void bottom_func_merge(SQLFunctionCtx *pCtx) {
// remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary
buildTopBotStruct(pInput, pCtx);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1);
+ assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1);
STopBotInfo *pOutput = getTopBotOutputInfo(pCtx);
@@ -2403,16 +2395,16 @@ static void bottom_func_second_merge(SQLFunctionCtx *pCtx) {
SET_VAL(pCtx, pInput->num, pOutput->num);
if (pOutput->num > 0) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
}
}
static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
// data in temporary list is less than the required number of results, not enough qualified number of results
- STopBotInfo *pRes = pResInfo->interResultBuf;
+ STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo);
if (pRes->num == 0) { // no result
assert(pResInfo->hasResult != DATA_SET_FLAG);
// TODO:
@@ -2443,8 +2435,8 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
}
// in the first round, get the min-max value of all involved data
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SPercentileInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX);
SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX);
pInfo->numOfElems = 0;
@@ -2455,18 +2447,28 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
static void percentile_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SPercentileInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
// the first stage, only acquire the min/max value
if (pInfo->stage == 0) {
if (pCtx->preAggVals.isSet) {
- if (GET_DOUBLE_VAL(&pInfo->minval) > pCtx->preAggVals.statis.min) {
- SET_DOUBLE_VAL(&pInfo->minval, (double)pCtx->preAggVals.statis.min);
+ double tmin = 0.0, tmax = 0.0;
+ if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) {
+ tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min);
+ tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max);
+ } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) {
+ tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min);
+ tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max);
+ } else {
+ assert(true);
+ }
+ if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) {
+ SET_DOUBLE_VAL(&pInfo->minval, tmin);
}
- if (GET_DOUBLE_VAL(&pInfo->maxval) < pCtx->preAggVals.statis.max) {
- SET_DOUBLE_VAL(&pInfo->maxval, (double)pCtx->preAggVals.statis.max);
+ if (GET_DOUBLE_VAL(&pInfo->maxval) < tmax) {
+ SET_DOUBLE_VAL(&pInfo->maxval, tmax);
}
pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull);
@@ -2536,9 +2538,9 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
return;
}
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
- SPercentileInfo *pInfo = (SPercentileInfo *)pResInfo->interResultBuf;
+ SPercentileInfo *pInfo = (SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo);
if (pInfo->stage == 0) {
// TODO extract functions
@@ -2585,8 +2587,8 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
static void percentile_finalizer(SQLFunctionCtx *pCtx) {
double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey;
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- tMemBucket * pMemBucket = ((SPercentileInfo *)pResInfo->interResultBuf)->pMemBucket;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ tMemBucket * pMemBucket = ((SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo))->pMemBucket;
if (pMemBucket->total > 0) { // check for null
*(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v);
@@ -2599,8 +2601,8 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) {
}
static void percentile_next_step(SQLFunctionCtx *pCtx) {
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SPercentileInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
if (pInfo->stage == 0) {
// all data are null, set it completed
@@ -2617,12 +2619,12 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) {
//////////////////////////////////////////////////////////////////////////////////
static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
- if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) {
+ if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) {
return (SAPercentileInfo*) pCtx->aOutputBuf;
} else {
- return pResInfo->interResultBuf;
+ return GET_ROWCELL_INTERBUF(pResInfo);
}
}
@@ -2641,7 +2643,7 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx) {
static void apercentile_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
SAPercentileInfo *pInfo = getAPerctInfo(pCtx);
for (int32_t i = 0; i < pCtx->size; ++i) {
@@ -2694,8 +2696,8 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
return;
}
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SAPercentileInfo *pInfo = getAPerctInfo(pCtx); // pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SAPercentileInfo *pInfo = getAPerctInfo(pCtx);
double v = 0;
switch (pCtx->inputType) {
@@ -2726,8 +2728,8 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
}
static void apercentile_func_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ assert(pCtx->stableQuery);
SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx);
@@ -2784,7 +2786,7 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) {
pOutput->pHisto = pRes;
}
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
pResInfo->hasResult = DATA_SET_FLAG;
SET_VAL(pCtx, 1, 1);
}
@@ -2792,8 +2794,8 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) {
static void apercentile_finalizer(SQLFunctionCtx *pCtx) {
double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i64Key : pCtx->param[0].dKey;
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SAPercentileInfo *pOutput = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo);
if (pCtx->currentStage == SECONDARY_STAGE_MERGE) {
if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null
@@ -2830,8 +2832,8 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) {
return false;
}
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SLeastsquareInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
// 2*3 matrix
pInfo->startVal = pCtx->param[0].dKey;
@@ -2857,8 +2859,8 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) {
}
static void leastsquares_function(SQLFunctionCtx *pCtx) {
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SLeastsquareInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
double(*param)[3] = pInfo->mat;
double x = pInfo->startVal;
@@ -2928,8 +2930,8 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) {
return;
}
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SLeastsquareInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
double(*param)[3] = pInfo->mat;
@@ -2978,8 +2980,8 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) {
static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
// no data in query
- SResultInfo * pResInfo = GET_RES_INFO(pCtx);
- SLeastsquareInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
+ SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
if (pInfo->num == 0) {
if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) {
@@ -3044,7 +3046,7 @@ static void col_project_function(SQLFunctionCtx *pCtx) {
}
static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0
return;
}
@@ -3476,7 +3478,7 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) {
return false;
}
- SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf;
+ SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
// this is the server-side setup function in client-side, the secondary merge do not need this procedure
if (pCtx->currentStage == SECONDARY_STAGE_MERGE) {
@@ -3491,8 +3493,8 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) {
}
static void spread_function(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SSpreadInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
int32_t numOfElems = 0;
@@ -3558,8 +3560,8 @@ static void spread_function(SQLFunctionCtx *pCtx) {
}
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo));
}
}
@@ -3571,8 +3573,8 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) {
SET_VAL(pCtx, 1, 1);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SSpreadInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
double val = 0.0;
if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) {
@@ -3601,16 +3603,16 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) {
pResInfo->hasResult = DATA_SET_FLAG;
pInfo->hasResult = DATA_SET_FLAG;
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo));
}
}
void spread_func_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ assert(pCtx->stableQuery);
- SSpreadInfo *pResData = pResInfo->interResultBuf;
+ SSpreadInfo *pResData = GET_ROWCELL_INTERBUF(pResInfo);
int32_t notNullElems = 0;
for (int32_t i = 0; i < pCtx->size; ++i) {
@@ -3634,7 +3636,7 @@ void spread_func_merge(SQLFunctionCtx *pCtx) {
}
if (notNullElems > 0) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo));
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo));
pResInfo->hasResult = DATA_SET_FLAG;
}
}
@@ -3665,7 +3667,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) {
* here we do not check the input data types, because in case of metric query,
* the type of intermediate data is binary
*/
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pCtx->currentStage == SECONDARY_STAGE_MERGE) {
assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY);
@@ -3680,7 +3682,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) {
assert((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE) ||
(pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP));
- SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf;
+ SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
if (pInfo->hasResult != DATA_SET_FLAG) {
setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes);
return;
@@ -3704,8 +3706,8 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) {
return false;
}
- SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes;
- STwaInfo * pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes;
+ STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->lastKey = INT64_MIN;
pInfo->type = pCtx->inputType;
@@ -3744,8 +3746,8 @@ static void twa_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- STwaInfo * pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo);
int32_t i = 0;
@@ -3798,7 +3800,7 @@ static void twa_function(SQLFunctionCtx *pCtx) {
pResInfo->hasResult = DATA_SET_FLAG;
}
- if (pResInfo->superTableQ) {
+ if (pCtx->stableQuery) {
memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo));
}
@@ -3815,8 +3817,8 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
TSKEY *primaryKey = pCtx->ptsList;
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- STwaInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
if (pInfo->lastKey == INT64_MIN) {
pInfo->lastKey = pCtx->nStartQueryTimestamp;
@@ -3838,14 +3840,13 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
// pCtx->numOfIteratedElems += 1;
pResInfo->hasResult = DATA_SET_FLAG;
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(STwaInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo));
}
}
static void twa_func_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ assert(pCtx->stableQuery);
STwaInfo *pBuf = (STwaInfo *)pCtx->aOutputBuf;
char * indicator = pCtx->aInputElemBuf;
@@ -3885,16 +3886,16 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) {
*/
void twa_function_copy(SQLFunctionCtx *pCtx) {
assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
- memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes);
+ memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes);
pResInfo->hasResult = ((STwaInfo *)pCtx->aInputElemBuf)->hasResult;
}
void twa_function_finalizer(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
- STwaInfo *pInfo = (STwaInfo *)pResInfo->interResultBuf;
+ STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo);
assert(pInfo->EKey >= pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult);
if (pInfo->hasResult != DATA_SET_FLAG) {
@@ -3922,8 +3923,8 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) {
*/
static void interp_function(SQLFunctionCtx *pCtx) {
// at this point, the value is existed, return directly
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SInterpInfoDetail* pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SInterpInfoDetail* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
if (pCtx->size == 1) {
char *pData = GET_INPUT_CHAR(pCtx);
@@ -3977,7 +3978,7 @@ static void interp_function(SQLFunctionCtx *pCtx) {
if (isNull(data1, srcType) || isNull(data2, srcType)) {
setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes);
} else {
- taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point);
+ taosGetLinearInterpolationVal(pCtx->outputType, &point1, &point2, &point);
}
} else if (srcType == TSDB_DATA_TYPE_FLOAT) {
point1.val = data1;
@@ -3986,7 +3987,7 @@ static void interp_function(SQLFunctionCtx *pCtx) {
if (isNull(data1, srcType) || isNull(data2, srcType)) {
setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes);
} else {
- taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point);
+ taosGetLinearInterpolationVal(pCtx->outputType, &point1, &point2, &point);
}
} else {
@@ -3998,7 +3999,6 @@ static void interp_function(SQLFunctionCtx *pCtx) {
}
}
}
-
}
SET_VAL(pCtx, pCtx->size, 1);
@@ -4009,8 +4009,8 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) {
return false; // not initialized since it has been initialized
}
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- STSCompInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->pTSBuf = tsBufCreate(false, pCtx->order);
pInfo->pTSBuf->tsOrder = pCtx->order;
@@ -4018,18 +4018,18 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) {
}
static void ts_comp_function(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- STSBuf * pTSbuf = ((STSCompInfo *)(pResInfo->interResultBuf))->pTSBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf;
const char *input = GET_INPUT_CHAR(pCtx);
// primary ts must be existed, so no need to check its existance
if (pCtx->order == TSDB_ORDER_ASC) {
- tsBufAppend(pTSbuf, 0, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE);
+ tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE);
} else {
for (int32_t i = pCtx->size - 1; i >= 0; --i) {
char *d = GET_INPUT_CHAR_INDEX(pCtx, i);
- tsBufAppend(pTSbuf, 0, &pCtx->tag, d, TSDB_KEYSIZE);
+ tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE);
}
}
@@ -4043,21 +4043,21 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) {
return;
}
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- STSCompInfo *pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
STSBuf *pTSbuf = pInfo->pTSBuf;
- tsBufAppend(pTSbuf, 0, &pCtx->tag, pData, TSDB_KEYSIZE);
+ tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, pData, TSDB_KEYSIZE);
SET_VAL(pCtx, pCtx->size, 1);
pResInfo->hasResult = DATA_SET_FLAG;
}
static void ts_comp_finalize(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
- STSCompInfo *pInfo = pResInfo->interResultBuf;
+ STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
STSBuf * pTSbuf = pInfo->pTSBuf;
tsBufFlush(pTSbuf);
@@ -4106,8 +4106,8 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) {
return false;
}
- SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes;
- SRateInfo * pInfo = pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes;
+ SRateInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->CorrectionValue = 0;
pInfo->firstKey = INT64_MIN;
@@ -4126,8 +4126,8 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) {
static void rate_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
TSKEY *primaryKey = pCtx->ptsList;
tscDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull);
@@ -4190,8 +4190,8 @@ static void rate_function(SQLFunctionCtx *pCtx) {
}
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo));
}
}
@@ -4202,8 +4202,8 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) {
}
// NOTE: keep the intermediate result into the interResultBuf
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
TSKEY *primaryKey = pCtx->ptsList;
int64_t v = 0;
@@ -4247,20 +4247,18 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) {
pResInfo->hasResult = DATA_SET_FLAG;
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo));
}
}
static void rate_func_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ assert(pCtx->stableQuery);
tscDebug("rate_func_merge() size:%d", pCtx->size);
- //SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
SRateInfo *pBuf = (SRateInfo *)pCtx->aOutputBuf;
char *indicator = pCtx->aInputElemBuf;
@@ -4293,8 +4291,8 @@ static void rate_func_merge(SQLFunctionCtx *pCtx) {
static void rate_func_copy(SQLFunctionCtx *pCtx) {
assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY);
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes);
pResInfo->hasResult = ((SRateInfo*)pCtx->aInputElemBuf)->hasResult;
SRateInfo* pRateInfo = (SRateInfo*)pCtx->aInputElemBuf;
@@ -4305,8 +4303,8 @@ static void rate_func_copy(SQLFunctionCtx *pCtx) {
static void rate_finalizer(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
tscDebug("%p isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d",
pCtx, pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult);
@@ -4331,8 +4329,8 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) {
static void irate_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
TSKEY *primaryKey = pCtx->ptsList;
tscDebug("%p irate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull);
@@ -4394,8 +4392,8 @@ static void irate_function(SQLFunctionCtx *pCtx) {
}
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo));
}
}
@@ -4406,8 +4404,8 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) {
}
// NOTE: keep the intermediate result into the interResultBuf
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
TSKEY *primaryKey = pCtx->ptsList;
int64_t v = 0;
@@ -4443,16 +4441,16 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) {
pResInfo->hasResult = DATA_SET_FLAG;
// keep the data into the final output buffer for super table query since this execution may be the last one
- if (pResInfo->superTableQ) {
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo));
+ if (pCtx->stableQuery) {
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo));
}
}
static void do_sumrate_merge(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- assert(pResInfo->superTableQ);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ assert(pCtx->stableQuery);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
char * input = GET_INPUT_CHAR(pCtx);
for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) {
@@ -4476,7 +4474,7 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) {
if (DATA_SET_FLAG == pRateInfo->hasResult) {
pResInfo->hasResult = DATA_SET_FLAG;
SET_VAL(pCtx, pRateInfo->num, 1);
- memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo));
+ memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo));
}
}
@@ -4491,10 +4489,10 @@ static void sumrate_func_second_merge(SQLFunctionCtx *pCtx) {
}
static void sumrate_finalizer(SQLFunctionCtx *pCtx) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf;
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+ SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
- tscDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pResInfo->superTableQ, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult);
+ tscDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pCtx->stableQuery, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult);
if (pRateInfo->hasResult != DATA_SET_FLAG) {
setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c
index b4c3f3549b1576b3ff83300d4f10d9e07778530d..44dffab56f90e2d88604ee96a6cc1ddc8a5cfbef 100644
--- a/src/client/src/tscLocal.c
+++ b/src/client/src/tscLocal.c
@@ -49,82 +49,6 @@ typedef struct SCreateBuilder {
} SCreateBuilder;
static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength);
-static int32_t getToStringLength(const char *pData, int32_t length, int32_t type) {
- char buf[512] = {0};
-
- int32_t len = 0;
- int32_t MAX_BOOL_TYPE_LENGTH = 5; // max(strlen("true"), strlen("false"));
- switch (type) {
- case TSDB_DATA_TYPE_BINARY:
- return length;
- case TSDB_DATA_TYPE_NCHAR:
- return length;
- case TSDB_DATA_TYPE_DOUBLE: {
- double dv = 0;
- dv = GET_DOUBLE_VAL(pData);
- len = sprintf(buf, "%lf", dv);
- if (strncasecmp("nan", buf, 3) == 0) {
- len = 4;
- }
- } break;
- case TSDB_DATA_TYPE_FLOAT: {
- float fv = 0;
- fv = GET_FLOAT_VAL(pData);
- len = sprintf(buf, "%f", fv);
- if (strncasecmp("nan", buf, 3) == 0) {
- len = 4;
- }
- } break;
- case TSDB_DATA_TYPE_TIMESTAMP:
- case TSDB_DATA_TYPE_BIGINT:
- len = sprintf(buf, "%" PRId64, *(int64_t *)pData);
- break;
- case TSDB_DATA_TYPE_BOOL:
- len = MAX_BOOL_TYPE_LENGTH;
- break;
- default:
- len = sprintf(buf, "%d", *(int32_t *)pData);
- break;
- };
- return len;
-}
-
-/*
- * we need to convert all data into string, so we need to sprintf all kinds of
- * non-string data into string, and record its length to get the right
- * maximum length. The length may be less or greater than its original binary length:
- * For example:
- * length((short) 1) == 1, less than sizeof(short)
- * length((uint64_t) 123456789011) > 12, greater than sizsof(uint64_t)
- */
-static int32_t tscMaxLengthOfTagsFields(SSqlObj *pSql) {
- STableMeta *pMeta = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->pTableMeta;
-
- if (pMeta->tableType == TSDB_SUPER_TABLE || pMeta->tableType == TSDB_NORMAL_TABLE ||
- pMeta->tableType == TSDB_STREAM_TABLE) {
- return 0;
- }
-
- char * pTagValue = tsGetTagsValue(pMeta);
- SSchema *pTagsSchema = tscGetTableTagSchema(pMeta);
-
- int32_t len = getToStringLength(pTagValue, pTagsSchema[0].bytes, pTagsSchema[0].type);
-
- pTagValue += pTagsSchema[0].bytes;
- int32_t numOfTags = tscGetNumOfTags(pMeta);
-
- for (int32_t i = 1; i < numOfTags; ++i) {
- int32_t tLen = getToStringLength(pTagValue, pTagsSchema[i].bytes, pTagsSchema[i].type);
- if (len < tLen) {
- len = tLen;
- }
-
- pTagValue += pTagsSchema[i].bytes;
- }
-
- return len;
-}
-
static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
SSqlRes *pRes = &pSql->res;
@@ -186,8 +110,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
return 0;
}
- // the following is handle display tags value for meters created according to metric
- char *pTagValue = tsGetTagsValue(pMeta);
+ // the following is handle display tags for table created according to super table
for (int32_t i = numOfRows; i < totalNumOfRows; ++i) {
// field name
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
@@ -219,8 +142,6 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i;
const char *src = "TAG";
STR_WITH_MAXSIZE_TO_VARSTR(target, src, pField->bytes);
-
- pTagValue += pSchema[i].bytes;
}
return 0;
@@ -286,10 +207,10 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) {
const int32_t TYPE_COLUMN_LENGTH = 16;
const int32_t NOTE_COLUMN_MIN_LENGTH = 8;
- int32_t noteFieldLen = tscMaxLengthOfTagsFields(pSql);
- if (noteFieldLen == 0) {
- noteFieldLen = NOTE_COLUMN_MIN_LENGTH;
- }
+ int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH;//tscMaxLengthOfTagsFields(pSql);
+// if (noteFieldLen == 0) {
+// noteFieldLen = NOTE_COLUMN_MIN_LENGTH;
+// }
int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen);
tscFieldInfoUpdateOffset(pQueryInfo);
diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c
index 44ccb2471aa980d789c8557aca7981b6981df02f..e18ad0b4b0133c7cd01050f03559449303566be2 100644
--- a/src/client/src/tscLocalMerge.c
+++ b/src/client/src/tscLocalMerge.c
@@ -97,14 +97,14 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
pCtx->param[2].i64Key = pQueryInfo->order.order;
pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
pCtx->param[1].i64Key = pQueryInfo->order.orderColId;
+ } else if (functionId == TSDB_FUNC_APERCT) {
+ pCtx->param[0].i64Key = pExpr->param[0].i64Key;
+ pCtx->param[0].nType = pExpr->param[0].nType;
}
- SResultInfo *pResInfo = &pReducer->pResInfo[i];
- pResInfo->bufLen = pExpr->interBytes;
- pResInfo->interResultBuf = calloc(1, (size_t) pResInfo->bufLen);
-
- pCtx->resultInfo = &pReducer->pResInfo[i];
- pCtx->resultInfo->superTableQ = true;
+ pCtx->interBufBytes = pExpr->interBytes;
+ pCtx->resultInfo = calloc(1, pCtx->interBufBytes + sizeof(SResultRowCellInfo));
+ pCtx->stableQuery = true;
}
int16_t n = 0;
@@ -227,7 +227,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (ds == NULL) {
tscError("%p failed to create merge structure", pSql);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
- taosTFree(pReducer);
+ tfree(pReducer);
return;
}
@@ -254,7 +254,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (ds->filePage.num == 0) { // no data in this flush, the index does not increase
tscDebug("%p flush data is empty, ignore %d flush record", pSql, idx);
- taosTFree(ds);
+ tfree(ds);
continue;
}
@@ -264,7 +264,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
// no data actually, no need to merge result.
if (idx == 0) {
- taosTFree(pReducer);
+ tfree(pReducer);
return;
}
@@ -272,7 +272,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
SCompareParam *param = malloc(sizeof(SCompareParam));
if (param == NULL) {
- taosTFree(pReducer);
+ tfree(pReducer);
return;
}
@@ -286,8 +286,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator);
if (pReducer->pLoserTree == NULL || pRes->code != 0) {
- taosTFree(param);
- taosTFree(pReducer);
+ tfree(param);
+ tfree(pReducer);
return;
}
@@ -330,14 +330,14 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL ||
/*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
- taosTFree(pReducer->pTempBuffer);
- taosTFree(pReducer->discardData);
- taosTFree(pReducer->pResultBuf);
- taosTFree(pReducer->pFinalRes);
- taosTFree(pReducer->prevRowOfInput);
- taosTFree(pReducer->pLoserTree);
- taosTFree(param);
- taosTFree(pReducer);
+ tfree(pReducer->pTempBuffer);
+ tfree(pReducer->discardData);
+ tfree(pReducer->pResultBuf);
+ tfree(pReducer->pFinalRes);
+ tfree(pReducer->prevRowOfInput);
+ tfree(pReducer->pLoserTree);
+ tfree(param);
+ tfree(pReducer);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
return;
}
@@ -345,7 +345,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
size_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo);
pReducer->pTempBuffer->num = 0;
- pReducer->pResInfo = calloc(numOfCols, sizeof(SResultInfo));
tscCreateResPointerInfo(pRes, pQueryInfo);
tscInitSqlContext(pCmd, pReducer, pDesc);
@@ -489,47 +488,40 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
tscDebug("%p waiting for delete procedure, status: %d", pSql, status);
}
- pLocalReducer->pFillInfo = taosDestoryFillInfo(pLocalReducer->pFillInfo);
+ pLocalReducer->pFillInfo = taosDestroyFillInfo(pLocalReducer->pFillInfo);
if (pLocalReducer->pCtx != NULL) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i];
tVariantDestroy(&pCtx->tag);
+ tfree(pCtx->resultInfo);
+
if (pCtx->tagInfo.pTagCtxList != NULL) {
- taosTFree(pCtx->tagInfo.pTagCtxList);
+ tfree(pCtx->tagInfo.pTagCtxList);
}
}
- taosTFree(pLocalReducer->pCtx);
+ tfree(pLocalReducer->pCtx);
}
- taosTFree(pLocalReducer->prevRowOfInput);
-
- taosTFree(pLocalReducer->pTempBuffer);
- taosTFree(pLocalReducer->pResultBuf);
-
- if (pLocalReducer->pResInfo != NULL) {
- size_t num = tscSqlExprNumOfExprs(pQueryInfo);
- for (int32_t i = 0; i < num; ++i) {
- taosTFree(pLocalReducer->pResInfo[i].interResultBuf);
- }
+ tfree(pLocalReducer->prevRowOfInput);
- taosTFree(pLocalReducer->pResInfo);
- }
+ tfree(pLocalReducer->pTempBuffer);
+ tfree(pLocalReducer->pResultBuf);
if (pLocalReducer->pLoserTree) {
- taosTFree(pLocalReducer->pLoserTree->param);
- taosTFree(pLocalReducer->pLoserTree);
+ tfree(pLocalReducer->pLoserTree->param);
+ tfree(pLocalReducer->pLoserTree);
}
- taosTFree(pLocalReducer->pFinalRes);
- taosTFree(pLocalReducer->discardData);
+ tfree(pLocalReducer->pFinalRes);
+ tfree(pLocalReducer->discardData);
tscLocalReducerEnvDestroy(pLocalReducer->pExtMemBuffer, pLocalReducer->pDesc, pLocalReducer->resColModel,
pLocalReducer->numOfVnode);
for (int32_t i = 0; i < pLocalReducer->numOfBuffer; ++i) {
- taosTFree(pLocalReducer->pLocalDataSrc[i]);
+ tfree(pLocalReducer->pLocalDataSrc[i]);
}
pLocalReducer->numOfBuffer = 0;
@@ -596,7 +588,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm
}
*pOrderDesc = tOrderDesCreate(orderColIndexList, numOfGroupByCols, pModel, pQueryInfo->order.order);
- taosTFree(orderColIndexList);
+ tfree(orderColIndexList);
if (*pOrderDesc == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -698,7 +690,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
pg *= 2;
}
- size_t numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups;
+ size_t numOfSubs = pSql->subState.numOfSub;
+ assert(numOfSubs <= pTableMetaInfo->vgroupList->numOfVgroups);
for (int32_t i = 0; i < numOfSubs; ++i) {
(*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pg, pModel);
(*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL;
@@ -706,7 +699,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
- taosTFree(pSchema);
+ tfree(pSchema);
return pRes->code;
}
@@ -743,7 +736,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
}
*pFinalModel = createColumnModel(pSchema, (int32_t)size, capacity);
- taosTFree(pSchema);
+ tfree(pSchema);
return TSDB_CODE_SUCCESS;
}
@@ -763,7 +756,7 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe
pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]);
}
- taosTFree(pMemBuffer);
+ tfree(pMemBuffer);
}
/**
@@ -985,10 +978,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
pBeforeFillData->num = 0;
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
- taosTFree(pResPages[i]);
+ tfree(pResPages[i]);
}
- taosTFree(pResPages);
+ tfree(pResPages);
}
static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) {
@@ -1071,7 +1064,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx)
continue;
}
- SResultInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
+ SResultRowCellInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
if (maxOutput < pResInfo->numOfRes) {
maxOutput = pResInfo->numOfRes;
}
@@ -1252,10 +1245,11 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur
return true;
}
-void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { // reset output buffer to the beginning
- for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
- pLocalReducer->pCtx[i].aOutputBuf =
- pLocalReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pLocalReducer->resColModel->capacity;
+void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {// reset output buffer to the beginning
+ size_t t = tscSqlExprNumOfExprs(pQueryInfo);
+ for (int32_t i = 0; i < t; ++i) {
+ SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
+ pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + pExpr->offset * pLocalReducer->resColModel->capacity;
}
memset(pLocalReducer->pResultBuf, 0, pLocalReducer->nResultBufSize + sizeof(tFilePage));
@@ -1500,8 +1494,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) {
if (pLocalReducer->discard && sameGroup) {
pLocalReducer->hasUnprocessedRow = false;
tmpBuffer->num = 0;
- } else {
- // current row does not belongs to the previous group, so it is not be handled yet.
+ } else { // current row does not belongs to the previous group, so it is not be handled yet.
pLocalReducer->hasUnprocessedRow = true;
}
diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index ac34d26d2fe747773a550dd76ac7e967a592199e..c9115a8324721c0262a0ccaf54572631d89e423a 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -702,7 +702,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st
}
int32_t code = TSDB_CODE_TSC_INVALID_SQL;
- char * tmpTokenBuf = calloc(1, 4096); // used for deleting Escape character: \\, \', \"
+ char * tmpTokenBuf = calloc(1, 16*1024); // used for deleting Escape character: \\, \', \"
if (NULL == tmpTokenBuf) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
@@ -1309,7 +1309,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) {
if ((!pCmd->parseFinished) && (!initial)) {
tscDebug("%p resume to parse sql: %s", pSql, pCmd->curSql);
}
-
+
ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE);
if (TSDB_CODE_SUCCESS != ret) {
return ret;
@@ -1406,7 +1406,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
assert(taos_errno(pSql) == code);
taos_free_result(pSql);
- taosTFree(pSupporter);
+ tfree(pSupporter);
fclose(fp);
pParentSql->res.code = code;
@@ -1445,7 +1445,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
char *tokenBuf = calloc(1, 4096);
- while ((readLen = taosGetline(&line, &n, fp)) != -1) {
+ while ((readLen = tgetline(&line, &n, fp)) != -1) {
if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
line[--readLen] = 0;
}
@@ -1470,7 +1470,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
}
}
- taosTFree(tokenBuf);
+ tfree(tokenBuf);
free(line);
if (count > 0) {
@@ -1483,7 +1483,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
} else {
taos_free_result(pSql);
- taosTFree(pSupporter);
+ tfree(pSupporter);
fclose(fp);
pParentSql->fp = pParentSql->fetchFp;
@@ -1513,7 +1513,7 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) {
pSql->res.code = TAOS_SYSTEM_ERROR(errno);
tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code));
- taosTFree(pSupporter)
+ tfree(pSupporter)
tscQueueAsyncRes(pSql);
return;
diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c
index eb6843b0e4fd844c9d13da7f55dca47008f62a79..acc5acd786bfe00036e539a104da04af1485a263 100644
--- a/src/client/src/tscProfile.c
+++ b/src/client/src/tscProfile.c
@@ -222,7 +222,7 @@ void tscKillStream(STscObj *pObj, uint32_t killId) {
}
int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
- SCMHeartBeatMsg *pHeartbeat = pMsg;
+ SHeartBeatMsg *pHeartbeat = pMsg;
int allocedQueriesNum = pHeartbeat->numOfQueries;
int allocedStreamsNum = pHeartbeat->numOfStreams;
@@ -277,7 +277,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
}
int32_t msgLen = pHeartbeat->numOfQueries * sizeof(SQueryDesc) + pHeartbeat->numOfStreams * sizeof(SStreamDesc) +
- sizeof(SCMHeartBeatMsg);
+ sizeof(SHeartBeatMsg);
pHeartbeat->connId = htonl(pObj->connId);
pHeartbeat->numOfQueries = htonl(pHeartbeat->numOfQueries);
pHeartbeat->numOfStreams = htonl(pHeartbeat->numOfStreams);
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index e2573f7e19303966e9f17a2731444bf9fca7ab87..579e5f541023de677264eb6d2fb9c66701d8b3c2 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -16,6 +16,7 @@
#define _BSD_SOURCE
#define _XOPEN_SOURCE 500
#define _DEFAULT_SOURCE
+#define _GNU_SOURCE
#include "os.h"
#include "qAst.h"
@@ -65,7 +66,6 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq
static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len);
static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength);
-static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName);
static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult);
static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes,
@@ -80,9 +80,9 @@ static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo);
static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd);
-static int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
+static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
-static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
+static int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem);
@@ -114,7 +114,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString);
static int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex);
static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo);
static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
-static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate);
+static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate);
static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex);
@@ -417,7 +417,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
char* pMsg = pCmd->payload;
- SCMCfgDnodeMsg* pCfg = (SCMCfgDnodeMsg*)pMsg;
+ SCfgDnodeMsg* pCfg = (SCfgDnodeMsg*)pMsg;
pDCL->a[0].n = strdequote(pDCL->a[0].z);
strncpy(pCfg->ep, pDCL->a[0].z, pDCL->a[0].n);
@@ -616,14 +616,20 @@ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) {
return false;
}
-int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
+int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
const char* msg1 = "invalid query expression";
const char* msg2 = "interval cannot be less than 10 ms";
+ const char* msg3 = "sliding cannot be used without interval";
+
+ SSqlCmd* pCmd = &pSql->cmd;
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
if (pQuerySql->interval.type == 0 || pQuerySql->interval.n == 0) {
+ if (pQuerySql->sliding.n > 0) {
+ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
+ }
return TSDB_CODE_SUCCESS;
}
@@ -656,7 +662,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
return TSDB_CODE_TSC_INVALID_SQL;
}
- if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
+ if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@@ -709,7 +715,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
return TSDB_CODE_TSC_INVALID_SQL;
}
- if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
+ if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@@ -767,13 +773,15 @@ int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQue
return TSDB_CODE_SUCCESS;
}
-int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
+int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
const char* msg0 = "sliding value too small";
const char* msg1 = "sliding value no larger than the interval value";
const char* msg2 = "sliding value can not less than 1% of interval value";
- const char* msg3 = "does not support sliding when interval is natual month/year";
+ const char* msg3 = "does not support sliding when interval is natural month/year";
+ const char* msg4 = "sliding not support yet in ordinary query";
const static int32_t INTERVAL_SLIDING_FACTOR = 100;
+ SSqlCmd* pCmd = &pSql->cmd;
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
@@ -806,6 +814,10 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
+ if (pQueryInfo->interval.sliding != pQueryInfo->interval.interval && pSql->pStream == NULL) {
+ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
+ }
+
return TSDB_CODE_SUCCESS;
}
@@ -877,22 +889,13 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
int32_t nLen = 0;
for (int32_t i = 0; i < pFieldList->nField; ++i) {
- if (pFieldList->p[i].bytes == 0) {
+ TAOS_FIELD* pField = &pFieldList->p[i];
+
+ if (pField->bytes == 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
return false;
}
- nLen += pFieldList->p[i].bytes;
- }
-
- // max row length must be less than TSDB_MAX_BYTES_PER_ROW
- if (nLen > TSDB_MAX_BYTES_PER_ROW) {
- invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
- return false;
- }
- // field name must be unique
- for (int32_t i = 0; i < pFieldList->nField; ++i) {
- TAOS_FIELD* pField = &pFieldList->p[i];
if (pField->type < TSDB_DATA_TYPE_BOOL || pField->type > TSDB_DATA_TYPE_NCHAR) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
return false;
@@ -909,10 +912,19 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
return false;
}
+ // field name must be unique
if (has(pFieldList, i + 1, pFieldList->p[i].name) == true) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
return false;
}
+
+ nLen += pField->bytes;
+ }
+
+ // max row length must be less than TSDB_MAX_BYTES_PER_ROW
+ if (nLen > TSDB_MAX_BYTES_PER_ROW) {
+ invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
+ return false;
}
return true;
@@ -1211,7 +1223,7 @@ static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex*
tscColumnListInsert(pQueryInfo->colList, &tsCol);
}
static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) {
- const char* msg1 = "invalid column name, or illegal column type";
+ const char* msg1 = "invalid column name, illegal column type, or columns in arithmetic expression from two tables";
const char* msg2 = "invalid arithmetic expression in select clause";
const char* msg3 = "tag columns can not be used in arithmetic expression";
const char* msg4 = "columns from different table mixed up in arithmetic expression";
@@ -1625,16 +1637,15 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
}
static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc,
- char* aliasName, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) {
+ const char* name, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) {
const char* msg1 = "not support column types";
int16_t type = 0;
int16_t bytes = 0;
- char columnName[TSDB_COL_NAME_LEN] = {0};
int32_t functionID = cvtFunc.execFuncId;
if (functionID == TSDB_FUNC_SPREAD) {
- int32_t t1 = pSchema[pColIndex->columnIndex].type;
+ int32_t t1 = pSchema->type;
if (t1 == TSDB_DATA_TYPE_BINARY || t1 == TSDB_DATA_TYPE_NCHAR || t1 == TSDB_DATA_TYPE_BOOL) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
return -1;
@@ -1643,18 +1654,12 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
bytes = tDataTypeDesc[type].nSize;
}
} else {
- type = pSchema[pColIndex->columnIndex].type;
- bytes = pSchema[pColIndex->columnIndex].bytes;
+ type = pSchema->type;
+ bytes = pSchema->bytes;
}
- if (aliasName != NULL) {
- tstrncpy(columnName, aliasName, sizeof(columnName));
- } else {
- getRevisedName(columnName, cvtFunc.originFuncId, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name);
- }
-
SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false);
- tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName));
+ tstrncpy(pExpr->aliasName, name, tListLen(pExpr->aliasName));
if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != functionID) {
pExpr->colInfo.flag |= TSDB_COL_NULL;
@@ -1674,7 +1679,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
// if it is not in the final result, do not add it
SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex);
if (finalResult) {
- insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, columnName, pExpr);
+ insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, pExpr->aliasName, pExpr);
} else {
tscColumnListInsert(pQueryInfo->colList, &(ids.ids[0]));
}
@@ -1682,6 +1687,23 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
return TSDB_CODE_SUCCESS;
}
+void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken) {
+ if (pItem->aliasName != NULL) {
+ tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN);
+ } else {
+ char uname[TSDB_COL_NAME_LEN] = {0};
+ int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN);
+ tstrncpy(uname, pToken->z, len);
+
+ int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1;
+ char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1] = {0};
+
+ snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname);
+
+ tstrncpy(name, tmp, TSDB_COL_NAME_LEN);
+ }
+}
+
int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult) {
STableMetaInfo* pTableMetaInfo = NULL;
int32_t optr = pItem->pNode->nSQLOptr;
@@ -1939,8 +1961,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
SColumnIndex index = COLUMN_INDEX_INITIALIZER;
- if (pParamElem->pNode->nSQLOptr == TK_ALL) {
- // select table.*
+ if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.*
SStrToken tmpToken = pParamElem->pNode->colInfo;
if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
@@ -1950,9 +1971,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
+ char name[TSDB_COL_NAME_LEN] = {0};
for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) {
index.columnIndex = j;
- if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex++, &index, finalResult) != 0) {
+ SStrToken t = {.z = pSchema[j].name, .n = (uint32_t)strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)};
+ setResultColName(name, pItem, cvtFunc.originFuncId, &t);
+
+ if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, finalResult) != 0) {
return TSDB_CODE_TSC_INVALID_SQL;
}
}
@@ -1963,14 +1988,18 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
}
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
- SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
// functions can not be applied to tags
if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
- if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex + i, &index, finalResult) != 0) {
+ char name[TSDB_COL_NAME_LEN] = {0};
+
+ SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex);
+ setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo);
+
+ if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex + i, &index, finalResult) != 0) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@@ -2007,7 +2036,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) {
SColumnIndex index = {.tableIndex = j, .columnIndex = i};
- if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex, &index, finalResult) != 0) {
+
+ char name[TSDB_COL_NAME_LEN] = {0};
+ SStrToken t = {.z = pSchema[i].name, .n = (uint32_t)strnlen(pSchema[i].name, TSDB_COL_NAME_LEN)};
+ setResultColName(name, pItem, cvtFunc.originFuncId, &t);
+
+ if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@@ -2240,10 +2274,6 @@ void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLengt
}
}
-void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName) {
- snprintf(resultFieldName, maxLen, "%s(%s)", aAggs[functionId].aName, columnName);
-}
-
static bool isTablenameToken(SStrToken* token) {
SStrToken tmpToken = *token;
SStrToken tableToken = {0};
@@ -2728,7 +2758,8 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) {
int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd) {
const char* msg1 = "too many columns in group by clause";
const char* msg2 = "invalid column name in group by clause";
-// const char* msg3 = "group by columns must belong to one table";
+ const char* msg3 = "columns from one table allowed as group by columns";
+ const char* msg4 = "join query does not support group by";
const char* msg7 = "not support group by expression";
const char* msg8 = "not allowed column type for group by";
const char* msg9 = "tags not allowed for table query";
@@ -2749,6 +2780,10 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
+ if (pQueryInfo->numOfTables > 1) {
+ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
+ }
+
STableMeta* pTableMeta = NULL;
SSchema* pSchema = NULL;
SSchema s = tscGetTbnameColumnSchema();
@@ -2764,7 +2799,11 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
- tableIndex = index.tableIndex;
+ if (tableIndex == COLUMN_INDEX_INITIAL_VAL) {
+ tableIndex = index.tableIndex;
+ } else if (tableIndex != index.tableIndex) {
+ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
+ }
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
pTableMeta = pTableMetaInfo->pTableMeta;
@@ -2803,7 +2842,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
tscColumnListInsert(pTableMetaInfo->tagColList, &index);
} else {
// check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by
- if (pSchema->type > TSDB_DATA_TYPE_BINARY) {
+ if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8);
}
@@ -3349,7 +3388,8 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) {
return TSDB_CODE_SUCCESS;
}
-static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) {
+static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList,
+ int32_t* type, uint64_t* uid) {
if (pExpr->nSQLOptr == TK_ID) {
if (*type == NON_ARITHMEIC_EXPR) {
*type = NORMAL_ARITHMETIC;
@@ -3398,13 +3438,22 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer
}
// Not supported data type in arithmetic expression
+ uint64_t id = -1;
for(int32_t i = 0; i < inc; ++i) {
SSqlExpr* p1 = tscSqlExprGet(pQueryInfo, i + outputIndex);
int16_t t = p1->resType;
if (t == TSDB_DATA_TYPE_BINARY || t == TSDB_DATA_TYPE_NCHAR || t == TSDB_DATA_TYPE_BOOL || t == TSDB_DATA_TYPE_TIMESTAMP) {
return TSDB_CODE_TSC_INVALID_SQL;
}
+
+ if (i == 0) {
+ id = p1->uid;
+ } else if (id != p1->uid){
+ return TSDB_CODE_TSC_INVALID_SQL;
+ }
}
+
+ *uid = id;
}
return TSDB_CODE_SUCCESS;
@@ -3416,13 +3465,16 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI
}
tSQLExpr* pLeft = pExpr->pLeft;
+ uint64_t uidLeft = 0;
+ uint64_t uidRight = 0;
+
if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) {
int32_t ret = validateArithmeticSQLExpr(pCmd, pLeft, pQueryInfo, pList, type);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
} else {
- int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type);
+ int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type, &uidLeft);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
@@ -3435,10 +3487,15 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI
return ret;
}
} else {
- int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type);
+ int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type, &uidRight);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
+
+ // the expression not from the same table, return error
+ if (uidLeft != uidRight && uidLeft != 0 && uidRight != 0) {
+ return TSDB_CODE_TSC_INVALID_SQL;
+ }
}
return TSDB_CODE_SUCCESS;
@@ -3900,7 +3957,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen);
if (ret != TSDB_CODE_SUCCESS) {
taosStringBuilderDestroy(&sb1);
- taosTFree(segments);
+ tfree(segments);
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
return ret;
@@ -3913,7 +3970,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
pQueryInfo->tagCond.tbnameCond.cond = strdup(str);
taosStringBuilderDestroy(&sb1);
- taosTFree(segments);
+ tfree(segments);
return TSDB_CODE_SUCCESS;
}
@@ -4424,8 +4481,8 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) {
int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema* pSchema) {
const char* msg0 = "only support order by primary timestamp";
const char* msg1 = "invalid column name";
- const char* msg2 = "only support order by primary timestamp and queried column";
- const char* msg3 = "only support order by primary timestamp and first tag in groupby clause";
+ const char* msg2 = "only support order by primary timestamp or queried column";
+ const char* msg3 = "only support order by primary timestamp or first tag in groupby clause";
setDefaultOrderInfo(pQueryInfo);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
@@ -5186,7 +5243,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn
return TSDB_CODE_SUCCESS;
}
-static int32_t setKeepOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
+static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
const char* msg = "invalid number of options";
pMsg->daysToKeep = htonl(-1);
@@ -5217,7 +5274,7 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo*
return TSDB_CODE_SUCCESS;
}
-static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) {
+static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) {
const char* msg = "invalid time precision";
pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default
@@ -5241,7 +5298,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBIn
return TSDB_CODE_SUCCESS;
}
-static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
+static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
pMsg->maxTables = htonl(-1); // max tables can not be set anymore
pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize);
pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks);
@@ -5255,10 +5312,11 @@ static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
pMsg->replications = pCreateDb->replica;
pMsg->quorum = pCreateDb->quorum;
pMsg->ignoreExist = pCreateDb->ignoreExists;
+ pMsg->update = pCreateDb->update;
}
int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) {
- SCMCreateDbMsg* pMsg = (SCMCreateDbMsg*)(pCmd->payload);
+ SCreateDbMsg* pMsg = (SCreateDbMsg *)(pCmd->payload);
setCreateDBOption(pMsg, pCreateDbSql);
if (setKeepOption(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) {
@@ -5281,20 +5339,26 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau
if (pParentQueryInfo->groupbyExpr.numOfGroupCols > 0) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, subClauseIndex);
+ SSqlExpr* pExpr = NULL;
+
size_t size = taosArrayGetSize(pQueryInfo->exprList);
-
- SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, (int32_t)size - 1);
+ if (size > 0) {
+ pExpr = tscSqlExprGet(pQueryInfo, (int32_t)size - 1);
+ }
- if (pExpr->functionId != TSDB_FUNC_TAG) {
- STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
- int16_t columnInfo = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
- SColumnIndex index = {.tableIndex = 0, .columnIndex = columnInfo};
- SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
+ if (pExpr == NULL || pExpr->functionId != TSDB_FUNC_TAG) {
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pParentQueryInfo, tableIndex);
+
+ int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
+
+ SSchema* pTagSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, colId);
+ int16_t colIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, colId);
+ SColumnIndex index = {.tableIndex = 0, .columnIndex = colIndex};
+
+ char* name = pTagSchema->name;
+ int16_t type = pTagSchema->type;
+ int16_t bytes = pTagSchema->bytes;
- int16_t type = pSchema[index.columnIndex].type;
- int16_t bytes = pSchema[index.columnIndex].bytes;
- char* name = pSchema[index.columnIndex].name;
-
pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true);
pExpr->colInfo.flag = TSDB_COL_TAG;
@@ -5770,7 +5834,7 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
}
// can only perform the parameters based on the macro definitation
-int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) {
+int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) {
char msg[512] = {0};
if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) {
@@ -6094,7 +6158,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
}
// set interval value
- if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
+ if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
} else {
if ((pQueryInfo->interval.interval > 0) &&
@@ -6302,7 +6366,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
}
// set interval value
- if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
+ if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
} else {
if ((pQueryInfo->interval.interval > 0) &&
diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c
index dfd707344c5ab3ec42b49859937f604fa477fe10..441ad8d8382c667c164f7c797fc344cb9ab9118d 100644
--- a/src/client/src/tscSchemaUtil.c
+++ b/src/client/src/tscSchemaUtil.c
@@ -118,7 +118,7 @@ SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex)
}
// TODO for large number of columns, employ the binary search method
-SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
+SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
STableComInfo tinfo = tscGetTableInfo(pTableMeta);
for(int32_t i = 0; i < tinfo.numOfColumns + tinfo.numOfTags; ++i) {
@@ -140,7 +140,7 @@ struct SSchema tscGetTbnameColumnSchema() {
strcpy(s.name, TSQL_TBNAME_L);
return s;
}
-static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo *vgroupInfo) {
+static void tscInitCorVgroupInfo(SCorVgroupInfo *corVgroupInfo, SVgroupInfo *vgroupInfo) {
corVgroupInfo->version = 0;
corVgroupInfo->inUse = 0;
corVgroupInfo->numOfEps = vgroupInfo->numOfEps;
@@ -166,7 +166,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
pTableMeta->id.tid = pTableMetaMsg->tid;
pTableMeta->id.uid = pTableMetaMsg->uid;
- SCMVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo;
+ SVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo;
pVgroupInfo->numOfEps = pTableMetaMsg->vgroup.numOfEps;
pVgroupInfo->vgId = pTableMetaMsg->vgroup.vgId;
@@ -177,7 +177,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
pVgroupInfo->epAddr[i].port = pEpMsg->port;
}
- tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo);
+ tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, pVgroupInfo);
pTableMeta->sversion = pTableMetaMsg->sversion;
pTableMeta->tversion = pTableMetaMsg->tversion;
@@ -197,28 +197,6 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
return pTableMeta;
}
-/**
- * the TableMeta data format in memory is as follows:
- *
- * +--------------------+
- * |STableMeta Body data| sizeof(STableMeta)
- * +--------------------+
- * |Schema data | numOfTotalColumns * sizeof(SSchema)
- * +--------------------+
- * |Tags data | tag_col_1.bytes + tag_col_2.bytes + ....
- * +--------------------+
- *
- * @param pTableMeta
- * @return
- */
-char* tsGetTagsValue(STableMeta* pTableMeta) {
- int32_t offset = 0;
-// int32_t numOfTotalCols = pTableMeta->numOfColumns + pTableMeta->numOfTags;
-// uint32_t offset = sizeof(STableMeta) + numOfTotalCols * sizeof(SSchema);
-
- return ((char*)pTableMeta + offset);
-}
-
// todo refactor
UNUSED_FUNC static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) {
for (int32_t i = 0; i < num; ++i) {
diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c
index a0841fa2344682facb4a6dc8666716e38a31ea1d..e97e95a31779b5c5a6eab2be1b3a0ade425dc6c5 100644
--- a/src/client/src/tscServer.c
+++ b/src/client/src/tscServer.c
@@ -48,7 +48,7 @@ static int32_t getWaitingTimeInterval(int32_t count) {
return initial * (2<<(count - 2));
}
-static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) {
+static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) {
assert(pSql != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0);
SRpcEpSet* pEpSet = &pSql->epSet;
@@ -100,7 +100,7 @@ void tscUpdateMgmtEpSet(SRpcEpSet *pEpSet) {
tscMgmtEpSet.epSet = *pEpSet;
taosCorEndWrite(&tscMgmtEpSet.version);
}
-static void tscDumpEpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) {
+static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) {
if (pVgroupInfo == NULL) { return;}
taosCorBeginRead(&pVgroupInfo->version);
int8_t inUse = pVgroupInfo->inUse;
@@ -117,14 +117,14 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) {
SSqlCmd *pCmd = &pObj->cmd;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;}
- SCMCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo;
+ SCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo;
taosCorBeginWrite(&pVgroupInfo->version);
tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse);
pVgroupInfo->inUse = pEpSet->inUse;
pVgroupInfo->numOfEps = pEpSet->numOfEps;
for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) {
- taosTFree(pVgroupInfo->epAddr[i].fqdn);
+ tfree(pVgroupInfo->epAddr[i].fqdn);
pVgroupInfo->epAddr[i].fqdn = strndup(pEpSet->fqdn[i], tListLen(pEpSet->fqdn[i]));
pVgroupInfo->epAddr[i].port = pEpSet->port[i];
}
@@ -150,7 +150,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
if (pObj == NULL) return;
if (pObj != pObj->signature) {
- tscError("heart beat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature);
+ tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature);
return;
}
@@ -158,12 +158,12 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
SSqlRes *pRes = &pSql->res;
if (code == 0) {
- SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp;
- SRpcEpSet * epSet = &pRsp->epSet;
+ SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp;
+ SRpcEpSet * epSet = &pRsp->epSet;
if (epSet->numOfEps > 0) {
tscEpSetHtons(epSet);
tscUpdateMgmtEpSet(epSet);
- }
+ }
pSql->pTscObj->connId = htonl(pRsp->connId);
@@ -175,44 +175,44 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId));
}
} else {
- tscDebug("heartbeat failed, code:%s", tstrerror(code));
+ tscDebug("%p heartbeat failed, code:%s", pObj->pHb, tstrerror(code));
}
if (pObj->pHb != NULL) {
int32_t waitingDuring = tsShellActivityTimer * 500;
- tscDebug("%p start heartbeat in %dms", pSql, waitingDuring);
+ tscDebug("%p send heartbeat in %dms", pSql, waitingDuring);
- taosTmrReset(tscProcessActivityTimer, waitingDuring, pObj, tscTmr, &pObj->pTimer);
+ taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer);
} else {
tscDebug("%p start to close tscObj:%p, not send heartbeat again", pSql, pObj);
}
}
void tscProcessActivityTimer(void *handle, void *tmrId) {
- STscObj *pObj = (STscObj *)handle;
- if (pObj == NULL || pObj->signature != pObj) {
- return;
- }
+ int64_t rid = (int64_t) handle;
+ STscObj *pObj = taosAcquireRef(tscRefId, rid);
+ if (pObj == NULL) return;
SSqlObj* pHB = pObj->pHb;
- if (pObj->pTimer != tmrId || pHB == NULL) {
- return;
- }
void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE));
if (p == NULL) {
tscWarn("%p HB object has been released already", pHB);
+ taosReleaseRef(tscRefId, pObj->rid);
return;
}
assert(*pHB->self == pHB);
+ pHB->retry = 0;
int32_t code = tscProcessSql(pHB);
taosCacheRelease(tscObjCache, (void**) &p, false);
if (code != TSDB_CODE_SUCCESS) {
tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code));
}
+
+ taosReleaseRef(tscRefId, rid);
}
int tscSendMsgToServer(SSqlObj *pSql) {
@@ -237,7 +237,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
.pCont = pMsg,
.contLen = pSql->cmd.payloadLen,
.ahandle = pSql,
- .handle = &pSql->pRpcCtx,
+ .handle = NULL,
.code = 0
};
@@ -245,7 +245,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
// Otherwise, the pSql object may have been released already during the response function, which is
// processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely
// cause crash.
- rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg);
+ pSql->rpcRid = rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg);
return TSDB_CODE_SUCCESS;
}
@@ -265,7 +265,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
SSqlCmd *pCmd = &pSql->cmd;
assert(*pSql->self == pSql);
- pSql->pRpcCtx = NULL;
+ pSql->rpcRid = -1;
if (pObj->signature != pObj) {
tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
@@ -481,14 +481,25 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
int32_t vgIndex = pTableMetaInfo->vgroupIndex;
-
- SVgroupsInfo* pVgroupInfo = pTableMetaInfo->vgroupList;
- assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups);
+ if (pTableMetaInfo->pVgroupTables == NULL) {
+ SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList;
+ assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups);
+
+ pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId);
+ tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d", pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex);
+ } else {
+ int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables);
+ assert(vgIndex >= 0 && vgIndex < numOfVgroups);
+
+ SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex);
- pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId);
+ pRetrieveMsg->header.vgId = htonl(pTableIdList->vgInfo.vgId);
+ tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d", pSql, pTableIdList->vgInfo.vgId, vgIndex);
+ }
} else {
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId);
+ tscDebug("%p build fetch msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgroupInfo.vgId);
}
pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg);
@@ -538,11 +549,28 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);
int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo));
-
- size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
+
+ size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
int32_t exprSize = (int32_t)(sizeof(SSqlFuncMsg) * numOfExprs);
-
- return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + 4096;
+
+ int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0;
+
+ int32_t tableSerialize = 0;
+ STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+ if (pTableMetaInfo->pVgroupTables != NULL) {
+ size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables);
+
+ int32_t totalTables = 0;
+ for (int32_t i = 0; i < numOfGroups; ++i) {
+ SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i);
+ totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList);
+ }
+
+ tableSerialize = totalTables * sizeof(STableIdInfo);
+ }
+
+ return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize +
+ tableSerialize + 4096;
}
static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) {
@@ -552,7 +580,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) {
- SCMVgroupInfo* pVgroupInfo = NULL;
+ SVgroupInfo* pVgroupInfo = NULL;
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
int32_t index = pTableMetaInfo->vgroupIndex;
assert(index >= 0);
@@ -662,12 +690,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->limit = htobe64(pQueryInfo->limit.limit);
pQueryMsg->offset = htobe64(pQueryInfo->limit.offset);
pQueryMsg->numOfCols = htons((int16_t)taosArrayGetSize(pQueryInfo->colList));
- pQueryMsg->interval.interval = htobe64(pQueryInfo->interval.interval);
- pQueryMsg->interval.sliding = htobe64(pQueryInfo->interval.sliding);
+ pQueryMsg->interval.interval = htobe64(pQueryInfo->interval.interval);
+ pQueryMsg->interval.sliding = htobe64(pQueryInfo->interval.sliding);
pQueryMsg->interval.offset = htobe64(pQueryInfo->interval.offset);
pQueryMsg->interval.intervalUnit = pQueryInfo->interval.intervalUnit;
- pQueryMsg->interval.slidingUnit = pQueryInfo->interval.slidingUnit;
- pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit;
+ pQueryMsg->interval.slidingUnit = pQueryInfo->interval.slidingUnit;
+ pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit;
pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols);
pQueryMsg->numOfTags = htonl(numOfTags);
pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType);
@@ -846,36 +874,20 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
// compressed ts block
pQueryMsg->tsOffset = htonl((int32_t)(pMsg - pCmd->payload));
- int32_t tsLen = 0;
- int32_t numOfBlocks = 0;
if (pQueryInfo->tsBuf != NULL) {
- STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, pTableMetaInfo->vgroupIndex);
- assert(QUERY_IS_JOIN_QUERY(pQueryInfo->type) && pBlockInfo != NULL); // this query should not be sent
-
- // todo refactor
- if (fseek(pQueryInfo->tsBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
- int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f));
- tscError("%p: fseek failed: %s", pSql, tstrerror(code));
- return code;
- }
-
- size_t s = fread(pMsg, 1, pBlockInfo->compLen, pQueryInfo->tsBuf->f);
- if (s != pBlockInfo->compLen) {
- int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f));
- tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
+ // note: here used the index instead of actual vnode id.
+ int32_t vnodeIndex = pTableMetaInfo->vgroupIndex;
+ int32_t code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsLen, &pQueryMsg->tsNumOfBlocks);
+ if (code != TSDB_CODE_SUCCESS) {
return code;
}
- pMsg += pBlockInfo->compLen;
- tsLen = pBlockInfo->compLen;
- numOfBlocks = pBlockInfo->numOfBlocks;
- }
+ pMsg += pQueryMsg->tsLen;
- pQueryMsg->tsLen = htonl(tsLen);
- pQueryMsg->tsNumOfBlocks = htonl(numOfBlocks);
- if (pQueryInfo->tsBuf != NULL) {
pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder);
+ pQueryMsg->tsLen = htonl(pQueryMsg->tsLen);
+ pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks);
}
int32_t msgLen = (int32_t)(pMsg - pCmd->payload);
@@ -892,10 +904,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMCreateDbMsg);
+ pCmd->payloadLen = sizeof(SCreateDbMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DB;
- SCMCreateDbMsg *pCreateDbMsg = (SCMCreateDbMsg*)pCmd->payload;
+ SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload;
assert(pCmd->numOfClause == 1);
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
@@ -906,13 +918,13 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMCreateDnodeMsg);
+ pCmd->payloadLen = sizeof(SCreateDnodeMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMCreateDnodeMsg *pCreate = (SCMCreateDnodeMsg *)pCmd->payload;
+ SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload;
strncpy(pCreate->ep, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n);
pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DNODE;
@@ -922,13 +934,13 @@ int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMCreateAcctMsg);
+ pCmd->payloadLen = sizeof(SCreateAcctMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMCreateAcctMsg *pAlterMsg = (SCMCreateAcctMsg *)pCmd->payload;
+ SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload;
SStrToken *pName = &pInfo->pDCLInfo->user.user;
SStrToken *pPwd = &pInfo->pDCLInfo->user.passwd;
@@ -967,14 +979,14 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMCreateUserMsg);
+ pCmd->payloadLen = sizeof(SCreateUserMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMCreateUserMsg *pAlterMsg = (SCMCreateUserMsg*)pCmd->payload;
+ SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload;
SUserInfo *pUser = &pInfo->pDCLInfo->user;
strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n);
@@ -999,21 +1011,21 @@ int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMCfgDnodeMsg);
+ pCmd->payloadLen = sizeof(SCfgDnodeMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_CONFIG_DNODE;
return TSDB_CODE_SUCCESS;
}
int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMDropDbMsg);
+ pCmd->payloadLen = sizeof(SDropDbMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMDropDbMsg *pDropDbMsg = (SCMDropDbMsg*)pCmd->payload;
+ SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
tstrncpy(pDropDbMsg->db, pTableMetaInfo->name, sizeof(pDropDbMsg->db));
@@ -1043,13 +1055,13 @@ int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMDropDnodeMsg);
+ pCmd->payloadLen = sizeof(SDropDnodeMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload;
+ SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
tstrncpy(pDrop->ep, pTableMetaInfo->name, sizeof(pDrop->ep));
pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE;
@@ -1059,7 +1071,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMDropUserMsg);
+ pCmd->payloadLen = sizeof(SDropUserMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_USER;
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
@@ -1067,7 +1079,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload;
+ SDropUserMsg * pDropMsg = (SDropUserMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user));
@@ -1076,7 +1088,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMDropUserMsg);
+ pCmd->payloadLen = sizeof(SDropUserMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_ACCT;
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
@@ -1084,7 +1096,7 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload;
+ SDropUserMsg * pDropMsg = (SDropUserMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user));
@@ -1093,14 +1105,14 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMUseDbMsg);
+ pCmd->payloadLen = sizeof(SUseDbMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMUseDbMsg *pUseDbMsg = (SCMUseDbMsg*)pCmd->payload;
+ SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
strcpy(pUseDbMsg->db, pTableMetaInfo->name);
pCmd->msgType = TSDB_MSG_TYPE_CM_USE_DB;
@@ -1112,14 +1124,14 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
STscObj *pObj = pSql->pTscObj;
SSqlCmd *pCmd = &pSql->cmd;
pCmd->msgType = TSDB_MSG_TYPE_CM_SHOW;
- pCmd->payloadLen = sizeof(SCMShowMsg) + 100;
+ pCmd->payloadLen = sizeof(SShowMsg) + 100;
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMShowMsg *pShowMsg = (SCMShowMsg*)pCmd->payload;
+ SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
size_t nameLen = strlen(pTableMetaInfo->name);
@@ -1146,13 +1158,13 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pShowMsg->payloadLen = htons(pEpAddr->n);
}
- pCmd->payloadLen = sizeof(SCMShowMsg) + pShowMsg->payloadLen;
+ pCmd->payloadLen = sizeof(SShowMsg) + pShowMsg->payloadLen;
return TSDB_CODE_SUCCESS;
}
int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMKillQueryMsg);
+ pCmd->payloadLen = sizeof(SKillQueryMsg);
switch (pCmd->command) {
case TSDB_SQL_KILL_QUERY:
@@ -1264,8 +1276,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
- return minMsgSize() + sizeof(SCMAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) +
- TSDB_EXTRA_PAYLOAD_SIZE;
+ return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE;
}
int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
@@ -1284,7 +1295,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMAlterTableMsg *pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload;
+ SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload;
tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pAlterTableMsg->db);
strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name);
@@ -1333,10 +1344,10 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) {
int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
- pCmd->payloadLen = sizeof(SCMAlterDbMsg);
+ pCmd->payloadLen = sizeof(SAlterDbMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_ALTER_DB;
- SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload;
+ SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
tstrncpy(pAlterDbMsg->db, pTableMetaInfo->name, sizeof(pAlterDbMsg->db));
@@ -1461,14 +1472,14 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
STscObj *pObj = pSql->pTscObj;
SSqlCmd *pCmd = &pSql->cmd;
pCmd->msgType = TSDB_MSG_TYPE_CM_CONNECT;
- pCmd->payloadLen = sizeof(SCMConnectMsg);
+ pCmd->payloadLen = sizeof(SConnectMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMConnectMsg *pConnect = (SCMConnectMsg*)pCmd->payload;
+ SConnectMsg *pConnect = (SConnectMsg*)pCmd->payload;
// TODO refactor full_name
char *db; // ugly code to move the space
@@ -1490,11 +1501,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
- SCMTableInfoMsg* pInfoMsg = (SCMTableInfoMsg *)pCmd->payload;
+ STableInfoMsg *pInfoMsg = (STableInfoMsg *)pCmd->payload;
strcpy(pInfoMsg->tableId, pTableMetaInfo->name);
pInfoMsg->createFlag = htons(pSql->cmd.autoCreated ? 1 : 0);
- char* pMsg = (char*)pInfoMsg + sizeof(SCMTableInfoMsg);
+ char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg);
size_t len = htonl(pCmd->tagData.dataLen);
if (pSql->cmd.autoCreated) {
@@ -1513,7 +1524,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
/**
* multi table meta req pkg format:
- * | SMgmtHead | SCMMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ......
+ * | SMgmtHead | SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ......
* no used 4B
**/
int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
@@ -1531,16 +1542,16 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SMgmtHead *pMgmt = (SMgmtHead *)(pCmd->payload + tsRpcHeadSize);
memset(pMgmt->db, 0, TSDB_TABLE_FNAME_LEN); // server don't need the db
- SCMMultiTableInfoMsg *pInfoMsg = (SCMMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead));
+ SMultiTableInfoMsg *pInfoMsg = (SMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead));
pInfoMsg->numOfTables = htonl((int32_t)pCmd->count);
if (pCmd->payloadLen > 0) {
memcpy(pInfoMsg->tableIds, tmpData, pCmd->payloadLen);
}
- taosTFree(tmpData);
+ tfree(tmpData);
- pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SCMMultiTableInfoMsg);
+ pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SMultiTableInfoMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_TABLES_META;
assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize);
@@ -1585,12 +1596,12 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
char* pMsg = pCmd->payload;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
-
- SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pMsg;
+
+ SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg;
pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables);
- pMsg += sizeof(SCMSTableVgroupMsg);
-
- for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
+ pMsg += sizeof(SSTableVgroupMsg);
+
+ for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i);
size_t size = sizeof(pTableMetaInfo->name);
tstrncpy(pMsg, pTableMetaInfo->name, size);
@@ -1623,14 +1634,17 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
numOfStreams++;
}
- int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SCMHeartBeatMsg) + 100;
+ int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100;
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) {
pthread_mutex_unlock(&pObj->mutex);
- tscError("%p failed to malloc for heartbeat msg", pSql);
+ tscError("%p failed to create heartbeat msg", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
- SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload;
+ // TODO the expired hb and client can not be identified by server till now.
+ SHeartBeatMsg *pHeartbeat = (SHeartBeatMsg *)pCmd->payload;
+ tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer));
+
pHeartbeat->numOfQueries = numOfQueries;
pHeartbeat->numOfStreams = numOfStreams;
@@ -1704,8 +1718,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name,
strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000);
-
- // todo handle out of memory case
+
if (pTableMetaInfo->pTableMeta == NULL) {
free(pTableMeta);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -1719,7 +1732,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
/**
* multi table meta rsp pkg format:
- * | STaosRsp | ieType | SCMMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2
+ * | STaosRsp | ieType | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2
* |...... 1B 1B 4B
**/
int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
@@ -1736,9 +1749,9 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
rsp++;
- SCMMultiTableInfoMsg *pInfo = (SCMMultiTableInfoMsg *)rsp;
+ SMultiTableInfoMsg *pInfo = (SMultiTableInfoMsg *)rsp;
totalNum = htonl(pInfo->numOfTables);
- rsp += sizeof(SCMMultiTableInfoMsg);
+ rsp += sizeof(SMultiTableInfoMsg);
for (i = 0; i < totalNum; i++) {
SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp;
@@ -1830,10 +1843,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
SSqlRes* pRes = &pSql->res;
// NOTE: the order of several table must be preserved.
- SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pRes->pRsp;
+ SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp;
pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables);
- char* pMsg = pRes->pRsp + sizeof(SCMSTableVgroupRspMsg);
-
+ char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg);
+
// master sqlObj locates in param
SSqlObj* parent = pSql->param;
assert(parent != NULL);
@@ -1845,18 +1858,18 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
SVgroupsMsg * pVgroupMsg = (SVgroupsMsg *) pMsg;
pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups);
- size_t size = sizeof(SCMVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg);
+ size_t size = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg);
- size_t vgroupsz = sizeof(SCMVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo);
+ size_t vgroupsz = sizeof(SVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo);
pInfo->vgroupList = calloc(1, vgroupsz);
assert(pInfo->vgroupList != NULL);
pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups;
for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) {
//just init, no need to lock
- SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j];
+ SVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j];
- SCMVgroupMsg *vmsg = &pVgroupMsg->vgroups[j];
+ SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j];
pVgroups->vgId = htonl(vmsg->vgId);
pVgroups->numOfEps = vmsg->numOfEps;
@@ -1878,10 +1891,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
* current process do not use the cache at all
*/
int tscProcessShowRsp(SSqlObj *pSql) {
- STableMetaMsg * pMetaMsg;
- SCMShowRsp *pShow;
- SSchema * pSchema;
- char key[20];
+ STableMetaMsg *pMetaMsg;
+ SShowRsp * pShow;
+ SSchema * pSchema;
+ char key[20];
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
@@ -1890,7 +1903,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
- pShow = (SCMShowRsp *)pRes->pRsp;
+ pShow = (SShowRsp *)pRes->pRsp;
pShow->qhandle = htobe64(pShow->qhandle);
pRes->qhandle = pShow->qhandle;
@@ -1943,7 +1956,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput;
tscFieldInfoUpdateOffset(pQueryInfo);
- taosTFree(pTableMeta);
+ tfree(pTableMeta);
return 0;
}
@@ -1968,7 +1981,7 @@ static void createHBObj(STscObj* pObj) {
pSql->cmd.command = pQueryInfo->command;
if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) {
- taosTFree(pSql);
+ tfree(pSql);
return;
}
@@ -1983,11 +1996,12 @@ static void createHBObj(STscObj* pObj) {
}
int tscProcessConnectRsp(SSqlObj *pSql) {
- char temp[TSDB_TABLE_FNAME_LEN * 2];
STscObj *pObj = pSql->pTscObj;
SSqlRes *pRes = &pSql->res;
- SCMConnectRsp *pConnect = (SCMConnectRsp *)pRes->pRsp;
+ char temp[TSDB_TABLE_FNAME_LEN * 2] = {0};
+
+ SConnectRsp *pConnect = (SConnectRsp *)pRes->pRsp;
tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId)); // copy acctId from response
int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db);
@@ -2005,7 +2019,9 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
pObj->connId = htonl(pConnect->connId);
createHBObj(pObj);
- taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer);
+
+ //launch a timer to send heartbeat to maintain the connection and send status to mnode
+ taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pObj->rid, tscTmr, &pObj->pTimer);
return 0;
}
@@ -2271,7 +2287,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i);
STableMeta *pTableMeta = taosCacheAcquireByData(tscMetaCache, pMInfo->pTableMeta);
- tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList);
+ tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables);
}
if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) {
diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index 8cac9b3398a0bc4569b52acc7858bbc1e50de542..5f8a2eb6b764983a529ff58ffb84dc47447c281b 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -161,6 +161,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
registerSqlObj(pSql);
tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
+ pObj->rid = taosAddRef(tscRefId, pObj);
return pSql;
}
@@ -278,9 +279,9 @@ void taos_close(TAOS *taos) {
SSqlObj* pHb = pObj->pHb;
if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
- if (pHb->pRpcCtx != NULL) { // wait for rsp from dnode
- rpcCancelRequest(pHb->pRpcCtx);
- pHb->pRpcCtx = NULL;
+ if (pHb->rpcRid > 0) { // wait for rsp from dnode
+ rpcCancelRequest(pHb->rpcRid);
+ pHb->rpcRid = -1;
}
tscDebug("%p HB is freed", pHb);
@@ -296,7 +297,8 @@ void taos_close(TAOS *taos) {
}
tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
- tscCloseTscObj(pObj);
+
+ taosRemoveRef(tscRefId, pObj->rid);
}
void waitForQueryRsp(void *param, TAOS_RES *tres, int code) {
@@ -564,7 +566,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
pRes->rspType = 0;
pSql->subState.numOfSub = 0;
- taosTFree(pSql->pSubs);
+ tfree(pSql->pSubs);
assert(pSql->fp == NULL);
@@ -746,9 +748,9 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
assert(pSubObj->self == (SSqlObj**) p);
pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
- if (pSubObj->pRpcCtx != NULL) {
- rpcCancelRequest(pSubObj->pRpcCtx);
- pSubObj->pRpcCtx = NULL;
+ if (pSubObj->rpcRid > 0) {
+ rpcCancelRequest(pSubObj->rpcRid);
+ pSubObj->rpcRid = -1;
}
tscQueueAsyncRes(pSubObj);
@@ -773,7 +775,7 @@ void taos_stop_query(TAOS_RES *res) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
- assert(pSql->pRpcCtx == NULL);
+ assert(pSql->rpcRid <= 0);
tscKillSTableQuery(pSql);
} else {
if (pSql->cmd.command < TSDB_SQL_LOCAL) {
@@ -782,9 +784,9 @@ void taos_stop_query(TAOS_RES *res) {
* reset and freed in the processMsgFromServer function, and causes the invalid
* write problem for rpcCancelRequest.
*/
- if (pSql->pRpcCtx != NULL) {
- rpcCancelRequest(pSql->pRpcCtx);
- pSql->pRpcCtx = NULL;
+ if (pSql->rpcRid > 0) {
+ rpcCancelRequest(pSql->rpcRid);
+ pSql->rpcRid = -1;
}
tscQueueAsyncRes(pSql);
@@ -892,7 +894,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
if (sqlLen > tsMaxSQLStringLen) {
tscError("%p sql too long", pSql);
pRes->code = TSDB_CODE_TSC_INVALID_SQL;
- taosTFree(pSql);
+ tfree(pSql);
return pRes->code;
}
@@ -901,7 +903,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscError("%p failed to malloc sql string buffer", pSql);
tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(pSql), pObj);
- taosTFree(pSql);
+ tfree(pSql);
return pRes->code;
}
diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c
index 0f67911bbea992503979e5de019e4e10d3bf3c14..68c3bcae165050863cc4bf9c92a1510581531c3a 100644
--- a/src/client/src/tscStream.c
+++ b/src/client/src/tscStream.c
@@ -273,7 +273,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false);
tscFreeSqlResult(pSql);
- taosTFree(pSql->pSubs);
+ tfree(pSql->pSubs);
pSql->subState.numOfSub = 0;
pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList);
tscSetNextLaunchTimer(pStream, pSql);
@@ -617,6 +617,6 @@ void taos_close_stream(TAOS_STREAM *handle) {
pStream->pSql = NULL;
taos_free_result(pSql);
- taosTFree(pStream);
+ tfree(pStream);
}
}
diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c
index 6b615c3a9b14f97f57013f476b60acc0db53db53..eb32e2490a3aa1a363be632a31e051ad2ca7a6a8 100644
--- a/src/client/src/tscSubquery.c
+++ b/src/client/src/tscSubquery.c
@@ -23,7 +23,6 @@
#include "tscSubquery.h"
#include "tschemautil.h"
#include "tsclient.h"
-#include "tscSubquery.h"
typedef struct SInsertSupporter {
SSqlObj* pSql;
@@ -33,11 +32,26 @@ typedef struct SInsertSupporter {
static void freeJoinSubqueryObj(SSqlObj* pSql);
static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql);
-static bool tsCompare(int32_t order, int64_t left, int64_t right) {
+static int32_t tsCompare(int32_t order, int64_t left, int64_t right) {
+ if (left == right) {
+ return 0;
+ }
+
if (order == TSDB_ORDER_ASC) {
- return left < right;
+ return left < right? -1:1;
} else {
- return left > right;
+ return left > right? -1:1;
+ }
+}
+
+static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) {
+ while (tsBufNextPos(pTSBuf)) {
+ STSElem el1 = tsBufGetElem(pTSBuf);
+
+ int32_t res = tVariantCompare(el1.tag, tag1);
+ if (res != 0) { // it is a record with new tag
+ return;
+ }
}
}
@@ -52,13 +66,15 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
SLimitVal* pLimit = &pQueryInfo->limit;
int32_t order = pQueryInfo->order.order;
-
+
SQueryInfo* pSubQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[0]->cmd, 0);
SQueryInfo* pSubQueryInfo2 = tscGetQueryInfoDetail(&pSql->pSubs[1]->cmd, 0);
-
+
pSubQueryInfo1->tsBuf = output1;
pSubQueryInfo2->tsBuf = output2;
+ TSKEY st = taosGetTimestampUs();
+
// no result generated, return directly
if (pSupporter1->pTSBuf == NULL || pSupporter2->pTSBuf == NULL) {
tscDebug("%p at least one ts-comp is empty, 0 for secondary query after ts blocks intersecting", pSql);
@@ -87,58 +103,74 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
int64_t numOfInput1 = 1;
int64_t numOfInput2 = 1;
- while (1) {
- STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf);
- STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf);
+ while(1) {
+ STSElem elem = tsBufGetElem(pSupporter1->pTSBuf);
-#ifdef _DEBUG_VIEW
- tscInfo("%" PRId64 ", tags:%"PRId64" \t %" PRId64 ", tags:%"PRId64, elem1.ts, elem1.tag.i64Key, elem2.ts, elem2.tag.i64Key);
-#endif
+ // no data in pSupporter1 anymore, jump out of loop
+ if (!tsBufIsValidElem(&elem)) {
+ break;
+ }
- int32_t res = tVariantCompare(&elem1.tag, &elem2.tag);
- if (res == -1 || (res == 0 && tsCompare(order, elem1.ts, elem2.ts))) {
- if (!tsBufNextPos(pSupporter1->pTSBuf)) {
- break;
- }
+ // find the data in supporter2 with the same tag value
+ STSElem e2 = tsBufFindElemStartPosByTag(pSupporter2->pTSBuf, elem.tag);
- numOfInput1++;
- } else if ((res > 0) || (res == 0 && tsCompare(order, elem2.ts, elem1.ts))) {
- if (!tsBufNextPos(pSupporter2->pTSBuf)) {
- break;
- }
+ /**
+ * there are elements in pSupporter2 with the same tag, continue
+ */
+ tVariant tag1 = {0};
+ tVariantAssign(&tag1, elem.tag);
- numOfInput2++;
- } else {
- /*
- * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the
- * final results which is acquired after the secondry merge of in the client.
- */
- if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
- if (win->skey > elem1.ts) {
- win->skey = elem1.ts;
+ if (tsBufIsValidElem(&e2)) {
+ while (1) {
+ STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf);
+ STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf);
+
+ // data with current are exhausted
+ if (!tsBufIsValidElem(&elem1) || tVariantCompare(elem1.tag, &tag1) != 0) {
+ break;
}
-
- if (win->ekey < elem1.ts) {
- win->ekey = elem1.ts;
+
+ if (!tsBufIsValidElem(&elem2) || tVariantCompare(elem2.tag, &tag1) != 0) { // ignore all records with the same tag
+ skipRemainValue(pSupporter1->pTSBuf, &tag1);
+ break;
}
-
- tsBufAppend(output1, elem1.vnode, &elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts));
- tsBufAppend(output2, elem2.vnode, &elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts));
- } else {
- pLimit->offset -= 1;
- }
- if (!tsBufNextPos(pSupporter1->pTSBuf)) {
- break;
- }
+ /*
+ * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the
+ * final results which is acquired after the secondary merge of in the client.
+ */
+ int32_t re = tsCompare(order, elem1.ts, elem2.ts);
+ if (re < 0) {
+ tsBufNextPos(pSupporter1->pTSBuf);
+ numOfInput1++;
+ } else if (re > 0) {
+ tsBufNextPos(pSupporter2->pTSBuf);
+ numOfInput2++;
+ } else {
+ if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
+ if (win->skey > elem1.ts) {
+ win->skey = elem1.ts;
+ }
+
+ if (win->ekey < elem1.ts) {
+ win->ekey = elem1.ts;
+ }
+
+ tsBufAppend(output1, elem1.id, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts));
+ tsBufAppend(output2, elem2.id, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts));
+ } else {
+ pLimit->offset -= 1;//offset apply to projection?
+ }
- numOfInput1++;
+ tsBufNextPos(pSupporter1->pTSBuf);
+ numOfInput1++;
- if (!tsBufNextPos(pSupporter2->pTSBuf)) {
- break;
+ tsBufNextPos(pSupporter2->pTSBuf);
+ numOfInput2++;
+ }
}
-
- numOfInput2++;
+ } else { // no data in pSupporter2, ignore current data in pSupporter2
+ skipRemainValue(pSupporter1->pTSBuf, &tag1);
}
}
@@ -158,9 +190,11 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
tsBufDestroy(pSupporter1->pTSBuf);
tsBufDestroy(pSupporter2->pTSBuf);
- tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " for secondary query after ts blocks "
- "intersecting, skey:%" PRId64 ", ekey:%" PRId64, pSql, numOfInput1, numOfInput2, output1->numOfTotal,
- win->skey, win->ekey);
+ TSKEY et = taosGetTimestampUs();
+ tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks "
+ "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elapsed time:%" PRId64 " us",
+ pSql, numOfInput1, numOfInput2, output1->numOfTotal, output1->numOfGroups, win->skey, win->ekey,
+ tsBufGetNumOfGroup(output1), et - st);
return output1->numOfTotal;
}
@@ -216,7 +250,12 @@ static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) {
pSupporter->f = NULL;
}
- taosTFree(pSupporter->pIdTagList);
+ if (pSupporter->pVgroupTables != NULL) {
+ taosArrayDestroy(pSupporter->pVgroupTables);
+ pSupporter->pVgroupTables = NULL;
+ }
+
+ tfree(pSupporter->pIdTagList);
tscTagCondRelease(&pSupporter->tagCond);
free(pSupporter);
}
@@ -240,6 +279,68 @@ static UNUSED_FUNC bool needSecondaryQuery(SQueryInfo* pQueryInfo) {
return false;
}
+static void filterVgroupTables(SQueryInfo* pQueryInfo, SArray* pVgroupTables) {
+ int32_t num = 0;
+ int32_t* list = NULL;
+ tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list);
+
+ // The virtual node, of which all tables are disqualified after the timestamp intersection,
+ // is removed to avoid next stage query.
+ // TODO: If tables from some vnodes are not qualified for next stage query, discard them.
+ for (int32_t k = 0; k < taosArrayGetSize(pVgroupTables);) {
+ SVgroupTableInfo* p = taosArrayGet(pVgroupTables, k);
+
+ bool found = false;
+ for (int32_t f = 0; f < num; ++f) {
+ if (p->vgInfo.vgId == list[f]) {
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ tscRemoveVgroupTableGroup(pVgroupTables, k);
+ } else {
+ k++;
+ }
+ }
+
+ assert(taosArrayGetSize(pVgroupTables) > 0);
+ TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
+
+ tfree(list);
+}
+
+static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupTables) {
+ int32_t num = 0;
+ int32_t* list = NULL;
+ tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list);
+
+ size_t numOfGroups = taosArrayGetSize(pVgroupTables);
+
+ SArray* pNew = taosArrayInit(num, sizeof(SVgroupTableInfo));
+
+ SVgroupTableInfo info;
+ for (int32_t i = 0; i < num; ++i) {
+ int32_t vnodeId = list[i];
+
+ for (int32_t j = 0; j < numOfGroups; ++j) {
+ SVgroupTableInfo* p1 = taosArrayGet(pVgroupTables, j);
+ if (p1->vgInfo.vgId == vnodeId) {
+ tscVgroupTableCopy(&info, p1);
+ break;
+ }
+ }
+
+ taosArrayPush(pNew, &info);
+ }
+
+ tfree(list);
+ TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
+
+ return pNew;
+}
+
/*
* launch secondary stage query to fetch the result that contains timestamp in set
*/
@@ -305,7 +406,6 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
// set the second stage sub query for join process
TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE);
-
memcpy(&pQueryInfo->interval, &pSupporter->interval, sizeof(pQueryInfo->interval));
tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond);
@@ -313,24 +413,27 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
pQueryInfo->colList = pSupporter->colList;
pQueryInfo->exprList = pSupporter->exprList;
pQueryInfo->fieldsInfo = pSupporter->fieldsInfo;
+ pQueryInfo->groupbyExpr = pSupporter->groupInfo;
+
+ assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1);
+
+ tscFieldInfoUpdateOffset(pQueryInfo);
+
+ STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+ pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables;
pSupporter->exprList = NULL;
pSupporter->colList = NULL;
+ pSupporter->pVgroupTables = NULL;
memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo));
-
- SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
- assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
-
- tscFieldInfoUpdateOffset(pNewQueryInfo);
-
- STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0);
-
+ memset(&pSupporter->groupInfo, 0, sizeof(SSqlGroupbyExpr));
+
/*
* When handling the projection query, the offset value will be modified for table-table join, which is changed
* during the timestamp intersection.
*/
pSupporter->limit = pQueryInfo->limit;
- pNewQueryInfo->limit = pSupporter->limit;
+ pQueryInfo->limit = pSupporter->limit;
SColumnIndex index = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
SSchema* s = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, 0);
@@ -345,7 +448,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
tscAddSpecialColumnForSelect(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL);
tscPrintSelectClause(pNew, 0);
- tscFieldInfoUpdateOffset(pNewQueryInfo);
+ tscFieldInfoUpdateOffset(pQueryInfo);
pExpr = tscSqlExprGet(pQueryInfo, 0);
}
@@ -356,14 +459,25 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
// set the tag column id for executor to extract correct tag value
- pExpr->param[0].i64Key = colId;
+ pExpr->param[0] = (tVariant) {.i64Key = colId, .nType = TSDB_DATA_TYPE_BIGINT, .nLen = sizeof(int64_t)};
pExpr->numOfParams = 1;
}
- size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList);
+ if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+ assert(pTableMetaInfo->pVgroupTables != NULL);
+ if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
+ SArray* p = buildVgroupTableByResult(pQueryInfo, pTableMetaInfo->pVgroupTables);
+ tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables);
+ pTableMetaInfo->pVgroupTables = p;
+ } else {
+ filterVgroupTables(pQueryInfo, pTableMetaInfo->pVgroupTables);
+ }
+ }
+
+ size_t numOfCols = taosArrayGetSize(pQueryInfo->colList);
tscDebug("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s",
- pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, taosArrayGetSize(pNewQueryInfo->exprList),
- numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name);
+ pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, taosArrayGetSize(pQueryInfo->exprList),
+ numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name);
}
//prepare the subqueries object failed, abort
@@ -409,7 +523,7 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) {
assert(pSqlObj->subState.numOfRemain > 0);
if (atomic_sub_fetch_32(&pSqlObj->subState.numOfRemain, 1) <= 0) {
- tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code);
+ tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code));
freeJoinSubqueryObj(pSqlObj);
}
}
@@ -418,20 +532,40 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) {
static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) {
assert(pQueryInfo->window.skey <= win->skey && pQueryInfo->window.ekey >= win->ekey);
pQueryInfo->window = *win;
+
+
}
-int32_t tscCompareTidTags(const void* p1, const void* p2) {
- const STidTags* t1 = (const STidTags*) varDataVal(p1);
- const STidTags* t2 = (const STidTags*) varDataVal(p2);
+int32_t tidTagsCompar(const void* p1, const void* p2) {
+ const STidTags* t1 = (const STidTags*) (p1);
+ const STidTags* t2 = (const STidTags*) (p2);
if (t1->vgId != t2->vgId) {
return (t1->vgId > t2->vgId) ? 1 : -1;
}
- if (t1->tid != t2->tid) {
- return (t1->tid > t2->tid) ? 1 : -1;
+ tstr* tag1 = (tstr*) t1->tag;
+ tstr* tag2 = (tstr*) t2->tag;
+
+ if (tag1->len != tag2->len) {
+ return (tag1->len > tag2->len)? 1: -1;
+ }
+
+ return strncmp(tag1->data, tag2->data, tag1->len);
+}
+
+int32_t tagValCompar(const void* p1, const void* p2) {
+ const STidTags* t1 = (const STidTags*) varDataVal(p1);
+ const STidTags* t2 = (const STidTags*) varDataVal(p2);
+
+ tstr* tag1 = (tstr*) t1->tag;
+ tstr* tag2 = (tstr*) t2->tag;
+
+ if (tag1->len != tag2->len) {
+ return (tag1->len > tag2->len)? 1: -1;
}
- return 0;
+
+ return memcmp(tag1->data, tag2->data, tag1->len);
}
void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) {
@@ -449,7 +583,7 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr
SVgroupTableInfo info = {{0}};
for (int32_t m = 0; m < pvg->numOfVgroups; ++m) {
if (tt->vgId == pvg->vgroups[m].vgId) {
- tscSCMVgroupInfoCopy(&info.vgInfo, &pvg->vgroups[m]);
+ tscSVgroupInfoCopy(&info.vgInfo, &pvg->vgroups[m]);
break;
}
}
@@ -457,27 +591,40 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr
vgTables = taosArrayInit(4, sizeof(STableIdInfo));
info.itemList = vgTables;
+
+ if (taosArrayGetSize(result) > 0) {
+ SVgroupTableInfo* prevGroup = taosArrayGet(result, taosArrayGetSize(result) - 1);
+ tscDebug("%p vgId:%d, tables:%"PRId64, pSql, prevGroup->vgInfo.vgId, taosArrayGetSize(prevGroup->itemList));
+ }
+
taosArrayPush(result, &info);
}
- tscDebug("%p tid:%d, uid:%"PRIu64",vgId:%d added for vnode query", pSql, tt->tid, tt->uid, tt->vgId)
STableIdInfo item = {.uid = tt->uid, .tid = tt->tid, .key = INT64_MIN};
taosArrayPush(vgTables, &item);
+
+ tscTrace("%p tid:%d, uid:%"PRIu64",vgId:%d added", pSql, tt->tid, tt->uid, tt->vgId);
prev = tt;
}
pTableMetaInfo->pVgroupTables = result;
pTableMetaInfo->vgroupIndex = 0;
+
+ if (taosArrayGetSize(result) > 0) {
+ SVgroupTableInfo* g = taosArrayGet(result, taosArrayGetSize(result) - 1);
+ tscDebug("%p vgId:%d, tables:%"PRId64, pSql, g->vgInfo.vgId, taosArrayGetSize(g->itemList));
+ }
}
static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) {
SSqlCmd* pCmd = &pSql->cmd;
tscClearSubqueryInfo(pCmd);
tscFreeSqlResult(pSql);
-
+
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
+ assert(pQueryInfo->numOfTables == 1);
+
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-
tscInitQueryInfo(pQueryInfo);
TSDB_QUERY_CLEAR_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY);
@@ -524,13 +671,7 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj*
tscProcessSql(pSql);
}
-static bool checkForDuplicateTagVal(SQueryInfo* pQueryInfo, SJoinSupporter* p1, SSqlObj* pPSqlObj) {
- STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-
- SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);// todo: tags mismatch, tags not completed
- SColumn *pCol = taosArrayGetP(pTableMetaInfo->tagColList, 0);
- SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex];
-
+static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSqlObj* pPSqlObj) {
for(int32_t i = 1; i < p1->num; ++i) {
STidTags* prev = (STidTags*) varDataVal(p1->pIdTagList + (i - 1) * p1->tagSize);
STidTags* p = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize);
@@ -547,24 +688,26 @@ static bool checkForDuplicateTagVal(SQueryInfo* pQueryInfo, SJoinSupporter* p1,
}
static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray** s1, SArray** s2) {
- tscDebug("%p all subqueries retrieve complete, do tags match", pParentSql);
-
SJoinSupporter* p1 = pParentSql->pSubs[0]->param;
SJoinSupporter* p2 = pParentSql->pSubs[1]->param;
- qsort(p1->pIdTagList, p1->num, p1->tagSize, tscCompareTidTags);
- qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags);
+ tscDebug("%p all subquery retrieve complete, do tags match, %d, %d", pParentSql, p1->num, p2->num);
+
+ // sort according to the tag value
+ qsort(p1->pIdTagList, p1->num, p1->tagSize, tagValCompar);
+ qsort(p2->pIdTagList, p2->num, p2->tagSize, tagValCompar);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid);
- SSchema* pColSchema = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
+ SSchema* pColSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
// int16_t for padding
- *s1 = taosArrayInit(p1->num, p1->tagSize - sizeof(int16_t));
- *s2 = taosArrayInit(p2->num, p2->tagSize - sizeof(int16_t));
+ int32_t size = p1->tagSize - sizeof(int16_t);
+ *s1 = taosArrayInit(p1->num, size);
+ *s2 = taosArrayInit(p2->num, size);
- if (!(checkForDuplicateTagVal(pQueryInfo, p1, pParentSql) && checkForDuplicateTagVal(pQueryInfo, p2, pParentSql))) {
+ if (!(checkForDuplicateTagVal(pColSchema, p1, pParentSql) && checkForDuplicateTagVal(pColSchema, p2, pParentSql))) {
return TSDB_CODE_QRY_DUP_JOIN_KEY;
}
@@ -590,6 +733,27 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar
}
}
+ // reorganize the tid-tag value according to both the vgroup id and tag values
+ // sort according to the tag value
+ size_t t1 = taosArrayGetSize(*s1);
+ size_t t2 = taosArrayGetSize(*s2);
+
+ qsort((*s1)->pData, t1, size, tidTagsCompar);
+ qsort((*s2)->pData, t2, size, tidTagsCompar);
+
+#if 0
+ for(int32_t k = 0; k < t1; ++k) {
+ STidTags* p = (*s1)->pData + size * k;
+ printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data);
+ }
+
+ for(int32_t k = 0; k < t1; ++k) {
+ STidTags* p = (*s2)->pData + size * k;
+ printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data);
+ }
+#endif
+
+ tscDebug("%p tags match complete, result: %"PRId64", %"PRId64, pParentSql, t1, t2);
return TSDB_CODE_SUCCESS;
}
@@ -689,11 +853,15 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
}
if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) { // no results,return.
+ assert(pParentSql->fp != tscJoinQueryCallback);
+
tscDebug("%p tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", pParentSql);
freeJoinSubqueryObj(pParentSql);
// set no result command
pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
+ assert(pParentSql->fp != tscJoinQueryCallback);
+
(*pParentSql->fp)(pParentSql->param, pParentSql, 0);
} else {
// proceed to for ts_comp query
@@ -708,6 +876,12 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
STableMetaInfo* pTableMetaInfo2 = tscGetMetaInfo(pQueryInfo2, 0);
tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo2, s2);
+ SSqlObj* psub1 = pParentSql->pSubs[0];
+ ((SJoinSupporter*)psub1->param)->pVgroupTables = tscVgroupTableInfoClone(pTableMetaInfo1->pVgroupTables);
+
+ SSqlObj* psub2 = pParentSql->pSubs[1];
+ ((SJoinSupporter*)psub2->param)->pVgroupTables = tscVgroupTableInfoClone(pTableMetaInfo2->pVgroupTables);
+
pParentSql->subState.numOfSub = 2;
pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub;
@@ -766,9 +940,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
pSupporter->pTSBuf = pBuf;
} else {
assert(pQueryInfo->numOfTables == 1); // for subquery, only one
- STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-
- tsBufMerge(pSupporter->pTSBuf, pBuf, pTableMetaInfo->vgroupIndex);
+ tsBufMerge(pSupporter->pTSBuf, pBuf);
tsBufDestroy(pBuf);
}
@@ -835,6 +1007,8 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
// launch the query the retrieve actual results from vnode along with the filtered timestamp
SQueryInfo* pPQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex);
updateQueryTimeRange(pPQueryInfo, &win);
+
+ //update the vgroup that involved in real data query
tscLaunchRealSubqueries(pParentSql);
}
@@ -868,20 +1042,28 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
assert(pQueryInfo->numOfTables == 1);
// for projection query, need to try next vnode if current vnode is exhausted
- if ((++pTableMetaInfo->vgroupIndex) < pTableMetaInfo->vgroupList->numOfVgroups) {
- pState->numOfRemain = 1;
- pState->numOfSub = 1;
+ int32_t numOfVgroups = 0; // TODO refactor
+ if (pTableMetaInfo->pVgroupTables != NULL) {
+ numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables);
+ } else {
+ numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
+ }
+ if ((++pTableMetaInfo->vgroupIndex) < numOfVgroups) {
+ tscDebug("%p no result in current vnode anymore, try next vnode, vgIndex:%d", pSql, pTableMetaInfo->vgroupIndex);
pSql->cmd.command = TSDB_SQL_SELECT;
pSql->fp = tscJoinQueryCallback;
- tscProcessSql(pSql);
+ tscProcessSql(pSql);
return;
+ } else {
+ tscDebug("%p no result in current subquery anymore", pSql);
}
}
- if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) {
- tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pParentSql->subState.numOfRemain, pState->numOfSub);
+ assert(pState->numOfRemain > 0);
+ if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
+ tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pState->numOfRemain, pState->numOfSub);
return;
}
@@ -893,62 +1075,66 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
}
// update the records for each subquery in parent sql object.
+ bool stableQuery = tscIsTwoStageSTableQuery(pQueryInfo, 0);
for (int32_t i = 0; i < pState->numOfSub; ++i) {
if (pParentSql->pSubs[i] == NULL) {
+ tscDebug("%p %p sub:%d not retrieve data", pParentSql, NULL, i);
continue;
}
SSqlRes* pRes1 = &pParentSql->pSubs[i]->res;
- pRes1->numOfClauseTotal += pRes1->numOfRows;
- }
- // data has retrieved to client, build the join results
- tscBuildResFromSubqueries(pParentSql);
-}
-
-static SJoinSupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t numOfFetch) {
- int32_t notInvolved = 0;
- SJoinSupporter* pSupporter = NULL;
- SSubqueryState* pState = &pSql->subState;
-
- for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
- if (pSql->pSubs[i] == NULL) {
- notInvolved++;
+ if (pRes1->row > 0 && pRes1->numOfRows > 0) {
+ tscDebug("%p sub:%p index:%d numOfRows:%"PRId64" total:%"PRId64 " (not retrieve)", pParentSql, pParentSql->pSubs[i], i,
+ pRes1->numOfRows, pRes1->numOfTotal);
+ assert(pRes1->row < pRes1->numOfRows);
} else {
- pSupporter = (SJoinSupporter*)pSql->pSubs[i]->param;
+ if (!stableQuery) {
+ pRes1->numOfClauseTotal += pRes1->numOfRows;
+ }
+
+ tscDebug("%p sub:%p index:%d numOfRows:%"PRId64" total:%"PRId64, pParentSql, pParentSql->pSubs[i], i,
+ pRes1->numOfRows, pRes1->numOfTotal);
}
}
-
- pState->numOfRemain = numOfFetch;
- return pSupporter;
+
+ // data has retrieved to client, build the join results
+ tscBuildResFromSubqueries(pParentSql);
}
-void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
+void tscFetchDatablockForSubquery(SSqlObj* pSql) {
assert(pSql->subState.numOfSub >= 1);
int32_t numOfFetch = 0;
- bool hasData = true;
+ bool hasData = true;
+ bool reachLimit = false;
+
+ // if the subquery is NULL, it does not involved in the final result generation
for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
- // if the subquery is NULL, it does not involved in the final result generation
SSqlObj* pSub = pSql->pSubs[i];
if (pSub == NULL) {
continue;
}
-
+
SSqlRes *pRes = &pSub->res;
+
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSub->cmd, 0);
if (!tscHasReachLimitation(pQueryInfo, pRes)) {
if (pRes->row >= pRes->numOfRows) {
+ // no data left in current result buffer
hasData = false;
+ // The current query is completed for the active vnode, try next vnode if exists
+ // If it is completed, no need to fetch anymore.
if (!pRes->completed) {
numOfFetch++;
}
}
} else { // has reach the limitation, no data anymore
if (pRes->row >= pRes->numOfRows) {
- hasData = false;
+ reachLimit = true;
+ hasData = false;
break;
}
}
@@ -958,29 +1144,113 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
if (hasData) {
tscBuildResFromSubqueries(pSql);
return;
- } else if (numOfFetch <= 0) {
+ }
+
+ // If at least one subquery is completed in current vnode, try the next vnode in case of multi-vnode
+ // super table projection query.
+ if (reachLimit) {
pSql->res.completed = true;
freeJoinSubqueryObj(pSql);
-
+
if (pSql->res.code == TSDB_CODE_SUCCESS) {
(*pSql->fp)(pSql->param, pSql, 0);
} else {
tscQueueAsyncRes(pSql);
}
-
+
+ return;
+ }
+
+ if (numOfFetch <= 0) {
+ bool tryNextVnode = false;
+
+ bool orderedPrjQuery = false;
+ for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
+ SSqlObj* pSub = pSql->pSubs[i];
+ if (pSub == NULL) {
+ continue;
+ }
+
+ SQueryInfo* p = tscGetQueryInfoDetail(&pSub->cmd, 0);
+ orderedPrjQuery = tscNonOrderedProjectionQueryOnSTable(p, 0);
+ if (orderedPrjQuery) {
+ break;
+ }
+ }
+
+ // get the number of subquery that need to retrieve the next vnode.
+ if (orderedPrjQuery) {
+ for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
+ SSqlObj* pSub = pSql->pSubs[i];
+ if (pSub != NULL && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) {
+ pSql->subState.numOfRemain++;
+ }
+ }
+ }
+
+ for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
+ SSqlObj* pSub = pSql->pSubs[i];
+ if (pSub == NULL) {
+ continue;
+ }
+
+ SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSub->cmd, 0);
+
+ if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && pSub->res.row >= pSub->res.numOfRows &&
+ pSub->res.completed) {
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+ assert(pQueryInfo->numOfTables == 1);
+
+ // for projection query, need to try next vnode if current vnode is exhausted
+ int32_t numOfVgroups = 0; // TODO refactor
+ if (pTableMetaInfo->pVgroupTables != NULL) {
+ numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables);
+ } else {
+ numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
+ }
+
+ if ((++pTableMetaInfo->vgroupIndex) < numOfVgroups) {
+ tscDebug("%p no result in current vnode anymore, try next vnode, vgIndex:%d", pSub,
+ pTableMetaInfo->vgroupIndex);
+ pSub->cmd.command = TSDB_SQL_SELECT;
+ pSub->fp = tscJoinQueryCallback;
+
+ tscProcessSql(pSub);
+ tryNextVnode = true;
+ } else {
+ tscDebug("%p no result in current subquery anymore", pSub);
+ }
+ }
+ }
+
+ if (tryNextVnode) {
+ return;
+ }
+
+ pSql->res.completed = true;
+ freeJoinSubqueryObj(pSql);
+
+ if (pSql->res.code == TSDB_CODE_SUCCESS) {
+ (*pSql->fp)(pSql->param, pSql, 0);
+ } else {
+ tscQueueAsyncRes(pSql);
+ }
+
return;
}
// TODO multi-vnode retrieve for projection query with limitation has bugs, since the global limiation is not handled
+ // retrieve data from current vnode.
tscDebug("%p retrieve data from %d subqueries", pSql, numOfFetch);
- SJoinSupporter* pSupporter = tscUpdateSubqueryStatus(pSql, numOfFetch);
-
+ SJoinSupporter* pSupporter = NULL;
+ pSql->subState.numOfRemain = numOfFetch;
+
for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
SSqlObj* pSql1 = pSql->pSubs[i];
if (pSql1 == NULL) {
continue;
}
-
+
SSqlRes* pRes1 = &pSql1->res;
SSqlCmd* pCmd1 = &pSql1->cmd;
@@ -1013,7 +1283,6 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) {
SSqlCmd* pCmd = &pSql->cmd;
SSqlRes* pRes = &pSql->res;
- tscDebug("%p all subquery response, retrieve data for subclause:%d", pSql, pCmd->clauseIndex);
// the column transfer support struct has been built
if (pRes->pColumnIndex != NULL) {
@@ -1109,21 +1378,23 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
return;
}
- // wait for the other subqueries response from vnode
- if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) {
- return;
+
+ STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+
+ // In case of consequence query from other vnode, do not wait for other query response here.
+ if (!(pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0))) {
+ if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) {
+ return;
+ }
}
tscSetupOutputColumnIndex(pParentSql);
- STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
/**
* if the query is a continue query (vgroupIndex > 0 for projection query) for next vnode, do the retrieval of
* data instead of returning to its invoker
*/
if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
- pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub; // reset the record value
-
pSql->fp = joinRetrieveFinalResCallback; // continue retrieve data
pSql->cmd.command = TSDB_SQL_FETCH;
tscProcessSql(pSql);
@@ -1192,6 +1463,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
+ pSupporter->groupInfo = pNewQueryInfo->groupbyExpr;
+ memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr));
+
pNew->cmd.numOfCols = 0;
pNewQueryInfo->interval.interval = 0;
pSupporter->limit = pNewQueryInfo->limit;
@@ -1212,17 +1486,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
assert(pTagCond->joinInfo.hasJoin);
int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid);
- SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
-
- // get the tag colId column index
- int32_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta);
- SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
- for(int32_t i = 0; i < numOfTags; ++i) {
- if (pSchema[i].colId == tagColId) {
- colIndex.columnIndex = i;
- break;
- }
- }
+ SSchema* s = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId);
+
+ colIndex.columnIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, tagColId);
int16_t bytes = 0;
int16_t type = 0;
@@ -1303,6 +1569,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) {
tscDebug("%p start subquery, total:%d", pSql, pQueryInfo->numOfTables);
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
+
SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, i);
if (pSupporter == NULL) { // failed to create support struct, abort current query
@@ -1357,8 +1624,8 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) {
SRetrieveSupport* pSupport = pSub->param;
- taosTFree(pSupport->localBuffer);
- taosTFree(pSupport);
+ tfree(pSupport->localBuffer);
+ tfree(pSupport);
taos_free_result(pSub);
}
@@ -1386,14 +1653,20 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
SSubqueryState *pState = &pSql->subState;
- pState->numOfSub = pTableMetaInfo->vgroupList->numOfVgroups;
+ pState->numOfSub = 0;
+ if (pTableMetaInfo->pVgroupTables == NULL) {
+ pState->numOfSub = pTableMetaInfo->vgroupList->numOfVgroups;
+ } else {
+ pState->numOfSub = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables);
+ }
+
assert(pState->numOfSub > 0);
int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize);
if (ret != 0) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscQueueAsyncRes(pSql);
- taosTFree(pMemoryBuf);
+ tfree(pMemoryBuf);
return ret;
}
@@ -1402,7 +1675,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
tscDebug("%p retrieved query data from %d vnode(s)", pSql, pState->numOfSub);
if (pSql->pSubs == NULL) {
- taosTFree(pSql->pSubs);
+ tfree(pSql->pSubs);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pState->numOfSub);
@@ -1427,7 +1700,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
trs->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage));
if (trs->localBuffer == NULL) {
tscError("%p failed to malloc buffer for local buffer, orderOfSub:%d, reason:%s", pSql, i, strerror(errno));
- taosTFree(trs);
+ tfree(trs);
break;
}
@@ -1438,8 +1711,8 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL);
if (pNew == NULL) {
tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno));
- taosTFree(trs->localBuffer);
- taosTFree(trs);
+ tfree(trs->localBuffer);
+ tfree(trs);
break;
}
@@ -1493,8 +1766,8 @@ static void tscFreeRetrieveSup(SSqlObj *pSql) {
// SSqlObj *pParentSql = trsupport->pParentSql;
// assert(pSql == pParentSql->pSubs[index]);
- taosTFree(trsupport->localBuffer);
- taosTFree(trsupport);
+ tfree(trsupport->localBuffer);
+ tfree(trsupport);
}
static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows);
@@ -1518,8 +1791,8 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in
SSqlObj *pParentSql = trsupport->pParentSql;
int32_t subqueryIndex = trsupport->subqueryIndex;
- STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
- SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
+ STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
+ SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]);
@@ -1721,7 +1994,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0);
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
- SCMVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
+ SVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
@@ -1832,7 +2105,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1);
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
- SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex];
+ SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex];
// stable query killed or other subquery failed, all query stopped
if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
@@ -1894,7 +2167,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
pParentObj->res.code = pSql->res.code;
}
- taosTFree(pSupporter);
+ tfree(pSupporter);
if (atomic_sub_fetch_32(&pParentObj->subState.numOfRemain, 1) > 0) {
return;
@@ -2017,7 +2290,7 @@ static char* getResultBlockPosition(SSqlCmd* pCmd, SSqlRes* pRes, int32_t column
assert(pInfo->pSqlExpr != NULL);
*bytes = pInfo->pSqlExpr->resBytes;
- char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows;
+ char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + pRes->row * (*bytes);
return pData;
}
@@ -2029,14 +2302,17 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) {
int32_t numOfRes = INT32_MAX;
for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
- if (pSql->pSubs[i] == NULL) {
+ SSqlObj* pSub = pSql->pSubs[i];
+ if (pSub == NULL) {
continue;
}
- numOfRes = (int32_t)(MIN(numOfRes, pSql->pSubs[i]->res.numOfRows));
+ int32_t remain = (int32_t)(pSub->res.numOfRows - pSub->res.row);
+ numOfRes = (int32_t)(MIN(numOfRes, remain));
}
- if (numOfRes == 0) {
+ if (numOfRes == 0) { // no result any more, free all subquery objects
+ freeJoinSubqueryObj(pSql);
return;
}
@@ -2059,14 +2335,23 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) {
size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
for(int32_t i = 0; i < numOfExprs; ++i) {
SColumnIndex* pIndex = &pRes->pColumnIndex[i];
- SSqlRes *pRes1 = &pSql->pSubs[pIndex->tableIndex]->res;
- SSqlCmd *pCmd1 = &pSql->pSubs[pIndex->tableIndex]->cmd;
+ SSqlRes* pRes1 = &pSql->pSubs[pIndex->tableIndex]->res;
+ SSqlCmd* pCmd1 = &pSql->pSubs[pIndex->tableIndex]->cmd;
char* pData = getResultBlockPosition(pCmd1, pRes1, pIndex->columnIndex, &bytes);
memcpy(data, pData, bytes * numOfRes);
data += bytes * numOfRes;
- pRes1->row = numOfRes;
+ }
+
+ for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
+ SSqlObj* pSub = pSql->pSubs[i];
+ if (pSub == NULL) {
+ continue;
+ }
+
+ pSub->res.row += numOfRes;
+ assert(pSub->res.row <= pSub->res.numOfRows);
}
pRes->numOfRows = numOfRes;
@@ -2085,6 +2370,8 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo);
+ pRes->numOfCols = (int16_t)numOfExprs;
+
pRes->tsrow = calloc(numOfExprs, POINTER_BYTES);
pRes->buffer = calloc(numOfExprs, POINTER_BYTES);
pRes->length = calloc(numOfExprs, sizeof(int32_t));
@@ -2155,7 +2442,7 @@ TAOS_ROW doSetResultRowData(SSqlObj *pSql, bool finalResult) {
assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows);
if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker
- taosTFree(pRes->tsrow);
+ tfree(pRes->tsrow);
return pRes->tsrow;
}
diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c
index 47c2d35a75cba4787e00c645e2f242ae24788df7..01046a2840aab2e8cdfb27682778f572d3179dfb 100644
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@@ -36,6 +36,7 @@ void * tscTmr;
void * tscQhandle;
void * tscCheckDiskUsageTmr;
int tsInsertHeadSize;
+int tscRefId = -1;
int tscNumOfThreads;
@@ -79,7 +80,7 @@ int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeCon
void taos_init_imp(void) {
- char temp[128];
+ char temp[128] = {0};
errno = TSDB_CODE_SUCCESS;
srand(taosGetTimestampSec());
@@ -103,7 +104,6 @@ void taos_init_imp(void) {
taosReadGlobalCfg();
taosCheckGlobalCfg();
- taosPrintGlobalCfg();
tscDebug("starting to initialize TAOS client ...");
tscDebug("Local End Point is:%s", tsLocalEp);
@@ -146,29 +146,44 @@ void taos_init_imp(void) {
tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj");
}
+ tscRefId = taosOpenRef(200, tscCloseTscObj);
+
+ // in other language APIs, taos_cleanup is not available yet.
+ // So, to make sure taos_cleanup will be invoked to clean up the allocated
+ // resource to suppress the valgrind warning.
+ atexit(taos_cleanup);
tscDebug("client is initialized successfully");
}
void taos_init() { pthread_once(&tscinit, taos_init_imp); }
-void taos_cleanup() {
- if (tscMetaCache != NULL) {
- taosCacheCleanup(tscMetaCache);
- tscMetaCache = NULL;
+// this function may be called by user or system, or by both simultaneously.
+void taos_cleanup(void) {
+ tscDebug("start to cleanup client environment");
- taosCacheCleanup(tscObjCache);
- tscObjCache = NULL;
+ void* m = tscMetaCache;
+ if (m != NULL && atomic_val_compare_exchange_ptr(&tscMetaCache, m, 0) == m) {
+ taosCacheCleanup(m);
}
-
- if (tscQhandle != NULL) {
- taosCleanUpScheduler(tscQhandle);
- tscQhandle = NULL;
+
+ m = tscObjCache;
+ if (m != NULL && atomic_val_compare_exchange_ptr(&tscObjCache, m, 0) == m) {
+ taosCacheCleanup(m);
+ }
+
+ m = tscQhandle;
+ if (m != NULL && atomic_val_compare_exchange_ptr(&tscQhandle, m, 0) == m) {
+ taosCleanUpScheduler(m);
}
+ taosCloseRef(tscRefId);
taosCleanupKeywordsTable();
taosCloseLog();
-
- taosTmrCleanUp(tscTmr);
+
+ m = tscTmr;
+ if (m != NULL && atomic_val_compare_exchange_ptr(&tscTmr, m, 0) == m) {
+ taosTmrCleanUp(m);
+ }
}
static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index b60bf958a84bcefd8fb14970923f4af1074b7a6f..bb9725a74432ac2c06adc86513b62ee32b4b8125 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -245,7 +245,7 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) {
}
pQueryInfo->fillType = TSDB_FILL_NONE;
- taosTFree(pQueryInfo->fillVal);
+ tfree(pQueryInfo->fillVal);
}
int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
@@ -259,7 +259,7 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
// not enough memory
if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) {
- taosTFree(pRes->tsrow);
+ tfree(pRes->tsrow);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
return pRes->code;
}
@@ -271,24 +271,24 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
static void tscDestroyResPointerInfo(SSqlRes* pRes) {
if (pRes->buffer != NULL) { // free all buffers containing the multibyte string
for (int i = 0; i < pRes->numOfCols; i++) {
- taosTFree(pRes->buffer[i]);
+ tfree(pRes->buffer[i]);
}
pRes->numOfCols = 0;
}
- taosTFree(pRes->pRsp);
+ tfree(pRes->pRsp);
- taosTFree(pRes->tsrow);
- taosTFree(pRes->length);
- taosTFree(pRes->buffer);
+ tfree(pRes->tsrow);
+ tfree(pRes->length);
+ tfree(pRes->buffer);
- taosTFree(pRes->pGroupRec);
- taosTFree(pRes->pColumnIndex);
+ tfree(pRes->pGroupRec);
+ tfree(pRes->pColumnIndex);
if (pRes->pArithSup != NULL) {
- taosTFree(pRes->pArithSup->data);
- taosTFree(pRes->pArithSup);
+ tfree(pRes->pArithSup->data);
+ tfree(pRes->pArithSup);
}
pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free
@@ -305,11 +305,11 @@ static void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) {
freeQueryInfoImpl(pQueryInfo);
clearAllTableMetaInfo(pQueryInfo, (const char*)addr, removeFromCache);
- taosTFree(pQueryInfo);
+ tfree(pQueryInfo);
}
pCmd->numOfClause = 0;
- taosTFree(pCmd->pQueryInfo);
+ tfree(pCmd->pQueryInfo);
}
void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) {
@@ -338,34 +338,6 @@ void tscFreeSqlResult(SSqlObj* pSql) {
memset(&pSql->res, 0, sizeof(SSqlRes));
}
-void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
- if (pSql == NULL || pSql->signature != pSql) {
- return;
- }
-
- SSqlCmd* pCmd = &pSql->cmd;
- int32_t cmd = pCmd->command;
- if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
- cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
- tscRemoveFromSqlList(pSql);
- }
-
- // pSql->sqlstr will be used by tscBuildQueryStreamDesc
-// if (pObj->signature == pObj) {
- //pthread_mutex_lock(&pObj->mutex);
- taosTFree(pSql->sqlstr);
- //pthread_mutex_unlock(&pObj->mutex);
-// }
-
- tscFreeSqlResult(pSql);
-
- taosTFree(pSql->pSubs);
- pSql->subState.numOfSub = 0;
- pSql->self = 0;
-
- tscResetSqlCmdObj(pCmd, false);
-}
-
static void tscFreeSubobj(SSqlObj* pSql) {
if (pSql->subState.numOfSub == 0) {
return;
@@ -404,7 +376,7 @@ void tscFreeRegisteredSqlObj(void *pSql) {
tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref);
if (ref == 0) {
tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj);
- tscCloseTscObj(pTscObj);
+ taosRemoveRef(tscRefId, pTscObj->rid);
}
}
@@ -415,14 +387,14 @@ void tscFreeTableMetaHelper(void *pTableMeta) {
assert(numOfEps >= 0 && numOfEps <= TSDB_MAX_REPLICA);
for(int32_t i = 0; i < numOfEps; ++i) {
- taosTFree(p->vgroupInfo.epAddr[i].fqdn);
+ tfree(p->vgroupInfo.epAddr[i].fqdn);
}
int32_t numOfEps1 = p->corVgroupInfo.numOfEps;
assert(numOfEps1 >= 0 && numOfEps1 <= TSDB_MAX_REPLICA);
for(int32_t i = 0; i < numOfEps1; ++i) {
- taosTFree(p->corVgroupInfo.epAddr[i].fqdn);
+ tfree(p->corVgroupInfo.epAddr[i].fqdn);
}
}
@@ -434,22 +406,32 @@ void tscFreeSqlObj(SSqlObj* pSql) {
tscDebug("%p start to free sqlObj", pSql);
pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
+
tscFreeSubobj(pSql);
- tscPartiallyFreeSqlObj(pSql);
+ SSqlCmd* pCmd = &pSql->cmd;
+ int32_t cmd = pCmd->command;
+ if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
+ cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
+ tscRemoveFromSqlList(pSql);
+ }
pSql->signature = NULL;
pSql->fp = NULL;
-
- SSqlCmd* pCmd = &pSql->cmd;
+ tfree(pSql->sqlstr);
+
+ tfree(pSql->pSubs);
+ pSql->subState.numOfSub = 0;
+ pSql->self = 0;
+
+ tscFreeSqlResult(pSql);
+ tscResetSqlCmdObj(pCmd, false);
memset(pCmd->payload, 0, (size_t)pCmd->allocSize);
- taosTFree(pCmd->payload);
+ tfree(pCmd->payload);
pCmd->allocSize = 0;
- taosTFree(pSql->sqlstr);
tsem_destroy(&pSql->rspSem);
-
free(pSql);
}
@@ -458,15 +440,15 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
return;
}
- taosTFree(pDataBlock->pData);
- taosTFree(pDataBlock->params);
+ tfree(pDataBlock->pData);
+ tfree(pDataBlock->params);
// free the refcount for metermeta
if (pDataBlock->pTableMeta != NULL) {
taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false);
}
- taosTFree(pDataBlock);
+ tfree(pDataBlock);
}
SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes,
@@ -741,7 +723,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
taosHashCleanup(pVnodeDataBlockHashList);
tscDestroyBlockArrayList(pVnodeDataBlockList);
- taosTFree(dataBuf->pData);
+ tfree(dataBuf->pData);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
@@ -786,8 +768,8 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
}
// TODO: all subqueries should be freed correctly before close this connection.
-void tscCloseTscObj(STscObj* pObj) {
- assert(pObj != NULL);
+void tscCloseTscObj(void *param) {
+ STscObj *pObj = param;
pObj->signature = NULL;
taosTmrStopA(&(pObj->pTimer));
@@ -801,7 +783,7 @@ void tscCloseTscObj(STscObj* pObj) {
pthread_mutex_destroy(&pObj->mutex);
tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p);
- taosTFree(pObj);
+ tfree(pObj);
}
bool tscIsInsertData(char* sqlstr) {
@@ -965,12 +947,12 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) {
if (pInfo->pArithExprInfo != NULL) {
tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL);
- taosTFree(pInfo->pArithExprInfo);
+ tfree(pInfo->pArithExprInfo);
}
}
taosArrayDestroy(pFieldInfo->internalField);
- taosTFree(pFieldInfo->final);
+ tfree(pFieldInfo->final);
memset(pFieldInfo, 0, sizeof(SFieldInfo));
}
@@ -1086,7 +1068,7 @@ void* sqlExprDestroy(SSqlExpr* pExpr) {
tVariantDestroy(&pExpr->param[i]);
}
- taosTFree(pExpr);
+ tfree(pExpr);
return NULL;
}
@@ -1121,6 +1103,8 @@ int32_t tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepco
}
*p1 = *pExpr;
+ memset(p1->param, 0, sizeof(tVariant) * tListLen(p1->param));
+
for (int32_t j = 0; j < pExpr->numOfParams; ++j) {
tVariantAssign(&p1->param[j], &pExpr->param[j]);
}
@@ -1184,11 +1168,11 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) {
static void destroyFilterInfo(SColumnFilterInfo* pFilterInfo, int32_t numOfFilters) {
for(int32_t i = 0; i < numOfFilters; ++i) {
if (pFilterInfo[i].filterstr) {
- taosTFree(pFilterInfo[i].pz);
+ tfree(pFilterInfo[i].pz);
}
}
- taosTFree(pFilterInfo);
+ tfree(pFilterInfo);
}
SColumn* tscColumnClone(const SColumn* src) {
@@ -1476,7 +1460,7 @@ void tscTagCondRelease(STagCond* pTagCond) {
size_t s = taosArrayGetSize(pTagCond->pCond);
for (int32_t i = 0; i < s; ++i) {
SCond* p = taosArrayGet(pTagCond->pCond, i);
- taosTFree(p->cond);
+ tfree(p->cond);
}
taosArrayDestroy(pTagCond->pCond);
@@ -1663,11 +1647,12 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) {
if (pQueryInfo->groupbyExpr.columnInfo != NULL) {
taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo);
pQueryInfo->groupbyExpr.columnInfo = NULL;
+ pQueryInfo->groupbyExpr.numOfGroupCols = 0;
}
pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf);
- taosTFree(pQueryInfo->fillVal);
+ tfree(pQueryInfo->fillVal);
}
void tscClearSubqueryInfo(SSqlCmd* pCmd) {
@@ -1678,23 +1663,71 @@ void tscClearSubqueryInfo(SSqlCmd* pCmd) {
}
void tscFreeVgroupTableInfo(SArray* pVgroupTables) {
- if (pVgroupTables != NULL) {
- size_t num = taosArrayGetSize(pVgroupTables);
- for (size_t i = 0; i < num; i++) {
- SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i);
+ if (pVgroupTables == NULL) {
+ return;
+ }
- for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
- taosTFree(pInfo->vgInfo.epAddr[j].fqdn);
- }
+ size_t num = taosArrayGetSize(pVgroupTables);
+ for (size_t i = 0; i < num; i++) {
+ SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i);
- taosArrayDestroy(pInfo->itemList);
+ for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
+ tfree(pInfo->vgInfo.epAddr[j].fqdn);
}
- taosArrayDestroy(pVgroupTables);
+
+ taosArrayDestroy(pInfo->itemList);
+ }
+
+ taosArrayDestroy(pVgroupTables);
+}
+
+void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) {
+ assert(pVgroupTable != NULL && index >= 0);
+
+ size_t size = taosArrayGetSize(pVgroupTable);
+ assert(size > index);
+
+ SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTable, index);
+ for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
+ tfree(pInfo->vgInfo.epAddr[j].fqdn);
}
+
+ taosArrayDestroy(pInfo->itemList);
+ taosArrayRemove(pVgroupTable, index);
+}
+
+void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) {
+ memset(info, 0, sizeof(SVgroupTableInfo));
+
+ info->vgInfo = pInfo->vgInfo;
+ for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
+ info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn);
+ }
+
+ info->itemList = taosArrayClone(pInfo->itemList);
+}
+
+SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) {
+ if (pVgroupTables == NULL) {
+ return NULL;
+ }
+
+ size_t num = taosArrayGetSize(pVgroupTables);
+ SArray* pa = taosArrayInit(num, sizeof(SVgroupTableInfo));
+
+ SVgroupTableInfo info;
+ for (size_t i = 0; i < num; i++) {
+ SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i);
+ tscVgroupTableCopy(&info, pInfo);
+
+ taosArrayPush(pa, &info);
+ }
+
+ return pa;
}
void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) {
- tscDebug("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables);
+ tscDebug("%p unref %d tables in the tableMeta cache", address, pQueryInfo->numOfTables);
for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
@@ -1704,11 +1737,11 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool rem
free(pTableMetaInfo);
}
- taosTFree(pQueryInfo->pTableMetaInfo);
+ tfree(pQueryInfo->pTableMetaInfo);
}
STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta,
- SVgroupsInfo* vgroupList, SArray* pTagCols) {
+ SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables) {
void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES);
if (pAlloc == NULL) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -1734,6 +1767,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList);
}
+ // TODO handle malloc failure
pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES);
if (pTableMetaInfo->tagColList == NULL) {
return NULL;
@@ -1742,13 +1776,15 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
if (pTagCols != NULL) {
tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1);
}
+
+ pTableMetaInfo->pVgroupTables = tscVgroupTableInfoClone(pVgroupTables);
pQueryInfo->numOfTables += 1;
return pTableMetaInfo;
}
STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) {
- return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL);
+ return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL, NULL);
}
void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) {
@@ -1822,7 +1858,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
assert(pSql->cmd.clauseIndex == 0);
STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0);
- tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL);
+ tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL);
registerSqlObj(pNew);
return pNew;
@@ -1987,14 +2023,16 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
STableMeta* pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup
assert(pTableMeta != NULL);
- pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList);
+ pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList,
+ pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables);
} else { // transfer the ownership of pTableMeta to the newly create sql object.
STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0);
STableMeta* pPrevTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pPrevInfo->pTableMeta);
SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList;
- pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList);
+ pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList,
+ pTableMetaInfo->pVgroupTables);
}
if (pFinalInfo->pTableMeta == NULL) {
@@ -2106,6 +2144,21 @@ int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) {
}
}
+int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId) {
+ int32_t numOfTags = tscGetNumOfTags(pTableMeta);
+
+ SSchema* pSchema = tscGetTableTagSchema(pTableMeta);
+ for(int32_t i = 0; i < numOfTags; ++i) {
+ if (pSchema[i].colId == colId) {
+ return i;
+ }
+ }
+
+ // can not reach here
+ assert(0);
+ return INT16_MIN;
+}
+
bool tscIsUpdateQuery(SSqlObj* pSql) {
if (pSql == NULL || pSql->signature != pSql) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
@@ -2283,7 +2336,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) {
pRes->numOfTotal = num;
- taosTFree(pSql->pSubs);
+ tfree(pSql->pSubs);
pSql->subState.numOfSub = 0;
pSql->fp = fp;
@@ -2375,7 +2428,7 @@ SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) {
return NULL;
}
- size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups;
+ size_t size = sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupList->numOfVgroups;
SVgroupsInfo* pNew = calloc(1, size);
if (pNew == NULL) {
return NULL;
@@ -2384,9 +2437,9 @@ SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) {
pNew->numOfVgroups = vgroupList->numOfVgroups;
for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) {
- SCMVgroupInfo* pNewVInfo = &pNew->vgroups[i];
+ SVgroupInfo* pNewVInfo = &pNew->vgroups[i];
- SCMVgroupInfo* pvInfo = &vgroupList->vgroups[i];
+ SVgroupInfo* pvInfo = &vgroupList->vgroups[i];
pNewVInfo->vgId = pvInfo->vgId;
pNewVInfo->numOfEps = pvInfo->numOfEps;
@@ -2405,21 +2458,22 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) {
}
for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) {
- SCMVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i];
+ SVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i];
for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) {
- taosTFree(pVgroupInfo->epAddr[j].fqdn);
+ tfree(pVgroupInfo->epAddr[j].fqdn);
}
}
- taosTFree(vgroupList);
+ tfree(vgroupList);
return NULL;
}
-void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src) {
+void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src) {
dst->vgId = src->vgId;
dst->numOfEps = src->numOfEps;
for(int32_t i = 0; i < dst->numOfEps; ++i) {
+ tfree(dst->epAddr[i].fqdn);
dst->epAddr[i].port = src->epAddr[i].port;
dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn);
}
diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h
index cc4afeb3f850785ffb18a972961668f91061c6a4..8d4949d9b4364fd1e8c70cbb883aa56468724108 100644
--- a/src/common/inc/tdataformat.h
+++ b/src/common/inc/tdataformat.h
@@ -80,7 +80,7 @@ typedef struct {
#define schemaFLen(s) ((s)->flen)
#define schemaVLen(s) ((s)->vlen)
#define schemaColAt(s, i) ((s)->columns + i)
-#define tdFreeSchema(s) taosTFree((s))
+#define tdFreeSchema(s) tfree((s))
STSchema *tdDupSchema(STSchema *pSchema);
int tdEncodeSchema(void **buf, STSchema *pSchema);
@@ -119,6 +119,33 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes);
STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
+// ----------------- Semantic timestamp key definition
+typedef uint64_t TKEY;
+
+#define TKEY_INVALID UINT64_MAX
+#define TKEY_NULL TKEY_INVALID
+#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
+#define TKEY_DELETE_FLAG (((TKEY)1) << 62)
+#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG))
+
+#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
+#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0)
+#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG)
+#define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
+#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
+
+static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) {
+ TSKEY key1 = tdGetKey(*(TKEY *)tkey1);
+ TSKEY key2 = tdGetKey(*(TKEY *)tkey2);
+
+ if (key1 < key2) {
+ return -1;
+ } else if (key1 > key2) {
+ return 1;
+ } else {
+ return 0;
+ }
+}
// ----------------- Data row structure
/* A data row, the format is like below:
@@ -129,6 +156,8 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
* +----------+----------+---------------------------------+---------------------------------+
* | len | sversion | First part | Second part |
* +----------+----------+---------------------------------+---------------------------------+
+ *
+ * NOTE: timestamp in this row structure is TKEY instead of TSKEY
*/
typedef void *SDataRow;
@@ -137,11 +166,13 @@ typedef void *SDataRow;
#define dataRowLen(r) (*(uint16_t *)(r))
#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))
#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
-#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
+#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r)))
+#define dataRowKey(r) tdGetKey(dataRowTKey(r))
#define dataRowSetLen(r, l) (dataRowLen(r) = (l))
#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v))
#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
+#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r))
SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
void tdFreeDataRow(SDataRow row);
@@ -154,16 +185,18 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i
int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row));
- switch (type) {
- case TSDB_DATA_TYPE_BINARY:
- case TSDB_DATA_TYPE_NCHAR:
- *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
- memcpy(ptr, value, varDataTLen(value));
- dataRowLen(row) += varDataTLen(value);
- break;
- default:
+ if (IS_VAR_DATA_TYPE(type)) {
+ *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
+ memcpy(ptr, value, varDataTLen(value));
+ dataRowLen(row) += varDataTLen(value);
+ } else {
+ if (offset == 0) {
+ ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP);
+ TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
+ memcpy(POINTER_SHIFT(row, toffset), (void *)(&tvalue), TYPE_BYTES[type]);
+ } else {
memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
- break;
+ }
}
return 0;
@@ -171,12 +204,10 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i
// NOTE: offset here including the header size
static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
- switch (type) {
- case TSDB_DATA_TYPE_BINARY:
- case TSDB_DATA_TYPE_NCHAR:
- return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
- default:
- return POINTER_SHIFT(row, offset);
+ if (IS_VAR_DATA_TYPE(type)) {
+ return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
+ } else {
+ return POINTER_SHIFT(row, offset);
}
}
@@ -196,7 +227,6 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints);
-void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows);
void dataColSetOffset(SDataCol *pCol, int nEle);
bool isNEleNull(SDataCol *pCol, int nEle);
@@ -204,28 +234,20 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
// Get the data pointer from a column-wised data
static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
- switch (pCol->type) {
- case TSDB_DATA_TYPE_BINARY:
- case TSDB_DATA_TYPE_NCHAR:
- return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
- break;
-
- default:
- return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
- break;
+ if (IS_VAR_DATA_TYPE(pCol->type)) {
+ return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
+ } else {
+ return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
}
}
static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
ASSERT(rows > 0);
- switch (pDataCol->type) {
- case TSDB_DATA_TYPE_BINARY:
- case TSDB_DATA_TYPE_NCHAR:
- return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
- break;
- default:
- return TYPE_BYTES[pDataCol->type] * rows;
+ if (IS_VAR_DATA_TYPE(pDataCol->type)) {
+ return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
+ } else {
+ return TYPE_BYTES[pDataCol->type] * rows;
}
}
@@ -243,9 +265,14 @@ typedef struct {
} SDataCols;
#define keyCol(pCols) (&((pCols)->cols[0])) // Key column
-#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)]
-#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
-#define dataColsKeyLast(pCols) ((pCols->numOfRows == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
+#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]
+#define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx))
+#define dataColsTKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0))
+#define dataColsKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0))
+#define dataColsTKeyLast(pCols) \
+ (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1))
+#define dataColsKeyLast(pCols) \
+ (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
void tdResetDataCols(SDataCols *pCols);
@@ -253,10 +280,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
void tdFreeDataCols(SDataCols *pCols);
void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
-void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!!
int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
-void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
- int limit2, int tRows);
// ----------------- K-V data row structure
/*
@@ -284,7 +308,7 @@ typedef struct {
#define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r))
#define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset)
#define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i))
-#define kvRowFree(r) taosTFree(r)
+#define kvRowFree(r) tfree(r)
#define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r))
SKVRow tdKVRowDup(SKVRow row);
diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h
index 515115c323294a67318b5eb1dd17660e651d09f8..4087f638a95b063fcf6081db2556758643ef1c9a 100644
--- a/src/common/inc/tglobal.h
+++ b/src/common/inc/tglobal.h
@@ -44,13 +44,18 @@ extern int32_t tsMaxShellConns;
extern int32_t tsShellActivityTimer;
extern uint32_t tsMaxTmrCtrl;
extern float tsNumOfThreadsPerCore;
-extern float tsRatioOfQueryThreads;
+extern int32_t tsNumOfCommitThreads;
+extern float tsRatioOfQueryThreads; // todo remove it
extern int8_t tsDaylight;
extern char tsTimezone[];
extern char tsLocale[];
-extern char tsCharset[]; // default encode string
+extern char tsCharset[]; // default encode string
extern int32_t tsEnableCoreFile;
extern int32_t tsCompressMsgSize;
+extern char tsTempDir[];
+
+//query buffer management
+extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing
// client
extern int32_t tsTableMetaKeepTimer;
@@ -84,6 +89,7 @@ extern int16_t tsWAL;
extern int32_t tsFsyncPeriod;
extern int32_t tsReplications;
extern int32_t tsQuorum;
+extern int32_t tsUpdate;
// balance
extern int32_t tsEnableBalance;
@@ -180,13 +186,13 @@ extern int32_t debugFlag;
#define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
-void taosInitGlobalCfg();
-bool taosCheckGlobalCfg();
-void taosSetAllDebugFlag();
-bool taosCfgDynamicOptions(char *msg);
-int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
-bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
-
+void taosInitGlobalCfg();
+int32_t taosCheckGlobalCfg();
+void taosSetAllDebugFlag();
+bool taosCfgDynamicOptions(char *msg);
+int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
+bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c
index 28289b051e4ef32e1b1e22847df584238857002b..f21205479396ba606a1212f30350df2e0b3f59b5 100644
--- a/src/common/src/tdataformat.c
+++ b/src/common/src/tdataformat.c
@@ -18,6 +18,9 @@
#include "tcoding.h"
#include "wchar.h"
+static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
+ int limit2, int tRows);
+
/**
* Duplicate the schema and return a new object
*/
@@ -94,7 +97,7 @@ int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {
void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) {
if (pBuilder) {
- taosTFree(pBuilder->columns);
+ tfree(pBuilder->columns);
}
}
@@ -202,7 +205,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE;
pDataCol->len = 0;
- if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
+ if (IS_VAR_DATA_TYPE(pDataCol->type)) {
pDataCol->dataOff = (VarDataOffsetT *)(*pBuf);
pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints);
pDataCol->spaceSize = pDataCol->bytes * maxPoints;
@@ -215,60 +218,29 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
}
}
+// value from timestamp should be TKEY here instead of TSKEY
void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) {
ASSERT(pCol != NULL && value != NULL);
- switch (pCol->type) {
- case TSDB_DATA_TYPE_BINARY:
- case TSDB_DATA_TYPE_NCHAR:
- // set offset
- pCol->dataOff[numOfRows] = pCol->len;
- // Copy data
- memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
- // Update the length
- pCol->len += varDataTLen(value);
- break;
- default:
- ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
- memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
- pCol->len += pCol->bytes;
- break;
- }
-}
-
-void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows) {
- int pointsLeft = numOfRows - pointsToPop;
-
- ASSERT(pointsLeft > 0);
-
- if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) {
- ASSERT(pCol->len > 0);
- VarDataOffsetT toffset = pCol->dataOff[pointsToPop];
- pCol->len = pCol->len - toffset;
- ASSERT(pCol->len > 0);
- memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len);
- dataColSetOffset(pCol, pointsLeft);
+ if (IS_VAR_DATA_TYPE(pCol->type)) {
+ // set offset
+ pCol->dataOff[numOfRows] = pCol->len;
+ // Copy data
+ memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
+ // Update the length
+ pCol->len += varDataTLen(value);
} else {
ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
- pCol->len = TYPE_BYTES[pCol->type] * pointsLeft;
- memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len);
+ memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
+ pCol->len += pCol->bytes;
}
}
bool isNEleNull(SDataCol *pCol, int nEle) {
- switch (pCol->type) {
- case TSDB_DATA_TYPE_BINARY:
- case TSDB_DATA_TYPE_NCHAR:
- for (int i = 0; i < nEle; i++) {
- if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
- }
- return true;
- default:
- for (int i = 0; i < nEle; i++) {
- if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
- }
- return true;
+ for (int i = 0; i < nEle; i++) {
+ if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
}
+ return true;
}
void dataColSetNullAt(SDataCol *pCol, int index) {
@@ -367,8 +339,8 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
void tdFreeDataCols(SDataCols *pCols) {
if (pCols) {
- taosTFree(pCols->buf);
- taosTFree(pCols->cols);
+ tfree(pCols->buf);
+ tfree(pCols->cols);
free(pCols);
}
}
@@ -390,7 +362,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize;
pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf)));
- if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
+ if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
ASSERT(pDataCols->cols[i].dataOff != NULL);
pRet->cols[i].dataOff =
(int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf)));
@@ -400,7 +372,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
pRet->cols[i].len = pDataCols->cols[i].len;
if (pDataCols->cols[i].len > 0) {
memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
- if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
+ if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints);
}
}
@@ -420,58 +392,54 @@ void tdResetDataCols(SDataCols *pCols) {
}
void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) {
- ASSERT(dataColsKeyLast(pCols) < dataRowKey(row));
+ ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row));
int rcol = 0;
int dcol = 0;
- while (dcol < pCols->numOfCols) {
- SDataCol *pDataCol = &(pCols->cols[dcol]);
- if (rcol >= schemaNCols(pSchema)) {
- dataColSetNullAt(pDataCol, pCols->numOfRows);
- dcol++;
- continue;
+ if (dataRowDeleted(row)) {
+ for (; dcol < pCols->numOfCols; dcol++) {
+ SDataCol *pDataCol = &(pCols->cols[dcol]);
+ if (dcol == 0) {
+ dataColAppendVal(pDataCol, dataRowTuple(row), pCols->numOfRows, pCols->maxPoints);
+ } else {
+ dataColSetNullAt(pDataCol, pCols->numOfRows);
+ }
}
+ } else {
+ while (dcol < pCols->numOfCols) {
+ SDataCol *pDataCol = &(pCols->cols[dcol]);
+ if (rcol >= schemaNCols(pSchema)) {
+ dataColSetNullAt(pDataCol, pCols->numOfRows);
+ dcol++;
+ continue;
+ }
- STColumn *pRowCol = schemaColAt(pSchema, rcol);
- if (pRowCol->colId == pDataCol->colId) {
- void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset+TD_DATA_ROW_HEAD_SIZE);
- dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
- dcol++;
- rcol++;
- } else if (pRowCol->colId < pDataCol->colId) {
- rcol++;
- } else {
- dataColSetNullAt(pDataCol, pCols->numOfRows);
- dcol++;
+ STColumn *pRowCol = schemaColAt(pSchema, rcol);
+ if (pRowCol->colId == pDataCol->colId) {
+ void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE);
+ dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
+ dcol++;
+ rcol++;
+ } else if (pRowCol->colId < pDataCol->colId) {
+ rcol++;
+ } else {
+ dataColSetNullAt(pDataCol, pCols->numOfRows);
+ dcol++;
+ }
}
}
pCols->numOfRows++;
}
-// Pop pointsToPop points from the SDataCols
-void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
- int pointsLeft = pCols->numOfRows - pointsToPop;
- if (pointsLeft <= 0) {
- tdResetDataCols(pCols);
- return;
- }
-
- for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
- SDataCol *pCol = pCols->cols + iCol;
- dataColPopPoints(pCol, pointsToPop, pCols->numOfRows);
- }
- pCols->numOfRows = pointsLeft;
-}
-
int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
- ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
ASSERT(target->numOfCols == source->numOfCols);
SDataCols *pTarget = NULL;
if (dataColsKeyLast(target) < dataColsKeyFirst(source)) { // No overlap
+ ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
for (int i = 0; i < rowsToMerge; i++) {
for (int j = 0; j < source->numOfCols; j++) {
if (source->cols[j].len > 0) {
@@ -499,17 +467,23 @@ _err:
return -1;
}
-void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) {
+// src2 data has more priority than src1
+static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
+ int limit2, int tRows) {
tdResetDataCols(target);
ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);
while (target->numOfRows < tRows) {
if (*iter1 >= limit1 && *iter2 >= limit2) break;
- TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
- TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
+ TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1);
+ TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
+ TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2);
+ TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);
- if (key1 <= key2) {
+ ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
+
+ if (key1 < key2) {
for (int i = 0; i < src1->numOfCols; i++) {
ASSERT(target->cols[i].type == src1->cols[i].type);
if (src1->cols[i].len > 0) {
@@ -520,19 +494,23 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi
target->numOfRows++;
(*iter1)++;
- if (key1 == key2) (*iter2)++;
- } else {
- for (int i = 0; i < src2->numOfCols; i++) {
- ASSERT(target->cols[i].type == src2->cols[i].type);
- if (src2->cols[i].len > 0) {
- dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
- target->maxPoints);
+ } else if (key1 >= key2) {
+ if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
+ for (int i = 0; i < src2->numOfCols; i++) {
+ ASSERT(target->cols[i].type == src2->cols[i].type);
+ if (src2->cols[i].len > 0) {
+ dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
+ target->maxPoints);
+ }
}
+ target->numOfRows++;
}
- target->numOfRows++;
(*iter2)++;
+ if (key1 == key2) (*iter1)++;
}
+
+ ASSERT(target->numOfRows <= target->maxPoints);
}
}
@@ -691,8 +669,8 @@ int tdInitKVRowBuilder(SKVRowBuilder *pBuilder) {
}
void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder) {
- taosTFree(pBuilder->pColIdx);
- taosTFree(pBuilder->buf);
+ tfree(pBuilder->pColIdx);
+ tfree(pBuilder->buf);
}
void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) {
diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c
index c24ba490ba7f4cb25ba032b0404790d68540c826..4495c3d9288a5df0b8944aa444625c143b5c7670 100644
--- a/src/common/src/tglobal.c
+++ b/src/common/src/tglobal.c
@@ -45,19 +45,21 @@ int32_t tsEnableTelemetryReporting = 1;
char tsEmail[TSDB_FQDN_LEN] = {0};
// common
-int32_t tsRpcTimer = 1000;
-int32_t tsRpcMaxTime = 600; // seconds;
-int32_t tsMaxShellConns = 5000;
+int32_t tsRpcTimer = 1000;
+int32_t tsRpcMaxTime = 600; // seconds;
+int32_t tsMaxShellConns = 5000;
int32_t tsMaxConnections = 5000;
-int32_t tsShellActivityTimer = 3; // second
-float tsNumOfThreadsPerCore = 1.0;
-float tsRatioOfQueryThreads = 0.5;
-int8_t tsDaylight = 0;
+int32_t tsShellActivityTimer = 3; // second
+float tsNumOfThreadsPerCore = 1.0f;
+int32_t tsNumOfCommitThreads = 1;
+float tsRatioOfQueryThreads = 0.5f;
+int8_t tsDaylight = 0;
char tsTimezone[TSDB_TIMEZONE_LEN] = {0};
char tsLocale[TSDB_LOCALE_LEN] = {0};
char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string
int32_t tsEnableCoreFile = 0;
int32_t tsMaxBinaryDisplayWidth = 30;
+char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/";
/*
* denote if the server needs to compress response message at the application layer to client, including query rsp,
@@ -99,6 +101,12 @@ float tsStreamComputDelayRatio = 0.1f;
int32_t tsProjectExecInterval = 10000; // every 10sec, the projection will be executed once
int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance
+// the maximum allowed query buffer size during query processing for each data node.
+// -1 no limit (default)
+// 0 no query allowed, queries are disabled
+// positive value (in MB)
+int32_t tsQueryBufferSize = -1;
+
// db parameters
int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS;
@@ -113,6 +121,7 @@ int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL;
int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION;
int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION;
+int32_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION;
int32_t tsMaxVgroupsPerDb = 0;
int32_t tsMinTablePerVnode = TSDB_TABLES_STEP;
int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
@@ -210,6 +219,8 @@ int32_t (*monitorStartSystemFp)() = NULL;
void (*monitorStopSystemFp)() = NULL;
void (*monitorExecuteSQLFp)(char *sql) = NULL;
+char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"};
+
static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
void taosSetAllDebugFlag() {
@@ -416,6 +427,16 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
+ cfg.option = "numOfCommitThreads";
+ cfg.ptr = &tsNumOfCommitThreads;
+ cfg.valType = TAOS_CFG_VTYPE_INT32;
+ cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
+ cfg.minValue = 1;
+ cfg.maxValue = 100;
+ cfg.ptrLength = 0;
+ cfg.unitType = TAOS_CFG_UTYPE_NONE;
+ taosInitConfigOption(cfg);
+
cfg.option = "ratioOfQueryThreads";
cfg.ptr = &tsRatioOfQueryThreads;
cfg.valType = TAOS_CFG_VTYPE_FLOAT;
@@ -676,7 +697,7 @@ static void doInitGlobalConfig(void) {
cfg.minValue = TSDB_MIN_CACHE_BLOCK_SIZE;
cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE;
cfg.ptrLength = 0;
- cfg.unitType = TAOS_CFG_UTYPE_Mb;
+ cfg.unitType = TAOS_CFG_UTYPE_MB;
taosInitConfigOption(cfg);
cfg.option = "blocks";
@@ -779,6 +800,16 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
+ cfg.option = "update";
+ cfg.ptr = &tsUpdate;
+ cfg.valType = TAOS_CFG_VTYPE_INT32;
+ cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+ cfg.minValue = TSDB_MIN_DB_UPDATE;
+ cfg.maxValue = TSDB_MAX_DB_UPDATE;
+ cfg.ptrLength = 0;
+ cfg.unitType = TAOS_CFG_UTYPE_NONE;
+ taosInitConfigOption(cfg);
+
cfg.option = "mqttHostName";
cfg.ptr = tsMqttHostName;
cfg.valType = TAOS_CFG_VTYPE_STRING;
@@ -839,6 +870,16 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
+ cfg.option = "queryBufferSize";
+ cfg.ptr = &tsQueryBufferSize;
+ cfg.valType = TAOS_CFG_VTYPE_INT32;
+ cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+ cfg.minValue = -1;
+ cfg.maxValue = 500000000000.0f;
+ cfg.ptrLength = 0;
+ cfg.unitType = TAOS_CFG_UTYPE_BYTE;
+ taosInitConfigOption(cfg);
+
// locale & charset
cfg.option = "timezone";
cfg.ptr = tsTimezone;
@@ -1294,13 +1335,23 @@ static void doInitGlobalConfig(void) {
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
+
+ cfg.option = "tempDir";
+ cfg.ptr = tsTempDir;
+ cfg.valType = TAOS_CFG_VTYPE_STRING;
+ cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
+ cfg.minValue = 0;
+ cfg.maxValue = 0;
+ cfg.ptrLength = tListLen(tsTempDir);
+ cfg.unitType = TAOS_CFG_UTYPE_NONE;
+ taosInitConfigOption(cfg);
}
void taosInitGlobalCfg() {
pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig);
}
-bool taosCheckGlobalCfg() {
+int32_t taosCheckGlobalCfg() {
char fqdn[TSDB_FQDN_LEN];
uint16_t port;
@@ -1359,7 +1410,9 @@ bool taosCheckGlobalCfg() {
tsSyncPort = tsServerPort + TSDB_PORT_SYNC;
tsHttpPort = tsServerPort + TSDB_PORT_HTTP;
- return true;
+ taosPrintGlobalCfg();
+
+ return 0;
}
int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) {
diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c
index 005def6dc597361436b03c15535840af2bd3461e..6dd065382265cb09e26f8d1b6f86c4b5b3f5110b 100644
--- a/src/common/src/tvariant.c
+++ b/src/common/src/tvariant.c
@@ -108,7 +108,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
break;
}
case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method
- pVar->pz = calloc(len, sizeof(char));
+ pVar->pz = calloc(len + 1, sizeof(char));
memcpy(pVar->pz, pz, len);
pVar->nLen = (int32_t)len;
break;
@@ -125,7 +125,7 @@ void tVariantDestroy(tVariant *pVar) {
if (pVar == NULL) return;
if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) {
- taosTFree(pVar->pz);
+ tfree(pVar->pz);
pVar->nLen = 0;
}
@@ -144,21 +144,24 @@ void tVariantDestroy(tVariant *pVar) {
void tVariantAssign(tVariant *pDst, const tVariant *pSrc) {
if (pSrc == NULL || pDst == NULL) return;
- *pDst = *pSrc;
-
+ pDst->nType = pSrc->nType;
if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) {
- int32_t len = pSrc->nLen + 1;
- if (pSrc->nType == TSDB_DATA_TYPE_NCHAR) {
- len = len * TSDB_NCHAR_SIZE;
- }
-
- pDst->pz = calloc(1, len);
- memcpy(pDst->pz, pSrc->pz, len);
+ int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE;
+ char* p = realloc(pDst->pz, len);
+ assert(p);
+
+ memset(p, 0, len);
+ pDst->pz = p;
+
+ memcpy(pDst->pz, pSrc->pz, pSrc->nLen);
+ pDst->nLen = pSrc->nLen;
return;
+
}
- // this is only for string array
- if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) {
+ if (pSrc->nType >= TSDB_DATA_TYPE_BOOL && pSrc->nType <= TSDB_DATA_TYPE_DOUBLE) {
+ pDst->i64Key = pSrc->i64Key;
+ } else if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) { // this is only for string array
size_t num = taosArrayGetSize(pSrc->arr);
pDst->arr = taosArrayInit(num, sizeof(char*));
for(size_t i = 0; i < num; i++) {
@@ -166,8 +169,6 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) {
char* n = strdup(p);
taosArrayPush(pDst->arr, &n);
}
-
- return;
}
pDst->nLen = tDataTypeDesc[pDst->nType].nSize;
diff --git a/src/connector/C#/TDengineDriver.cs b/src/connector/C#/TDengineDriver.cs
new file mode 100644
index 0000000000000000000000000000000000000000..b6f143e1813d60c1ac4ae8356efdca4929c51345
--- /dev/null
+++ b/src/connector/C#/TDengineDriver.cs
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+
+namespace TDengineDriver
+{
+ enum TDengineDataType {
+ TSDB_DATA_TYPE_NULL = 0, // 1 bytes
+ TSDB_DATA_TYPE_BOOL = 1, // 1 bytes
+ TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes
+ TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes
+ TSDB_DATA_TYPE_INT = 4, // 4 bytes
+ TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes
+ TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes
+ TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes
+ TSDB_DATA_TYPE_BINARY = 8, // string
+ TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes
+ TSDB_DATA_TYPE_NCHAR = 10 // unicode string
+ }
+
+ enum TDengineInitOption
+ {
+ TSDB_OPTION_LOCALE = 0,
+ TSDB_OPTION_CHARSET = 1,
+ TSDB_OPTION_TIMEZONE = 2,
+ TDDB_OPTION_CONFIGDIR = 3,
+ TDDB_OPTION_SHELL_ACTIVITY_TIMER = 4
+ }
+
+ class TDengineMeta
+ {
+ public string name;
+ public short size;
+ public byte type;
+ public string TypeName()
+ {
+ switch ((TDengineDataType)type)
+ {
+ case TDengineDataType.TSDB_DATA_TYPE_BOOL:
+ return "BOOLEAN";
+ case TDengineDataType.TSDB_DATA_TYPE_TINYINT:
+ return "BYTE";
+ case TDengineDataType.TSDB_DATA_TYPE_SMALLINT:
+ return "SHORT";
+ case TDengineDataType.TSDB_DATA_TYPE_INT:
+ return "INT";
+ case TDengineDataType.TSDB_DATA_TYPE_BIGINT:
+ return "LONG";
+ case TDengineDataType.TSDB_DATA_TYPE_FLOAT:
+ return "FLOAT";
+ case TDengineDataType.TSDB_DATA_TYPE_DOUBLE:
+ return "DOUBLE";
+ case TDengineDataType.TSDB_DATA_TYPE_BINARY:
+ return "STRING";
+ case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP:
+ return "TIMESTAMP";
+ case TDengineDataType.TSDB_DATA_TYPE_NCHAR:
+ return "NCHAR";
+ default:
+ return "undefine";
+ }
+ }
+ }
+
+ class TDengine
+ {
+ public const int TSDB_CODE_SUCCESS = 0;
+
+ [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)]
+ static extern public void Init();
+
+ [DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)]
+ static extern public void Cleanup();
+
+ [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)]
+ static extern public void Options(int option, string value);
+
+ [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)]
+ static extern public IntPtr Connect(string ip, string user, string password, string db, short port);
+
+ [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)]
+ static extern private IntPtr taos_errstr(IntPtr res);
+ static public string Error(IntPtr res)
+ {
+ IntPtr errPtr = taos_errstr(res);
+ return Marshal.PtrToStringAnsi(errPtr);
+ }
+
+ [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)]
+ static extern public int ErrorNo(IntPtr res);
+
+ [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)]
+ static extern public IntPtr Query(IntPtr conn, string sqlstr);
+
+ [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)]
+ static extern public int AffectRows(IntPtr res);
+
+ [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)]
+ static extern public int FieldCount(IntPtr res);
+
+ [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)]
+ static extern private IntPtr taos_fetch_fields(IntPtr res);
+ static public List FetchFields(IntPtr res)
+ {
+ const int fieldSize = 68;
+
+ List metas = new List();
+ if (res == IntPtr.Zero)
+ {
+ return metas;
+ }
+
+ int fieldCount = FieldCount(res);
+ IntPtr fieldsPtr = taos_fetch_fields(res);
+
+ for (int i = 0; i < fieldCount; ++i)
+ {
+ int offset = i * fieldSize;
+
+ TDengineMeta meta = new TDengineMeta();
+ meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset);
+ meta.type = Marshal.ReadByte(fieldsPtr + offset + 65);
+ meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66);
+ metas.Add(meta);
+ }
+
+ return metas;
+ }
+
+ [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)]
+ static extern public IntPtr FetchRows(IntPtr res);
+
+ [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)]
+ static extern public IntPtr FreeResult(IntPtr res);
+
+ [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)]
+ static extern public int Close(IntPtr taos);
+ }
+}
\ No newline at end of file
diff --git a/src/connector/go b/src/connector/go
index 8d7bf743852897110cbdcc7c4322cd7a74d4167b..8c58c512b6acda8bcdfa48fdc7140227b5221766 160000
--- a/src/connector/go
+++ b/src/connector/go
@@ -1 +1 @@
-Subproject commit 8d7bf743852897110cbdcc7c4322cd7a74d4167b
+Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766
diff --git a/src/connector/hivemq-tdengine-extension b/src/connector/hivemq-tdengine-extension
new file mode 160000
index 0000000000000000000000000000000000000000..b62a26ecc164a310104df57691691b237e091c89
--- /dev/null
+++ b/src/connector/hivemq-tdengine-extension
@@ -0,0 +1 @@
+Subproject commit b62a26ecc164a310104df57691691b237e091c89
diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml
index 3f6ebeff03f538e4963540e015c7fd4b310acc42..893290868ddc4d87e7052e2fe27a13461c02508d 100755
--- a/src/connector/jdbc/deploy-pom.xml
+++ b/src/connector/jdbc/deploy-pom.xml
@@ -5,7 +5,7 @@
com.taosdata.jdbc
taos-jdbcdriver
- 2.0.6
+ 2.0.9
jar
JDBCDriver
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java
index ac0e4eb84aa0fbcc8162e68d94de00cb2f4e79f5..294544ed4742205f1aaae61a52f1a42c02bec030 100644
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java
@@ -52,7 +52,6 @@ public class TSDBConnection implements Connection {
public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException {
this.dbMetaData = meta;
-
connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST),
Integer.parseInt(info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "0")),
info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), info.getProperty(TSDBDriver.PROPERTY_KEY_USER),
@@ -197,12 +196,13 @@ public class TSDBConnection implements Connection {
}
public SQLWarning getWarnings() throws SQLException {
+ //todo: implement getWarnings according to the warning messages returned from TDengine
throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG);
}
public void clearWarnings() throws SQLException {
// left blank to support HikariCP connection
- //todo: implement getWarnings according to the warning messages returned from TDengine
+ //todo: implement clearWarnings according to the warning messages returned from TDengine
}
public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException {
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java
index c1d9d2af8e5a5c24dcfed6039e3ce06530b95276..f4dee67adf03a3474f582dbe662a8c5e988e3fab 100644
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java
@@ -96,7 +96,7 @@ public class TSDBDatabaseMetaData implements java.sql.DatabaseMetaData {
}
public int getDriverMajorVersion() {
- return 0;
+ return 2;
}
public int getDriverMinorVersion() {
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
index 97d93fb0a18fa887465583bce1492c8305faaec5..63c42ca399b6098a5fa8c34d6fafeb51bdc8f588 100755
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
@@ -14,13 +14,9 @@
*****************************************************************************/
package com.taosdata.jdbc;
-
import java.io.*;
-
import java.sql.*;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
+import java.util.*;
import java.util.logging.Logger;
/**
@@ -44,76 +40,53 @@ import java.util.logging.Logger;
*/
public class TSDBDriver implements java.sql.Driver {
-
@Deprecated
private static final String URL_PREFIX1 = "jdbc:TSDB://";
private static final String URL_PREFIX = "jdbc:TAOS://";
- /**
- * Key used to retrieve the database value from the properties instance passed
- * to the driver.
- */
- public static final String PROPERTY_KEY_DBNAME = "dbname";
-
/**
* Key used to retrieve the host value from the properties instance passed to
* the driver.
*/
public static final String PROPERTY_KEY_HOST = "host";
- /**
- * Key used to retrieve the password value from the properties instance passed
- * to the driver.
- */
- public static final String PROPERTY_KEY_PASSWORD = "password";
-
/**
* Key used to retrieve the port number value from the properties instance
* passed to the driver.
*/
public static final String PROPERTY_KEY_PORT = "port";
-
+ /**
+ * Key used to retrieve the database value from the properties instance passed
+ * to the driver.
+ */
+ public static final String PROPERTY_KEY_DBNAME = "dbname";
/**
* Key used to retrieve the user value from the properties instance passed to
* the driver.
*/
public static final String PROPERTY_KEY_USER = "user";
-
-
+ /**
+ * Key used to retrieve the password value from the properties instance passed
+ * to the driver.
+ */
+ public static final String PROPERTY_KEY_PASSWORD = "password";
/**
* Key for the configuration file directory of TSDB client in properties instance
*/
public static final String PROPERTY_KEY_CONFIG_DIR = "cfgdir";
-
/**
* Key for the timezone used by the TSDB client in properties instance
*/
public static final String PROPERTY_KEY_TIME_ZONE = "timezone";
-
/**
* Key for the locale used by the TSDB client in properties instance
*/
public static final String PROPERTY_KEY_LOCALE = "locale";
-
-
/**
* Key for the char encoding used by the TSDB client in properties instance
*/
public static final String PROPERTY_KEY_CHARSET = "charset";
- public static final String PROPERTY_KEY_PROTOCOL = "protocol";
-
-
- /**
- * Index for port coming out of parseHostPortPair().
- */
- public final static int PORT_NUMBER_INDEX = 1;
-
- /**
- * Index for host coming out of parseHostPortPair().
- */
- public final static int HOST_NAME_INDEX = 0;
-
private TSDBDatabaseMetaData dbMetaData = null;
static {
@@ -169,9 +142,11 @@ public class TSDBDriver implements java.sql.Driver {
}
public Connection connect(String url, Properties info) throws SQLException {
- if (url == null) {
+ if (url == null)
throw new SQLException(TSDBConstants.WrapErrMsg("url is not set!"));
- }
+
+ if (!acceptsURL(url))
+ return null;
Properties props = null;
if ((props = parseURL(url, info)) == null) {
@@ -179,7 +154,10 @@ public class TSDBDriver implements java.sql.Driver {
}
//load taos.cfg start
- if (info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null && info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null) {
+ if ((info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null ||
+ info.getProperty(TSDBDriver.PROPERTY_KEY_HOST).isEmpty()) && (
+ info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null ||
+ info.getProperty(TSDBDriver.PROPERTY_KEY_PORT).isEmpty())) {
File cfgDir = loadConfigDir(info.getProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR));
File cfgFile = cfgDir.listFiles((dir, name) -> "taos.cfg".equalsIgnoreCase(name))[0];
List endpoints = loadConfigEndpoints(cfgFile);
@@ -190,7 +168,9 @@ public class TSDBDriver implements java.sql.Driver {
}
try {
- TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE), (String) props.get(PROPERTY_KEY_CHARSET),
+ TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR),
+ (String) props.get(PROPERTY_KEY_LOCALE),
+ (String) props.get(PROPERTY_KEY_CHARSET),
(String) props.get(PROPERTY_KEY_TIME_ZONE));
Connection newConn = new TSDBConnection(props, this.dbMetaData);
return newConn;
@@ -208,43 +188,15 @@ public class TSDBDriver implements java.sql.Driver {
}
/**
- * Parses hostPortPair in the form of [host][:port] into an array, with the
- * element of index HOST_NAME_INDEX being the host (or null if not specified),
- * and the element of index PORT_NUMBER_INDEX being the port (or null if not
- * specified).
- *
- * @param hostPortPair host and port in form of of [host][:port]
- * @return array containing host and port as Strings
- * @throws SQLException if a parse error occurs
+ * @param url the URL of the database
+ * @return true if this driver understands the given URL;
+ * false otherwise
+ * @throws SQLException if a database access error occurs or the url is {@code null}
*/
- protected static String[] parseHostPortPair(String hostPortPair) throws SQLException {
- String[] splitValues = new String[2];
-
- int portIndex = hostPortPair.indexOf(":");
-
- String hostname = null;
-
- if (portIndex != -1) {
- if ((portIndex + 1) < hostPortPair.length()) {
- String portAsString = hostPortPair.substring(portIndex + 1);
- hostname = hostPortPair.substring(0, portIndex);
-
- splitValues[HOST_NAME_INDEX] = hostname;
-
- splitValues[PORT_NUMBER_INDEX] = portAsString;
- } else {
- throw new SQLException(TSDBConstants.WrapErrMsg("port is not proper!"));
- }
- } else {
- splitValues[HOST_NAME_INDEX] = hostPortPair;
- splitValues[PORT_NUMBER_INDEX] = null;
- }
-
- return splitValues;
- }
-
public boolean acceptsURL(String url) throws SQLException {
- return (url != null && url.length() > 0 && url.trim().length() > 0) && url.startsWith(URL_PREFIX);
+ if (url == null)
+ throw new SQLException(TSDBConstants.WrapErrMsg("url is null"));
+ return (url != null && url.length() > 0 && url.trim().length() > 0) && (url.startsWith(URL_PREFIX) || url.startsWith(URL_PREFIX1));
}
public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException {
@@ -252,15 +204,17 @@ public class TSDBDriver implements java.sql.Driver {
info = new Properties();
}
- if ((url != null) && (url.startsWith(URL_PREFIX) || url.startsWith(URL_PREFIX1))) {
+ if (acceptsURL(url)) {
info = parseURL(url, info);
}
DriverPropertyInfo hostProp = new DriverPropertyInfo(PROPERTY_KEY_HOST, info.getProperty(PROPERTY_KEY_HOST));
- hostProp.required = true;
+ hostProp.required = false;
+ hostProp.description = "Hostname";
DriverPropertyInfo portProp = new DriverPropertyInfo(PROPERTY_KEY_PORT, info.getProperty(PROPERTY_KEY_PORT, TSDBConstants.DEFAULT_PORT));
portProp.required = false;
+ portProp.description = "Port";
DriverPropertyInfo dbProp = new DriverPropertyInfo(PROPERTY_KEY_DBNAME, info.getProperty(PROPERTY_KEY_DBNAME));
dbProp.required = false;
@@ -268,9 +222,11 @@ public class TSDBDriver implements java.sql.Driver {
DriverPropertyInfo userProp = new DriverPropertyInfo(PROPERTY_KEY_USER, info.getProperty(PROPERTY_KEY_USER));
userProp.required = true;
+ userProp.description = "User";
DriverPropertyInfo passwordProp = new DriverPropertyInfo(PROPERTY_KEY_PASSWORD, info.getProperty(PROPERTY_KEY_PASSWORD));
passwordProp.required = true;
+ passwordProp.description = "Password";
DriverPropertyInfo[] propertyInfo = new DriverPropertyInfo[5];
propertyInfo[0] = hostProp;
@@ -283,20 +239,68 @@ public class TSDBDriver implements java.sql.Driver {
}
/**
- * example: jdbc:TSDB://127.0.0.1:0/db?user=root&password=your_password
+ * example: jdbc:TAOS://127.0.0.1:0/db?user=root&password=your_password
*/
- public Properties parseURL(String url, Properties defaults) throws java.sql.SQLException {
+ public Properties parseURL(String url, Properties defaults) {
Properties urlProps = (defaults != null) ? defaults : new Properties();
- if (url == null) {
+ if (url == null || url.length() <= 0 || url.trim().length() <= 0)
return null;
- }
-
- if (!url.startsWith(URL_PREFIX) && !url.startsWith(URL_PREFIX1)) {
+ if (!url.startsWith(URL_PREFIX) && !url.startsWith(URL_PREFIX1))
return null;
- }
+ // parse properties
String urlForMeta = url;
+ int beginningOfSlashes = url.indexOf("//");
+ int index = url.indexOf("?");
+ if (index != -1) {
+ String paramString = url.substring(index + 1, url.length());
+ url = url.substring(0, index);
+ StringTokenizer queryParams = new StringTokenizer(paramString, "&");
+ while (queryParams.hasMoreElements()) {
+ String parameterValuePair = queryParams.nextToken();
+ int indexOfEqual = parameterValuePair.indexOf("=");
+ String parameter = null;
+ String value = null;
+ if (indexOfEqual != -1) {
+ parameter = parameterValuePair.substring(0, indexOfEqual);
+ if (indexOfEqual + 1 < parameterValuePair.length()) {
+ value = parameterValuePair.substring(indexOfEqual + 1);
+ }
+ }
+ if ((value != null && value.length() > 0) && (parameter != null && parameter.length() > 0)) {
+ urlProps.setProperty(parameter, value);
+ }
+ }
+ }
+ // parse Product Name
+ String dbProductName = url.substring(0, beginningOfSlashes);
+ dbProductName = dbProductName.substring(dbProductName.indexOf(":") + 1);
+ dbProductName = dbProductName.substring(0, dbProductName.indexOf(":"));
+ // parse dbname
+ url = url.substring(beginningOfSlashes + 2);
+ int indexOfSlash = url.indexOf("/");
+ if (indexOfSlash != -1) {
+ if (indexOfSlash + 1 < url.length()) {
+ urlProps.setProperty(TSDBDriver.PROPERTY_KEY_DBNAME, url.substring(indexOfSlash + 1));
+ }
+ url = url.substring(0, indexOfSlash);
+ }
+ // parse port
+ int indexOfColon = url.indexOf(":");
+ if (indexOfColon != -1) {
+ if (indexOfColon + 1 < url.length()) {
+ urlProps.setProperty(TSDBDriver.PROPERTY_KEY_PORT, url.substring(indexOfColon + 1));
+ }
+ url = url.substring(0, indexOfColon);
+ }
+ if (url != null && url.length() > 0 && url.trim().length() > 0) {
+ urlProps.setProperty(TSDBDriver.PROPERTY_KEY_HOST, url);
+ }
+ this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, urlProps.getProperty(TSDBDriver.PROPERTY_KEY_USER));
+
+ /*
+ String urlForMeta = url;
String dbProductName = url.substring(url.indexOf(":") + 1);
dbProductName = dbProductName.substring(0, dbProductName.indexOf(":"));
int beginningOfSlashes = url.indexOf("//");
@@ -345,11 +349,11 @@ public class TSDBDriver implements java.sql.Driver {
user = urlProps.getProperty(PROPERTY_KEY_USER).toString();
this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, user);
-
+*/
return urlProps;
}
- public void setPropertyValue(Properties property, String[] keyValuePair) {
+ private void setPropertyValue(Properties property, String[] keyValuePair) {
switch (keyValuePair[0].toLowerCase()) {
case PROPERTY_KEY_USER:
property.setProperty(PROPERTY_KEY_USER, keyValuePair[1]);
@@ -372,13 +376,12 @@ public class TSDBDriver implements java.sql.Driver {
}
}
-
public int getMajorVersion() {
- return 1;
+ return 2;
}
public int getMinorVersion() {
- return 1;
+ return 0;
}
public boolean jdbcCompliant() {
@@ -389,33 +392,4 @@ public class TSDBDriver implements java.sql.Driver {
return null;
}
- /**
- * Returns the host property
- *
- * @param props the java.util.Properties instance to retrieve the hostname from.
- * @return the host
- */
- public String host(Properties props) {
- return props.getProperty(PROPERTY_KEY_HOST, "localhost");
- }
-
- /**
- * Returns the port number property
- *
- * @param props the properties to get the port number from
- * @return the port number
- */
- public int port(Properties props) {
- return Integer.parseInt(props.getProperty(PROPERTY_KEY_PORT, TSDBConstants.DEFAULT_PORT));
- }
-
- /**
- * Returns the database property from props
- *
- * @param props the Properties to look for the database property.
- * @return the database name.
- */
- public String database(Properties props) {
- return props.getProperty(PROPERTY_KEY_DBNAME);
- }
}
diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
index 0cd185de50d858aec78b60acc4055c1ae99a4cb5..edc160e323d2dc9c885b5404545a6d0689857a31 100755
--- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
+++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
@@ -111,8 +111,8 @@ public class TSDBJNIConnector {
* @throws SQLException
*/
public long executeQuery(String sql) throws SQLException {
- // close previous result set if the user forgets to invoke the
- // free method to close previous result set.
+ // close previous result set if the user forgets to invoke the
+ // free method to close previous result set.
if (!this.isResultsetClosed) {
freeResultSet(taosResultSetPointer);
}
@@ -122,23 +122,23 @@ public class TSDBJNIConnector {
pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos);
} catch (Exception e) {
e.printStackTrace();
- this.freeResultSet(pSql);
+ this.freeResultSetImp(this.taos, pSql);
throw new SQLException(TSDBConstants.WrapErrMsg("Unsupported encoding"));
}
-
+
int code = this.getErrCode(pSql);
if (code != 0) {
affectedRows = -1;
String msg = this.getErrMsg(pSql);
-
- this.freeResultSet(pSql);
+
+ this.freeResultSetImp(this.taos, pSql);
throw new SQLException(TSDBConstants.WrapErrMsg(msg), "", code);
}
// Try retrieving result set for the executed SQL using the current connection pointer.
taosResultSetPointer = this.getResultSetImp(this.taos, pSql);
isResultsetClosed = (taosResultSetPointer == TSDBConstants.JNI_NULL_POINTER);
-
+
return pSql;
}
@@ -171,11 +171,11 @@ public class TSDBJNIConnector {
}
private native long getResultSetImp(long connection, long pSql);
-
+
public boolean isUpdateQuery(long pSql) {
- return isUpdateQueryImp(this.taos, pSql) == 1? true:false;
+ return isUpdateQueryImp(this.taos, pSql) == 1 ? true : false;
}
-
+
private native long isUpdateQueryImp(long connection, long pSql);
/**
@@ -191,7 +191,7 @@ public class TSDBJNIConnector {
res = this.freeResultSetImp(this.taos, result);
taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER;
}
-
+
isResultsetClosed = true;
return res;
}
@@ -274,7 +274,7 @@ public class TSDBJNIConnector {
* Consume a subscription
*/
long consume(long subscription) {
- return this.consumeImp(subscription);
+ return this.consumeImp(subscription);
}
private native long consumeImp(long subscription);
diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java
index a0981063a5052f04b849b2187a78352a2c2560be..8adcdefb2974d5817793297091eee1c2f40a52b9 100644
--- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java
+++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java
@@ -1,47 +1,210 @@
package com.taosdata.jdbc;
+import org.junit.BeforeClass;
import org.junit.Test;
-import java.sql.SQLException;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.sql.*;
import java.util.Properties;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
public class TSDBDriverTest {
+ private static final String[] validURLs = {
+ "jdbc:TAOS://localhost:0",
+ "jdbc:TAOS://localhost",
+ "jdbc:TAOS://localhost:6030/test",
+ "jdbc:TAOS://localhost:6030",
+ "jdbc:TAOS://localhost:6030/",
+ "jdbc:TSDB://localhost:6030",
+ "jdbc:TSDB://localhost:6030/",
+ "jdbc:TAOS://127.0.0.1:0/db?user=root&password=taosdata",
+ "jdbc:TAOS://:",
+ "jdbc:TAOS://:/",
+ "jdbc:TAOS://:/test",
+ "jdbc:TAOS://localhost:0/?user=root&password=taosdata"
+ };
+ private static boolean islibLoaded = false;
+ private static boolean isTaosdActived;
+
+ private Connection conn;
+
+ @BeforeClass
+ public static void before() {
+ String osName = System.getProperty("os.name").toLowerCase();
+ if (!osName.equals("linux"))
+ return;
+ // try to load taos lib
+ try {
+ System.loadLibrary("taos");
+ islibLoaded = true;
+ } catch (UnsatisfiedLinkError error) {
+ System.out.println("load tdengine lib failed.");
+ error.printStackTrace();
+ }
+ // check taosd is activated
+ try {
+ String[] cmd = {"/bin/bash", "-c", "ps -ef | grep taosd | grep -v \"grep\""};
+ Process exec = Runtime.getRuntime().exec(cmd);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(exec.getInputStream()));
+ int lineCnt = 0;
+ while (reader.readLine() != null) {
+ lineCnt++;
+ }
+ if (lineCnt > 0)
+ isTaosdActived = true;
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ try {
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testConnectWithJdbcURL() {
+ final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata";
+ try {
+ if (islibLoaded && isTaosdActived) {
+ conn = DriverManager.getConnection(url);
+ assertNotNull("failure - connection should not be null", conn);
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ fail("failure - should not throw Exception");
+ }
+ }
+
+ @Test
+ public void testConnectWithProperties() {
+ final String jdbcUrl = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata";
+ Properties connProps = new Properties();
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ try {
+ if (islibLoaded && isTaosdActived) {
+ conn = DriverManager.getConnection(jdbcUrl, connProps);
+ assertNotNull("failure - connection should not be null", conn);
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ fail("failure - should not throw Exception");
+ }
+ }
+
@Test
- public void urlParserTest() throws SQLException {
+ public void testConnectWithConfigFile() {
+ String jdbcUrl = "jdbc:TAOS://:/log?user=root&password=taosdata";
+ Properties connProps = new Properties();
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ try {
+ if (islibLoaded && isTaosdActived) {
+ conn = DriverManager.getConnection(jdbcUrl, connProps);
+ assertNotNull("failure - connection should not be null", conn);
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ fail("failure - should not throw Exception");
+ }
+ }
+
+ @Test(expected = SQLException.class)
+ public void testAcceptsURL() throws SQLException {
+ Driver driver = new TSDBDriver();
+ for (String url : validURLs) {
+ assertTrue("failure - acceptsURL(\" " + url + " \") should be true", driver.acceptsURL(url));
+ }
+ driver.acceptsURL(null);
+ fail("acceptsURL throws exception when parameter is null");
+ }
+
+ @Test
+ public void testParseURL() {
TSDBDriver driver = new TSDBDriver();
- String url = "jdbc:TSDB://127.0.0.1:0/db";
-
- Properties properties = new Properties();
- driver.parseURL(url, properties);
- assertEquals(properties.get("host"), "127.0.0.1");
- assertEquals(properties.get("port"), "0");
- assertEquals(properties.get("dbname"), "db");
- assertEquals(properties.get("user"), "root");
- assertEquals(properties.get("password"), "your_password");
-
- url = "jdbc:TSDB://127.0.0.1:0/log?charset=UTF-8";
- properties = new Properties();
- driver.parseURL(url, properties);
- assertEquals(properties.get("host"), "127.0.0.1");
- assertEquals(properties.get("port"), "0");
- assertEquals(properties.get("dbname"), "log");
- assertEquals(properties.get("charset"), "UTF-8");
-
- url = "jdbc:TSDB://127.0.0.1:0/";
- properties = new Properties();
- driver.parseURL(url, properties);
- assertEquals(properties.get("host"), "127.0.0.1");
- assertEquals(properties.get("port"), "0");
- assertEquals(properties.get("dbname"), null);
-
- url = "jdbc:TSDB://127.0.0.1:0/db";
- properties = new Properties();
- driver.parseURL(url, properties);
- assertEquals(properties.get("host"), "127.0.0.1");
- assertEquals(properties.get("port"), "0");
- assertEquals(properties.get("dbname"), "db");
+
+ String url = "jdbc:TAOS://127.0.0.1:0/db?user=root&password=taosdata&charset=UTF-8";
+ Properties config = new Properties();
+ Properties actual = driver.parseURL(url, config);
+ assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.get("host"));
+ assertEquals("failure - port should be 0", "0", actual.get("port"));
+ assertEquals("failure - dbname should be db", "db", actual.get("dbname"));
+ assertEquals("failure - user should be root", "root", actual.get("user"));
+ assertEquals("failure - password should be taosdata", "taosdata", actual.get("password"));
+ assertEquals("failure - charset should be UTF-8", "UTF-8", actual.get("charset"));
+
+ url = "jdbc:TAOS://127.0.0.1:0";
+ config = new Properties();
+ actual = driver.parseURL(url, config);
+ assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.getProperty("host"));
+ assertEquals("failure - port should be 0", "0", actual.get("port"));
+ assertNull("failure - dbname should be null", actual.get("dbname"));
+
+ url = "jdbc:TAOS://127.0.0.1:0/db";
+ config = new Properties();
+ actual = driver.parseURL(url, config);
+ assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.getProperty("host"));
+ assertEquals("failure - port should be 0", "0", actual.get("port"));
+ assertEquals("failure - dbname should be db", "db", actual.get("dbname"));
+
+ url = "jdbc:TAOS://:/?";
+ config = new Properties();
+ config.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+ config.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
+ actual = driver.parseURL(url, config);
+ assertEquals("failure - user should be root", "root", actual.getProperty("user"));
+ assertEquals("failure - password should be taosdata", "taosdata", actual.getProperty("password"));
+ assertNull("failure - host should be null", actual.getProperty("host"));
+ assertNull("failure - port should be null", actual.getProperty("port"));
+ assertNull("failure - dbname should be null", actual.getProperty("dbname"));
+ }
+
+ @Test
+ public void testGetPropertyInfo() throws SQLException {
+ Driver driver = new TSDBDriver();
+ final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata";
+ Properties connProps = new Properties();
+ DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, connProps);
+ for (DriverPropertyInfo info : propertyInfo) {
+ if (info.name.equals(TSDBDriver.PROPERTY_KEY_HOST))
+ assertEquals("failure - host should be localhost", "localhost", info.value);
+ if (info.name.equals(TSDBDriver.PROPERTY_KEY_PORT))
+ assertEquals("failure - port should be 6030", "6030", info.value);
+ if (info.name.equals(TSDBDriver.PROPERTY_KEY_DBNAME))
+ assertEquals("failure - dbname should be test", "log", info.value);
+ if (info.name.equals(TSDBDriver.PROPERTY_KEY_USER))
+ assertEquals("failure - user should be root", "root", info.value);
+ if (info.name.equals(TSDBDriver.PROPERTY_KEY_PASSWORD))
+ assertEquals("failure - password should be root", "taosdata", info.value);
+ }
+ }
+
+ @Test
+ public void testGetMajorVersion() {
+ assertEquals("failure - getMajorVersion should be 2", 2, new TSDBDriver().getMajorVersion());
+ }
+
+ @Test
+ public void testGetMinorVersion() {
+ assertEquals("failure - getMinorVersion should be 0", 0, new TSDBDriver().getMinorVersion());
+ }
+
+ @Test
+ public void testJdbcCompliant() {
+ assertFalse("failure - jdbcCompliant should be false", new TSDBDriver().jdbcCompliant());
}
+
+ @Test
+ public void testGetParentLogger() throws SQLFeatureNotSupportedException {
+ assertNull("failure - getParentLogger should be be null", new TSDBDriver().getParentLogger());
+ }
+
}
\ No newline at end of file
diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c
index 1a99a84b8ebfe8ed503213299646da39b5b1d27a..1be7552a892e25cf44317a99fe52ff57689ad338 100644
--- a/src/cq/src/cqMain.c
+++ b/src/cq/src/cqMain.c
@@ -39,16 +39,16 @@
#define cTrace(...) { if (cqDebugFlag & DEBUG_TRACE) { taosPrintLog("CQ ", cqDebugFlag, __VA_ARGS__); }}
typedef struct {
- int vgId;
+ int32_t vgId;
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char db[TSDB_DB_NAME_LEN];
FCqWrite cqWrite;
void *ahandle;
- int num; // number of continuous streams
+ int32_t num; // number of continuous streams
struct SCqObj *pHead;
void *dbConn;
- int master;
+ int32_t master;
void *tmrCtrl;
pthread_mutex_t mutex;
} SCqContext;
@@ -57,7 +57,7 @@ typedef struct SCqObj {
tmr_h tmrId;
uint64_t uid;
int32_t tid; // table ID
- int rowSize; // bytes of a row
+ int32_t rowSize; // bytes of a row
char * sqlStr; // SQL string
STSchema * pSchema; // pointer to schema array
void * pStream;
@@ -115,7 +115,7 @@ void cqClose(void *handle) {
SCqObj *pTemp = pObj;
pObj = pObj->next;
tdFreeSchema(pTemp->pSchema);
- taosTFree(pTemp->sqlStr);
+ tfree(pTemp->sqlStr);
free(pTemp);
}
@@ -175,7 +175,7 @@ void cqStop(void *handle) {
pthread_mutex_unlock(&pContext->mutex);
}
-void *cqCreate(void *handle, uint64_t uid, int tid, char *sqlStr, STSchema *pSchema) {
+void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) {
SCqContext *pContext = handle;
SCqObj *pObj = calloc(sizeof(SCqObj), 1);
@@ -237,7 +237,7 @@ void cqDrop(void *handle) {
pthread_mutex_unlock(&pContext->mutex);
}
-static void doCreateStream(void *param, TAOS_RES *result, int code) {
+static void doCreateStream(void *param, TAOS_RES *result, int32_t code) {
SCqObj* pObj = (SCqObj*)param;
SCqContext* pContext = pObj->pContext;
SSqlObj* pSql = (SSqlObj*)result;
@@ -288,7 +288,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr);
- int size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize;
+ int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize;
char *buffer = calloc(size, 1);
SWalHead *pHead = (SWalHead *)buffer;
@@ -334,7 +334,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
pHead->version = 0;
// write into vnode write queue
- pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ);
+ pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ, NULL);
free(buffer);
}
diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c
index 1daee644a7effc5fa50fbc340ad7df8e2b5d70b0..e1114fc024054f7a4af32ca1497a63f7da942147 100644
--- a/src/cq/test/cqtest.c
+++ b/src/cq/test/cqtest.c
@@ -24,7 +24,7 @@
int64_t ver = 0;
void *pCq = NULL;
-int writeToQueue(void *pVnode, void *data, int type) {
+int writeToQueue(void *pVnode, void *data, int type, void *pMsg) {
return 0;
}
diff --git a/src/dnode/inc/dnodeCfg.h b/src/dnode/inc/dnodeCfg.h
new file mode 100644
index 0000000000000000000000000000000000000000..35d889646021a079abe18c01a26fa5891718aaa0
--- /dev/null
+++ b/src/dnode/inc/dnodeCfg.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_DNODE_CFG_H
+#define TDENGINE_DNODE_CFG_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t dnodeInitCfg();
+void dnodeCleanupCfg();
+void dnodeUpdateCfg(SDnodeCfg *cfg);
+int32_t dnodeGetDnodeId();
+void dnodeGetCfg(int32_t *dnodeId, char *clusterId);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/dnode/inc/dnodeEps.h b/src/dnode/inc/dnodeEps.h
new file mode 100644
index 0000000000000000000000000000000000000000..2a203498c1f270c7e456694a4e1e195cbd9022cd
--- /dev/null
+++ b/src/dnode/inc/dnodeEps.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_DNODE_EP_H
+#define TDENGINE_DNODE_EP_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taosmsg.h"
+
+int32_t dnodeInitEps();
+void dnodeCleanupEps();
+void dnodeUpdateEps(SDnodeEps *eps);
+void dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port);
+bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/dnode/inc/dnodeMInfos.h b/src/dnode/inc/dnodeMInfos.h
new file mode 100644
index 0000000000000000000000000000000000000000..9c3c85c47e2dbcc11c5b5a80fbf091bd93855149
--- /dev/null
+++ b/src/dnode/inc/dnodeMInfos.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_DNODE_MINFOS_H
+#define TDENGINE_DNODE_MINFOS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taosmsg.h"
+
+int32_t dnodeInitMInfos();
+void dnodeCleanupMInfos();
+void dnodeUpdateMInfos(SMnodeInfos *minfos);
+void dnodeUpdateEpSetForPeer(SRpcEpSet *epSet);
+void dnodeGetMInfos(SMnodeInfos *minfos);
+bool dnodeIsMasterEp(char *ep);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/dnode/inc/dnodeMPeer.h b/src/dnode/inc/dnodeMPeer.h
index 9a48703110c389f10ac623ccaeaa85420a32817f..00221baa221a411d614c1fa1bb3dc4525de2ed71 100644
--- a/src/dnode/inc/dnodeMPeer.h
+++ b/src/dnode/inc/dnodeMPeer.h
@@ -20,11 +20,11 @@
extern "C" {
#endif
-int32_t dnodeInitMnodePeer();
-void dnodeCleanupMnodePeer();
-int32_t dnodeAllocateMnodePqueue();
-void dnodeFreeMnodePqueue();
-void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg);
+int32_t dnodeInitMPeer();
+void dnodeCleanupMPeer();
+int32_t dnodeAllocateMPeerQueue();
+void dnodeFreeMPeerQueue();
+void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg);
#ifdef __cplusplus
}
diff --git a/src/dnode/inc/dnodeMRead.h b/src/dnode/inc/dnodeMRead.h
index 4e93838b7998850e1bec79f017fdfbd28f286a11..8a8e71227ddf3cd7b7c77ef664ac3d4c935889db 100644
--- a/src/dnode/inc/dnodeMRead.h
+++ b/src/dnode/inc/dnodeMRead.h
@@ -20,11 +20,11 @@
extern "C" {
#endif
-int32_t dnodeInitMnodeRead();
-void dnodeCleanupMnodeRead();
-int32_t dnodeAllocateMnodeRqueue();
-void dnodeFreeMnodeRqueue();
-void dnodeDispatchToMnodeReadQueue(SRpcMsg *rpcMsg);
+int32_t dnodeInitMRead();
+void dnodeCleanupMRead();
+int32_t dnodeAllocMReadQueue();
+void dnodeFreeMReadQueue();
+void dnodeDispatchToMReadQueue(SRpcMsg *rpcMsg);
#ifdef __cplusplus
}
diff --git a/src/dnode/inc/dnodeMWrite.h b/src/dnode/inc/dnodeMWrite.h
index 498fea81c59329b4d30874d36d10182b6e3ae54f..6a3d41bc81a03465a2e097bc711c83a8d8af362e 100644
--- a/src/dnode/inc/dnodeMWrite.h
+++ b/src/dnode/inc/dnodeMWrite.h
@@ -20,11 +20,11 @@
extern "C" {
#endif
-int32_t dnodeInitMnodeWrite();
-void dnodeCleanupMnodeWrite();
-int32_t dnodeAllocateMnodeWqueue();
-void dnodeFreeMnodeWqueue();
-void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg);
+int32_t dnodeInitMWrite();
+void dnodeCleanupMWrite();
+int32_t dnodeAllocMWritequeue();
+void dnodeFreeMWritequeue();
+void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg);
#ifdef __cplusplus
}
diff --git a/src/dnode/inc/dnodeMgmt.h b/src/dnode/inc/dnodeMgmt.h
index e8f4a0823f3cc9846b85804d323e73a8cfc0476e..2038ef5286b32522b11409ba5a253b33228b984d 100644
--- a/src/dnode/inc/dnodeMgmt.h
+++ b/src/dnode/inc/dnodeMgmt.h
@@ -20,6 +20,8 @@
extern "C" {
#endif
+#include "trpc.h"
+
int32_t dnodeInitMgmt();
void dnodeCleanupMgmt();
int32_t dnodeInitMgmtTimer();
@@ -35,8 +37,8 @@ void* dnodeGetVnodeTsdb(void *pVnode);
void dnodeReleaseVnode(void *pVnode);
void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
-void dnodeGetMnodeEpSetForPeer(void *epSet);
-void dnodeGetMnodeEpSetForShell(void *epSet);
+void dnodeGetEpSetForPeer(SRpcEpSet *epSet);
+void dnodeGetEpSetForShell(SRpcEpSet *epSet);
#ifdef __cplusplus
}
diff --git a/src/dnode/inc/dnodeVRead.h b/src/dnode/inc/dnodeVRead.h
index a1035200475259bd91757f934d35a0dd5a69b1fe..b3c3df80b2e201ff9f4b40ebfcceafba57de1366 100644
--- a/src/dnode/inc/dnodeVRead.h
+++ b/src/dnode/inc/dnodeVRead.h
@@ -20,9 +20,11 @@
extern "C" {
#endif
-int32_t dnodeInitVnodeRead();
-void dnodeCleanupVnodeRead();
-void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg);
+int32_t dnodeInitVRead();
+void dnodeCleanupVRead();
+void dnodeDispatchToVReadQueue(SRpcMsg *pMsg);
+void * dnodeAllocVReadQueue(void *pVnode);
+void dnodeFreeVReadQueue(void *rqueue);
#ifdef __cplusplus
}
diff --git a/src/dnode/inc/dnodeVWrite.h b/src/dnode/inc/dnodeVWrite.h
index 7da701a8e270239ee2f48e090138f048cc81f880..323405143fd10b7b4f73bac5103ad63ed870044f 100644
--- a/src/dnode/inc/dnodeVWrite.h
+++ b/src/dnode/inc/dnodeVWrite.h
@@ -20,9 +20,12 @@
extern "C" {
#endif
-int32_t dnodeInitVnodeWrite();
-void dnodeCleanupVnodeWrite();
-void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg);
+int32_t dnodeInitVWrite();
+void dnodeCleanupVWrite();
+void dnodeDispatchToVWriteQueue(SRpcMsg *pMsg);
+void * dnodeAllocVWriteQueue(void *pVnode);
+void dnodeFreeVWriteQueue(void *wqueue);
+void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code);
#ifdef __cplusplus
}
diff --git a/src/dnode/src/dnodeCfg.c b/src/dnode/src/dnodeCfg.c
new file mode 100644
index 0000000000000000000000000000000000000000..16d109a13a7600c5414b1f1edd175a8f15aa4cf0
--- /dev/null
+++ b/src/dnode/src/dnodeCfg.c
@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeCfg.h"
+
+static SDnodeCfg tsCfg = {0};
+static pthread_mutex_t tsCfgMutex;
+
+static int32_t dnodeReadCfg();
+static int32_t dnodeWriteCfg();
+static void dnodeResetCfg(SDnodeCfg *cfg);
+static void dnodePrintCfg(SDnodeCfg *cfg);
+
+int32_t dnodeInitCfg() {
+ pthread_mutex_init(&tsCfgMutex, NULL);
+ dnodeResetCfg(NULL);
+ int32_t ret = dnodeReadCfg();
+ if (ret == 0) {
+ dInfo("dnode cfg is initialized");
+ }
+ return ret;
+}
+
+void dnodeCleanupCfg() { pthread_mutex_destroy(&tsCfgMutex); }
+
+void dnodeUpdateCfg(SDnodeCfg *cfg) {
+ if (tsCfg.dnodeId != 0) return;
+ dnodeResetCfg(cfg);
+}
+
+int32_t dnodeGetDnodeId() {
+ int32_t dnodeId = 0;
+ pthread_mutex_lock(&tsCfgMutex);
+ dnodeId = tsCfg.dnodeId;
+ pthread_mutex_unlock(&tsCfgMutex);
+ return dnodeId;
+}
+
+void dnodeGetCfg(int32_t *dnodeId, char *clusterId) {
+ pthread_mutex_lock(&tsCfgMutex);
+ *dnodeId = tsCfg.dnodeId;
+ tstrncpy(clusterId, tsCfg.clusterId, TSDB_CLUSTER_ID_LEN);
+ pthread_mutex_unlock(&tsCfgMutex);
+}
+
+static void dnodeResetCfg(SDnodeCfg *cfg) {
+ if (cfg == NULL) return;
+ if (cfg->dnodeId == 0) return;
+
+ pthread_mutex_lock(&tsCfgMutex);
+ tsCfg.dnodeId = cfg->dnodeId;
+ tstrncpy(tsCfg.clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN);
+ dnodePrintCfg(cfg);
+ dnodeWriteCfg();
+ pthread_mutex_unlock(&tsCfgMutex);
+}
+
+static void dnodePrintCfg(SDnodeCfg *cfg) {
+ dInfo("dnodeId is set to %d, clusterId is set to %s", cfg->dnodeId, cfg->clusterId);
+}
+
+static int32_t dnodeReadCfg() {
+ int32_t len = 0;
+ int32_t maxLen = 200;
+ char * content = calloc(1, maxLen + 1);
+ cJSON * root = NULL;
+ FILE * fp = NULL;
+ SDnodeCfg cfg = {0};
+
+ char file[TSDB_FILENAME_LEN + 20] = {0};
+ sprintf(file, "%s/dnodeCfg.json", tsDnodeDir);
+
+ fp = fopen(file, "r");
+ if (!fp) {
+ dDebug("failed to read %s, file not exist", file);
+ goto PARSE_CFG_OVER;
+ }
+
+ len = fread(content, 1, maxLen, fp);
+ if (len <= 0) {
+ dError("failed to read %s, content is null", file);
+ goto PARSE_CFG_OVER;
+ }
+
+ content[len] = 0;
+ root = cJSON_Parse(content);
+ if (root == NULL) {
+ dError("failed to read %s, invalid json format", file);
+ goto PARSE_CFG_OVER;
+ }
+
+ cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId");
+ if (!dnodeId || dnodeId->type != cJSON_Number) {
+ dError("failed to read %s, dnodeId not found", file);
+ goto PARSE_CFG_OVER;
+ }
+ cfg.dnodeId = dnodeId->valueint;
+
+ cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
+ if (!clusterId || clusterId->type != cJSON_String) {
+ dError("failed to read %s, clusterId not found", file);
+ goto PARSE_CFG_OVER;
+ }
+ tstrncpy(cfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN);
+
+ dInfo("read file %s successed", file);
+
+PARSE_CFG_OVER:
+ if (content != NULL) free(content);
+ if (root != NULL) cJSON_Delete(root);
+ if (fp != NULL) fclose(fp);
+ terrno = 0;
+
+ dnodeResetCfg(&cfg);
+ return 0;
+}
+
+static int32_t dnodeWriteCfg() {
+ char file[TSDB_FILENAME_LEN + 20] = {0};
+ sprintf(file, "%s/dnodeCfg.json", tsDnodeDir);
+
+ FILE *fp = fopen(file, "w");
+ if (!fp) {
+ dError("failed to write %s, reason:%s", file, strerror(errno));
+ return -1;
+ }
+
+ int32_t len = 0;
+ int32_t maxLen = 200;
+ char * content = calloc(1, maxLen + 1);
+
+ len += snprintf(content + len, maxLen - len, "{\n");
+ len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", tsCfg.dnodeId);
+ len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%s\"\n", tsCfg.clusterId);
+ len += snprintf(content + len, maxLen - len, "}\n");
+
+ fwrite(content, 1, len, fp);
+ fflush(fp);
+ fclose(fp);
+ free(content);
+ terrno = 0;
+
+ dInfo("successed to write %s", file);
+ return 0;
+}
diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c
index 9b68fc1f6c7a4db266eeb36cc20459bc71870739..a9ee4ac649c2f4b2734ce7f0dd59004b08e2fb67 100644
--- a/src/dnode/src/dnodeCheck.c
+++ b/src/dnode/src/dnodeCheck.c
@@ -15,9 +15,7 @@
#define _DEFAULT_SOURCE
#include "os.h"
-#include "taosdef.h"
#include "tglobal.h"
-#include "mnode.h"
#include "dnodeInt.h"
#include "dnodeCheck.h"
@@ -30,8 +28,8 @@ typedef struct {
void (*stopFp)();
} SCheckItem;
-static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}};
-int64_t tsMinFreeMemSizeForStart = 0;
+static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}};
+int64_t tsMinFreeMemSizeForStart = 0;
static int bindTcpPort(int port) {
int serverSocket;
@@ -264,8 +262,6 @@ int32_t dnodeInitCheck() {
}
}
+ dInfo("dnode check is initialized");
return 0;
}
-
-
-
diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c
new file mode 100644
index 0000000000000000000000000000000000000000..9c90c391813f57024ecf090136dfa86f5e5f91e6
--- /dev/null
+++ b/src/dnode/src/dnodeEps.c
@@ -0,0 +1,283 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "hash.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeEps.h"
+
+static SDnodeEps *tsEps = NULL;
+static SHashObj * tsEpsHash = NULL;
+static pthread_mutex_t tsEpsMutex;
+
+static int32_t dnodeReadEps();
+static int32_t dnodeWriteEps();
+static void dnodeResetEps(SDnodeEps *eps);
+static void dnodePrintEps(SDnodeEps *eps);
+
+int32_t dnodeInitEps() {
+ pthread_mutex_init(&tsEpsMutex, NULL);
+ tsEpsHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
+ dnodeResetEps(NULL);
+ int32_t ret = dnodeReadEps();
+ if (ret == 0) {
+ dInfo("dnode eps is initialized");
+ }
+ return ret;
+}
+
+void dnodeCleanupEps() {
+ pthread_mutex_lock(&tsEpsMutex);
+ if (tsEps) {
+ free(tsEps);
+ tsEps = NULL;
+ }
+ if (tsEpsHash) {
+ taosHashCleanup(tsEpsHash);
+ tsEpsHash = NULL;
+ }
+ pthread_mutex_unlock(&tsEpsMutex);
+ pthread_mutex_destroy(&tsEpsMutex);
+}
+
+void dnodeUpdateEps(SDnodeEps *eps) {
+ if (eps == NULL) return;
+
+ eps->dnodeNum = htonl(eps->dnodeNum);
+ for (int32_t i = 0; i < eps->dnodeNum; ++i) {
+ eps->dnodeEps[i].dnodeId = htonl(eps->dnodeEps[i].dnodeId);
+ eps->dnodeEps[i].dnodePort = htons(eps->dnodeEps[i].dnodePort);
+ }
+
+ pthread_mutex_lock(&tsEpsMutex);
+ if (eps->dnodeNum != tsEps->dnodeNum) {
+ dnodeResetEps(eps);
+ dnodeWriteEps();
+ } else {
+ int32_t size = sizeof(SDnodeEps) + eps->dnodeNum * sizeof(SDnodeEp);
+ if (memcmp(eps, tsEps, size) != 0) {
+ dnodeResetEps(eps);
+ dnodeWriteEps();
+ }
+ }
+ pthread_mutex_unlock(&tsEpsMutex);
+}
+
+bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr) {
+ bool changed = false;
+ pthread_mutex_lock(&tsEpsMutex);
+ SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t));
+ if (ep != NULL) {
+ char epSaved[TSDB_EP_LEN + 1];
+ snprintf(epSaved, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort);
+ changed = strcmp(epstr, epSaved) != 0;
+ tstrncpy(epstr, epSaved, TSDB_EP_LEN);
+ }
+ pthread_mutex_unlock(&tsEpsMutex);
+ return changed;
+}
+
+void dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) {
+ pthread_mutex_lock(&tsEpsMutex);
+ SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t));
+ if (ep != NULL) {
+ if (port) *port = ep->dnodePort;
+ if (fqdn) tstrncpy(fqdn, ep->dnodeFqdn, TSDB_FQDN_LEN);
+ if (epstr) snprintf(epstr, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort);
+ }
+ pthread_mutex_unlock(&tsEpsMutex);
+}
+
+static void dnodeResetEps(SDnodeEps *eps) {
+ if (eps == NULL) {
+ int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp);
+ if (tsEps == NULL) {
+ tsEps = calloc(1, size);
+ } else {
+ tsEps->dnodeNum = 0;
+ }
+ } else {
+ assert(tsEps);
+
+ int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp) * eps->dnodeNum;
+ if (eps->dnodeNum > tsEps->dnodeNum) {
+ tsEps = realloc(tsEps, size);
+ }
+ memcpy(tsEps, eps, size);
+ dnodePrintEps(eps);
+ }
+
+ for (int32_t i = 0; i < tsEps->dnodeNum; ++i) {
+ SDnodeEp *ep = &tsEps->dnodeEps[i];
+ taosHashPut(tsEpsHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp));
+ }
+}
+
+static void dnodePrintEps(SDnodeEps *eps) {
+ dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum);
+ for (int32_t i = 0; i < eps->dnodeNum; i++) {
+ SDnodeEp *ep = &eps->dnodeEps[i];
+ dDebug("dnodeId:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort);
+ }
+}
+
+static int32_t dnodeReadEps() {
+ int32_t ret = -1;
+ int32_t len = 0;
+ int32_t maxLen = 30000;
+ char * content = calloc(1, maxLen + 1);
+ cJSON * root = NULL;
+ FILE * fp = NULL;
+ SDnodeEps *eps = NULL;
+
+ char file[TSDB_FILENAME_LEN + 20] = {0};
+ sprintf(file, "%s/dnodeEps.json", tsDnodeDir);
+
+ fp = fopen(file, "r");
+ if (!fp) {
+ dDebug("failed to read %s, file not exist", file);
+ goto PRASE_EPS_OVER;
+ }
+
+ len = fread(content, 1, maxLen, fp);
+ if (len <= 0) {
+ dError("failed to read %s, content is null", file);
+ goto PRASE_EPS_OVER;
+ }
+
+ content[len] = 0;
+ root = cJSON_Parse(content);
+ if (root == NULL) {
+ dError("failed to read %s, invalid json format", file);
+ goto PRASE_EPS_OVER;
+ }
+
+ cJSON *dnodeNum = cJSON_GetObjectItem(root, "dnodeNum");
+ if (!dnodeNum || dnodeNum->type != cJSON_Number) {
+ dError("failed to read %s, dnodeNum not found", file);
+ goto PRASE_EPS_OVER;
+ }
+
+ cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos");
+ if (!dnodeInfos || dnodeInfos->type != cJSON_Array) {
+ dError("failed to read %s, dnodeInfos not found", file);
+ goto PRASE_EPS_OVER;
+ }
+
+ int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos);
+ if (dnodeInfosSize != dnodeNum->valueint) {
+ dError("failed to read %s, dnodeInfos size:%d not matched dnodeNum:%d", file, dnodeInfosSize,
+ (int32_t)dnodeNum->valueint);
+ goto PRASE_EPS_OVER;
+ }
+
+ int32_t epsSize = sizeof(SDnodeEps) + dnodeInfosSize * sizeof(SDnodeEp);
+ eps = calloc(1, epsSize);
+ eps->dnodeNum = dnodeInfosSize;
+
+ for (int32_t i = 0; i < dnodeInfosSize; ++i) {
+ cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i);
+ if (dnodeInfo == NULL) break;
+
+ SDnodeEp *ep = &eps->dnodeEps[i];
+
+ cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId");
+ if (!dnodeId || dnodeId->type != cJSON_Number) {
+ dError("failed to read %s, dnodeId not found", file);
+ goto PRASE_EPS_OVER;
+ }
+ ep->dnodeId = dnodeId->valueint;
+
+ cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn");
+ if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
+ dError("failed to read %s, dnodeFqdn not found", file);
+ goto PRASE_EPS_OVER;
+ }
+ strncpy(ep->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
+
+ cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort");
+ if (!dnodePort || dnodePort->type != cJSON_Number) {
+ dError("failed to read %s, dnodePort not found", file);
+ goto PRASE_EPS_OVER;
+ }
+ ep->dnodePort = (uint16_t)dnodePort->valueint;
+ }
+
+ ret = 0;
+
+ dInfo("read file %s successed", file);
+ dnodePrintEps(eps);
+
+PRASE_EPS_OVER:
+ if (content != NULL) free(content);
+ if (root != NULL) cJSON_Delete(root);
+ if (fp != NULL) fclose(fp);
+ if (ret != 0) {
+ if (eps) free(eps);
+ eps = NULL;
+ }
+
+ dnodeResetEps(eps);
+ if (eps) free(eps);
+
+ dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort);
+
+ terrno = 0;
+ return 0;
+}
+
+static int32_t dnodeWriteEps() {
+ char file[TSDB_FILENAME_LEN + 20] = {0};
+ sprintf(file, "%s/dnodeEps.json", tsDnodeDir);
+
+ FILE *fp = fopen(file, "w");
+ if (!fp) {
+ dError("failed to write %s, reason:%s", file, strerror(errno));
+ return -1;
+ }
+
+ int32_t len = 0;
+ int32_t maxLen = 30000;
+ char * content = calloc(1, maxLen + 1);
+
+ len += snprintf(content + len, maxLen - len, "{\n");
+ len += snprintf(content + len, maxLen - len, " \"dnodeNum\": %d,\n", tsEps->dnodeNum);
+ len += snprintf(content + len, maxLen - len, " \"dnodeInfos\": [{\n");
+ for (int32_t i = 0; i < tsEps->dnodeNum; ++i) {
+ SDnodeEp *ep = &tsEps->dnodeEps[i];
+ len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", ep->dnodeId);
+ len += snprintf(content + len, maxLen - len, " \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn);
+ len += snprintf(content + len, maxLen - len, " \"dnodePort\": %u\n", ep->dnodePort);
+ if (i < tsEps->dnodeNum - 1) {
+ len += snprintf(content + len, maxLen - len, " },{\n");
+ } else {
+ len += snprintf(content + len, maxLen - len, " }]\n");
+ }
+ }
+ len += snprintf(content + len, maxLen - len, "}\n");
+
+ fwrite(content, 1, len, fp);
+ fflush(fp);
+ fclose(fp);
+ free(content);
+ terrno = 0;
+
+ dInfo("successed to write %s", file);
+ return 0;
+}
diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c
new file mode 100644
index 0000000000000000000000000000000000000000..cefe44aebe7f87803141ce3d75c45dca18463849
--- /dev/null
+++ b/src/dnode/src/dnodeMInfos.c
@@ -0,0 +1,288 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "mnode.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeMInfos.h"
+
+static SMnodeInfos tsMInfos;
+static SRpcEpSet tsMEpSet;
+static pthread_mutex_t tsMInfosMutex;
+
+static void dnodeResetMInfos(SMnodeInfos *minfos);
+static void dnodePrintMInfos(SMnodeInfos *minfos);
+static int32_t dnodeReadMInfos();
+static int32_t dnodeWriteMInfos();
+
+int32_t dnodeInitMInfos() {
+ pthread_mutex_init(&tsMInfosMutex, NULL);
+ dnodeResetMInfos(NULL);
+ int32_t ret = dnodeReadMInfos();
+ if (ret == 0) {
+ dInfo("dnode minfos is initialized");
+ }
+
+ return ret;
+}
+
+void dnodeCleanupMInfos() { pthread_mutex_destroy(&tsMInfosMutex); }
+
+void dnodeUpdateMInfos(SMnodeInfos *minfos) {
+ if (minfos->mnodeNum <= 0 || minfos->mnodeNum > 3) {
+ dError("invalid mnode infos, mnodeNum:%d", minfos->mnodeNum);
+ return;
+ }
+
+ for (int32_t i = 0; i < minfos->mnodeNum; ++i) {
+ SMnodeInfo *minfo = &minfos->mnodeInfos[i];
+ minfo->mnodeId = htonl(minfo->mnodeId);
+ if (minfo->mnodeId <= 0 || strlen(minfo->mnodeEp) <= 5) {
+ dError("invalid mnode info:%d, mnodeId:%d mnodeEp:%s", i, minfo->mnodeId, minfo->mnodeEp);
+ return;
+ }
+ }
+
+ pthread_mutex_lock(&tsMInfosMutex);
+ if (minfos->mnodeNum != tsMInfos.mnodeNum) {
+ dnodeResetMInfos(minfos);
+ dnodeWriteMInfos();
+ sdbUpdateAsync();
+ } else {
+ int32_t size = sizeof(SMnodeInfos);
+ if (memcmp(minfos, &tsMInfos, size) != 0) {
+ dnodeResetMInfos(minfos);
+ dnodeWriteMInfos();
+ sdbUpdateAsync();
+ }
+ }
+ pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+void dnodeUpdateEpSetForPeer(SRpcEpSet *ep) {
+ if (ep->numOfEps <= 0) {
+ dError("minfos is changed, but content is invalid, discard it");
+ return;
+ }
+
+ pthread_mutex_lock(&tsMInfosMutex);
+ dInfo("minfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse);
+ for (int i = 0; i < ep->numOfEps; ++i) {
+ ep->port[i] -= TSDB_PORT_DNODEDNODE;
+ dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]);
+ }
+ tsMEpSet = *ep;
+ pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+bool dnodeIsMasterEp(char *ep) {
+ pthread_mutex_lock(&tsMInfosMutex);
+ bool isMaster = strcmp(ep, tsMInfos.mnodeInfos[tsMEpSet.inUse].mnodeEp) == 0;
+ pthread_mutex_unlock(&tsMInfosMutex);
+
+ return isMaster;
+}
+
+void dnodeGetMInfos(SMnodeInfos *minfos) {
+ pthread_mutex_lock(&tsMInfosMutex);
+ memcpy(minfos, &tsMInfos, sizeof(SMnodeInfos));
+ for (int32_t i = 0; i < tsMInfos.mnodeNum; ++i) {
+ minfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId);
+ }
+ pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+void dnodeGetEpSetForPeer(SRpcEpSet *epSet) {
+ pthread_mutex_lock(&tsMInfosMutex);
+ *epSet = tsMEpSet;
+ for (int i = 0; i < epSet->numOfEps; ++i) {
+ epSet->port[i] += TSDB_PORT_DNODEDNODE;
+ }
+ pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+void dnodeGetEpSetForShell(SRpcEpSet *epSet) {
+ pthread_mutex_lock(&tsMInfosMutex);
+ *epSet = tsMEpSet;
+ pthread_mutex_unlock(&tsMInfosMutex);
+}
+
+static void dnodePrintMInfos(SMnodeInfos *minfos) {
+ dInfo("print mnode infos, mnodeNum:%d inUse:%d", minfos->mnodeNum, minfos->inUse);
+ for (int32_t i = 0; i < minfos->mnodeNum; i++) {
+ dInfo("mnode index:%d, %s", minfos->mnodeInfos[i].mnodeId, minfos->mnodeInfos[i].mnodeEp);
+ }
+}
+
+static void dnodeResetMInfos(SMnodeInfos *minfos) {
+ if (minfos == NULL) {
+ tsMEpSet.numOfEps = 1;
+ taosGetFqdnPortFromEp(tsFirst, tsMEpSet.fqdn[0], &tsMEpSet.port[0]);
+
+ if (strcmp(tsSecond, tsFirst) != 0) {
+ tsMEpSet.numOfEps = 2;
+ taosGetFqdnPortFromEp(tsSecond, tsMEpSet.fqdn[1], &tsMEpSet.port[1]);
+ }
+ return;
+ }
+
+ if (minfos->mnodeNum == 0) return;
+
+ int32_t size = sizeof(SMnodeInfos);
+ memcpy(&tsMInfos, minfos, size);
+
+ tsMEpSet.inUse = tsMInfos.inUse;
+ tsMEpSet.numOfEps = tsMInfos.mnodeNum;
+ for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) {
+ taosGetFqdnPortFromEp(tsMInfos.mnodeInfos[i].mnodeEp, tsMEpSet.fqdn[i], &tsMEpSet.port[i]);
+ }
+
+ dnodePrintMInfos(minfos);
+}
+
+static int32_t dnodeReadMInfos() {
+ int32_t len = 0;
+ int32_t maxLen = 2000;
+ char * content = calloc(1, maxLen + 1);
+ cJSON * root = NULL;
+ FILE * fp = NULL;
+ SMnodeInfos minfos = {0};
+
+ char file[TSDB_FILENAME_LEN + 20] = {0};
+ sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir);
+
+ fp = fopen(file, "r");
+ if (!fp) {
+ dDebug("failed to read %s, file not exist", file);
+ goto PARSE_MINFOS_OVER;
+ }
+
+ len = fread(content, 1, maxLen, fp);
+ if (len <= 0) {
+ dError("failed to read %s, content is null", file);
+ goto PARSE_MINFOS_OVER;
+ }
+
+ content[len] = 0;
+ root = cJSON_Parse(content);
+ if (root == NULL) {
+ dError("failed to read %s, invalid json format", file);
+ goto PARSE_MINFOS_OVER;
+ }
+
+ cJSON *inUse = cJSON_GetObjectItem(root, "inUse");
+ if (!inUse || inUse->type != cJSON_Number) {
+ dError("failed to read mnodeEpSet.json, inUse not found");
+ goto PARSE_MINFOS_OVER;
+ }
+ tsMInfos.inUse = inUse->valueint;
+
+ cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum");
+ if (!nodeNum || nodeNum->type != cJSON_Number) {
+ dError("failed to read mnodeEpSet.json, nodeNum not found");
+ goto PARSE_MINFOS_OVER;
+ }
+ minfos.mnodeNum = nodeNum->valueint;
+
+ cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
+ if (!nodeInfos || nodeInfos->type != cJSON_Array) {
+ dError("failed to read mnodeEpSet.json, nodeInfos not found");
+ goto PARSE_MINFOS_OVER;
+ }
+
+ int size = cJSON_GetArraySize(nodeInfos);
+ if (size != minfos.mnodeNum) {
+ dError("failed to read mnodeEpSet.json, nodeInfos size not matched");
+ goto PARSE_MINFOS_OVER;
+ }
+
+ for (int i = 0; i < size; ++i) {
+ cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
+ if (nodeInfo == NULL) continue;
+
+ cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
+ if (!nodeId || nodeId->type != cJSON_Number) {
+ dError("failed to read mnodeEpSet.json, nodeId not found");
+ goto PARSE_MINFOS_OVER;
+ }
+ minfos.mnodeInfos[i].mnodeId = nodeId->valueint;
+
+ cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
+ if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
+ dError("failed to read mnodeEpSet.json, nodeName not found");
+ goto PARSE_MINFOS_OVER;
+ }
+ strncpy(minfos.mnodeInfos[i].mnodeEp, nodeEp->valuestring, TSDB_EP_LEN);
+ }
+
+ dInfo("read file %s successed", file);
+ dnodePrintMInfos(&minfos);
+
+PARSE_MINFOS_OVER:
+ if (content != NULL) free(content);
+ if (root != NULL) cJSON_Delete(root);
+ if (fp != NULL) fclose(fp);
+ terrno = 0;
+
+ for (int32_t i = 0; i < minfos.mnodeNum; ++i) {
+ SMnodeInfo *mInfo = &minfos.mnodeInfos[i];
+ dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL);
+ }
+ dnodeResetMInfos(&minfos);
+ return 0;
+}
+
+static int32_t dnodeWriteMInfos() {
+ char file[TSDB_FILENAME_LEN + 20] = {0};
+ sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir);
+
+ FILE *fp = fopen(file, "w");
+ if (!fp) {
+ dError("failed to write %s, reason:%s", file, strerror(errno));
+ return -1;
+ }
+
+ int32_t len = 0;
+ int32_t maxLen = 2000;
+ char * content = calloc(1, maxLen + 1);
+
+ len += snprintf(content + len, maxLen - len, "{\n");
+ len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsMInfos.inUse);
+ len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsMInfos.mnodeNum);
+ len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
+ for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) {
+ len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMInfos.mnodeInfos[i].mnodeId);
+ len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", tsMInfos.mnodeInfos[i].mnodeEp);
+ if (i < tsMInfos.mnodeNum - 1) {
+ len += snprintf(content + len, maxLen - len, " },{\n");
+ } else {
+ len += snprintf(content + len, maxLen - len, " }]\n");
+ }
+ }
+ len += snprintf(content + len, maxLen - len, "}\n");
+
+ fwrite(content, 1, len, fp);
+ fflush(fp);
+ fclose(fp);
+ free(content);
+ terrno = 0;
+
+ dInfo("successed to write %s", file);
+ return 0;
+}
diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c
index 8414d79a9815287dfdfae3af5cc123304745c56d..05b37bd3388c22046d2a0dd1827655c97c2e40ad 100644
--- a/src/dnode/src/dnodeMPeer.c
+++ b/src/dnode/src/dnodeMPeer.c
@@ -35,44 +35,44 @@ typedef struct {
typedef struct {
int32_t curNum;
int32_t maxNum;
- SMPeerWorker *peerWorker;
+ SMPeerWorker *worker;
} SMPeerWorkerPool;
-static SMPeerWorkerPool tsMPeerPool;
+static SMPeerWorkerPool tsMPeerWP;
static taos_qset tsMPeerQset;
static taos_queue tsMPeerQueue;
-static void *dnodeProcessMnodePeerQueue(void *param);
+static void *dnodeProcessMPeerQueue(void *param);
-int32_t dnodeInitMnodePeer() {
+int32_t dnodeInitMPeer() {
tsMPeerQset = taosOpenQset();
- tsMPeerPool.maxNum = 1;
- tsMPeerPool.curNum = 0;
- tsMPeerPool.peerWorker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerPool.maxNum);
+ tsMPeerWP.maxNum = 1;
+ tsMPeerWP.curNum = 0;
+ tsMPeerWP.worker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerWP.maxNum);
- if (tsMPeerPool.peerWorker == NULL) return -1;
- for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
- SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+ if (tsMPeerWP.worker == NULL) return -1;
+ for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
+ SMPeerWorker *pWorker = tsMPeerWP.worker + i;
pWorker->workerId = i;
dDebug("dnode mpeer worker:%d is created", i);
}
- dDebug("dnode mpeer is opened, workers:%d qset:%p", tsMPeerPool.maxNum, tsMPeerQset);
+ dDebug("dnode mpeer is initialized, workers:%d qset:%p", tsMPeerWP.maxNum, tsMPeerQset);
return 0;
}
-void dnodeCleanupMnodePeer() {
- for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
- SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+void dnodeCleanupMPeer() {
+ for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
+ SMPeerWorker *pWorker = tsMPeerWP.worker + i;
if (pWorker->thread) {
taosQsetThreadResume(tsMPeerQset);
}
dDebug("dnode mpeer worker:%d is closed", i);
}
- for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
- SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+ for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
+ SMPeerWorker *pWorker = tsMPeerWP.worker + i;
dDebug("dnode mpeer worker:%d start to join", i);
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
@@ -84,61 +84,60 @@ void dnodeCleanupMnodePeer() {
taosCloseQset(tsMPeerQset);
tsMPeerQset = NULL;
- taosTFree(tsMPeerPool.peerWorker);
+ tfree(tsMPeerWP.worker);
}
-int32_t dnodeAllocateMnodePqueue() {
+int32_t dnodeAllocateMPeerQueue() {
tsMPeerQueue = taosOpenQueue();
if (tsMPeerQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
taosAddIntoQset(tsMPeerQset, tsMPeerQueue, NULL);
- for (int32_t i = tsMPeerPool.curNum; i < tsMPeerPool.maxNum; ++i) {
- SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+ for (int32_t i = tsMPeerWP.curNum; i < tsMPeerWP.maxNum; ++i) {
+ SMPeerWorker *pWorker = tsMPeerWP.worker + i;
pWorker->workerId = i;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
- if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodePeerQueue, pWorker) != 0) {
+ if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMPeerQueue, pWorker) != 0) {
dError("failed to create thread to process mpeer queue, reason:%s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
- tsMPeerPool.curNum = i + 1;
- dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerPool.maxNum);
+ tsMPeerWP.curNum = i + 1;
+ dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerWP.maxNum);
}
dDebug("dnode mpeer queue:%p is allocated", tsMPeerQueue);
return TSDB_CODE_SUCCESS;
}
-void dnodeFreeMnodePqueue() {
+void dnodeFreeMPeerQueue() {
dDebug("dnode mpeer queue:%p is freed", tsMPeerQueue);
taosCloseQueue(tsMPeerQueue);
tsMPeerQueue = NULL;
}
-void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg) {
+void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg) {
if (!mnodeIsRunning() || tsMPeerQueue == NULL) {
dnodeSendRedirectMsg(pMsg, false);
- rpcFreeCont(pMsg->pCont);
- return;
+ } else {
+ SMnodeMsg *pPeer = mnodeCreateMsg(pMsg);
+ taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer);
}
- SMnodeMsg *pPeer = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
- mnodeCreateMsg(pPeer, pMsg);
- taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer);
+ rpcFreeCont(pMsg->pCont);
}
-static void dnodeFreeMnodePeerMsg(SMnodeMsg *pPeer) {
+static void dnodeFreeMPeerMsg(SMnodeMsg *pPeer) {
mnodeCleanupMsg(pPeer);
taosFreeQitem(pPeer);
}
-static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) {
+static void dnodeSendRpcMPeerRsp(SMnodeMsg *pPeer, int32_t code) {
if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
SRpcMsg rpcRsp = {
@@ -149,10 +148,10 @@ static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) {
};
rpcSendResponse(&rpcRsp);
- dnodeFreeMnodePeerMsg(pPeer);
+ dnodeFreeMPeerMsg(pPeer);
}
-static void *dnodeProcessMnodePeerQueue(void *param) {
+static void *dnodeProcessMPeerQueue(void *param) {
SMnodeMsg *pPeerMsg;
int32_t type;
void * unUsed;
@@ -165,7 +164,7 @@ static void *dnodeProcessMnodePeerQueue(void *param) {
dDebug("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]);
int32_t code = mnodeProcessPeerReq(pPeerMsg);
- dnodeSendRpcMnodePeerRsp(pPeerMsg, code);
+ dnodeSendRpcMPeerRsp(pPeerMsg, code);
}
return NULL;
diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c
index fdcbb5889f766ddebdb3f1e56ccffa0b5b129552..c14c7a81586d306ff1423e8dba91e0844293c407 100644
--- a/src/dnode/src/dnodeMRead.c
+++ b/src/dnode/src/dnodeMRead.c
@@ -35,46 +35,46 @@ typedef struct {
typedef struct {
int32_t curNum;
int32_t maxNum;
- SMReadWorker *readWorker;
+ SMReadWorker *worker;
} SMReadWorkerPool;
-static SMReadWorkerPool tsMReadPool;
+static SMReadWorkerPool tsMReadWP;
static taos_qset tsMReadQset;
static taos_queue tsMReadQueue;
-static void *dnodeProcessMnodeReadQueue(void *param);
+static void *dnodeProcessMReadQueue(void *param);
-int32_t dnodeInitMnodeRead() {
+int32_t dnodeInitMRead() {
tsMReadQset = taosOpenQset();
- tsMReadPool.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2;
- tsMReadPool.maxNum = MAX(2, tsMReadPool.maxNum);
- tsMReadPool.maxNum = MIN(4, tsMReadPool.maxNum);
- tsMReadPool.curNum = 0;
- tsMReadPool.readWorker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadPool.maxNum);
+ tsMReadWP.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2;
+ tsMReadWP.maxNum = MAX(2, tsMReadWP.maxNum);
+ tsMReadWP.maxNum = MIN(4, tsMReadWP.maxNum);
+ tsMReadWP.curNum = 0;
+ tsMReadWP.worker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadWP.maxNum);
- if (tsMReadPool.readWorker == NULL) return -1;
- for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
- SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+ if (tsMReadWP.worker == NULL) return -1;
+ for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
+ SMReadWorker *pWorker = tsMReadWP.worker + i;
pWorker->workerId = i;
dDebug("dnode mread worker:%d is created", i);
}
- dDebug("dnode mread is opened, workers:%d qset:%p", tsMReadPool.maxNum, tsMReadQset);
+ dDebug("dnode mread is initialized, workers:%d qset:%p", tsMReadWP.maxNum, tsMReadQset);
return 0;
}
-void dnodeCleanupMnodeRead() {
- for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
- SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+void dnodeCleanupMRead() {
+ for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
+ SMReadWorker *pWorker = tsMReadWP.worker + i;
if (pWorker->thread) {
taosQsetThreadResume(tsMReadQset);
}
dDebug("dnode mread worker:%d is closed", i);
}
- for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
- SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+ for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
+ SMReadWorker *pWorker = tsMReadWP.worker + i;
dDebug("dnode mread worker:%d start to join", i);
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
@@ -86,64 +86,63 @@ void dnodeCleanupMnodeRead() {
taosCloseQset(tsMReadQset);
tsMReadQset = NULL;
- free(tsMReadPool.readWorker);
+ free(tsMReadWP.worker);
}
-int32_t dnodeAllocateMnodeRqueue() {
+int32_t dnodeAllocMReadQueue() {
tsMReadQueue = taosOpenQueue();
if (tsMReadQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
taosAddIntoQset(tsMReadQset, tsMReadQueue, NULL);
- for (int32_t i = tsMReadPool.curNum; i < tsMReadPool.maxNum; ++i) {
- SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+ for (int32_t i = tsMReadWP.curNum; i < tsMReadWP.maxNum; ++i) {
+ SMReadWorker *pWorker = tsMReadWP.worker + i;
pWorker->workerId = i;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
- if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeReadQueue, pWorker) != 0) {
+ if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMReadQueue, pWorker) != 0) {
dError("failed to create thread to process mread queue, reason:%s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
- tsMReadPool.curNum = i + 1;
- dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadPool.maxNum);
+ tsMReadWP.curNum = i + 1;
+ dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadWP.maxNum);
}
dDebug("dnode mread queue:%p is allocated", tsMReadQueue);
return TSDB_CODE_SUCCESS;
}
-void dnodeFreeMnodeRqueue() {
+void dnodeFreeMReadQueue() {
dDebug("dnode mread queue:%p is freed", tsMReadQueue);
taosCloseQueue(tsMReadQueue);
tsMReadQueue = NULL;
}
-void dnodeDispatchToMnodeReadQueue(SRpcMsg *pMsg) {
+void dnodeDispatchToMReadQueue(SRpcMsg *pMsg) {
if (!mnodeIsRunning() || tsMReadQueue == NULL) {
dnodeSendRedirectMsg(pMsg, true);
- rpcFreeCont(pMsg->pCont);
- return;
+ } else {
+ SMnodeMsg *pRead = mnodeCreateMsg(pMsg);
+ taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead);
}
- SMnodeMsg *pRead = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
- mnodeCreateMsg(pRead, pMsg);
- taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead);
+ rpcFreeCont(pMsg->pCont);
}
-static void dnodeFreeMnodeReadMsg(SMnodeMsg *pRead) {
+static void dnodeFreeMReadMsg(SMnodeMsg *pRead) {
mnodeCleanupMsg(pRead);
taosFreeQitem(pRead);
}
-static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) {
+static void dnodeSendRpcMReadRsp(SMnodeMsg *pRead, int32_t code) {
if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) {
// may be a auto create req, should put into write queue
- dnodeReprocessMnodeWriteMsg(pRead);
+ dnodeReprocessMWriteMsg(pRead);
return;
}
@@ -155,23 +154,23 @@ static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) {
};
rpcSendResponse(&rpcRsp);
- dnodeFreeMnodeReadMsg(pRead);
+ dnodeFreeMReadMsg(pRead);
}
-static void *dnodeProcessMnodeReadQueue(void *param) {
- SMnodeMsg *pReadMsg;
+static void *dnodeProcessMReadQueue(void *param) {
+ SMnodeMsg *pRead;
int32_t type;
void * unUsed;
-
+
while (1) {
- if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pReadMsg, &unUsed) == 0) {
+ if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pRead, &unUsed) == 0) {
dDebug("qset:%p, mnode read got no message from qset, exiting", tsMReadQset);
break;
}
- dDebug("%p, msg:%s will be processed in mread queue", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]);
- int32_t code = mnodeProcessRead(pReadMsg);
- dnodeSendRpcMnodeReadRsp(pReadMsg, code);
+ dDebug("%p, msg:%s will be processed in mread queue", pRead->rpcMsg.ahandle, taosMsg[pRead->rpcMsg.msgType]);
+ int32_t code = mnodeProcessRead(pRead);
+ dnodeSendRpcMReadRsp(pRead, code);
}
return NULL;
diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c
index 384a0fae75088197d7fb01dd67c6e1b9d38739cd..bde4b95bc6d0019178312cfa76a34bb381608097 100644
--- a/src/dnode/src/dnodeMWrite.c
+++ b/src/dnode/src/dnodeMWrite.c
@@ -36,45 +36,45 @@ typedef struct {
typedef struct {
int32_t curNum;
int32_t maxNum;
- SMWriteWorker *writeWorker;
+ SMWriteWorker *worker;
} SMWriteWorkerPool;
-static SMWriteWorkerPool tsMWritePool;
+static SMWriteWorkerPool tsMWriteWP;
static taos_qset tsMWriteQset;
static taos_queue tsMWriteQueue;
extern void * tsDnodeTmr;
-static void *dnodeProcessMnodeWriteQueue(void *param);
+static void *dnodeProcessMWriteQueue(void *param);
-int32_t dnodeInitMnodeWrite() {
+int32_t dnodeInitMWrite() {
tsMWriteQset = taosOpenQset();
- tsMWritePool.maxNum = 1;
- tsMWritePool.curNum = 0;
- tsMWritePool.writeWorker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWritePool.maxNum);
+ tsMWriteWP.maxNum = 1;
+ tsMWriteWP.curNum = 0;
+ tsMWriteWP.worker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWriteWP.maxNum);
- if (tsMWritePool.writeWorker == NULL) return -1;
- for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
- SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+ if (tsMWriteWP.worker == NULL) return -1;
+ for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
+ SMWriteWorker *pWorker = tsMWriteWP.worker + i;
pWorker->workerId = i;
dDebug("dnode mwrite worker:%d is created", i);
}
- dDebug("dnode mwrite is opened, workers:%d qset:%p", tsMWritePool.maxNum, tsMWriteQset);
+ dDebug("dnode mwrite is initialized, workers:%d qset:%p", tsMWriteWP.maxNum, tsMWriteQset);
return 0;
}
-void dnodeCleanupMnodeWrite() {
- for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
- SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+void dnodeCleanupMWrite() {
+ for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
+ SMWriteWorker *pWorker = tsMWriteWP.worker + i;
if (pWorker->thread) {
taosQsetThreadResume(tsMWriteQset);
}
dDebug("dnode mwrite worker:%d is closed", i);
}
- for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
- SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+ for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
+ SMWriteWorker *pWorker = tsMWriteWP.worker + i;
dDebug("dnode mwrite worker:%d start to join", i);
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
@@ -86,58 +86,56 @@ void dnodeCleanupMnodeWrite() {
taosCloseQset(tsMWriteQset);
tsMWriteQset = NULL;
- taosTFree(tsMWritePool.writeWorker);
+ tfree(tsMWriteWP.worker);
}
-int32_t dnodeAllocateMnodeWqueue() {
+int32_t dnodeAllocMWritequeue() {
tsMWriteQueue = taosOpenQueue();
if (tsMWriteQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
taosAddIntoQset(tsMWriteQset, tsMWriteQueue, NULL);
- for (int32_t i = tsMWritePool.curNum; i < tsMWritePool.maxNum; ++i) {
- SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+ for (int32_t i = tsMWriteWP.curNum; i < tsMWriteWP.maxNum; ++i) {
+ SMWriteWorker *pWorker = tsMWriteWP.worker + i;
pWorker->workerId = i;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
- if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeWriteQueue, pWorker) != 0) {
+ if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMWriteQueue, pWorker) != 0) {
dError("failed to create thread to process mwrite queue, reason:%s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
- tsMWritePool.curNum = i + 1;
- dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWritePool.maxNum);
+ tsMWriteWP.curNum = i + 1;
+ dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWriteWP.maxNum);
}
dDebug("dnode mwrite queue:%p is allocated", tsMWriteQueue);
return TSDB_CODE_SUCCESS;
}
-void dnodeFreeMnodeWqueue() {
+void dnodeFreeMWritequeue() {
dDebug("dnode mwrite queue:%p is freed", tsMWriteQueue);
taosCloseQueue(tsMWriteQueue);
tsMWriteQueue = NULL;
}
-void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) {
+void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) {
if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
dnodeSendRedirectMsg(pMsg, true);
- rpcFreeCont(pMsg->pCont);
- return;
+ } else {
+ SMnodeMsg *pWrite = mnodeCreateMsg(pMsg);
+ dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
+ taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
+ taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
}
- SMnodeMsg *pWrite = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
- mnodeCreateMsg(pWrite, pMsg);
-
- dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
- taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
- taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
+ rpcFreeCont(pMsg->pCont);
}
-static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
+static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) {
dDebug("app:%p:%p, msg:%s is freed from mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
@@ -145,12 +143,12 @@ static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
taosFreeQitem(pWrite);
}
-void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) {
+void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code) {
SMnodeMsg *pWrite = pMsg;
if (pWrite == NULL) return;
if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) {
- dnodeReprocessMnodeWriteMsg(pWrite);
+ dnodeReprocessMWriteMsg(pWrite);
return;
}
@@ -162,10 +160,10 @@ void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) {
};
rpcSendResponse(&rpcRsp);
- dnodeFreeMnodeWriteMsg(pWrite);
+ dnodeFreeMWriteMsg(pWrite);
}
-static void *dnodeProcessMnodeWriteQueue(void *param) {
+static void *dnodeProcessMWriteQueue(void *param) {
SMnodeMsg *pWrite;
int32_t type;
void * unUsed;
@@ -180,13 +178,13 @@ static void *dnodeProcessMnodeWriteQueue(void *param) {
taosMsg[pWrite->rpcMsg.msgType]);
int32_t code = mnodeProcessWrite(pWrite);
- dnodeSendRpcMnodeWriteRsp(pWrite, code);
+ dnodeSendRpcMWriteRsp(pWrite, code);
}
return NULL;
}
-void dnodeReprocessMnodeWriteMsg(void *pMsg) {
+void dnodeReprocessMWriteMsg(void *pMsg) {
SMnodeMsg *pWrite = pMsg;
if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
@@ -194,7 +192,7 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) {
taosMsg[pWrite->rpcMsg.msgType], pWrite->retry);
dnodeSendRedirectMsg(pMsg, true);
- dnodeFreeMnodeWriteMsg(pWrite);
+ dnodeFreeMWriteMsg(pWrite);
} else {
dDebug("app:%p:%p, msg:%s is reput into mwrite queue:%p, retry times:%d", pWrite->rpcMsg.ahandle, pWrite,
taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue, pWrite->retry);
@@ -203,12 +201,12 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) {
}
}
-static void dnodeDoDelayReprocessMnodeWriteMsg(void *param, void *tmrId) {
- dnodeReprocessMnodeWriteMsg(param);
+static void dnodeDoDelayReprocessMWriteMsg(void *param, void *tmrId) {
+ dnodeReprocessMWriteMsg(param);
}
-void dnodeDelayReprocessMnodeWriteMsg(void *pMsg) {
+void dnodeDelayReprocessMWriteMsg(void *pMsg) {
SMnodeMsg *mnodeMsg = pMsg;
void *unUsed = NULL;
- taosTmrReset(dnodeDoDelayReprocessMnodeWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed);
+ taosTmrReset(dnodeDoDelayReprocessMWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed);
}
diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c
index 97e6f2ce6debe7ffc273ca103e3f55576da0eb31..f4c0ee565e4246e0d8a5c4fe4d67a590721e8de2 100644
--- a/src/dnode/src/dnodeMain.c
+++ b/src/dnode/src/dnodeMain.c
@@ -19,11 +19,15 @@
#include "tutil.h"
#include "tconfig.h"
#include "tglobal.h"
+#include "twal.h"
#include "dnode.h"
#include "dnodeInt.h"
#include "dnodeMgmt.h"
#include "dnodePeer.h"
#include "dnodeModule.h"
+#include "dnodeEps.h"
+#include "dnodeMInfos.h"
+#include "dnodeCfg.h"
#include "dnodeCheck.h"
#include "dnodeVRead.h"
#include "dnodeVWrite.h"
@@ -33,29 +37,35 @@
#include "dnodeShell.h"
#include "dnodeTelemetry.h"
+static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED;
+
static int32_t dnodeInitStorage();
-static void dnodeCleanupStorage();
-static void dnodeSetRunStatus(SDnodeRunStatus status);
-static void dnodeCheckDataDirOpenned(char *dir);
-static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED;
+static void dnodeCleanupStorage();
+static void dnodeSetRunStatus(SRunStatus status);
+static void dnodeCheckDataDirOpenned(char *dir);
static int32_t dnodeInitComponents();
-static void dnodeCleanupComponents(int32_t stepId);
-static int dnodeCreateDir(const char *dir);
+static void dnodeCleanupComponents(int32_t stepId);
+static int dnodeCreateDir(const char *dir);
typedef struct {
const char *const name;
- int (*init)();
- void (*cleanup)();
+ int32_t (*init)();
+ void (*cleanup)();
} SDnodeComponent;
static const SDnodeComponent tsDnodeComponents[] = {
{"storage", dnodeInitStorage, dnodeCleanupStorage},
+ {"dnodecfg", dnodeInitCfg, dnodeCleanupCfg},
+ {"dnodeeps", dnodeInitEps, dnodeCleanupEps},
+ {"globalcfg" ,taosCheckGlobalCfg, NULL},
+ {"mnodeinfos",dnodeInitMInfos, dnodeCleanupMInfos},
+ {"wal", walInit, walCleanUp},
{"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
- {"vread", dnodeInitVnodeRead, dnodeCleanupVnodeRead},
- {"vwrite", dnodeInitVnodeWrite, dnodeCleanupVnodeWrite},
- {"mread", dnodeInitMnodeRead, dnodeCleanupMnodeRead},
- {"mwrite", dnodeInitMnodeWrite, dnodeCleanupMnodeWrite},
- {"mpeer", dnodeInitMnodePeer, dnodeCleanupMnodePeer},
+ {"vread", dnodeInitVRead, dnodeCleanupVRead},
+ {"vwrite", dnodeInitVWrite, dnodeCleanupVWrite},
+ {"mread", dnodeInitMRead, dnodeCleanupMRead},
+ {"mwrite", dnodeInitMWrite, dnodeCleanupMWrite},
+ {"mpeer", dnodeInitMPeer, dnodeCleanupMPeer},
{"client", dnodeInitClient, dnodeCleanupClient},
{"server", dnodeInitServer, dnodeCleanupServer},
{"mgmt", dnodeInitMgmt, dnodeCleanupMgmt},
@@ -75,7 +85,9 @@ static int dnodeCreateDir(const char *dir) {
static void dnodeCleanupComponents(int32_t stepId) {
for (int32_t i = stepId; i >= 0; i--) {
- tsDnodeComponents[i].cleanup();
+ if (tsDnodeComponents[i].cleanup) {
+ (*tsDnodeComponents[i].cleanup)();
+ }
}
}
@@ -92,7 +104,7 @@ static int32_t dnodeInitComponents() {
}
int32_t dnodeInitSystem() {
- dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_INITIALIZE);
+ dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE);
tscEmbedded = 1;
taosBlockSIGPIPE();
taosResolveCRC();
@@ -112,21 +124,20 @@ int32_t dnodeInitSystem() {
printf("failed to init log file\n");
}
- if (!taosReadGlobalCfg() || !taosCheckGlobalCfg()) {
+ if (!taosReadGlobalCfg()) {
taosPrintGlobalCfg();
dError("TDengine read global config failed");
return -1;
}
- taosPrintGlobalCfg();
- dInfo("start to initialize TDengine on %s", tsLocalEp);
+ dInfo("start to initialize TDengine");
if (dnodeInitComponents() != 0) {
return -1;
}
dnodeStartModules();
- dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_RUNING);
+ dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING);
dInfo("TDengine is initialized successfully");
@@ -134,20 +145,20 @@ int32_t dnodeInitSystem() {
}
void dnodeCleanUpSystem() {
- if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) {
- dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED);
+ if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) {
+ dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED);
dnodeCleanupComponents(sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]) - 1);
taos_cleanup();
taosCloseLog();
}
}
-SDnodeRunStatus dnodeGetRunStatus() {
- return tsDnodeRunStatus;
+SRunStatus dnodeGetRunStatus() {
+ return tsRunStatus;
}
-static void dnodeSetRunStatus(SDnodeRunStatus status) {
- tsDnodeRunStatus = status;
+static void dnodeSetRunStatus(SRunStatus status) {
+ tsRunStatus = status;
}
static void dnodeCheckDataDirOpenned(char *dir) {
@@ -198,7 +209,7 @@ static int32_t dnodeInitStorage() {
dnodeCheckDataDirOpenned(tsDnodeDir);
- dInfo("storage directory is initialized");
+ dInfo("dnode storage is initialized at %s", tsDnodeDir);
return 0;
}
diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c
index 968a8d9759e5618753996476b40efc3be77f7925..dcb48f7833abf921d1085e1cbb14b9cfbf2aa8b7 100644
--- a/src/dnode/src/dnodeMgmt.c
+++ b/src/dnode/src/dnodeMgmt.c
@@ -31,12 +31,13 @@
#include "mnode.h"
#include "dnodeInt.h"
#include "dnodeMgmt.h"
+#include "dnodeEps.h"
+#include "dnodeCfg.h"
+#include "dnodeMInfos.h"
#include "dnodeVRead.h"
#include "dnodeVWrite.h"
#include "dnodeModule.h"
-#define MPEER_CONTENT_LEN 2000
-
typedef struct {
pthread_t thread;
int32_t threadIndex;
@@ -46,23 +47,18 @@ typedef struct {
int32_t * vnodeList;
} SOpenVnodeThread;
-void * tsDnodeTmr = NULL;
-static void * tsStatusTimer = NULL;
-static uint32_t tsRebootTime;
-
-static SRpcEpSet tsDMnodeEpSet = {0};
-static SDMMnodeInfos tsDMnodeInfos = {0};
-static SDMDnodeCfg tsDnodeCfg = {0};
-static taos_qset tsMgmtQset = NULL;
-static taos_queue tsMgmtQueue = NULL;
-static pthread_t tsQthread;
-
-static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes);
-static bool dnodeReadMnodeInfos();
-static void dnodeSaveMnodeInfos();
-static void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg);
-static bool dnodeReadDnodeCfg();
-static void dnodeSaveDnodeCfg();
+typedef struct {
+ SRpcMsg rpcMsg;
+ char pCont[];
+} SMgmtMsg;
+
+void * tsDnodeTmr = NULL;
+static void * tsStatusTimer = NULL;
+static uint32_t tsRebootTime;
+static taos_qset tsMgmtQset = NULL;
+static taos_queue tsMgmtQueue = NULL;
+static pthread_t tsQthread;
+
static void dnodeProcessStatusRsp(SRpcMsg *pMsg);
static void dnodeSendStatusMsg(void *handle, void *tmrId);
static void *dnodeProcessMgmtQueue(void *param);
@@ -74,7 +70,7 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg);
static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
-static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
+static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);
int32_t dnodeInitMgmt() {
@@ -86,28 +82,8 @@ int32_t dnodeInitMgmt() {
dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg;
dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP, dnodeProcessStatusRsp);
- dnodeReadDnodeCfg();
tsRebootTime = taosGetTimestampSec();
- if (!dnodeReadMnodeInfos()) {
- memset(&tsDMnodeEpSet, 0, sizeof(SRpcEpSet));
- memset(&tsDMnodeInfos, 0, sizeof(SDMMnodeInfos));
-
- tsDMnodeEpSet.numOfEps = 1;
- taosGetFqdnPortFromEp(tsFirst, tsDMnodeEpSet.fqdn[0], &tsDMnodeEpSet.port[0]);
-
- if (strcmp(tsSecond, tsFirst) != 0) {
- tsDMnodeEpSet.numOfEps = 2;
- taosGetFqdnPortFromEp(tsSecond, tsDMnodeEpSet.fqdn[1], &tsDMnodeEpSet.port[1]);
- }
- } else {
- tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse;
- tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum;
- for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
- taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]);
- }
- }
-
int32_t code = vnodeInitResources();
if (code != TSDB_CODE_SUCCESS) {
dnodeCleanupMgmt();
@@ -201,38 +177,46 @@ void dnodeCleanupMgmt() {
vnodeCleanupResources();
}
-void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
- void *item;
+static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
+ int32_t size = sizeof(SMgmtMsg) + pMsg->contLen;
+ SMgmtMsg *pMgmt = taosAllocateQitem(size);
+ if (pMgmt == NULL) {
+ return TSDB_CODE_DND_OUT_OF_MEMORY;
+ }
- item = taosAllocateQitem(sizeof(SRpcMsg));
- if (item) {
- memcpy(item, pMsg, sizeof(SRpcMsg));
- taosWriteQitem(tsMgmtQueue, 1, item);
- } else {
- SRpcMsg rsp = {
- .handle = pMsg->handle,
- .pCont = NULL,
- .code = TSDB_CODE_DND_OUT_OF_MEMORY
- };
-
+ pMgmt->rpcMsg = *pMsg;
+ pMgmt->rpcMsg.pCont = pMgmt->pCont;
+ memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen);
+ taosWriteQitem(tsMgmtQueue, TAOS_QTYPE_RPC, pMgmt);
+
+ return TSDB_CODE_SUCCESS;
+}
+
+void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
+ int32_t code = dnodeWriteToMgmtQueue(pMsg);
+ if (code != TSDB_CODE_SUCCESS) {
+ SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
rpcSendResponse(&rsp);
- rpcFreeCont(pMsg->pCont);
}
+
+ rpcFreeCont(pMsg->pCont);
}
static void *dnodeProcessMgmtQueue(void *param) {
- SRpcMsg *pMsg;
- SRpcMsg rsp = {0};
- int type;
- void * handle;
+ SMgmtMsg *pMgmt;
+ SRpcMsg * pMsg;
+ SRpcMsg rsp = {0};
+ int32_t qtype;
+ void * handle;
while (1) {
- if (taosReadQitemFromQset(tsMgmtQset, &type, (void **) &pMsg, &handle) == 0) {
+ if (taosReadQitemFromQset(tsMgmtQset, &qtype, (void **)&pMgmt, &handle) == 0) {
dDebug("qset:%p, dnode mgmt got no message from qset, exit", tsMgmtQset);
break;
}
- dDebug("%p, msg:%s will be processed", pMsg->ahandle, taosMsg[pMsg->msgType]);
+ pMsg = &pMgmt->rpcMsg;
+ dDebug("%p, msg:%p:%s will be processed", pMsg->ahandle, pMgmt, taosMsg[pMsg->msgType]);
if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
} else {
@@ -240,10 +224,9 @@ static void *dnodeProcessMgmtQueue(void *param) {
}
rsp.handle = pMsg->handle;
- rsp.pCont = NULL;
+ rsp.pCont = NULL;
rpcSendResponse(&rsp);
- rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
@@ -381,7 +364,7 @@ static void dnodeCloseVnodes() {
}
static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
- SMDCreateVnodeMsg *pCreate = rpcMsg->pCont;
+ SCreateVnodeMsg *pCreate = rpcMsg->pCont;
pCreate->cfg.vgId = htonl(pCreate->cfg.vgId);
pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion);
pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables);
@@ -404,7 +387,7 @@ static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
}
static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
- SMDCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);
+ SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg);
void *pVnode = vnodeAcquire(pCreate->cfg.vgId);
if (pVnode != NULL) {
@@ -418,7 +401,7 @@ static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
}
static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
- SMDAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);
+ SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg);
void *pVnode = vnodeAcquire(pAlter->cfg.vgId);
if (pVnode != NULL) {
@@ -433,14 +416,14 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
}
static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
- SMDDropVnodeMsg *pDrop = rpcMsg->pCont;
+ SDropVnodeMsg *pDrop = rpcMsg->pCont;
pDrop->vgId = htonl(pDrop->vgId);
return vnodeDrop(pDrop->vgId);
}
static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
-// SMDAlterStreamMsg *pStream = pCont;
+// SAlterStreamMsg *pStream = pCont;
// pStream->uid = htobe64(pStream->uid);
// pStream->stime = htobe64(pStream->stime);
// pStream->vnode = htonl(pStream->vnode);
@@ -453,12 +436,12 @@ static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
}
static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
- SMDCfgDnodeMsg *pCfg = pMsg->pCont;
+ SCfgDnodeMsg *pCfg = pMsg->pCont;
return taosCfgDynamicOptions(pCfg->config);
}
static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
- SMDCreateMnodeMsg *pCfg = pMsg->pCont;
+ SCreateMnodeMsg *pCfg = pMsg->pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId);
if (pCfg->dnodeId != dnodeGetDnodeId()) {
dError("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
@@ -470,10 +453,10 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED;
}
- dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.nodeNum);
- for (int i = 0; i < pCfg->mnodes.nodeNum; ++i) {
- pCfg->mnodes.nodeInfos[i].nodeId = htonl(pCfg->mnodes.nodeInfos[i].nodeId);
- dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.nodeInfos[i].nodeId, pCfg->mnodes.nodeInfos[i].nodeEp);
+ dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum);
+ for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) {
+ pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId);
+ dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp);
}
dnodeStartMnode(&pCfg->mnodes);
@@ -481,34 +464,6 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
return TSDB_CODE_SUCCESS;
}
-void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet) {
- if (pEpSet->numOfEps <= 0) {
- dError("mnode EP list for peer is changed, but content is invalid, discard it");
- return;
- }
-
- dInfo("mnode EP list for peer is changed, numOfEps:%d inUse:%d", pEpSet->numOfEps, pEpSet->inUse);
- for (int i = 0; i < pEpSet->numOfEps; ++i) {
- pEpSet->port[i] -= TSDB_PORT_DNODEDNODE;
- dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]);
- }
-
- tsDMnodeEpSet = *pEpSet;
-}
-
-void dnodeGetMnodeEpSetForPeer(void *epSetRaw) {
- SRpcEpSet *epSet = epSetRaw;
- *epSet = tsDMnodeEpSet;
-
- for (int i=0; inumOfEps; ++i)
- epSet->port[i] += TSDB_PORT_DNODEDNODE;
-}
-
-void dnodeGetMnodeEpSetForShell(void *epSetRaw) {
- SRpcEpSet *epSet = epSetRaw;
- *epSet = tsDMnodeEpSet;
-}
-
static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
if (pMsg->code != TSDB_CODE_SUCCESS) {
dError("status rsp is received, error:%s", tstrerror(pMsg->code));
@@ -516,202 +471,24 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
return;
}
- SDMStatusRsp *pStatusRsp = pMsg->pCont;
- SDMMnodeInfos *pMnodes = &pStatusRsp->mnodes;
- if (pMnodes->nodeNum <= 0) {
- dError("status msg is invalid, num of ips is %d", pMnodes->nodeNum);
- taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
- return;
- }
+ SStatusRsp *pStatusRsp = pMsg->pCont;
+ SMnodeInfos *minfos = &pStatusRsp->mnodes;
+ dnodeUpdateMInfos(minfos);
- SDMDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
- pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
+ SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
+ pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
pCfg->moduleStatus = htonl(pCfg->moduleStatus);
pCfg->dnodeId = htonl(pCfg->dnodeId);
-
- for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
- SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
- pMnodeInfo->nodeId = htonl(pMnodeInfo->nodeId);
- }
+ dnodeUpdateCfg(pCfg);
vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
- // will not set mnode in status msg
- // dnodeProcessModuleStatus(pCfg->moduleStatus);
- dnodeUpdateDnodeCfg(pCfg);
+ SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess));
+ dnodeUpdateEps(pEps);
- dnodeUpdateMnodeInfos(pMnodes);
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
}
-static bool dnodeCheckMnodeInfos(SDMMnodeInfos *pMnodes) {
- if (pMnodes->nodeNum <= 0 || pMnodes->nodeNum > 3) {
- dError("invalid mnode infos, num:%d", pMnodes->nodeNum);
- return false;
- }
-
- for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
- SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
- if (pMnodeInfo->nodeId <= 0 || strlen(pMnodeInfo->nodeEp) <= 5) {
- dError("invalid mnode info:%d, nodeId:%d nodeEp:%s", i, pMnodeInfo->nodeId, pMnodeInfo->nodeEp);
- return false;
- }
- }
-
- return true;
-}
-
-static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
- bool mnodesChanged = (memcmp(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos)) != 0);
- bool mnodesNotInit = (tsDMnodeInfos.nodeNum == 0);
- if (!(mnodesChanged || mnodesNotInit)) return;
-
- if (!dnodeCheckMnodeInfos(pMnodes)) return;
-
- memcpy(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos));
- dInfo("mnode infos is changed, nodeNum:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse);
- for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
- dInfo("mnode index:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp);
- }
-
- tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse;
- tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum;
- for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
- taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]);
- }
-
- dnodeSaveMnodeInfos();
- sdbUpdateAsync();
-}
-
-static bool dnodeReadMnodeInfos() {
- char ipFile[TSDB_FILENAME_LEN*2] = {0};
-
- sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir);
- FILE *fp = fopen(ipFile, "r");
- if (!fp) {
- dDebug("failed to read mnodeEpSet.json, file not exist");
- return false;
- }
-
- bool ret = false;
- int maxLen = 2000;
- char *content = calloc(1, maxLen + 1);
- int len = fread(content, 1, maxLen, fp);
- if (len <= 0) {
- free(content);
- fclose(fp);
- dError("failed to read mnodeEpSet.json, content is null");
- return false;
- }
-
- content[len] = 0;
- cJSON* root = cJSON_Parse(content);
- if (root == NULL) {
- dError("failed to read mnodeEpSet.json, invalid json format");
- goto PARSE_OVER;
- }
-
- cJSON* inUse = cJSON_GetObjectItem(root, "inUse");
- if (!inUse || inUse->type != cJSON_Number) {
- dError("failed to read mnodeEpSet.json, inUse not found");
- goto PARSE_OVER;
- }
- tsDMnodeInfos.inUse = inUse->valueint;
-
- cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum");
- if (!nodeNum || nodeNum->type != cJSON_Number) {
- dError("failed to read mnodeEpSet.json, nodeNum not found");
- goto PARSE_OVER;
- }
- tsDMnodeInfos.nodeNum = nodeNum->valueint;
-
- cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
- if (!nodeInfos || nodeInfos->type != cJSON_Array) {
- dError("failed to read mnodeEpSet.json, nodeInfos not found");
- goto PARSE_OVER;
- }
-
- int size = cJSON_GetArraySize(nodeInfos);
- if (size != tsDMnodeInfos.nodeNum) {
- dError("failed to read mnodeEpSet.json, nodeInfos size not matched");
- goto PARSE_OVER;
- }
-
- for (int i = 0; i < size; ++i) {
- cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
- if (nodeInfo == NULL) continue;
-
- cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
- if (!nodeId || nodeId->type != cJSON_Number) {
- dError("failed to read mnodeEpSet.json, nodeId not found");
- goto PARSE_OVER;
- }
- tsDMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint;
-
- cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
- if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
- dError("failed to read mnodeEpSet.json, nodeName not found");
- goto PARSE_OVER;
- }
- strncpy(tsDMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
- }
-
- ret = true;
-
- dInfo("read mnode epSet successed, numOfEps:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse);
- for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
- dInfo("mnode:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp);
- }
-
-PARSE_OVER:
- free(content);
- cJSON_Delete(root);
- fclose(fp);
- return ret;
-}
-
-static void dnodeSaveMnodeInfos() {
- char ipFile[TSDB_FILENAME_LEN] = {0};
- sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir);
- FILE *fp = fopen(ipFile, "w");
- if (!fp) return;
-
- int32_t len = 0;
- int32_t maxLen = 2000;
- char * content = calloc(1, maxLen + 1);
-
- len += snprintf(content + len, maxLen - len, "{\n");
- len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsDMnodeInfos.inUse);
- len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsDMnodeInfos.nodeNum);
- len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
- for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
- len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsDMnodeInfos.nodeInfos[i].nodeId);
- len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", tsDMnodeInfos.nodeInfos[i].nodeEp);
- if (i < tsDMnodeInfos.nodeNum -1) {
- len += snprintf(content + len, maxLen - len, " },{\n");
- } else {
- len += snprintf(content + len, maxLen - len, " }]\n");
- }
- }
- len += snprintf(content + len, maxLen - len, "}\n");
-
- fwrite(content, 1, len, fp);
- fflush(fp);
- fclose(fp);
- free(content);
-
- dInfo("save mnode epSet successed");
-}
-
-char *dnodeGetMnodeMasterEp() {
- return tsDMnodeInfos.nodeInfos[tsDMnodeEpSet.inUse].nodeEp;
-}
-
-void* dnodeGetMnodeInfos() {
- return &tsDMnodeInfos;
-}
-
static void dnodeSendStatusMsg(void *handle, void *tmrId) {
if (tsDnodeTmr == NULL) {
dError("dnode timer is already released");
@@ -724,22 +501,21 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
return;
}
- int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
- SDMStatusMsg *pStatus = rpcMallocCont(contLen);
+ int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
+ SStatusMsg *pStatus = rpcMallocCont(contLen);
if (pStatus == NULL) {
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
dError("failed to malloc status message");
return;
}
- //strcpy(pStatus->dnodeName, tsDnodeName);
+ dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId);
+ pStatus->dnodeId = htonl(dnodeGetDnodeId());
pStatus->version = htonl(tsVersion);
- pStatus->dnodeId = htonl(tsDnodeCfg.dnodeId);
pStatus->lastReboot = htonl(tsRebootTime);
pStatus->numOfCores = htons((uint16_t) tsNumOfCores);
pStatus->diskAvailable = tsAvailDataDirGB;
pStatus->alternativeRole = (uint8_t) tsAlternativeRole;
- tstrncpy(pStatus->clusterId, tsDnodeCfg.clusterId, TSDB_CLUSTER_ID_LEN);
tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);
// fill cluster cfg parameters
@@ -759,7 +535,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);
vnodeBuildStatusMsg(pStatus);
- contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
+ contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
pStatus->openVnodes = htons(pStatus->openVnodes);
SRpcMsg rpcMsg = {
@@ -769,110 +545,19 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
};
SRpcEpSet epSet;
- dnodeGetMnodeEpSetForPeer(&epSet);
+ dnodeGetEpSetForPeer(&epSet);
dnodeSendMsgToDnode(&epSet, &rpcMsg);
}
-static bool dnodeReadDnodeCfg() {
- char dnodeCfgFile[TSDB_FILENAME_LEN*2] = {0};
-
- sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir);
-
- FILE *fp = fopen(dnodeCfgFile, "r");
- if (!fp) {
- dDebug("failed to read dnodeCfg.json, file not exist");
- return false;
- }
-
- bool ret = false;
- int maxLen = 100;
- char *content = calloc(1, maxLen + 1);
- int len = fread(content, 1, maxLen, fp);
- if (len <= 0) {
- free(content);
- fclose(fp);
- dError("failed to read dnodeCfg.json, content is null");
- return false;
- }
-
- content[len] = 0;
- cJSON* root = cJSON_Parse(content);
- if (root == NULL) {
- dError("failed to read dnodeCfg.json, invalid json format");
- goto PARSE_CFG_OVER;
- }
-
- cJSON* dnodeId = cJSON_GetObjectItem(root, "dnodeId");
- if (!dnodeId || dnodeId->type != cJSON_Number) {
- dError("failed to read dnodeCfg.json, dnodeId not found");
- goto PARSE_CFG_OVER;
- }
- tsDnodeCfg.dnodeId = dnodeId->valueint;
-
- cJSON* clusterId = cJSON_GetObjectItem(root, "clusterId");
- if (!clusterId || clusterId->type != cJSON_String) {
- dError("failed to read dnodeCfg.json, clusterId not found");
- goto PARSE_CFG_OVER;
- }
- tstrncpy(tsDnodeCfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN);
-
- ret = true;
-
- dInfo("read numOfVnodes successed, dnodeId:%d", tsDnodeCfg.dnodeId);
-
-PARSE_CFG_OVER:
- free(content);
- cJSON_Delete(root);
- fclose(fp);
- return ret;
-}
-
-static void dnodeSaveDnodeCfg() {
- char dnodeCfgFile[TSDB_FILENAME_LEN] = {0};
- sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir);
-
- FILE *fp = fopen(dnodeCfgFile, "w");
- if (!fp) return;
-
- int32_t len = 0;
- int32_t maxLen = 200;
- char * content = calloc(1, maxLen + 1);
-
- len += snprintf(content + len, maxLen - len, "{\n");
- len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", tsDnodeCfg.dnodeId);
- len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%s\"\n", tsDnodeCfg.clusterId);
- len += snprintf(content + len, maxLen - len, "}\n");
-
- fwrite(content, 1, len, fp);
- fflush(fp);
- fclose(fp);
- free(content);
-
- dInfo("save dnodeId successed");
-}
-
-void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) {
- if (tsDnodeCfg.dnodeId == 0) {
- dInfo("dnodeId is set to %d, clusterId is set to %s", pCfg->dnodeId, pCfg->clusterId);
- tsDnodeCfg.dnodeId = pCfg->dnodeId;
- tstrncpy(tsDnodeCfg.clusterId, pCfg->clusterId, TSDB_CLUSTER_ID_LEN);
- dnodeSaveDnodeCfg();
- }
-}
-
-int32_t dnodeGetDnodeId() {
- return tsDnodeCfg.dnodeId;
-}
-
void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
SRpcConnInfo connInfo = {0};
rpcGetConnInfo(rpcMsg->handle, &connInfo);
SRpcEpSet epSet = {0};
if (forShell) {
- dnodeGetMnodeEpSetForShell(&epSet);
+ dnodeGetEpSetForShell(&epSet);
} else {
- dnodeGetMnodeEpSetForPeer(&epSet);
+ dnodeGetEpSetForPeer(&epSet);
}
dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c
index 46376159c6782efde7adbd19c75af83aa4cde397..bd9500ba51226d138fd3fe52f027d144d289681b 100644
--- a/src/dnode/src/dnodeModule.c
+++ b/src/dnode/src/dnodeModule.c
@@ -114,6 +114,7 @@ int32_t dnodeInitModules() {
}
}
+ dInfo("dnode modules is initialized");
return 0;
}
@@ -146,8 +147,8 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) {
}
}
-bool dnodeStartMnode(void *pMnodes) {
- SDMMnodeInfos *mnodes = pMnodes;
+bool dnodeStartMnode(SMnodeInfos *minfos) {
+ SMnodeInfos *mnodes = minfos;
if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) {
dDebug("mnode module is already started, module status:%d", tsModuleStatus);
diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c
index 3bc2f7b48b319f3c9e6215a05463ebedc74035fa..afa712a965e8233baaf9d4f5f14abeee8e88cc38 100644
--- a/src/dnode/src/dnodePeer.c
+++ b/src/dnode/src/dnodePeer.c
@@ -19,6 +19,7 @@
* to dnode. All theses messages are handled from here
*/
+#define _DEFAULT_SOURCE
#include "os.h"
#include "taosmsg.h"
#include "tglobal.h"
@@ -28,20 +29,20 @@
#include "dnodeMgmt.h"
#include "dnodeVWrite.h"
#include "dnodeMPeer.h"
+#include "dnodeMInfos.h"
-extern void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet);
static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *);
static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg);
static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet);
-static void *tsDnodeServerRpc = NULL;
-static void *tsDnodeClientRpc = NULL;
+static void *tsServerRpc = NULL;
+static void *tsClientRpc = NULL;
int32_t dnodeInitServer() {
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVnodeWriteQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = dnodeDispatchToVnodeWriteQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = dnodeDispatchToVnodeWriteQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVnodeWriteQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVWriteQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = dnodeDispatchToVWriteQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = dnodeDispatchToVWriteQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVWriteQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeDispatchToMgmtQueue;
@@ -50,11 +51,11 @@ int32_t dnodeInitServer() {
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue;
dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToMgmtQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMnodePeerQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMnodePeerQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = dnodeDispatchToMnodePeerQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = dnodeDispatchToMnodePeerQueue;
- dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = dnodeDispatchToMnodePeerQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = dnodeDispatchToMPeerQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = dnodeDispatchToMPeerQueue;
+ dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = dnodeDispatchToMPeerQueue;
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
@@ -66,20 +67,20 @@ int32_t dnodeInitServer() {
rpcInit.connType = TAOS_CONN_SERVER;
rpcInit.idleTime = tsShellActivityTimer * 1000;
- tsDnodeServerRpc = rpcOpen(&rpcInit);
- if (tsDnodeServerRpc == NULL) {
+ tsServerRpc = rpcOpen(&rpcInit);
+ if (tsServerRpc == NULL) {
dError("failed to init inter-dnodes RPC server");
return -1;
}
- dInfo("inter-dnodes RPC server is opened");
+ dInfo("dnode inter-dnodes RPC server is initialized");
return 0;
}
void dnodeCleanupServer() {
- if (tsDnodeServerRpc) {
- rpcClose(tsDnodeServerRpc);
- tsDnodeServerRpc = NULL;
+ if (tsServerRpc) {
+ rpcClose(tsServerRpc);
+ tsServerRpc = NULL;
dInfo("inter-dnodes RPC server is closed");
}
}
@@ -93,7 +94,7 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
if (pMsg->pCont == NULL) return;
- if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
+ if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
rspMsg.code = TSDB_CODE_APP_NOT_READY;
rpcSendResponse(&rspMsg);
rpcFreeCont(pMsg->pCont);
@@ -131,27 +132,27 @@ int32_t dnodeInitClient() {
rpcInit.ckey = "key";
rpcInit.secret = secret;
- tsDnodeClientRpc = rpcOpen(&rpcInit);
- if (tsDnodeClientRpc == NULL) {
+ tsClientRpc = rpcOpen(&rpcInit);
+ if (tsClientRpc == NULL) {
dError("failed to init mnode rpc client");
return -1;
}
- dInfo("inter-dnodes rpc client is opened");
+ dInfo("dnode inter-dnodes rpc client is initialized");
return 0;
}
void dnodeCleanupClient() {
- if (tsDnodeClientRpc) {
- rpcClose(tsDnodeClientRpc);
- tsDnodeClientRpc = NULL;
- dInfo("inter-dnodes rpc client is closed");
+ if (tsClientRpc) {
+ rpcClose(tsClientRpc);
+ tsClientRpc = NULL;
+ dInfo("dnode inter-dnodes rpc client is closed");
}
}
static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) {
- dnodeUpdateMnodeEpSetForPeer(pEpSet);
+ dnodeUpdateEpSetForPeer(pEpSet);
}
if (dnodeProcessRspMsgFp[pMsg->msgType]) {
@@ -168,15 +169,15 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
}
void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
- rpcSendRequest(tsDnodeClientRpc, epSet, rpcMsg);
+ rpcSendRequest(tsClientRpc, epSet, rpcMsg);
}
void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
SRpcEpSet epSet = {0};
- dnodeGetMnodeEpSetForPeer(&epSet);
- rpcSendRecv(tsDnodeClientRpc, &epSet, rpcMsg, rpcRsp);
+ dnodeGetEpSetForPeer(&epSet);
+ rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp);
}
void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) {
- rpcSendRecv(tsDnodeClientRpc, epSet, rpcMsg, rpcRsp);
+ rpcSendRecv(tsClientRpc, epSet, rpcMsg, rpcRsp);
}
\ No newline at end of file
diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c
index 4c6c2100e048e9aaf23f4155d3c54604104a8f9b..89f657f78986b8e57c0b5e1dedb841c451db00ba 100644
--- a/src/dnode/src/dnodeShell.c
+++ b/src/dnode/src/dnodeShell.c
@@ -33,46 +33,46 @@
static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *);
static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey);
-static void * tsDnodeShellRpc = NULL;
-static int32_t tsDnodeQueryReqNum = 0;
-static int32_t tsDnodeSubmitReqNum = 0;
+static void * tsShellRpc = NULL;
+static int32_t tsQueryReqNum = 0;
+static int32_t tsSubmitReqNum = 0;
int32_t dnodeInitShell() {
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVnodeWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue;
// the following message shall be treated as mnode write
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMnodeWriteQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMnodeWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMWriteQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMWriteQueue;
// the following message shall be treated as mnode query
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMnodeReadQueue;
- dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMnodeReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue;
+ dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue;
int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore;
numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0);
@@ -91,24 +91,24 @@ int32_t dnodeInitShell() {
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.afp = dnodeRetrieveUserAuthInfo;
- tsDnodeShellRpc = rpcOpen(&rpcInit);
- if (tsDnodeShellRpc == NULL) {
+ tsShellRpc = rpcOpen(&rpcInit);
+ if (tsShellRpc == NULL) {
dError("failed to init shell rpc server");
return -1;
}
- dInfo("shell rpc server is opened");
+ dInfo("dnode shell rpc server is initialized");
return 0;
}
void dnodeCleanupShell() {
- if (tsDnodeShellRpc) {
- rpcClose(tsDnodeShellRpc);
- tsDnodeShellRpc = NULL;
+ if (tsShellRpc) {
+ rpcClose(tsShellRpc);
+ tsShellRpc = NULL;
}
}
-void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
+static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
SRpcMsg rpcMsg = {
.handle = pMsg->handle,
.pCont = NULL,
@@ -117,7 +117,7 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
if (pMsg->pCont == NULL) return;
- if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
+ if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
rpcMsg.code = TSDB_CODE_APP_NOT_READY;
rpcSendResponse(&rpcMsg);
@@ -126,9 +126,9 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
}
if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) {
- atomic_fetch_add_32(&tsDnodeQueryReqNum, 1);
+ atomic_fetch_add_32(&tsQueryReqNum, 1);
} else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
- atomic_fetch_add_32(&tsDnodeSubmitReqNum, 1);
+ atomic_fetch_add_32(&tsSubmitReqNum, 1);
} else {}
if ( dnodeProcessShellMsgFp[pMsg->msgType] ) {
@@ -146,12 +146,12 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey);
if (code != TSDB_CODE_APP_NOT_READY) return code;
- SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg));
+ SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg));
tstrncpy(pMsg->user, user, sizeof(pMsg->user));
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pMsg;
- rpcMsg.contLen = sizeof(SDMAuthMsg);
+ rpcMsg.contLen = sizeof(SAuthMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH;
dDebug("user:%s, send auth msg to mnodes", user);
@@ -161,7 +161,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
if (rpcRsp.code != 0) {
dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code));
} else {
- SDMAuthRsp *pRsp = rpcRsp.pCont;
+ SAuthRsp *pRsp = rpcRsp.pCont;
dDebug("user:%s, auth msg received from mnodes", user);
memcpy(secret, pRsp->secret, TSDB_KEY_LEN);
memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN);
@@ -176,8 +176,8 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char
void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) {
dDebug("vgId:%d, tid:%d send config table msg to mnode", vgId, tid);
- int32_t contLen = sizeof(SDMConfigTableMsg);
- SDMConfigTableMsg *pMsg = rpcMallocCont(contLen);
+ int32_t contLen = sizeof(SConfigTableMsg);
+ SConfigTableMsg *pMsg = rpcMallocCont(contLen);
pMsg->dnodeId = htonl(dnodeGetDnodeId());
pMsg->vgId = htonl(vgId);
@@ -211,12 +211,12 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) {
}
}
-SDnodeStatisInfo dnodeGetStatisInfo() {
- SDnodeStatisInfo info = {0};
- if (dnodeGetRunStatus() == TSDB_DNODE_RUN_STATUS_RUNING) {
+SStatisInfo dnodeGetStatisInfo() {
+ SStatisInfo info = {0};
+ if (dnodeGetRunStatus() == TSDB_RUN_STATUS_RUNING) {
info.httpReqNum = httpGetReqCount();
- info.queryReqNum = atomic_exchange_32(&tsDnodeQueryReqNum, 0);
- info.submitReqNum = atomic_exchange_32(&tsDnodeSubmitReqNum, 0);
+ info.queryReqNum = atomic_exchange_32(&tsQueryReqNum, 0);
+ info.submitReqNum = atomic_exchange_32(&tsSubmitReqNum, 0);
}
return info;
diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c
index 4fdc0b8a73b58efa801f33ea99ffb7d123a72dcf..e973f9901f19b7aa6f4d4958d88e01ba258a82bb 100644
--- a/src/dnode/src/dnodeTelemetry.c
+++ b/src/dnode/src/dnodeTelemetry.c
@@ -268,7 +268,7 @@ static void dnodeGetEmail(char* filepath) {
return;
}
- if (taosTRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) {
+ if (taosRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) {
dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
}
@@ -299,6 +299,7 @@ int32_t dnodeInitTelemetry() {
dTrace("failed to create telemetry thread, reason:%s", strerror(errno));
}
+ dInfo("dnode telemetry is initialized");
return 0;
}
diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c
index e61158ef30dddd6037af36c61be1fd522f8af10b..34df11adcce72a6a21718b81c1895c30c66bea1c 100644
--- a/src/dnode/src/dnodeVRead.c
+++ b/src/dnode/src/dnodeVRead.c
@@ -17,225 +17,184 @@
#include "os.h"
#include "taoserror.h"
#include "taosmsg.h"
-#include "tutil.h"
-#include "tqueue.h"
-#include "twal.h"
#include "tglobal.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
-#include "dnodeVRead.h"
+#include "tqueue.h"
#include "vnode.h"
+#include "dnodeInt.h"
typedef struct {
- pthread_t thread; // thread
- int32_t workerId; // worker ID
-} SReadWorker;
+ pthread_t thread; // thread
+ int32_t workerId; // worker ID
+} SVReadWorker;
typedef struct {
- int32_t max; // max number of workers
- int32_t min; // min number of workers
- int32_t num; // current number of workers
- SReadWorker *readWorker;
+ int32_t max; // max number of workers
+ int32_t min; // min number of workers
+ int32_t num; // current number of workers
+ SVReadWorker * worker;
pthread_mutex_t mutex;
-} SReadWorkerPool;
+} SVReadWorkerPool;
static void *dnodeProcessReadQueue(void *param);
-static void dnodeHandleIdleReadWorker(SReadWorker *);
// module global variable
-static SReadWorkerPool readPool;
-static taos_qset readQset;
+static SVReadWorkerPool tsVReadWP;
+static taos_qset tsVReadQset;
-int32_t dnodeInitVnodeRead() {
- readQset = taosOpenQset();
+int32_t dnodeInitVRead() {
+ tsVReadQset = taosOpenQset();
- readPool.min = tsNumOfCores;
- readPool.max = tsNumOfCores * tsNumOfThreadsPerCore;
- if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min;
- readPool.readWorker = (SReadWorker *)calloc(sizeof(SReadWorker), readPool.max);
- pthread_mutex_init(&readPool.mutex, NULL);
+ tsVReadWP.min = tsNumOfCores;
+ tsVReadWP.max = tsNumOfCores * tsNumOfThreadsPerCore;
+ if (tsVReadWP.max <= tsVReadWP.min * 2) tsVReadWP.max = 2 * tsVReadWP.min;
+ tsVReadWP.worker = (SVReadWorker *)calloc(sizeof(SVReadWorker), tsVReadWP.max);
+ pthread_mutex_init(&tsVReadWP.mutex, NULL);
- if (readPool.readWorker == NULL) return -1;
- for (int i = 0; i < readPool.max; ++i) {
- SReadWorker *pWorker = readPool.readWorker + i;
+ if (tsVReadWP.worker == NULL) return -1;
+ for (int i = 0; i < tsVReadWP.max; ++i) {
+ SVReadWorker *pWorker = tsVReadWP.worker + i;
pWorker->workerId = i;
}
- dInfo("dnode read is opened, min worker:%d max worker:%d", readPool.min, readPool.max);
+ dInfo("dnode vread is initialized, min worker:%d max worker:%d", tsVReadWP.min, tsVReadWP.max);
return 0;
}
-void dnodeCleanupVnodeRead() {
- for (int i = 0; i < readPool.max; ++i) {
- SReadWorker *pWorker = readPool.readWorker + i;
+void dnodeCleanupVRead() {
+ for (int i = 0; i < tsVReadWP.max; ++i) {
+ SVReadWorker *pWorker = tsVReadWP.worker + i;
if (pWorker->thread) {
- taosQsetThreadResume(readQset);
+ taosQsetThreadResume(tsVReadQset);
}
}
- for (int i = 0; i < readPool.max; ++i) {
- SReadWorker *pWorker = readPool.readWorker + i;
+ for (int i = 0; i < tsVReadWP.max; ++i) {
+ SVReadWorker *pWorker = tsVReadWP.worker + i;
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
}
}
- free(readPool.readWorker);
- taosCloseQset(readQset);
- pthread_mutex_destroy(&readPool.mutex);
+ free(tsVReadWP.worker);
+ taosCloseQset(tsVReadQset);
+ pthread_mutex_destroy(&tsVReadWP.mutex);
- dInfo("dnode read is closed");
+ dInfo("dnode vread is closed");
}
-void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) {
- int32_t queuedMsgNum = 0;
- int32_t leftLen = pMsg->contLen;
- char *pCont = (char *) pMsg->pCont;
+void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
+ int32_t queuedMsgNum = 0;
+ int32_t leftLen = pMsg->contLen;
+ char * pCont = (char *)pMsg->pCont;
while (leftLen > 0) {
- SMsgHead *pHead = (SMsgHead *) pCont;
- pHead->vgId = htonl(pHead->vgId);
+ SMsgHead *pHead = (SMsgHead *)pCont;
+ pHead->vgId = htonl(pHead->vgId);
pHead->contLen = htonl(pHead->contLen);
- taos_queue queue = vnodeAcquireRqueue(pHead->vgId);
-
- if (queue == NULL) {
- leftLen -= pHead->contLen;
- pCont -= pHead->contLen;
- continue;
+ void *pVnode = vnodeAcquire(pHead->vgId);
+ if (pVnode != NULL) {
+ int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
+ if (code == TSDB_CODE_SUCCESS) queuedMsgNum++;
+ vnodeRelease(pVnode);
}
- // put message into queue
- SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg));
- pRead->rpcMsg = *pMsg;
- pRead->pCont = pCont;
- pRead->contLen = pHead->contLen;
-
- // next vnode
leftLen -= pHead->contLen;
pCont -= pHead->contLen;
- queuedMsgNum++;
-
- taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead);
}
if (queuedMsgNum == 0) {
- SRpcMsg rpcRsp = {
- .handle = pMsg->handle,
- .pCont = NULL,
- .contLen = 0,
- .code = TSDB_CODE_VND_INVALID_VGROUP_ID,
- .msgType = 0
- };
+ SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID};
rpcSendResponse(&rpcRsp);
- rpcFreeCont(pMsg->pCont);
}
+
+ rpcFreeCont(pMsg->pCont);
}
-void *dnodeAllocateVnodeRqueue(void *pVnode) {
- pthread_mutex_lock(&readPool.mutex);
+void *dnodeAllocVReadQueue(void *pVnode) {
+ pthread_mutex_lock(&tsVReadWP.mutex);
taos_queue queue = taosOpenQueue();
if (queue == NULL) {
- pthread_mutex_unlock(&readPool.mutex);
+ pthread_mutex_unlock(&tsVReadWP.mutex);
return NULL;
}
- taosAddIntoQset(readQset, queue, pVnode);
+ taosAddIntoQset(tsVReadQset, queue, pVnode);
// spawn a thread to process queue
- if (readPool.num < readPool.max) {
+ if (tsVReadWP.num < tsVReadWP.max) {
do {
- SReadWorker *pWorker = readPool.readWorker + readPool.num;
+ SVReadWorker *pWorker = tsVReadWP.worker + tsVReadWP.num;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessReadQueue, pWorker) != 0) {
- dError("failed to create thread to process read queue, reason:%s", strerror(errno));
+ dError("failed to create thread to process vread vqueue since %s", strerror(errno));
}
pthread_attr_destroy(&thAttr);
- readPool.num++;
- dDebug("read worker:%d is launched, total:%d", pWorker->workerId, readPool.num);
- } while (readPool.num < readPool.min);
+ tsVReadWP.num++;
+ dDebug("dnode vread worker:%d is launched, total:%d", pWorker->workerId, tsVReadWP.num);
+ } while (tsVReadWP.num < tsVReadWP.min);
}
- pthread_mutex_unlock(&readPool.mutex);
- dDebug("pVnode:%p, read queue:%p is allocated", pVnode, queue);
+ pthread_mutex_unlock(&tsVReadWP.mutex);
+ dDebug("pVnode:%p, dnode vread queue:%p is allocated", pVnode, queue);
return queue;
}
-void dnodeFreeVnodeRqueue(void *rqueue) {
+void dnodeFreeVReadQueue(void *rqueue) {
taosCloseQueue(rqueue);
-
- // dynamically adjust the number of threads
}
-void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) {
+void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) {
SRpcMsg rpcRsp = {
- .handle = pRead->rpcMsg.handle,
+ .handle = pRead->rpcHandle,
.pCont = pRead->rspRet.rsp,
.contLen = pRead->rspRet.len,
.code = code,
};
rpcSendResponse(&rpcRsp);
- rpcFreeCont(pRead->rpcMsg.pCont);
vnodeRelease(pVnode);
}
-void dnodeDispatchNonRspMsg(void *pVnode, SReadMsg *pRead, int32_t code) {
- rpcFreeCont(pRead->rpcMsg.pCont);
+void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) {
vnodeRelease(pVnode);
}
static void *dnodeProcessReadQueue(void *param) {
- SReadMsg *pReadMsg;
- int type;
- void *pVnode;
+ SVReadMsg *pRead;
+ int32_t qtype;
+ void * pVnode;
while (1) {
- if (taosReadQitemFromQset(readQset, &type, (void **)&pReadMsg, &pVnode) == 0) {
- dDebug("qset:%p dnode read got no message from qset, exiting", readQset);
+ if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pRead, &pVnode) == 0) {
+ dDebug("qset:%p dnode vread got no message from qset, exiting", tsVReadQset);
break;
}
- dDebug("%p, msg:%s will be processed in vread queue, qtype:%d, msg:%p", pReadMsg->rpcMsg.ahandle,
- taosMsg[pReadMsg->rpcMsg.msgType], type, pReadMsg);
+ dDebug("%p, msg:%p:%s will be processed in vread queue, qtype:%d", pRead->rpcAhandle, pRead,
+ taosMsg[pRead->msgType], qtype);
- int32_t code = vnodeProcessRead(pVnode, pReadMsg);
+ int32_t code = vnodeProcessRead(pVnode, pRead);
- if (type == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) {
- dnodeSendRpcReadRsp(pVnode, pReadMsg, code);
+ if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) {
+ dnodeSendRpcVReadRsp(pVnode, pRead, code);
} else {
if (code == TSDB_CODE_QRY_HAS_RSP) {
- dnodeSendRpcReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code);
- } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client
- assert(pReadMsg->rpcMsg.handle == NULL || (pReadMsg->rpcMsg.handle != NULL && pReadMsg->rpcMsg.msgType == 5));
- dnodeDispatchNonRspMsg(pVnode, pReadMsg, code);
+ dnodeSendRpcVReadRsp(pVnode, pRead, pRead->code);
+ } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client
+ assert(pRead->rpcHandle == NULL || (pRead->rpcHandle != NULL && pRead->msgType == 5));
+ dnodeDispatchNonRspMsg(pVnode, pRead, code);
}
}
- taosFreeQitem(pReadMsg);
+ taosFreeQitem(pRead);
}
return NULL;
}
-
-
-UNUSED_FUNC
-static void dnodeHandleIdleReadWorker(SReadWorker *pWorker) {
- int32_t num = taosGetQueueNumber(readQset);
-
- if (num == 0 || (num <= readPool.min && readPool.num > readPool.min)) {
- readPool.num--;
- dDebug("read worker:%d is released, total:%d", pWorker->workerId, readPool.num);
- pthread_exit(NULL);
- } else {
- usleep(30000);
- sched_yield();
- }
-}
-
diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c
index f2740bf6b810197283926a602db4fc423067e231..e0345eb1f6f72c28a99b99546f2d0c2a96b8ed37 100644
--- a/src/dnode/src/dnodeVWrite.c
+++ b/src/dnode/src/dnodeVWrite.c
@@ -15,74 +15,55 @@
#define _DEFAULT_SOURCE
#include "os.h"
-#include "taosmsg.h"
#include "taoserror.h"
-#include "tutil.h"
+#include "taosmsg.h"
+#include "tglobal.h"
#include "tqueue.h"
-#include "trpc.h"
-#include "tsdb.h"
#include "twal.h"
-#include "tdataformat.h"
-#include "tglobal.h"
-#include "tsync.h"
#include "vnode.h"
#include "dnodeInt.h"
-#include "syncInt.h"
-#include "dnodeVWrite.h"
-#include "dnodeMgmt.h"
-
-typedef struct {
- taos_qall qall;
- taos_qset qset; // queue set
- pthread_t thread; // thread
- int32_t workerId; // worker ID
-} SWriteWorker;
typedef struct {
- SRspRet rspRet;
- int32_t processedCount;
- int32_t code;
- void *pCont;
- int32_t contLen;
- SRpcMsg rpcMsg;
-} SWriteMsg;
+ taos_qall qall;
+ taos_qset qset; // queue set
+ int32_t workerId; // worker ID
+ pthread_t thread; // thread
+} SVWriteWorker;
typedef struct {
- int32_t max; // max number of workers
- int32_t nextId; // from 0 to max-1, cyclic
- SWriteWorker *writeWorker;
+ int32_t max; // max number of workers
+ int32_t nextId; // from 0 to max-1, cyclic
+ SVWriteWorker * worker;
pthread_mutex_t mutex;
-} SWriteWorkerPool;
+} SVWriteWorkerPool;
-static void *dnodeProcessWriteQueue(void *param);
-static void dnodeHandleIdleWorker(SWriteWorker *pWorker);
+static SVWriteWorkerPool tsVWriteWP;
+static void *dnodeProcessVWriteQueue(void *param);
-SWriteWorkerPool wWorkerPool;
+int32_t dnodeInitVWrite() {
+ tsVWriteWP.max = tsNumOfCores;
+ tsVWriteWP.worker = (SVWriteWorker *)tcalloc(sizeof(SVWriteWorker), tsVWriteWP.max);
+ if (tsVWriteWP.worker == NULL) return -1;
+ pthread_mutex_init(&tsVWriteWP.mutex, NULL);
-int32_t dnodeInitVnodeWrite() {
- wWorkerPool.max = tsNumOfCores;
- wWorkerPool.writeWorker = (SWriteWorker *)calloc(sizeof(SWriteWorker), wWorkerPool.max);
- if (wWorkerPool.writeWorker == NULL) return -1;
- pthread_mutex_init(&wWorkerPool.mutex, NULL);
-
- for (int32_t i = 0; i < wWorkerPool.max; ++i) {
- wWorkerPool.writeWorker[i].workerId = i;
+ for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
+ tsVWriteWP.worker[i].workerId = i;
}
- dInfo("dnode write is opened, max worker %d", wWorkerPool.max);
+ dInfo("dnode vwrite is initialized, max worker %d", tsVWriteWP.max);
return 0;
}
-void dnodeCleanupVnodeWrite() {
- for (int32_t i = 0; i < wWorkerPool.max; ++i) {
- SWriteWorker *pWorker = wWorkerPool.writeWorker + i;
+void dnodeCleanupVWrite() {
+ for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
+ SVWriteWorker *pWorker = tsVWriteWP.worker + i;
if (pWorker->thread) {
taosQsetThreadResume(pWorker->qset);
}
}
- for (int32_t i = 0; i < wWorkerPool.max; ++i) {
- SWriteWorker *pWorker = wWorkerPool.writeWorker + i;
+ for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
+ SVWriteWorker *pWorker = tsVWriteWP.worker + i;
if (pWorker->thread) {
pthread_join(pWorker->thread, NULL);
taosFreeQall(pWorker->qall);
@@ -90,52 +71,51 @@ void dnodeCleanupVnodeWrite() {
}
}
- pthread_mutex_destroy(&wWorkerPool.mutex);
- free(wWorkerPool.writeWorker);
- dInfo("dnode write is closed");
+ pthread_mutex_destroy(&tsVWriteWP.mutex);
+ tfree(tsVWriteWP.worker);
+ dInfo("dnode vwrite is closed");
}
-void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) {
- char *pCont = (char *)pMsg->pCont;
+void dnodeDispatchToVWriteQueue(SRpcMsg *pRpcMsg) {
+ int32_t code;
+ char *pCont = pRpcMsg->pCont;
- if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
+ if (pRpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
SMsgDesc *pDesc = (SMsgDesc *)pCont;
pDesc->numOfVnodes = htonl(pDesc->numOfVnodes);
pCont += sizeof(SMsgDesc);
}
- SMsgHead *pHead = (SMsgHead *) pCont;
- pHead->vgId = htonl(pHead->vgId);
- pHead->contLen = htonl(pHead->contLen);
-
- taos_queue queue = vnodeAcquireWqueue(pHead->vgId);
- if (queue) {
- // put message into queue
- SWriteMsg *pWrite = (SWriteMsg *)taosAllocateQitem(sizeof(SWriteMsg));
- pWrite->rpcMsg = *pMsg;
- pWrite->pCont = pCont;
- pWrite->contLen = pHead->contLen;
+ SMsgHead *pMsg = (SMsgHead *)pCont;
+ pMsg->vgId = htonl(pMsg->vgId);
+ pMsg->contLen = htonl(pMsg->contLen);
- taosWriteQitem(queue, TAOS_QTYPE_RPC, pWrite);
+ void *pVnode = vnodeAcquire(pMsg->vgId);
+ if (pVnode == NULL) {
+ code = TSDB_CODE_VND_INVALID_VGROUP_ID;
} else {
- SRpcMsg rpcRsp = {
- .handle = pMsg->handle,
- .pCont = NULL,
- .contLen = 0,
- .code = TSDB_CODE_VND_INVALID_VGROUP_ID,
- .msgType = 0
- };
+ SWalHead *pHead = (SWalHead *)(pCont - sizeof(SWalHead));
+ pHead->msgType = pRpcMsg->msgType;
+ pHead->version = 0;
+ pHead->len = pMsg->contLen;
+ code = vnodeWriteToWQueue(pVnode, pHead, TAOS_QTYPE_RPC, pRpcMsg);
+ }
+
+ if (code != TSDB_CODE_SUCCESS) {
+ SRpcMsg rpcRsp = {.handle = pRpcMsg->handle, .code = code};
rpcSendResponse(&rpcRsp);
- rpcFreeCont(pMsg->pCont);
}
+
+ vnodeRelease(pVnode);
+ rpcFreeCont(pRpcMsg->pCont);
}
-void *dnodeAllocateVnodeWqueue(void *pVnode) {
- pthread_mutex_lock(&wWorkerPool.mutex);
- SWriteWorker *pWorker = wWorkerPool.writeWorker + wWorkerPool.nextId;
- void *queue = taosOpenQueue();
+void *dnodeAllocVWriteQueue(void *pVnode) {
+ pthread_mutex_lock(&tsVWriteWP.mutex);
+ SVWriteWorker *pWorker = tsVWriteWP.worker + tsVWriteWP.nextId;
+ taos_queue *queue = taosOpenQueue();
if (queue == NULL) {
- pthread_mutex_unlock(&wWorkerPool.mutex);
+ pthread_mutex_unlock(&tsVWriteWP.mutex);
return NULL;
}
@@ -143,7 +123,7 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) {
pWorker->qset = taosOpenQset();
if (pWorker->qset == NULL) {
taosCloseQueue(queue);
- pthread_mutex_unlock(&wWorkerPool.mutex);
+ pthread_mutex_unlock(&tsVWriteWP.mutex);
return NULL;
}
@@ -152,45 +132,43 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) {
if (pWorker->qall == NULL) {
taosCloseQset(pWorker->qset);
taosCloseQueue(queue);
- pthread_mutex_unlock(&wWorkerPool.mutex);
+ pthread_mutex_unlock(&tsVWriteWP.mutex);
return NULL;
}
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
- if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessWriteQueue, pWorker) != 0) {
- dError("failed to create thread to process read queue, reason:%s", strerror(errno));
+ if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessVWriteQueue, pWorker) != 0) {
+ dError("failed to create thread to process vwrite queue since %s", strerror(errno));
taosFreeQall(pWorker->qall);
taosCloseQset(pWorker->qset);
taosCloseQueue(queue);
queue = NULL;
} else {
- dDebug("write worker:%d is launched", pWorker->workerId);
- wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
+ dDebug("dnode vwrite worker:%d is launched", pWorker->workerId);
+ tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max;
}
pthread_attr_destroy(&thAttr);
} else {
taosAddIntoQset(pWorker->qset, queue, pVnode);
- wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
+ tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max;
}
- pthread_mutex_unlock(&wWorkerPool.mutex);
- dDebug("pVnode:%p, write queue:%p is allocated", pVnode, queue);
+ pthread_mutex_unlock(&tsVWriteWP.mutex);
+ dDebug("pVnode:%p, dnode vwrite queue:%p is allocated", pVnode, queue);
return queue;
}
-void dnodeFreeVnodeWqueue(void *wqueue) {
+void dnodeFreeVWriteQueue(void *wqueue) {
taosCloseQueue(wqueue);
-
- // dynamically adjust the number of threads
}
-void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {
- SWriteMsg *pWrite = (SWriteMsg *)param;
- if (pWrite == NULL) return;
+void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code) {
+ if (param == NULL) return;
+ SVWriteMsg *pWrite = param;
if (code < 0) pWrite->code = code;
int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1);
@@ -198,86 +176,61 @@ void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {
if (count <= 1) return;
SRpcMsg rpcRsp = {
- .handle = pWrite->rpcMsg.handle,
+ .handle = pWrite->rpcHandle,
.pCont = pWrite->rspRet.rsp,
.contLen = pWrite->rspRet.len,
.code = pWrite->code,
};
rpcSendResponse(&rpcRsp);
- rpcFreeCont(pWrite->rpcMsg.pCont);
taosFreeQitem(pWrite);
vnodeRelease(pVnode);
}
-static void *dnodeProcessWriteQueue(void *param) {
- SWriteWorker *pWorker = (SWriteWorker *)param;
- SWriteMsg * pWrite;
- SWalHead * pHead;
- int32_t numOfMsgs;
- int type;
- void * pVnode, *item;
- SRspRet * pRspRet;
+static void *dnodeProcessVWriteQueue(void *param) {
+ SVWriteWorker *pWorker = param;
+ SVWriteMsg * pWrite;
+ void * pVnode;
+ int32_t numOfMsgs;
+ int32_t qtype;
- dDebug("write worker:%d is running", pWorker->workerId);
+ dDebug("dnode vwrite worker:%d is running", pWorker->workerId);
while (1) {
numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode);
if (numOfMsgs == 0) {
- dDebug("qset:%p, dnode write got no message from qset, exiting", pWorker->qset);
+ dDebug("qset:%p, dnode vwrite got no message from qset, exiting", pWorker->qset);
break;
}
+ bool forceFsync = false;
for (int32_t i = 0; i < numOfMsgs; ++i) {
- pWrite = NULL;
- pRspRet = NULL;
- taosGetQitem(pWorker->qall, &type, &item);
- if (type == TAOS_QTYPE_RPC) {
- pWrite = (SWriteMsg *)item;
- pRspRet = &pWrite->rspRet;
- pHead = (SWalHead *)(pWrite->pCont - sizeof(SWalHead));
- pHead->msgType = pWrite->rpcMsg.msgType;
- pHead->version = 0;
- pHead->len = pWrite->contLen;
- dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle,
- taosMsg[pWrite->rpcMsg.msgType]);
- } else if (type == TAOS_QTYPE_CQ) {
- pHead = (SWalHead *)((char*)item + sizeof(SSyncHead));
- dTrace("%p, CQ wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType],
- pHead->version);
- } else {
- pHead = (SWalHead *)item;
- dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType],
- pHead->version);
- }
+ taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite);
+ dTrace("%p, msg:%p:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite->rpcAhandle, pWrite,
+ taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version);
- int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet);
- dTrace("%p, msg:%s is processed in vwrite queue, version:%" PRIu64 ", result:%s", pHead, taosMsg[pHead->msgType],
- pHead->version, tstrerror(code));
+ pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet);
+ if (pWrite->code <= 0) pWrite->processedCount = 1;
+ if (pWrite->pHead->msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true;
- if (pWrite) {
- pWrite->rpcMsg.code = code;
- if (code <= 0) pWrite->processedCount = 1;
- }
+ dTrace("msg:%p is processed in vwrite queue, result:%s", pWrite, tstrerror(pWrite->code));
}
- walFsync(vnodeGetWal(pVnode));
+ walFsync(vnodeGetWal(pVnode), forceFsync);
// browse all items, and process them one by one
taosResetQitems(pWorker->qall);
for (int32_t i = 0; i < numOfMsgs; ++i) {
- taosGetQitem(pWorker->qall, &type, &item);
- if (type == TAOS_QTYPE_RPC) {
- pWrite = (SWriteMsg *)item;
- dnodeSendRpcVnodeWriteRsp(pVnode, item, pWrite->rpcMsg.code);
- } else if (type == TAOS_QTYPE_FWD) {
- pHead = (SWalHead *)item;
- vnodeConfirmForward(pVnode, pHead->version, 0);
- taosFreeQitem(item);
+ taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite);
+ if (qtype == TAOS_QTYPE_RPC) {
+ dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code);
+ } else if (qtype == TAOS_QTYPE_FWD) {
+ vnodeConfirmForward(pVnode, pWrite->pHead->version, 0);
+ taosFreeQitem(pWrite);
vnodeRelease(pVnode);
} else {
- taosFreeQitem(item);
+ taosFreeQitem(pWrite);
vnodeRelease(pVnode);
}
}
@@ -285,19 +238,3 @@ static void *dnodeProcessWriteQueue(void *param) {
return NULL;
}
-
-UNUSED_FUNC
-static void dnodeHandleIdleWorker(SWriteWorker *pWorker) {
- int32_t num = taosGetQueueNumber(pWorker->qset);
-
- if (num > 0) {
- usleep(30000);
- sched_yield();
- } else {
- taosFreeQall(pWorker->qall);
- taosCloseQset(pWorker->qset);
- pWorker->qset = NULL;
- dDebug("write worker:%d is released", pWorker->workerId);
- pthread_exit(NULL);
- }
-}
diff --git a/src/inc/dnode.h b/src/inc/dnode.h
index e84545be1753f3fedb4ee78acf397c91d824ad8b..9454b97e388e6d51b52e7fd6c1fe586b1e5bdbbd 100644
--- a/src/inc/dnode.h
+++ b/src/inc/dnode.h
@@ -21,29 +21,31 @@ extern "C" {
#endif
#include "trpc.h"
+#include "taosmsg.h"
typedef struct {
int32_t queryReqNum;
int32_t submitReqNum;
int32_t httpReqNum;
-} SDnodeStatisInfo;
+} SStatisInfo;
typedef enum {
- TSDB_DNODE_RUN_STATUS_INITIALIZE,
- TSDB_DNODE_RUN_STATUS_RUNING,
- TSDB_DNODE_RUN_STATUS_STOPPED
-} SDnodeRunStatus;
+ TSDB_RUN_STATUS_INITIALIZE,
+ TSDB_RUN_STATUS_RUNING,
+ TSDB_RUN_STATUS_STOPPED
+} SRunStatus;
-SDnodeRunStatus dnodeGetRunStatus();
-SDnodeStatisInfo dnodeGetStatisInfo();
+SRunStatus dnodeGetRunStatus();
+SStatisInfo dnodeGetStatisInfo();
bool dnodeIsFirstDeploy();
-char * dnodeGetMnodeMasterEp();
-void dnodeGetMnodeEpSetForPeer(void *epSet);
-void dnodeGetMnodeEpSetForShell(void *epSet);
-void * dnodeGetMnodeInfos();
+bool dnodeIsMasterEp(char *ep);
+void dnodeGetEpSetForPeer(SRpcEpSet *epSet);
+void dnodeGetEpSetForShell(SRpcEpSet *epSet);
int32_t dnodeGetDnodeId();
-bool dnodeStartMnode(void *pModes);
+void dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port);
+bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr);
+bool dnodeStartMnode(SMnodeInfos *minfos);
void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg);
@@ -51,21 +53,21 @@ void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp);
void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet);
void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid);
-void *dnodeAllocateVnodeWqueue(void *pVnode);
-void dnodeFreeVnodeWqueue(void *queue);
-void *dnodeAllocateVnodeRqueue(void *pVnode);
-void dnodeFreeVnodeRqueue(void *rqueue);
-void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code);
+void *dnodeAllocVWriteQueue(void *pVnode);
+void dnodeFreeVWriteQueue(void *wqueue);
+void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code);
+void *dnodeAllocVReadQueue(void *pVnode);
+void dnodeFreeVReadQueue(void *rqueue);
-int32_t dnodeAllocateMnodePqueue();
-void dnodeFreeMnodePqueue();
-int32_t dnodeAllocateMnodeRqueue();
-void dnodeFreeMnodeRqueue();
-int32_t dnodeAllocateMnodeWqueue();
-void dnodeFreeMnodeWqueue();
-void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code);
-void dnodeReprocessMnodeWriteMsg(void *pMsg);
-void dnodeDelayReprocessMnodeWriteMsg(void *pMsg);
+int32_t dnodeAllocateMPeerQueue();
+void dnodeFreeMPeerQueue();
+int32_t dnodeAllocMReadQueue();
+void dnodeFreeMReadQueue();
+int32_t dnodeAllocMWritequeue();
+void dnodeFreeMWritequeue();
+void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code);
+void dnodeReprocessMWriteMsg(void *pMsg);
+void dnodeDelayReprocessMWriteMsg(void *pMsg);
void dnodeSendStatusMsgToMnode();
diff --git a/src/inc/mnode.h b/src/inc/mnode.h
index 5bef7402e30a0cc9f1964cc2430e0b1f5141590c..bdc30b0c46ced0961715bd48623fdb9e52fb440e 100644
--- a/src/inc/mnode.h
+++ b/src/inc/mnode.h
@@ -35,24 +35,26 @@ typedef struct {
} SMnodeRsp;
typedef struct SMnodeMsg {
- SRpcMsg rpcMsg;
+ struct SAcctObj * pAcct;
+ struct SDnodeObj *pDnode;
+ struct SUserObj * pUser;
+ struct SDbObj * pDb;
+ struct SVgObj * pVgroup;
+ struct STableObj *pTable;
+ struct SSTableObj*pSTable;
SMnodeRsp rpcRsp;
int8_t received;
int8_t successed;
int8_t expected;
int8_t retry;
+ int32_t incomingTs;
int32_t code;
void * pObj;
- struct SAcctObj * pAcct;
- struct SDnodeObj *pDnode;
- struct SUserObj * pUser;
- struct SDbObj * pDb;
- struct SVgObj * pVgroup;
- struct STableObj *pTable;
- struct SSuperTableObj *pSTable;
+ SRpcMsg rpcMsg;
+ char pCont[];
} SMnodeMsg;
-void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg);
+void * mnodeCreateMsg(SRpcMsg *pRpcMsg);
int32_t mnodeInitMsg(SMnodeMsg *pMsg);
void mnodeCleanupMsg(SMnodeMsg *pMsg);
diff --git a/src/inc/query.h b/src/inc/query.h
index 0c18f85dc31bae5e77bae7228d5390a8d32df07a..5e1de77889cc469566cc94b729c55622e5462bd6 100644
--- a/src/inc/query.h
+++ b/src/inc/query.h
@@ -78,7 +78,6 @@ int32_t qKillQuery(qinfo_t qinfo);
int32_t qQueryCompleted(qinfo_t qinfo);
-
/**
* destroy query info structure
* @param qHandle
diff --git a/src/inc/taos.h b/src/inc/taos.h
index 315313734753de73bf477b1f67783a45c38c87c9..2c6454ced1911de603f120c79af5237c2f064c48 100644
--- a/src/inc/taos.h
+++ b/src/inc/taos.h
@@ -64,7 +64,7 @@ typedef struct taosField {
#endif
DLL_EXPORT void taos_init();
-DLL_EXPORT void taos_cleanup();
+DLL_EXPORT void taos_cleanup(void);
DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...);
DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
DLL_EXPORT void taos_close(TAOS *taos);
diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index aee60da20147c674979540aa73da5ac101fae7f0..875092b88da1aa03b7eb017e14fff3148f3644a8 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -75,6 +75,7 @@ extern const int32_t TYPE_BYTES[11];
#define TSDB_DATA_SMALLINT_NULL 0x8000
#define TSDB_DATA_INT_NULL 0x80000000L
#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L
+#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL
#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN
#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN
@@ -363,6 +364,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_MAX_WAL_LEVEL 2
#define TSDB_DEFAULT_WAL_LEVEL 1
+#define TSDB_MIN_DB_UPDATE 0
+#define TSDB_MAX_DB_UPDATE 1
+#define TSDB_DEFAULT_DB_UPDATE_OPTION 0
+
#define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond
#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second
@@ -424,42 +429,47 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_PORT_DNODEDNODE 5
#define TSDB_PORT_SYNC 10
#define TSDB_PORT_HTTP 11
-#define TSDB_PORT_ARBITRATOR 12
+#define TSDB_PORT_ARBITRATOR 12
+
+#define TSDB_MAX_WAL_SIZE (1024*1024)
-#define TAOS_QTYPE_RPC 0
-#define TAOS_QTYPE_FWD 1
-#define TAOS_QTYPE_WAL 2
-#define TAOS_QTYPE_CQ 3
-#define TAOS_QTYPE_QUERY 4
+typedef enum {
+ TAOS_QTYPE_RPC = 0,
+ TAOS_QTYPE_FWD = 1,
+ TAOS_QTYPE_WAL = 2,
+ TAOS_QTYPE_CQ = 3,
+ TAOS_QTYPE_QUERY = 4
+} EQType;
typedef enum {
- TSDB_SUPER_TABLE = 0, // super table
- TSDB_CHILD_TABLE = 1, // table created from super table
- TSDB_NORMAL_TABLE = 2, // ordinary table
- TSDB_STREAM_TABLE = 3, // table created from stream computing
- TSDB_TABLE_MAX = 4
+ TSDB_SUPER_TABLE = 0, // super table
+ TSDB_CHILD_TABLE = 1, // table created from super table
+ TSDB_NORMAL_TABLE = 2, // ordinary table
+ TSDB_STREAM_TABLE = 3, // table created from stream computing
+ TSDB_TABLE_MAX = 4
} ETableType;
typedef enum {
- TSDB_MOD_MNODE,
- TSDB_MOD_HTTP,
- TSDB_MOD_MONITOR,
- TSDB_MOD_MQTT,
- TSDB_MOD_MAX
+ TSDB_MOD_MNODE = 0,
+ TSDB_MOD_HTTP = 1,
+ TSDB_MOD_MONITOR = 2,
+ TSDB_MOD_MQTT = 3,
+ TSDB_MOD_MAX = 4
} EModuleType;
- typedef enum {
- TSDB_CHECK_ITEM_NETWORK,
- TSDB_CHECK_ITEM_MEM,
- TSDB_CHECK_ITEM_CPU,
- TSDB_CHECK_ITEM_DISK,
- TSDB_CHECK_ITEM_OS,
- TSDB_CHECK_ITEM_ACCESS,
- TSDB_CHECK_ITEM_VERSION,
- TSDB_CHECK_ITEM_DATAFILE,
- TSDB_CHECK_ITEM_MAX
- } ECheckItemType;
-
+typedef enum {
+ TSDB_CHECK_ITEM_NETWORK,
+ TSDB_CHECK_ITEM_MEM,
+ TSDB_CHECK_ITEM_CPU,
+ TSDB_CHECK_ITEM_DISK,
+ TSDB_CHECK_ITEM_OS,
+ TSDB_CHECK_ITEM_ACCESS,
+ TSDB_CHECK_ITEM_VERSION,
+ TSDB_CHECK_ITEM_DATAFILE,
+ TSDB_CHECK_ITEM_MAX
+} ECheckItemType;
+
+extern char *qtypeStr[];
#ifdef __cplusplus
}
diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h
index bb111d2da0da75e6a3e3812ac21364f9a18fb6f3..1919747a0b019239e5d6b717921c6a957728d455 100644
--- a/src/inc/taoserror.h
+++ b/src/inc/taoserror.h
@@ -28,7 +28,7 @@ extern "C" {
#else
#define TAOS_DEFINE_ERROR(name, mod, code, msg) static const int32_t name = (0x80000000 | ((mod)<<16) | (code));
#endif
-
+
#define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code))
#define TAOS_SUCCEEDED(err) ((err) >= 0)
#define TAOS_FAILED(err) ((err) < 0)
@@ -37,7 +37,7 @@ const char* tstrerror(int32_t err);
int32_t* taosGetErrno();
#define terrno (*taosGetErrno())
-
+
#define TSDB_CODE_SUCCESS 0
#ifdef TAOS_ERROR_C
@@ -74,6 +74,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_COM_MEMORY_CORRUPTED, 0, 0x0101, "Memory cor
TAOS_DEFINE_ERROR(TSDB_CODE_COM_OUT_OF_MEMORY, 0, 0x0102, "Out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_COM_INVALID_CFG_MSG, 0, 0x0103, "Invalid config message")
TAOS_DEFINE_ERROR(TSDB_CODE_COM_FILE_CORRUPTED, 0, 0x0104, "Data file corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, 0, 0x0105, "Ref out of memory")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, 0, 0x0106, "too many Ref Objs")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, 0, 0x0107, "Ref ID is removed")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID, 0, 0x0108, "Invalid Ref ID")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_ALREADY_EXIST, 0, 0x0109, "Ref is already there")
+TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, 0, 0x010A, "Ref is not there")
//client
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_SQL, 0, 0x0200, "Invalid SQL statement")
@@ -175,6 +181,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, 0, 0x0383, "Invalid da
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN, 0, 0x0384, "Cannot delete monitor database")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, 0, 0x0385, "Too many databases for account")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, 0, 0x0386, "Database not available")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY, 0, 0x0387, "Database unsynced")
// dnode
TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, 0, 0x0400, "Message not processed")
@@ -182,7 +189,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, 0, 0x0401, "Dnode out
TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, 0, 0x0402, "No permission for disk files in dnode")
TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, 0, 0x0403, "Invalid message length")
-// vnode
+// vnode
TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, 0, 0x0500, "Action in progress")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_MSG_NOT_PROCESSED, 0, 0x0501, "Message not processed")
TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_NEED_REPROCESSED, 0, 0x0502, "Action need to be reprocessed")
@@ -230,6 +237,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, 0, 0x070B, "Query buffer limit has reached")
// grant
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired")
@@ -253,6 +261,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, 0, 0x0902, "Invalid Sy
// wal
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, 0, 0x1001, "WAL file is corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, 0, 0x1002, "WAL size exceeds limit")
// http
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE, 0, 0x1100, "http server is not onlin")
diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h
index 600347c44fdb2a89e03bf24413f22240bb2815f2..743616e08ed7601650bd144e6f4a6f1d9fc72230 100644
--- a/src/inc/taosmsg.h
+++ b/src/inc/taosmsg.h
@@ -106,6 +106,10 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" )
+
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "network-test" )
+
+
#ifndef TAOS_MESSAGE_C
TSDB_MSG_TYPE_MAX // 105
#endif
@@ -291,7 +295,7 @@ typedef struct {
SSchema schema[];
// tagVal is padded after schema
// char tagVal[];
-} SCMAlterTableMsg;
+} SAlterTableMsg;
typedef struct {
SMsgHead head;
@@ -308,12 +312,12 @@ typedef struct {
} SUpdateTableTagValMsg;
typedef struct {
- char clientVersion[TSDB_VERSION_LEN];
- char msgVersion[TSDB_VERSION_LEN];
- char db[TSDB_TABLE_FNAME_LEN];
- char appName[TSDB_APPNAME_LEN];
+ char clientVersion[TSDB_VERSION_LEN];
+ char msgVersion[TSDB_VERSION_LEN];
+ char db[TSDB_TABLE_FNAME_LEN];
+ char appName[TSDB_APPNAME_LEN];
int32_t pid;
-} SCMConnectMsg;
+} SConnectMsg;
typedef struct {
char acctId[TSDB_ACCT_LEN];
@@ -324,7 +328,7 @@ typedef struct {
int8_t reserved2;
int32_t connId;
SRpcEpSet epSet;
-} SCMConnectRsp;
+} SConnectRsp;
typedef struct {
int32_t maxUsers;
@@ -344,18 +348,18 @@ typedef struct {
char user[TSDB_USER_LEN];
char pass[TSDB_KEY_LEN];
SAcctCfg cfg;
-} SCMCreateAcctMsg, SCMAlterAcctMsg;
+} SCreateAcctMsg, SAlterAcctMsg;
typedef struct {
char user[TSDB_USER_LEN];
-} SCMDropUserMsg, SCMDropAcctMsg;
+} SDropUserMsg, SDropAcctMsg;
typedef struct {
char user[TSDB_USER_LEN];
char pass[TSDB_KEY_LEN];
int8_t privilege;
int8_t flag;
-} SCMCreateUserMsg, SCMAlterUserMsg;
+} SCreateUserMsg, SAlterUserMsg;
typedef struct {
int32_t contLen;
@@ -370,11 +374,11 @@ typedef struct {
int32_t vgId;
uint64_t uid;
char tableId[TSDB_TABLE_FNAME_LEN];
-} SMDDropSTableMsg;
+} SDropSTableMsg;
typedef struct {
int32_t vgId;
-} SMDDropVnodeMsg;
+} SDropVnodeMsg;
typedef struct SColIndex {
int16_t colId; // column id
@@ -540,12 +544,14 @@ typedef struct {
int8_t replications;
int8_t quorum;
int8_t ignoreExist;
-} SCMCreateDbMsg, SCMAlterDbMsg;
+ int8_t update;
+ int8_t reserve[9];
+} SCreateDbMsg, SAlterDbMsg;
typedef struct {
char db[TSDB_TABLE_FNAME_LEN];
uint8_t ignoreNotExists;
-} SCMDropDbMsg, SCMUseDbMsg;
+} SDropDbMsg, SUseDbMsg;
// IMPORTANT: sizeof(SVnodeStatisticInfo) should not exceed
// TSDB_FILE_HEADER_LEN/4 - TSDB_FILE_HEADER_VERSION_SIZE
@@ -560,7 +566,7 @@ typedef struct {
typedef struct {
int32_t vgId;
int8_t accessState;
-} SDMVgroupAccess;
+} SVgroupAccess;
typedef struct {
int32_t dnodeId;
@@ -568,18 +574,29 @@ typedef struct {
uint32_t numOfVnodes;
char clusterId[TSDB_CLUSTER_ID_LEN];
char reserved[16];
-} SDMDnodeCfg;
+} SDnodeCfg;
typedef struct {
- int32_t nodeId;
- char nodeEp[TSDB_EP_LEN];
-} SDMMnodeInfo;
+ int32_t dnodeId;
+ uint16_t dnodePort;
+ char dnodeFqdn[TSDB_FQDN_LEN];
+} SDnodeEp;
typedef struct {
- int8_t inUse;
- int8_t nodeNum;
- SDMMnodeInfo nodeInfos[TSDB_MAX_REPLICA];
-} SDMMnodeInfos;
+ int32_t dnodeNum;
+ SDnodeEp dnodeEps[];
+} SDnodeEps;
+
+typedef struct {
+ int32_t mnodeId;
+ char mnodeEp[TSDB_EP_LEN];
+} SMnodeInfo;
+
+typedef struct {
+ int8_t inUse;
+ int8_t mnodeNum;
+ SMnodeInfo mnodeInfos[TSDB_MAX_REPLICA];
+} SMnodeInfos;
typedef struct {
int32_t numOfMnodes; // tsNumOfMnodes
@@ -611,13 +628,13 @@ typedef struct {
uint8_t reserve2[15];
SClusterCfg clusterCfg;
SVnodeLoad load[];
-} SDMStatusMsg;
+} SStatusMsg;
typedef struct {
- SDMMnodeInfos mnodes;
- SDMDnodeCfg dnodeCfg;
- SDMVgroupAccess vgAccess[];
-} SDMStatusRsp;
+ SMnodeInfos mnodes;
+ SDnodeCfg dnodeCfg;
+ SVgroupAccess vgAccess[];
+} SStatusRsp;
typedef struct {
uint32_t vgId;
@@ -639,55 +656,56 @@ typedef struct {
int8_t replications;
int8_t wals;
int8_t quorum;
- int8_t reserved[16];
-} SMDVnodeCfg;
+ int8_t update;
+ int8_t reserved[15];
+} SVnodeCfg;
typedef struct {
int32_t nodeId;
char nodeEp[TSDB_EP_LEN];
-} SMDVnodeDesc;
+} SVnodeDesc;
typedef struct {
- char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
- SMDVnodeCfg cfg;
- SMDVnodeDesc nodes[TSDB_MAX_REPLICA];
-} SMDCreateVnodeMsg, SMDAlterVnodeMsg;
+ char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+ SVnodeCfg cfg;
+ SVnodeDesc nodes[TSDB_MAX_REPLICA];
+} SCreateVnodeMsg, SAlterVnodeMsg;
typedef struct {
char tableId[TSDB_TABLE_FNAME_LEN];
int16_t createFlag;
char tags[];
-} SCMTableInfoMsg;
+} STableInfoMsg;
typedef struct {
int32_t numOfTables;
char tableIds[];
-} SCMMultiTableInfoMsg;
+} SMultiTableInfoMsg;
-typedef struct SCMSTableVgroupMsg {
+typedef struct SSTableVgroupMsg {
int32_t numOfTables;
-} SCMSTableVgroupMsg, SCMSTableVgroupRspMsg;
+} SSTableVgroupMsg, SSTableVgroupRspMsg;
typedef struct {
int32_t vgId;
int8_t numOfEps;
SEpAddr1 epAddr[TSDB_MAX_REPLICA];
-} SCMVgroupInfo;
+} SVgroupInfo;
typedef struct {
int32_t vgId;
int8_t numOfEps;
SEpAddrMsg epAddr[TSDB_MAX_REPLICA];
-} SCMVgroupMsg;
+} SVgroupMsg;
typedef struct {
int32_t numOfVgroups;
- SCMVgroupInfo vgroups[];
+ SVgroupInfo vgroups[];
} SVgroupsInfo;
typedef struct {
int32_t numOfVgroups;
- SCMVgroupMsg vgroups[];
+ SVgroupMsg vgroups[];
} SVgroupsMsg;
typedef struct STableMetaMsg {
@@ -702,7 +720,7 @@ typedef struct STableMetaMsg {
int16_t tversion;
int32_t tid;
uint64_t uid;
- SCMVgroupMsg vgroup;
+ SVgroupMsg vgroup;
SSchema schema[];
} STableMetaMsg;
@@ -728,38 +746,38 @@ typedef struct {
char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
uint16_t payloadLen;
char payload[];
-} SCMShowMsg;
+} SShowMsg;
-typedef struct SCMShowRsp {
+typedef struct SShowRsp {
uint64_t qhandle;
STableMetaMsg tableMeta;
-} SCMShowRsp;
+} SShowRsp;
typedef struct {
- char ep[TSDB_EP_LEN]; // end point, hostname:port
-} SCMCreateDnodeMsg, SCMDropDnodeMsg;
+ char ep[TSDB_EP_LEN]; // end point, hostname:port
+} SCreateDnodeMsg, SDropDnodeMsg;
typedef struct {
int32_t dnodeId;
char dnodeEp[TSDB_EP_LEN]; // end point, hostname:port
- SDMMnodeInfos mnodes;
-} SMDCreateMnodeMsg;
+ SMnodeInfos mnodes;
+} SCreateMnodeMsg;
typedef struct {
int32_t dnodeId;
int32_t vgId;
int32_t tid;
-} SDMConfigTableMsg;
+} SConfigTableMsg;
typedef struct {
uint32_t dnodeId;
int32_t vgId;
-} SDMConfigVnodeMsg;
+} SConfigVnodeMsg;
typedef struct {
char ep[TSDB_EP_LEN]; // end point, hostname:port
char config[64];
-} SMDCfgDnodeMsg, SCMCfgDnodeMsg;
+} SCfgDnodeMsg;
typedef struct {
char sql[TSDB_SHOW_SQL_LEN];
@@ -781,13 +799,14 @@ typedef struct {
} SStreamDesc;
typedef struct {
+ char clientVer[TSDB_VERSION_LEN];
uint32_t connId;
int32_t pid;
int32_t numOfQueries;
int32_t numOfStreams;
char appName[TSDB_APPNAME_LEN];
char pData[];
-} SCMHeartBeatMsg;
+} SHeartBeatMsg;
typedef struct {
uint32_t queryId;
@@ -797,11 +816,11 @@ typedef struct {
uint32_t connId;
int8_t killConnection;
SRpcEpSet epSet;
-} SCMHeartBeatRsp;
+} SHeartBeatRsp;
typedef struct {
char queryId[TSDB_KILL_MSG_LEN + 1];
-} SCMKillQueryMsg, SCMKillStreamMsg, SCMKillConnMsg;
+} SKillQueryMsg, SKillStreamMsg, SKillConnMsg;
typedef struct {
int32_t vnode;
@@ -810,7 +829,7 @@ typedef struct {
uint64_t stime; // stream starting time
int32_t status;
char tableId[TSDB_TABLE_FNAME_LEN];
-} SMDAlterStreamMsg;
+} SAlterStreamMsg;
typedef struct {
char user[TSDB_USER_LEN];
@@ -818,7 +837,7 @@ typedef struct {
char encrypt;
char secret[TSDB_KEY_LEN];
char ckey[TSDB_KEY_LEN];
-} SDMAuthMsg, SDMAuthRsp;
+} SAuthMsg, SAuthRsp;
#pragma pack(pop)
diff --git a/src/inc/tcq.h b/src/inc/tcq.h
index 32b75674c3278b3273fd4b98dd645f4168543155..7a0727f1b8dd7cdea1a815c174b9f85004eedf87 100644
--- a/src/inc/tcq.h
+++ b/src/inc/tcq.h
@@ -21,10 +21,10 @@ extern "C" {
#include "tdataformat.h"
-typedef int (*FCqWrite)(void *ahandle, void *pHead, int type);
+typedef int32_t (*FCqWrite)(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
typedef struct {
- int vgId;
+ int32_t vgId;
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char db[TSDB_DB_NAME_LEN];
@@ -42,12 +42,12 @@ void cqStart(void *handle);
void cqStop(void *handle);
// cqCreate is called by TSDB to start an instance of CQ
-void *cqCreate(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema);
+void *cqCreate(void *handle, uint64_t uid, int32_t sid, char *sqlStr, STSchema *pSchema);
// cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate
void cqDrop(void *handle);
-extern int cqDebugFlag;
+extern int32_t cqDebugFlag;
#ifdef __cplusplus
diff --git a/src/inc/trpc.h b/src/inc/trpc.h
index bdee917b5e8203743a79ca27d8fbc987569c35ab..e430a43807bc6fb0dd5c48d8a912223c1ba015af 100644
--- a/src/inc/trpc.h
+++ b/src/inc/trpc.h
@@ -83,13 +83,13 @@ void rpcClose(void *);
void *rpcMallocCont(int contLen);
void rpcFreeCont(void *pCont);
void *rpcReallocCont(void *ptr, int contLen);
-void rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg);
+int64_t rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg);
void rpcSendResponse(const SRpcMsg *pMsg);
void rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet);
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
int rpcReportProgress(void *pConn, char *pCont, int contLen);
-void rpcCancelRequest(void *pContext);
+void rpcCancelRequest(int64_t rid);
#ifdef __cplusplus
}
diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h
index 85f9b3bdc7f1cf806309f597ffad6955151b4c36..d7515a14956d7029d909241b2985023d33e58622 100644
--- a/src/inc/tsdb.h
+++ b/src/inc/tsdb.h
@@ -65,6 +65,7 @@ typedef struct {
int32_t maxRowsPerFileBlock; // maximum rows per file block
int8_t precision;
int8_t compression;
+ int8_t update;
} STsdbCfg;
// --------- TSDB REPOSITORY USAGE STATISTICS
@@ -163,6 +164,12 @@ typedef struct STsdbQueryCond {
SColumnInfo *colList;
} STsdbQueryCond;
+typedef struct SMemRef {
+ int32_t ref;
+ void *mem;
+ void *imem;
+} SMemRef;
+
typedef struct SDataBlockInfo {
STimeWindow window;
int32_t rows;
@@ -192,7 +199,7 @@ typedef struct {
* @param qinfo query info handle from query processor
* @return
*/
-TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo);
+TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo, SMemRef* pRef);
/**
* Get the last row of the given query time window for all the tables in STableGroupInfo object.
@@ -204,7 +211,7 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
* @param tableInfo table list.
* @return
*/
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo);
+TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo, SMemRef* pRef);
/**
* get the queried table object list
@@ -222,7 +229,7 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
* @return
*/
TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
- void *qinfo);
+ void *qinfo, SMemRef* pRef);
/**
* move to next block if exists
@@ -314,6 +321,10 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
*/
void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage);
+int tsdbInitCommitQueue(int nthreads);
+void tsdbDestroyCommitQueue();
+int tsdbSyncCommit(TSDB_REPO_T *repo);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/inc/tsync.h b/src/inc/tsync.h
index ca0f70d104d603d176d89dd5b92979433f390466..d57433eba9bacb622081032f4ed6a2a72406248d 100644
--- a/src/inc/tsync.h
+++ b/src/inc/tsync.h
@@ -51,9 +51,9 @@ typedef struct {
} SSyncCfg;
typedef struct {
- int selfIndex;
- uint32_t nodeId[TAOS_SYNC_MAX_REPLICA];
- int role[TAOS_SYNC_MAX_REPLICA];
+ int32_t selfIndex;
+ uint32_t nodeId[TAOS_SYNC_MAX_REPLICA];
+ int32_t role[TAOS_SYNC_MAX_REPLICA];
} SNodesRole;
/*
@@ -68,10 +68,10 @@ typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uin
// get the wal file from index or after
// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
-typedef int (*FGetWalInfo)(void *ahandle, char *name, uint32_t *index);
+typedef int32_t (*FGetWalInfo)(void *ahandle, char *fileName, int64_t *fileId);
-// when a forward pkt is received, call this to handle data
-typedef int (*FWriteToCache)(void *ahandle, void *pHead, int type);
+// when a forward pkt is received, call this to handle data
+typedef int32_t (*FWriteToCache)(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
// when forward is confirmed by peer, master call this API to notify app
typedef void (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code);
@@ -83,48 +83,47 @@ typedef void (*FNotifyRole)(void *ahandle, int8_t role);
typedef void (*FNotifyFlowCtrl)(void *ahandle, int32_t mseconds);
// when data file is synced successfully, notity app
-typedef int (*FNotifyFileSynced)(void *ahandle, uint64_t fversion);
+typedef int32_t (*FNotifyFileSynced)(void *ahandle, uint64_t fversion);
typedef struct {
- int32_t vgId; // vgroup ID
- uint64_t version; // initial version
- SSyncCfg syncCfg; // configuration from mgmt
- char path[128]; // path to the file
-
- void *ahandle; // handle provided by APP
- FGetFileInfo getFileInfo;
- FGetWalInfo getWalInfo;
- FWriteToCache writeToCache;
- FConfirmForward confirmForward;
- FNotifyRole notifyRole;
- FNotifyFlowCtrl notifyFlowCtrl;
+ int32_t vgId; // vgroup ID
+ uint64_t version; // initial version
+ SSyncCfg syncCfg; // configuration from mgmt
+ char path[128]; // path to the file
+ void * ahandle; // handle provided by APP
+ FGetFileInfo getFileInfo;
+ FGetWalInfo getWalInfo;
+ FWriteToCache writeToCache;
+ FConfirmForward confirmForward;
+ FNotifyRole notifyRole;
+ FNotifyFlowCtrl notifyFlowCtrl;
FNotifyFileSynced notifyFileSynced;
} SSyncInfo;
-typedef void* tsync_h;
+typedef void *tsync_h;
int32_t syncInit();
void syncCleanUp();
-tsync_h syncStart(const SSyncInfo *);
-void syncStop(tsync_h shandle);
-int32_t syncReconfig(tsync_h shandle, const SSyncCfg *);
-int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype);
-void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code);
-void syncRecover(tsync_h shandle); // recover from other nodes:
-int syncGetNodesRole(tsync_h shandle, SNodesRole *);
+int64_t syncStart(const SSyncInfo *);
+void syncStop(int64_t rid);
+int32_t syncReconfig(int64_t rid, const SSyncCfg *);
+int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype);
+void syncConfirmForward(int64_t rid, uint64_t version, int32_t code);
+void syncRecover(int64_t rid); // recover from other nodes:
+int32_t syncGetNodesRole(int64_t rid, SNodesRole *);
-extern char *syncRole[];
+extern char *syncRole[];
//global configurable parameters
-extern int tsMaxSyncNum;
-extern int tsSyncTcpThreads;
-extern int tsMaxWatchFiles;
-extern int tsSyncTimer;
-extern int tsMaxFwdInfo;
-extern int sDebugFlag;
-extern char tsArbitrator[];
-extern uint16_t tsSyncPort;
+extern int32_t tsMaxSyncNum;
+extern int32_t tsSyncTcpThreads;
+extern int32_t tsMaxWatchFiles;
+extern int32_t tsSyncTimer;
+extern int32_t tsMaxFwdInfo;
+extern int32_t sDebugFlag;
+extern char tsArbitrator[];
+extern uint16_t tsSyncPort;
#ifdef __cplusplus
}
diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h
index a94cdaad15a5c2abe35b7a02502ff8ae95102cd9..32f2a2d32ea9000b098b0feadaf5b4cf9e6e8fc9 100644
--- a/src/inc/ttokendef.h
+++ b/src/inc/ttokendef.h
@@ -114,114 +114,115 @@
#define TK_FSYNC 95
#define TK_COMP 96
#define TK_PRECISION 97
-#define TK_LP 98
-#define TK_RP 99
-#define TK_TAGS 100
-#define TK_USING 101
-#define TK_AS 102
-#define TK_COMMA 103
-#define TK_NULL 104
-#define TK_SELECT 105
-#define TK_UNION 106
-#define TK_ALL 107
-#define TK_FROM 108
-#define TK_VARIABLE 109
-#define TK_INTERVAL 110
-#define TK_FILL 111
-#define TK_SLIDING 112
-#define TK_ORDER 113
-#define TK_BY 114
-#define TK_ASC 115
-#define TK_DESC 116
-#define TK_GROUP 117
-#define TK_HAVING 118
-#define TK_LIMIT 119
-#define TK_OFFSET 120
-#define TK_SLIMIT 121
-#define TK_SOFFSET 122
-#define TK_WHERE 123
-#define TK_NOW 124
-#define TK_RESET 125
-#define TK_QUERY 126
-#define TK_ADD 127
-#define TK_COLUMN 128
-#define TK_TAG 129
-#define TK_CHANGE 130
-#define TK_SET 131
-#define TK_KILL 132
-#define TK_CONNECTION 133
-#define TK_STREAM 134
-#define TK_COLON 135
-#define TK_ABORT 136
-#define TK_AFTER 137
-#define TK_ATTACH 138
-#define TK_BEFORE 139
-#define TK_BEGIN 140
-#define TK_CASCADE 141
-#define TK_CLUSTER 142
-#define TK_CONFLICT 143
-#define TK_COPY 144
-#define TK_DEFERRED 145
-#define TK_DELIMITERS 146
-#define TK_DETACH 147
-#define TK_EACH 148
-#define TK_END 149
-#define TK_EXPLAIN 150
-#define TK_FAIL 151
-#define TK_FOR 152
-#define TK_IGNORE 153
-#define TK_IMMEDIATE 154
-#define TK_INITIALLY 155
-#define TK_INSTEAD 156
-#define TK_MATCH 157
-#define TK_KEY 158
-#define TK_OF 159
-#define TK_RAISE 160
-#define TK_REPLACE 161
-#define TK_RESTRICT 162
-#define TK_ROW 163
-#define TK_STATEMENT 164
-#define TK_TRIGGER 165
-#define TK_VIEW 166
-#define TK_COUNT 167
-#define TK_SUM 168
-#define TK_AVG 169
-#define TK_MIN 170
-#define TK_MAX 171
-#define TK_FIRST 172
-#define TK_LAST 173
-#define TK_TOP 174
-#define TK_BOTTOM 175
-#define TK_STDDEV 176
-#define TK_PERCENTILE 177
-#define TK_APERCENTILE 178
-#define TK_LEASTSQUARES 179
-#define TK_HISTOGRAM 180
-#define TK_DIFF 181
-#define TK_SPREAD 182
-#define TK_TWA 183
-#define TK_INTERP 184
-#define TK_LAST_ROW 185
-#define TK_RATE 186
-#define TK_IRATE 187
-#define TK_SUM_RATE 188
-#define TK_SUM_IRATE 189
-#define TK_AVG_RATE 190
-#define TK_AVG_IRATE 191
-#define TK_TBID 192
-#define TK_SEMI 193
-#define TK_NONE 194
-#define TK_PREV 195
-#define TK_LINEAR 196
-#define TK_IMPORT 197
-#define TK_METRIC 198
-#define TK_TBNAME 199
-#define TK_JOIN 200
-#define TK_METRICS 201
-#define TK_STABLE 202
-#define TK_INSERT 203
-#define TK_INTO 204
-#define TK_VALUES 205
+#define TK_UPDATE 98
+#define TK_LP 99
+#define TK_RP 100
+#define TK_TAGS 101
+#define TK_USING 102
+#define TK_AS 103
+#define TK_COMMA 104
+#define TK_NULL 105
+#define TK_SELECT 106
+#define TK_UNION 107
+#define TK_ALL 108
+#define TK_FROM 109
+#define TK_VARIABLE 110
+#define TK_INTERVAL 111
+#define TK_FILL 112
+#define TK_SLIDING 113
+#define TK_ORDER 114
+#define TK_BY 115
+#define TK_ASC 116
+#define TK_DESC 117
+#define TK_GROUP 118
+#define TK_HAVING 119
+#define TK_LIMIT 120
+#define TK_OFFSET 121
+#define TK_SLIMIT 122
+#define TK_SOFFSET 123
+#define TK_WHERE 124
+#define TK_NOW 125
+#define TK_RESET 126
+#define TK_QUERY 127
+#define TK_ADD 128
+#define TK_COLUMN 129
+#define TK_TAG 130
+#define TK_CHANGE 131
+#define TK_SET 132
+#define TK_KILL 133
+#define TK_CONNECTION 134
+#define TK_STREAM 135
+#define TK_COLON 136
+#define TK_ABORT 137
+#define TK_AFTER 138
+#define TK_ATTACH 139
+#define TK_BEFORE 140
+#define TK_BEGIN 141
+#define TK_CASCADE 142
+#define TK_CLUSTER 143
+#define TK_CONFLICT 144
+#define TK_COPY 145
+#define TK_DEFERRED 146
+#define TK_DELIMITERS 147
+#define TK_DETACH 148
+#define TK_EACH 149
+#define TK_END 150
+#define TK_EXPLAIN 151
+#define TK_FAIL 152
+#define TK_FOR 153
+#define TK_IGNORE 154
+#define TK_IMMEDIATE 155
+#define TK_INITIALLY 156
+#define TK_INSTEAD 157
+#define TK_MATCH 158
+#define TK_KEY 159
+#define TK_OF 160
+#define TK_RAISE 161
+#define TK_REPLACE 162
+#define TK_RESTRICT 163
+#define TK_ROW 164
+#define TK_STATEMENT 165
+#define TK_TRIGGER 166
+#define TK_VIEW 167
+#define TK_COUNT 168
+#define TK_SUM 169
+#define TK_AVG 170
+#define TK_MIN 171
+#define TK_MAX 172
+#define TK_FIRST 173
+#define TK_LAST 174
+#define TK_TOP 175
+#define TK_BOTTOM 176
+#define TK_STDDEV 177
+#define TK_PERCENTILE 178
+#define TK_APERCENTILE 179
+#define TK_LEASTSQUARES 180
+#define TK_HISTOGRAM 181
+#define TK_DIFF 182
+#define TK_SPREAD 183
+#define TK_TWA 184
+#define TK_INTERP 185
+#define TK_LAST_ROW 186
+#define TK_RATE 187
+#define TK_IRATE 188
+#define TK_SUM_RATE 189
+#define TK_SUM_IRATE 190
+#define TK_AVG_RATE 191
+#define TK_AVG_IRATE 192
+#define TK_TBID 193
+#define TK_SEMI 194
+#define TK_NONE 195
+#define TK_PREV 196
+#define TK_LINEAR 197
+#define TK_IMPORT 198
+#define TK_METRIC 199
+#define TK_TBNAME 200
+#define TK_JOIN 201
+#define TK_METRICS 202
+#define TK_STABLE 203
+#define TK_INSERT 204
+#define TK_INTO 205
+#define TK_VALUES 206
#define TK_SPACE 300
#define TK_COMMENT 301
diff --git a/src/inc/twal.h b/src/inc/twal.h
index 92204abd7d34a9ee2eebf1b74c2e8d58b9599f17..8dd3a8a91209e840abeb9560f94a52ce362492a9 100644
--- a/src/inc/twal.h
+++ b/src/inc/twal.h
@@ -19,42 +19,53 @@
extern "C" {
#endif
-#define TAOS_WAL_NOLOG 0
-#define TAOS_WAL_WRITE 1
-#define TAOS_WAL_FSYNC 2
-
+typedef enum {
+ TAOS_WAL_NOLOG = 0,
+ TAOS_WAL_WRITE = 1,
+ TAOS_WAL_FSYNC = 2
+} EWalType;
+
+typedef enum {
+ TAOS_WAL_NOT_KEEP = 0,
+ TAOS_WAL_KEEP = 1
+} EWalKeep;
+
typedef struct {
- int8_t msgType;
- int8_t reserved[3];
- int32_t len;
- uint64_t version;
- uint32_t signature;
- uint32_t cksum;
- char cont[];
+ int8_t msgType;
+ int8_t sver;
+ int8_t reserved[2];
+ int32_t len;
+ uint64_t version;
+ uint32_t signature;
+ uint32_t cksum;
+ char cont[];
} SWalHead;
typedef struct {
- int8_t walLevel; // wal level
- int32_t fsyncPeriod; // millisecond
- int8_t wals; // number of WAL files;
- int8_t keep; // keep the wal file when closed
+ int32_t vgId;
+ int32_t fsyncPeriod; // millisecond
+ EWalType walLevel; // wal level
+ EWalKeep keep; // keep the wal file when closed
} SWalCfg;
-typedef void* twalh; // WAL HANDLE
-typedef int (*FWalWrite)(void *ahandle, void *pHead, int type);
-
-twalh walOpen(const char *path, const SWalCfg *pCfg);
-int walAlter(twalh pWal, const SWalCfg *pCfg);
-void walClose(twalh);
-int walRenew(twalh);
-int walWrite(twalh, SWalHead *);
-void walFsync(twalh);
-int walRestore(twalh, void *pVnode, FWalWrite writeFp);
-int walGetWalFile(twalh, char *name, uint32_t *index);
-int64_t walGetVersion(twalh);
+typedef void * twalh; // WAL HANDLE
+typedef int32_t FWalWrite(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
-extern int wDebugFlag;
+int32_t walInit();
+void walCleanUp();
+twalh walOpen(char *path, SWalCfg *pCfg);
+int32_t walAlter(twalh pWal, SWalCfg *pCfg);
+void walStop(twalh);
+void walClose(twalh);
+int32_t walRenew(twalh);
+void walRemoveOneOldFile(twalh);
+void walRemoveAllOldFiles(twalh);
+int32_t walWrite(twalh, SWalHead *);
+void walFsync(twalh, bool forceFsync);
+int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp);
+int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId);
+uint64_t walGetVersion(twalh);
#ifdef __cplusplus
}
diff --git a/src/inc/vnode.h b/src/inc/vnode.h
index fdce4d62794075bd2e7027b125780fbd7a2deaed..018e96e1938336809fc8829be42ed3e946c6ac98 100644
--- a/src/inc/vnode.h
+++ b/src/inc/vnode.h
@@ -20,6 +20,8 @@
extern "C" {
#endif
+#include "twal.h"
+
typedef enum _VN_STATUS {
TAOS_VN_STATUS_INIT,
TAOS_VN_STATUS_READY,
@@ -29,44 +31,57 @@ typedef enum _VN_STATUS {
} EVnStatus;
typedef struct {
- int len;
- void *rsp;
- void *qhandle; //used by query and retrieve msg
+ int32_t len;
+ void * rsp;
+ void * qhandle; // used by query and retrieve msg
} SRspRet;
typedef struct {
+ int32_t code;
+ int32_t contLen;
+ void * rpcHandle;
+ void * rpcAhandle;
+ void * qhandle;
+ int8_t qtype;
+ int8_t msgType;
+ SRspRet rspRet;
+ char pCont[];
+} SVReadMsg;
+
+typedef struct {
+ int32_t code;
+ int32_t processedCount;
+ void * rpcHandle;
+ void * rpcAhandle;
SRspRet rspRet;
- void *pCont;
- int32_t contLen;
- SRpcMsg rpcMsg;
-} SReadMsg;
+ char reserveForSync[16];
+ SWalHead pHead[];
+} SVWriteMsg;
extern char *vnodeStatus[];
-int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg);
+int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg);
int32_t vnodeDrop(int32_t vgId);
int32_t vnodeOpen(int32_t vgId, char *rootDir);
-int32_t vnodeAlter(void *pVnode, SMDCreateVnodeMsg *pVnodeCfg);
+int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg);
int32_t vnodeClose(int32_t vgId);
void* vnodeAcquire(int32_t vgId); // add refcount
-void* vnodeAcquireRqueue(int32_t vgId); // add refCount, get read queue
-void* vnodeAcquireWqueue(int32_t vgId); // add recCount, get write queue
void vnodeRelease(void *pVnode); // dec refCount
void* vnodeGetWal(void *pVnode);
-int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item);
-int32_t vnodeCheckWrite(void *pVnode);
+int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam);
+int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam);
int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes);
void vnodeBuildStatusMsg(void *param);
void vnodeConfirmForward(void *param, uint64_t version, int32_t code);
-void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes);
+void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes);
int32_t vnodeInitResources();
void vnodeCleanupResources();
-int32_t vnodeProcessRead(void *pVnode, SReadMsg *pReadMsg);
-int32_t vnodeCheckRead(void *pVnode);
+int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam);
+int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead);
#ifdef __cplusplus
}
diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h
index f508d186083c84482080f2f8fe251173733f1366..d65c943e28d7d5a63aba4fc1839a4ba9cf744746 100644
--- a/src/kit/shell/inc/shell.h
+++ b/src/kit/shell/inc/shell.h
@@ -60,7 +60,7 @@ typedef struct SShellArguments {
extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments);
extern TAOS* shellInit(SShellArguments* args);
extern void* shellLoopQuery(void* arg);
-extern void taos_error(TAOS_RES* tres);
+extern void taos_error(TAOS_RES* tres, int64_t st);
extern int regex_match(const char* s, const char* reg, int cflags);
void shellReadCommand(TAOS* con, char command[]);
int32_t shellRunCommand(TAOS* con, char* command);
diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c
index ffe537dd91834247460f04c47efff5ebe8687d7e..995b56f341b36b84e413aec154ae1abcc832741d 100644
--- a/src/kit/shell/src/shellDarwin.c
+++ b/src/kit/shell/src/shellDarwin.c
@@ -229,8 +229,8 @@ void shellReadCommand(TAOS *con, char *command) {
printf("\n");
if (isReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
- taosTFree(cmd.buffer);
- taosTFree(cmd.command);
+ tfree(cmd.buffer);
+ tfree(cmd.command);
return;
} else {
updateBuffer(&cmd);
diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c
index 748b7e792982352de611227d82a844448d251ee2..22f01ac142c72603b9fa0595ce8c1fc89c626d7e 100644
--- a/src/kit/shell/src/shellEngine.c
+++ b/src/kit/shell/src/shellEngine.c
@@ -193,7 +193,7 @@ int32_t shellRunCommand(TAOS* con, char* command) {
history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE] == NULL ||
strcmp(command, history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE]) != 0) {
if (history.hist[history.hend] != NULL) {
- taosTFree(history.hist[history.hend]);
+ tfree(history.hist[history.hend]);
}
history.hist[history.hend] = strdup(command);
@@ -244,7 +244,7 @@ int32_t shellRunCommand(TAOS* con, char* command) {
}
*p++ = c;
- if (c == ';') {
+ if (c == ';' && quote == 0) {
c = *p;
*p = 0;
if (shellRunSingleCommand(con, cmd) < 0) {
@@ -296,7 +296,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
TAOS_RES* pSql = taos_query_h(con, command, &result);
if (taos_errno(pSql)) {
- taos_error(pSql);
+ taos_error(pSql, st);
return;
}
@@ -770,7 +770,7 @@ void read_history() {
return;
}
- while ((read_size = taosGetline(&line, &line_size, f)) != -1) {
+ while ((read_size = tgetline(&line, &line_size, f)) != -1) {
line[read_size - 1] = '\0';
history.hist[history.hend] = strdup(line);
@@ -800,16 +800,17 @@ void write_history() {
for (int i = history.hstart; i != history.hend;) {
if (history.hist[i] != NULL) {
fprintf(f, "%s\n", history.hist[i]);
- taosTFree(history.hist[i]);
+ tfree(history.hist[i]);
}
i = (i + 1) % MAX_HISTORY_SIZE;
}
fclose(f);
}
-void taos_error(TAOS_RES *tres) {
+void taos_error(TAOS_RES *tres, int64_t st) {
+ int64_t et = taosGetTimestampUs();
atomic_store_ptr(&result, 0);
- fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres));
+ fprintf(stderr, "\nDB error: %s (%.6fs)\n", taos_errstr(tres), (et - st) / 1E6);
taos_free_result(tres);
}
@@ -853,7 +854,7 @@ void source_file(TAOS *con, char *fptr) {
return;
}
- while ((read_len = taosGetline(&line, &line_len, f)) != -1) {
+ while ((read_len = tgetline(&line, &line_len, f)) != -1) {
if (read_len >= tsMaxSQLStringLen) continue;
line[--read_len] = '\0';
diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c
index 6f5ea33d79f09219f49ee4094de2c377648b2290..57f0f65be4315dda479f751cee20c833dc4b4c75 100644
--- a/src/kit/shell/src/shellLinux.c
+++ b/src/kit/shell/src/shellLinux.c
@@ -232,8 +232,8 @@ void shellReadCommand(TAOS *con, char *command) {
printf("\n");
if (isReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
- taosTFree(cmd.buffer);
- taosTFree(cmd.command);
+ tfree(cmd.buffer);
+ tfree(cmd.command);
return;
} else {
updateBuffer(&cmd);
@@ -351,7 +351,7 @@ void *shellLoopQuery(void *arg) {
reset_terminal_mode();
} while (shellRunCommand(con, command) == 0);
- taosTFree(command);
+ tfree(command);
exitShell();
pthread_cleanup_pop(1);
diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c
index 6cb7c669cc7a08434b2558588067d007b51b3595..2083ad3e9b7d5a101f35af4fe20dcd38957d5be2 100644
--- a/src/kit/shell/src/shellMain.c
+++ b/src/kit/shell/src/shellMain.c
@@ -80,7 +80,10 @@ int main(int argc, char* argv[]) {
shellParseArgument(argc, argv, &args);
if (args.netTestRole && args.netTestRole[0] != 0) {
- taosNetTest(args.host, (uint16_t)args.port, (uint16_t)args.endPort, args.pktLen, args.netTestRole);
+ taos_init();
+ CmdArguments cmdArgs;
+ memcpy(&cmdArgs, &args, sizeof(SShellArguments));
+ taosNetTest(&cmdArgs);
exit(0);
}
diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c
index 699e96428e201e998c1a4bb9cb8cd3c835e61364..60785ec004776aa39ab719de5ca0975922b70d10 100644
--- a/src/kit/taosdemo/taosdemo.c
+++ b/src/kit/taosdemo/taosdemo.c
@@ -475,6 +475,7 @@ typedef struct {
tsem_t mutex_sem;
int notFinished;
tsem_t lock_sem;
+ int counter;
} info;
typedef struct {
@@ -766,6 +767,7 @@ int main(int argc, char *argv[]) {
t_info->data_of_rate = rate;
t_info->end_table_id = i < b ? last + a : last + a - 1;
last = t_info->end_table_id + 1;
+ t_info->counter = 0;
tsem_init(&(t_info->mutex_sem), 0, 1);
t_info->notFinished = t_info->end_table_id - t_info->start_table_id + 1;
@@ -788,14 +790,14 @@ int main(int argc, char *argv[]) {
printf("ASYNC Insert with %d connections:\n", threads);
}
- fprintf(fp, "|%10.d | %10.2f | %10.2f | %10.4f |\n\n",
- ntables * nrecords_per_table, ntables * nrecords_per_table / t,
- (ntables * nrecords_per_table) / (t * nrecords_per_request),
+ fprintf(fp, "|%"PRIu64" | %10.2f | %10.2f | %10.4f |\n\n",
+ (int64_t)ntables * nrecords_per_table, ntables * nrecords_per_table / t,
+ ((int64_t)ntables * nrecords_per_table) / (t * nrecords_per_request),
t * 1000);
- printf("Spent %.4f seconds to insert %d records with %d record(s) per request: %.2f records/second\n",
- t, ntables * nrecords_per_table, nrecords_per_request,
- ntables * nrecords_per_table / t);
+ printf("Spent %.4f seconds to insert %"PRIu64" records with %d record(s) per request: %.2f records/second\n",
+ t, (int64_t)ntables * nrecords_per_table, nrecords_per_request,
+ (int64_t)ntables * nrecords_per_table / t);
for (int i = 0; i < threads; i++) {
info *t_info = infos + i;
@@ -955,7 +957,7 @@ void querySqlFile(TAOS* taos, char* sqlFile)
double t = getCurrentTime();
- while ((read_len = taosGetline(&line, &line_len, fp)) != -1) {
+ while ((read_len = tgetline(&line, &line_len, fp)) != -1) {
if (read_len >= MAX_SQL_SIZE) continue;
line[--read_len] = '\0';
@@ -1283,68 +1285,39 @@ void *syncWrite(void *sarg) {
void *asyncWrite(void *sarg) {
info *winfo = (info *)sarg;
-
- sTable *tb_infos = (sTable *)malloc(sizeof(sTable) * (winfo->end_table_id - winfo->start_table_id + 1));
-
- for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) {
- sTable *tb_info = tb_infos + tID - winfo->start_table_id;
- tb_info->data_type = winfo->datatype;
- tb_info->ncols_per_record = winfo->ncols_per_record;
- tb_info->taos = winfo->taos;
- sprintf(tb_info->tb_name, "%s.%s%d", winfo->db_name, winfo->tb_prefix, tID);
- tb_info->timestamp = winfo->start_time;
- tb_info->counter = 0;
- tb_info->target = winfo->nrecords_per_table;
- tb_info->len_of_binary = winfo->len_of_binary;
- tb_info->nrecords_per_request = winfo->nrecords_per_request;
- tb_info->mutex_sem = &(winfo->mutex_sem);
- tb_info->notFinished = &(winfo->notFinished);
- tb_info->lock_sem = &(winfo->lock_sem);
- tb_info->data_of_order = winfo->data_of_order;
- tb_info->data_of_rate = winfo->data_of_rate;
-
- /* char buff[BUFFER_SIZE] = "\0"; */
- /* sprintf(buff, "insert into %s values (0, 0)", tb_info->tb_name); */
- /* queryDB(tb_info->taos,buff); */
-
- taos_query_a(winfo->taos, "show databases", callBack, tb_info);
- }
+ taos_query_a(winfo->taos, "show databases", callBack, winfo);
tsem_wait(&(winfo->lock_sem));
- free(tb_infos);
return NULL;
}
void callBack(void *param, TAOS_RES *res, int code) {
- sTable *tb_info = (sTable *)param;
- char **datatype = tb_info->data_type;
- int ncols_per_record = tb_info->ncols_per_record;
- int len_of_binary = tb_info->len_of_binary;
- int64_t tmp_time = tb_info->timestamp;
-
- if (code < 0) {
- fprintf(stderr, "failed to insert data %d:reason; %s\n", code, taos_errstr(res));
- exit(EXIT_FAILURE);
- }
+ info* winfo = (info*)param;
+ char **datatype = winfo->datatype;
+ int ncols_per_record = winfo->ncols_per_record;
+ int len_of_binary = winfo->len_of_binary;
- // If finished;
- if (tb_info->counter >= tb_info->target) {
- tsem_wait(tb_info->mutex_sem);
- (*(tb_info->notFinished))--;
- if (*(tb_info->notFinished) == 0) tsem_post(tb_info->lock_sem);
- tsem_post(tb_info->mutex_sem);
+ int64_t tmp_time = winfo->start_time;
+ char *buffer = calloc(1, BUFFER_SIZE);
+ char *data = calloc(1, MAX_DATA_SIZE);
+ char *pstr = buffer;
+ pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, winfo->start_table_id);
+ if (winfo->counter >= winfo->nrecords_per_table) {
+ winfo->start_table_id++;
+ winfo->counter = 0;
+ }
+ if (winfo->start_table_id > winfo->end_table_id) {
+ tsem_post(&winfo->lock_sem);
+ free(buffer);
+ free(data);
+ taos_free_result(res);
return;
}
-
- char buffer[BUFFER_SIZE] = "\0";
- char data[MAX_DATA_SIZE];
- char *pstr = buffer;
- pstr += sprintf(pstr, "insert into %s values", tb_info->tb_name);
-
- for (int i = 0; i < tb_info->nrecords_per_request; i++) {
+
+ for (int i = 0; i < winfo->nrecords_per_request; i++) {
int rand_num = rand() % 100;
- if (tb_info->data_of_order ==1 && rand_num < tb_info->data_of_rate)
+ if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate)
{
int64_t d = tmp_time - rand() % 1000000 + rand_num;
generateData(data, datatype, ncols_per_record, d, len_of_binary);
@@ -1353,15 +1326,15 @@ void callBack(void *param, TAOS_RES *res, int code) {
generateData(data, datatype, ncols_per_record, tmp_time += 1000, len_of_binary);
}
pstr += sprintf(pstr, "%s", data);
- tb_info->counter++;
+ winfo->counter++;
- if (tb_info->counter >= tb_info->target) {
+ if (winfo->counter >= winfo->nrecords_per_table) {
break;
}
}
- tb_info->timestamp = tmp_time;
-
- taos_query_a(tb_info->taos, buffer, callBack, tb_info);
+ taos_query_a(winfo->taos, buffer, callBack, winfo);
+ free(buffer);
+ free(data);
taos_free_result(res);
}
diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h
index 4bc840f026d41dc4fe4e8d61c84d53140501bff6..6d3061c4269f54f0075a07ee75115f43e856479b 100644
--- a/src/mnode/inc/mnodeDef.h
+++ b/src/mnode/inc/mnodeDef.h
@@ -89,7 +89,7 @@ typedef struct STableObj {
int8_t type;
} STableObj;
-typedef struct SSuperTableObj {
+typedef struct SSTableObj {
STableObj info;
int8_t reserved0[9]; // for fill struct STableObj to 4byte align
int16_t nextColId;
@@ -104,7 +104,7 @@ typedef struct SSuperTableObj {
int32_t numOfTables;
SSchema * schema;
void * vgHash;
-} SSuperTableObj;
+} SSTableObj;
typedef struct {
STableObj info;
@@ -122,8 +122,8 @@ typedef struct {
int32_t refCount;
char* sql; //used by normal table
SSchema* schema; //used by normal table
- SSuperTableObj *superTable;
-} SChildTableObj;
+ SSTableObj*superTable;
+} SCTableObj;
typedef struct {
int32_t dnodeId;
@@ -172,7 +172,8 @@ typedef struct {
int8_t walLevel;
int8_t replications;
int8_t quorum;
- int8_t reserved[12];
+ int8_t update;
+ int8_t reserved[11];
} SDbCfg;
typedef struct SDbObj {
diff --git a/src/mnode/inc/mnodeProfile.h b/src/mnode/inc/mnodeProfile.h
index 1e5b1c0f9c7c1c220a88339f00d46538c78cf938..ee57c5da1f66b620e8624a74ba774643c589e818 100644
--- a/src/mnode/inc/mnodeProfile.h
+++ b/src/mnode/inc/mnodeProfile.h
@@ -45,7 +45,7 @@ void mnodeCleanupProfile();
SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port, int32_t pid, const char* app);
SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t port);
void mnodeReleaseConn(SConnObj *pConn);
-int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg);
+int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg);
#ifdef __cplusplus
}
diff --git a/src/mnode/inc/mnodeTable.h b/src/mnode/inc/mnodeTable.h
index ed0dbe4ecfc4b344052c70487c3b740d81eaa0f3..7c0077aa609843aa99e9f5114573aea8f7446fc1 100644
--- a/src/mnode/inc/mnodeTable.h
+++ b/src/mnode/inc/mnodeTable.h
@@ -29,8 +29,8 @@ int64_t mnodeGetChildTableNum();
void * mnodeGetTable(char *tableId);
void mnodeIncTableRef(void *pTable);
void mnodeDecTableRef(void *pTable);
-void * mnodeGetNextChildTable(void *pIter, SChildTableObj **pTable);
-void * mnodeGetNextSuperTable(void *pIter, SSuperTableObj **pTable);
+void * mnodeGetNextChildTable(void *pIter, SCTableObj **pTable);
+void * mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable);
void mnodeDropAllChildTables(SDbObj *pDropDb);
void mnodeDropAllSuperTables(SDbObj *pDropDb);
void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup);
diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h
index 7aa662b81cc08c90d6453ff5c73191ad81416073..0e6d9dfde463dc50810654921b2006fda212e496 100644
--- a/src/mnode/inc/mnodeVgroup.h
+++ b/src/mnode/inc/mnodeVgroup.h
@@ -43,8 +43,8 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle);
void mnodeAlterVgroup(SVgObj *pVgroup, void *ahandle);
int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_t *sid);
-void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
-void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
+void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable);
+void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable);
void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle);
void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
void mnodeSendAlterVgroupMsg(SVgObj *pVgroup);
diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c
index e161940a2b1a07572d9beff5222b8eb8d3829075..365cf656de283a53899ccd866682e8d0a06caaf8 100644
--- a/src/mnode/src/mnodeAcct.c
+++ b/src/mnode/src/mnodeAcct.c
@@ -34,7 +34,7 @@ static int32_t mnodeCreateRootAcct();
static int32_t mnodeAcctActionDestroy(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
pthread_mutex_destroy(&pAcct->mutex);
- taosTFree(pOper->pObj);
+ tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c
index 35b6a67ab24d5ee50a4f4da7bb902ab0d7232b7e..98587cf53d673b847a321ded9c369e2085195081 100644
--- a/src/mnode/src/mnodeCluster.c
+++ b/src/mnode/src/mnodeCluster.c
@@ -33,7 +33,7 @@ static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *
static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mnodeClusterActionDestroy(SSdbOper *pOper) {
- taosTFree(pOper->pObj);
+ tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c
index 8d7c267ab7cb328bb187518a88be3d344b0d53e2..3b3a21aeb18f08d9af20a17ee78dcba124bd00d7 100644
--- a/src/mnode/src/mnodeDb.c
+++ b/src/mnode/src/mnodeDb.c
@@ -41,7 +41,7 @@
static void * tsDbSdb = NULL;
static int32_t tsDbUpdateSize;
-static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg);
+static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *pMsg);
static int32_t mnodeDropDb(SMnodeMsg *newMsg);
static int32_t mnodeSetDbDropping(SDbObj *pDb);
static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
@@ -52,8 +52,8 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg);
static void mnodeDestroyDb(SDbObj *pDb) {
pthread_mutex_destroy(&pDb->mutex);
- taosTFree(pDb->vgList);
- taosTFree(pDb);
+ tfree(pDb->vgList);
+ tfree(pDb);
}
static int32_t mnodeDbActionDestroy(SSdbOper *pOper) {
@@ -319,6 +319,11 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
}
#endif
+ if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) {
+ mError("invalid db option update:%d valid range: [%d, %d]", pCfg->update, TSDB_MIN_DB_UPDATE, TSDB_MAX_DB_UPDATE);
+ return TSDB_CODE_MND_INVALID_DB_OPTION;
+ }
+
return TSDB_CODE_SUCCESS;
}
@@ -339,6 +344,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL;
if (pCfg->replications < 0) pCfg->replications = tsReplications;
if (pCfg->quorum < 0) pCfg->quorum = tsQuorum;
+ if (pCfg->update < 0) pCfg->update = tsUpdate;
}
static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
@@ -352,7 +358,7 @@ static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
return code;
}
-static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg) {
+static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *pMsg) {
int32_t code = acctCheck(pAcct, ACCT_GRANT_DB);
if (code != 0) return code;
@@ -391,14 +397,15 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg
.compression = pCreate->compression,
.walLevel = pCreate->walLevel,
.replications = pCreate->replications,
- .quorum = pCreate->quorum
+ .quorum = pCreate->quorum,
+ .update = pCreate->update
};
mnodeSetDefaultDbCfg(&pDb->cfg);
code = mnodeCheckDbCfg(&pDb->cfg);
if (code != TSDB_CODE_SUCCESS) {
- taosTFree(pDb);
+ tfree(pDb);
return code;
}
@@ -610,6 +617,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
+ pShow->bytes[cols] = 1;
+ pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
+ strcpy(pSchema[cols].name, "update");
+ pSchema[cols].bytes = htons(pShow->bytes[cols]);
+ cols++;
+
pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "status");
@@ -749,6 +762,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2);
cols++;
+ pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+ *(int8_t *)pWrite = pDb->cfg.update;
+ cols++;
+
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
if (pDb->status == TSDB_DB_STATUS_READY) {
const char *src = "ready";
@@ -805,7 +822,7 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb) {
}
static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) {
- SCMCreateDbMsg *pCreate = pMsg->rpcMsg.pCont;
+ SCreateDbMsg *pCreate = pMsg->rpcMsg.pCont;
pCreate->maxTables = htonl(pCreate->maxTables);
pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize);
pCreate->totalBlocks = htonl(pCreate->totalBlocks);
@@ -830,7 +847,7 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) {
return code;
}
-static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
+static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
SDbCfg newCfg = pDb->cfg;
int32_t maxTables = htonl(pAlter->maxTables);
int32_t cacheBlockSize = htonl(pAlter->cacheBlockSize);
@@ -848,6 +865,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
int8_t replications = pAlter->replications;
int8_t quorum = pAlter->quorum;
int8_t precision = pAlter->precision;
+ int8_t update = pAlter->update;
terrno = TSDB_CODE_SUCCESS;
@@ -950,6 +968,16 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
newCfg.quorum = quorum;
}
+ if (update >= 0 && update != pDb->cfg.update) {
+#if 0
+ mDebug("db:%s, update:%d change to %d", pDb->name, pDb->cfg.update, update);
+ newCfg.update = update;
+#else
+ mError("db:%s, can't alter update option", pDb->name);
+ terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
+#endif
+ }
+
return newCfg;
}
@@ -977,7 +1005,7 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
return TSDB_CODE_SUCCESS;
}
-static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) {
+static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) {
SDbCfg newCfg = mnodeGetAlterDbOption(pDb, pAlter);
if (terrno != TSDB_CODE_SUCCESS) {
return terrno;
@@ -1009,7 +1037,7 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) {
}
static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) {
- SCMAlterDbMsg *pAlter = pMsg->rpcMsg.pCont;
+ SAlterDbMsg *pAlter = pMsg->rpcMsg.pCont;
mDebug("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->rpcMsg.handle);
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pAlter->db);
@@ -1060,7 +1088,7 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) {
}
static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) {
- SCMDropDbMsg *pDrop = pMsg->rpcMsg.pCont;
+ SDropDbMsg *pDrop = pMsg->rpcMsg.pCont;
mDebug("db:%s, drop db msg is received from thandle:%p", pDrop->db, pMsg->rpcMsg.handle);
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pDrop->db);
diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c
index 1cd861e22303d5f6f0dcb75c7ef83cf66696578f..7e34e0837389e8a288ba70a2b18d4627f91219ae 100644
--- a/src/mnode/src/mnodeDnode.c
+++ b/src/mnode/src/mnodeDnode.c
@@ -39,11 +39,15 @@
#include "mnodeCluster.h"
int32_t tsAccessSquence = 0;
-static void *tsDnodeSdb = NULL;
+static void * tsDnodeSdb = NULL;
static int32_t tsDnodeUpdateSize = 0;
extern void * tsMnodeSdb;
extern void * tsVgroupSdb;
+static SDnodeEps*tsDnodeEps;
+static int32_t tsDnodeEpsSize;
+static pthread_mutex_t tsDnodeEpsMutex;
+
static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg);
static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg);
static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg);
@@ -59,6 +63,7 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo
static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole);
+static void mnodeUpdateDnodeEps();
static char* offlineReason[] = {
"",
@@ -83,7 +88,7 @@ static char* offlineReason[] = {
};
static int32_t mnodeDnodeActionDestroy(SSdbOper *pOper) {
- taosTFree(pOper->pObj);
+ tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
@@ -95,6 +100,9 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) {
pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED;
}
+ dnodeUpdateEp(pDnode->dnodeId, pDnode->dnodeEp, pDnode->dnodeFqdn, &pDnode->dnodePort);
+ mnodeUpdateDnodeEps();
+
mInfo("dnode:%d, fqdn:%s ep:%s port:%d, do insert action", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort);
return TSDB_CODE_SUCCESS;
}
@@ -107,6 +115,7 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
#endif
mnodeDropMnodeLocal(pDnode->dnodeId);
balanceAsyncNotify();
+ mnodeUpdateDnodeEps();
mDebug("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId);
return TSDB_CODE_SUCCESS;
@@ -121,6 +130,7 @@ static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) {
}
mnodeDecDnodeRef(pDnode);
+ mnodeUpdateDnodeEps();
return TSDB_CODE_SUCCESS;
}
@@ -152,12 +162,14 @@ static int32_t mnodeDnodeActionRestored() {
}
}
+ mnodeUpdateDnodeEps();
return TSDB_CODE_SUCCESS;
}
int32_t mnodeInitDnodes() {
SDnodeObj tObj;
tsDnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
+ pthread_mutex_init(&tsDnodeEpsMutex, NULL);
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_DNODE,
@@ -201,6 +213,9 @@ int32_t mnodeInitDnodes() {
void mnodeCleanupDnodes() {
sdbCloseTable(tsDnodeSdb);
+ pthread_mutex_destroy(&tsDnodeEpsMutex);
+ free(tsDnodeEps);
+ tsDnodeEps = NULL;
tsDnodeSdb = NULL;
}
@@ -299,7 +314,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_NO_RIGHTS;
}
- SCMCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont;
+ SCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont;
if (pCmCfgDnode->ep[0] == 0) {
tstrncpy(pCmCfgDnode->ep, tsLocalEp, TSDB_EP_LEN);
}
@@ -334,7 +349,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
mnodeDecDnodeRef(pDnode);
return code;
} else {
- SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg));
+ SCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SCfgDnodeMsg));
strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep);
strcpy(pMdCfgDnode->config, pCmCfgDnode->config);
@@ -343,7 +358,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
.code = 0,
.msgType = TSDB_MSG_TYPE_MD_CONFIG_DNODE,
.pCont = pMdCfgDnode,
- .contLen = sizeof(SMDCfgDnodeMsg)
+ .contLen = sizeof(SCfgDnodeMsg)
};
mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user);
@@ -418,9 +433,54 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) {
return 0;
}
+static int32_t mnodeGetDnodeEpsSize() {
+ pthread_mutex_lock(&tsDnodeEpsMutex);
+ int32_t size = tsDnodeEpsSize;
+ pthread_mutex_unlock(&tsDnodeEpsMutex);
+ return size;
+}
+
+static void mnodeGetDnodeEpsData(SDnodeEps *pEps, int32_t epsSize) {
+ pthread_mutex_lock(&tsDnodeEpsMutex);
+ if (epsSize == tsDnodeEpsSize) {
+ memcpy(pEps, tsDnodeEps, tsDnodeEpsSize);
+ }
+ pthread_mutex_unlock(&tsDnodeEpsMutex);
+}
+
+static void mnodeUpdateDnodeEps() {
+ pthread_mutex_lock(&tsDnodeEpsMutex);
+
+ int32_t totalDnodes = mnodeGetDnodesNum();
+ tsDnodeEpsSize = sizeof(SDnodeEps) + totalDnodes * sizeof(SDnodeEp);
+ free(tsDnodeEps);
+ tsDnodeEps = calloc(1, tsDnodeEpsSize);
+ tsDnodeEps->dnodeNum = htonl(totalDnodes);
+
+ SDnodeObj *pDnode = NULL;
+ void * pIter = NULL;
+ int32_t dnodesNum = 0;
+
+ while (1) {
+ pIter = mnodeGetNextDnode(pIter, &pDnode);
+ if (pDnode == NULL) break;
+ if (dnodesNum >= totalDnodes) break;
+
+ SDnodeEp *pEp = &tsDnodeEps->dnodeEps[dnodesNum];
+ dnodesNum++;
+ pEp->dnodeId = htonl(pDnode->dnodeId);
+ pEp->dnodePort = htons(pDnode->dnodePort);
+ tstrncpy(pEp->dnodeFqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN);
+ mnodeDecDnodeRef(pDnode);
+ }
+
+ sdbFreeIter(pIter);
+ pthread_mutex_unlock(&tsDnodeEpsMutex);
+}
+
static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
SDnodeObj *pDnode = NULL;
- SDMStatusMsg *pStatus = pMsg->rpcMsg.pCont;
+ SStatusMsg *pStatus = pMsg->rpcMsg.pCont;
pStatus->dnodeId = htonl(pStatus->dnodeId);
pStatus->moduleStatus = htonl(pStatus->moduleStatus);
pStatus->lastReboot = htonl(pStatus->lastReboot);
@@ -477,8 +537,11 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
}
int32_t openVnodes = htons(pStatus->openVnodes);
- int32_t contLen = sizeof(SDMStatusRsp) + openVnodes * sizeof(SDMVgroupAccess);
- SDMStatusRsp *pRsp = rpcMallocCont(contLen);
+ int32_t epsSize = mnodeGetDnodeEpsSize();
+ int32_t vgAccessSize = openVnodes * sizeof(SVgroupAccess);
+ int32_t contLen = sizeof(SStatusRsp) + vgAccessSize + epsSize;
+
+ SStatusRsp *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) {
mnodeDecDnodeRef(pDnode);
return TSDB_CODE_MND_OUT_OF_MEMORY;
@@ -488,8 +551,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt);
pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes);
tstrncpy(pRsp->dnodeCfg.clusterId, mnodeGetClusterId(), TSDB_CLUSTER_ID_LEN);
- SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp));
-
+ SVgroupAccess *pAccess = (SVgroupAccess *)((char *)pRsp + sizeof(SStatusRsp));
+
for (int32_t j = 0; j < openVnodes; ++j) {
SVnodeLoad *pVload = &pStatus->load[j];
pVload->vgId = htonl(pVload->vgId);
@@ -521,7 +584,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT;
}
- mDebug("dnode:%d, from offline to online", pDnode->dnodeId);
+ mInfo("dnode:%d, from offline to online", pDnode->dnodeId);
pDnode->status = TAOS_DN_STATUS_READY;
pDnode->offlineReason = TAOS_DN_OFF_ONLINE;
balanceSyncNotify();
@@ -539,6 +602,9 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
mnodeDecDnodeRef(pDnode);
+ SDnodeEps *pEps = (SDnodeEps *)((char *)pRsp + sizeof(SStatusRsp) + vgAccessSize);
+ mnodeGetDnodeEpsData(pEps, epsSize);
+
pMsg->rpcRsp.len = contLen;
pMsg->rpcRsp.rsp = pRsp;
@@ -589,7 +655,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) {
int32_t code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
int dnodeId = pDnode->dnodeId;
- taosTFree(pDnode);
+ tfree(pDnode);
mError("failed to create dnode:%d, reason:%s", dnodeId, tstrerror(code));
} else {
mLInfo("dnode:%d is created", pDnode->dnodeId);
@@ -645,7 +711,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
}
static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) {
- SCMCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont;
+ SCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont;
if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
return TSDB_CODE_MND_NO_RIGHTS;
@@ -655,7 +721,7 @@ static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) {
}
static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg) {
- SCMDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont;
+ SDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont;
if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) {
return TSDB_CODE_MND_NO_RIGHTS;
diff --git a/src/mnode/src/mnodeInt.c b/src/mnode/src/mnodeInt.c
index fb1b8741a9f8895c579ad0e549dc8f3c15f7618c..8431baf4f455a2c577321ec18ff3bfcd7fbe5d46 100644
--- a/src/mnode/src/mnodeInt.c
+++ b/src/mnode/src/mnodeInt.c
@@ -18,7 +18,7 @@
#include "taosmsg.h"
#include "taoserror.h"
#include "trpc.h"
-#include "tcache.h"
+#include "tqueue.h"
#include "mnode.h"
#include "dnode.h"
#include "mnodeDef.h"
@@ -34,8 +34,16 @@
#include "mnodeUser.h"
#include "mnodeVgroup.h"
-void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg) {
- pMsg->rpcMsg = *rpcMsg;
+void *mnodeCreateMsg(SRpcMsg *pRpcMsg) {
+ int32_t size = sizeof(SMnodeMsg) + pRpcMsg->contLen;
+ SMnodeMsg *pMsg = taosAllocateQitem(size);
+
+ pMsg->rpcMsg = *pRpcMsg;
+ pMsg->rpcMsg.pCont = pMsg->pCont;
+ pMsg->incomingTs = taosGetTimestampSec();
+ memcpy(pMsg->pCont, pRpcMsg->pCont, pRpcMsg->contLen);
+
+ return pMsg;
}
int32_t mnodeInitMsg(SMnodeMsg *pMsg) {
@@ -54,7 +62,9 @@ int32_t mnodeInitMsg(SMnodeMsg *pMsg) {
void mnodeCleanupMsg(SMnodeMsg *pMsg) {
if (pMsg != NULL) {
- if (pMsg->rpcMsg.pCont) rpcFreeCont(pMsg->rpcMsg.pCont);
+ if (pMsg->rpcMsg.pCont != pMsg->pCont) {
+ tfree(pMsg->rpcMsg.pCont);
+ }
if (pMsg->pUser) mnodeDecUserRef(pMsg->pUser);
if (pMsg->pDb) mnodeDecDbRef(pMsg->pDb);
if (pMsg->pVgroup) mnodeDecVgroupRef(pMsg->pVgroup);
diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c
index 2bb8a810566a676a4ae991bc79e5eb78234747c7..1f5ad42bdead75064c60f8626ce4916e7b08fd9f 100644
--- a/src/mnode/src/mnodeMain.c
+++ b/src/mnode/src/mnodeMain.c
@@ -96,9 +96,9 @@ int32_t mnodeStartSystem() {
return -1;
}
- dnodeAllocateMnodeWqueue();
- dnodeAllocateMnodeRqueue();
- dnodeAllocateMnodePqueue();
+ dnodeAllocMWritequeue();
+ dnodeAllocMReadQueue();
+ dnodeAllocateMPeerQueue();
if (mnodeInitComponents() != 0) {
return -1;
@@ -127,9 +127,9 @@ void mnodeCleanupSystem() {
mInfo("starting to clean up mnode");
tsMgmtIsRunning = false;
- dnodeFreeMnodeWqueue();
- dnodeFreeMnodeRqueue();
- dnodeFreeMnodePqueue();
+ dnodeFreeMWritequeue();
+ dnodeFreeMReadQueue();
+ dnodeFreeMPeerQueue();
mnodeCleanupTimer();
mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1);
diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c
index 89b2f50b731c90c1261957da93623ba16cebf2ab..092f246c139533f806ce1d311162e0516088d879 100644
--- a/src/mnode/src/mnodeMnode.c
+++ b/src/mnode/src/mnodeMnode.c
@@ -38,7 +38,7 @@ static void * tsMnodeSdb = NULL;
static int32_t tsMnodeUpdateSize = 0;
static SRpcEpSet tsMnodeEpSetForShell;
static SRpcEpSet tsMnodeEpSetForPeer;
-static SDMMnodeInfos tsMnodeInfos;
+static SMnodeInfos tsMnodeInfos;
static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
@@ -59,7 +59,7 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo
#endif
static int32_t mnodeMnodeActionDestroy(SSdbOper *pOper) {
- taosTFree(pOper->pObj);
+ tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
@@ -70,8 +70,9 @@ static int32_t mnodeMnodeActionInsert(SSdbOper *pOper) {
pDnode->isMgmt = true;
mnodeDecDnodeRef(pDnode);
-
- mInfo("mnode:%d, fqdn:%s ep:%s port:%d, do insert action", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort);
+
+ mInfo("mnode:%d, fqdn:%s ep:%s port:%u, do insert action", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp,
+ pDnode->dnodePort);
return TSDB_CODE_SUCCESS;
}
@@ -202,7 +203,7 @@ void mnodeUpdateMnodeEpSet() {
memset(&tsMnodeEpSetForShell, 0, sizeof(SRpcEpSet));
memset(&tsMnodeEpSetForPeer, 0, sizeof(SRpcEpSet));
- memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos));
+ memset(&tsMnodeInfos, 0, sizeof(SMnodeInfos));
int32_t index = 0;
void * pIter = NULL;
@@ -221,8 +222,8 @@ void mnodeUpdateMnodeEpSet() {
tsMnodeEpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE);
mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForPeer.fqdn[index], htons(tsMnodeEpSetForPeer.port[index]));
- tsMnodeInfos.nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
- strcpy(tsMnodeInfos.nodeInfos[index].nodeEp, pDnode->dnodeEp);
+ tsMnodeInfos.mnodeInfos[index].mnodeId = htonl(pMnode->mnodeId);
+ strcpy(tsMnodeInfos.mnodeInfos[index].mnodeEp, pDnode->dnodeEp);
if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
tsMnodeEpSetForShell.inUse = index;
@@ -238,7 +239,7 @@ void mnodeUpdateMnodeEpSet() {
mnodeDecMnodeRef(pMnode);
}
- tsMnodeInfos.nodeNum = index;
+ tsMnodeInfos.mnodeNum = index;
tsMnodeEpSetForShell.numOfEps = index;
tsMnodeEpSetForPeer.numOfEps = index;
@@ -260,19 +261,19 @@ void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) {
}
char* mnodeGetMnodeMasterEp() {
- return tsMnodeInfos.nodeInfos[tsMnodeInfos.inUse].nodeEp;
+ return tsMnodeInfos.mnodeInfos[tsMnodeInfos.inUse].mnodeEp;
}
void mnodeGetMnodeInfos(void *mnodeInfos) {
mnodeMnodeRdLock();
- *(SDMMnodeInfos *)mnodeInfos = tsMnodeInfos;
+ *(SMnodeInfos *)mnodeInfos = tsMnodeInfos;
mnodeMnodeUnLock();
}
static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) {
mDebug("dnode:%d, send create mnode msg to dnode %s", dnodeId, dnodeEp);
- SMDCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SMDCreateMnodeMsg));
+ SCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SCreateMnodeMsg));
if (pCreate == NULL) {
return TSDB_CODE_MND_OUT_OF_MEMORY;
} else {
@@ -280,21 +281,21 @@ static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) {
tstrncpy(pCreate->dnodeEp, dnodeEp, sizeof(pCreate->dnodeEp));
pCreate->mnodes = tsMnodeInfos;
bool found = false;
- for (int i = 0; i < pCreate->mnodes.nodeNum; ++i) {
- if (pCreate->mnodes.nodeInfos[i].nodeId == htonl(dnodeId)) {
+ for (int i = 0; i < pCreate->mnodes.mnodeNum; ++i) {
+ if (pCreate->mnodes.mnodeInfos[i].mnodeId == htonl(dnodeId)) {
found = true;
}
}
if (!found) {
- pCreate->mnodes.nodeInfos[pCreate->mnodes.nodeNum].nodeId = htonl(dnodeId);
- tstrncpy(pCreate->mnodes.nodeInfos[pCreate->mnodes.nodeNum].nodeEp, dnodeEp, sizeof(pCreate->dnodeEp));
- pCreate->mnodes.nodeNum++;
+ pCreate->mnodes.mnodeInfos[pCreate->mnodes.mnodeNum].mnodeId = htonl(dnodeId);
+ tstrncpy(pCreate->mnodes.mnodeInfos[pCreate->mnodes.mnodeNum].mnodeEp, dnodeEp, sizeof(pCreate->dnodeEp));
+ pCreate->mnodes.mnodeNum++;
}
}
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pCreate;
- rpcMsg.contLen = sizeof(SMDCreateMnodeMsg);
+ rpcMsg.contLen = sizeof(SCreateMnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_MD_CREATE_MNODE;
SRpcMsg rpcRsp = {0};
@@ -341,14 +342,14 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) {
}
if (code != TSDB_CODE_SUCCESS) {
- taosTFree(pMnode);
+ tfree(pMnode);
return;
}
code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("dnode:%d, failed to create mnode, ep:%s reason:%s", dnodeId, dnodeEp, tstrerror(code));
- taosTFree(pMnode);
+ tfree(pMnode);
}
}
diff --git a/src/mnode/src/mnodePeer.c b/src/mnode/src/mnodePeer.c
index 2a04f541c51483ccba93369a65061507f83ead23..e43f8c1b786e6126253cdb40c0184bc34dc4dab2 100644
--- a/src/mnode/src/mnodePeer.c
+++ b/src/mnode/src/mnodePeer.c
@@ -63,9 +63,9 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) {
for (int32_t i = 0; i < epSet->numOfEps; ++i) {
if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) {
epSet->inUse = (i + 1) % epSet->numOfEps;
- mDebug("mnode index:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
+ mDebug("mpeer:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
} else {
- mDebug("mnode index:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
+ mDebug("mpeer:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
}
}
diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c
index f8f99e22c6f437b1eece0d704c8c4551bc434110..12ac64854c6834ca0cd1509f9bfedd7b65a0900c 100644
--- a/src/mnode/src/mnodeProfile.c
+++ b/src/mnode/src/mnodeProfile.c
@@ -131,8 +131,8 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po
static void mnodeFreeConn(void *data) {
SConnObj *pConn = data;
- taosTFree(pConn->pQueries);
- taosTFree(pConn->pStreams);
+ tfree(pConn->pQueries);
+ tfree(pConn->pStreams);
mDebug("connId:%d, is destroyed", pConn->connId);
}
@@ -182,7 +182,7 @@ static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
// app name
pShow->bytes[cols] = TSDB_APPNAME_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
- strcpy(pSchema[cols].name, "app_name");
+ strcpy(pSchema[cols].name, "program");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@@ -280,7 +280,7 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi
}
// not thread safe, need optimized
-int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg) {
+int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg) {
pConn->numOfQueries = htonl(pHBMsg->numOfQueries);
if (pConn->numOfQueries > 0) {
if (pConn->pQueries == NULL) {
@@ -561,7 +561,7 @@ static int32_t mnodeProcessKillQueryMsg(SMnodeMsg *pMsg) {
SUserObj *pUser = pMsg->pUser;
if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
- SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
+ SKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
mInfo("kill query msg is received, queryId:%s", pKill->queryId);
const char delim = ':';
@@ -592,7 +592,7 @@ static int32_t mnodeProcessKillStreamMsg(SMnodeMsg *pMsg) {
SUserObj *pUser = pMsg->pUser;
if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
- SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
+ SKillQueryMsg *pKill = pMsg->rpcMsg.pCont;
mInfo("kill stream msg is received, streamId:%s", pKill->queryId);
const char delim = ':';
@@ -623,7 +623,7 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg) {
SUserObj *pUser = pMsg->pUser;
if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS;
- SCMKillConnMsg *pKill = pMsg->rpcMsg.pCont;
+ SKillConnMsg *pKill = pMsg->rpcMsg.pCont;
int32_t connId = atoi(pKill->queryId);
SConnObj *pConn = taosCacheAcquireByKey(tsMnodeConnCache, &connId, sizeof(int32_t));
if (pConn == NULL) {
diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c
index 14558485aa743a42bee016226dfc2e16a4f81d0c..12d977f32b224f9ce85db93fb4afc3d6f0e550f8 100644
--- a/src/mnode/src/mnodeSdb.c
+++ b/src/mnode/src/mnodeSdb.c
@@ -72,7 +72,7 @@ typedef struct {
ESyncRole role;
ESdbStatus status;
int64_t version;
- void * sync;
+ int64_t sync;
void * wal;
SSyncCfg cfg;
int32_t numOfTables;
@@ -98,8 +98,8 @@ static taos_qall tsSdbWriteQall;
static taos_queue tsSdbWriteQueue;
static SSdbWriteWorkerPool tsSdbPool;
-static int sdbWrite(void *param, void *data, int type);
-static int sdbWriteToQueue(void *param, void *data, int type);
+static int32_t sdbWrite(void *param, void *data, int32_t type, void *pMsg);
+static int32_t sdbWriteToQueue(void *param, void *data, int32_t type, void *pMsg);
static void * sdbWorkerFp(void *param);
static int32_t sdbInitWriteWorker();
static void sdbCleanupWriteWorker();
@@ -175,7 +175,7 @@ static void *sdbGetTableFromId(int32_t tableId) {
}
static int32_t sdbInitWal() {
- SWalCfg walCfg = {.walLevel = 2, .wals = 2, .keep = 1, .fsyncPeriod = 0};
+ SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0};
char temp[TSDB_FILENAME_LEN];
sprintf(temp, "%s/wal", tsMnodeDir);
tsSdbObj.wal = walOpen(temp, &walCfg);
@@ -212,7 +212,7 @@ static void sdbRestoreTables() {
}
void sdbUpdateMnodeRoles() {
- if (tsSdbObj.sync == NULL) return;
+ if (tsSdbObj.sync <= 0) return;
SNodesRole roles = {0};
syncGetNodesRole(tsSdbObj.sync, &roles);
@@ -237,8 +237,8 @@ static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint3
return 0;
}
-static int sdbGetWalInfo(void *ahandle, char *name, uint32_t *index) {
- return walGetWalFile(tsSdbObj.wal, name, index);
+static int32_t sdbGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) {
+ return walGetWalFile(tsSdbObj.wal, fileName, fileId);
}
static void sdbNotifyRole(void *ahandle, int8_t role) {
@@ -281,14 +281,21 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
((SSdbTable *)pOper->table)->tableName, pOper->pObj, sdbGetKeyStr(pOper->table, pHead->cont),
pHead->version, action, tstrerror(pOper->retCode));
if (action == SDB_ACTION_INSERT) {
- sdbDeleteHash(pOper->table, pOper);
+ // It's better to create a table in two stages, create it first and then set it success
+ //sdbDeleteHash(pOper->table, pOper);
+ SSdbOper oper = {
+ .type = SDB_OPER_GLOBAL,
+ .table = pOper->table,
+ .pObj = pOper->pObj
+ };
+ sdbDeleteRow(&oper);
}
}
if (pOper->writeCb != NULL) {
pOper->retCode = (*pOper->writeCb)(pMsg, pOper->retCode);
}
- dnodeSendRpcMnodeWriteRsp(pMsg, pOper->retCode);
+ dnodeSendRpcMWriteRsp(pMsg, pOper->retCode);
// if ahandle, means this func is called by sdb write
if (ahandle == NULL) {
@@ -305,7 +312,7 @@ void sdbUpdateAsync() {
}
void sdbUpdateSync(void *pMnodes) {
- SDMMnodeInfos *mnodes = pMnodes;
+ SMnodeInfos *mnodes = pMnodes;
if (!mnodeIsRunning()) {
mDebug("mnode not start yet, update sync config later");
return;
@@ -339,10 +346,10 @@ void sdbUpdateSync(void *pMnodes) {
syncCfg.replica = index;
mDebug("mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica);
} else {
- for (index = 0; index < mnodes->nodeNum; ++index) {
- SDMMnodeInfo *node = &mnodes->nodeInfos[index];
- syncCfg.nodeInfo[index].nodeId = node->nodeId;
- taosGetFqdnPortFromEp(node->nodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort);
+ for (index = 0; index < mnodes->mnodeNum; ++index) {
+ SMnodeInfo *node = &mnodes->mnodeInfos[index];
+ syncCfg.nodeInfo[index].nodeId = node->mnodeId;
+ taosGetFqdnPortFromEp(node->mnodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort);
syncCfg.nodeInfo[index].nodePort += TSDB_PORT_SYNC;
}
syncCfg.replica = index;
@@ -426,7 +433,7 @@ void sdbCleanUp() {
if (tsSdbObj.sync) {
syncStop(tsSdbObj.sync);
- tsSdbObj.sync = NULL;
+ tsSdbObj.sync = -1;
}
if (tsSdbObj.wal) {
@@ -568,7 +575,7 @@ static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) {
return TSDB_CODE_SUCCESS;
}
-static int sdbWrite(void *param, void *data, int type) {
+static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) {
SSdbOper *pOper = param;
SWalHead *pHead = data;
int32_t tableId = pHead->msgType / 10;
@@ -977,7 +984,7 @@ void sdbCleanupWriteWorker() {
}
sdbFreeWritequeue();
- taosTFree(tsSdbPool.writeWorker);
+ tfree(tsSdbPool.writeWorker);
mInfo("sdb write is closed");
}
@@ -1033,13 +1040,13 @@ void sdbFreeWritequeue() {
tsSdbWriteQueue = NULL;
}
-int sdbWriteToQueue(void *param, void *data, int type) {
+int32_t sdbWriteToQueue(void *param, void *data, int32_t qtype, void *pMsg) {
SWalHead *pHead = data;
- int size = sizeof(SWalHead) + pHead->len;
- SWalHead *pWal = (SWalHead *)taosAllocateQitem(size);
+ int32_t size = sizeof(SWalHead) + pHead->len;
+ SWalHead *pWal = taosAllocateQitem(size);
memcpy(pWal, pHead, size);
- taosWriteQitem(tsSdbWriteQueue, type, pWal);
+ taosWriteQitem(tsSdbWriteQueue, qtype, pWal);
return 0;
}
@@ -1074,7 +1081,7 @@ static void *sdbWorkerFp(void *param) {
pOper = NULL;
}
- int32_t code = sdbWrite(pOper, pHead, type);
+ int32_t code = sdbWrite(pOper, pHead, type, NULL);
if (code > 0) code = 0;
if (pOper) {
pOper->retCode = code;
@@ -1083,7 +1090,7 @@ static void *sdbWorkerFp(void *param) {
}
}
- walFsync(tsSdbObj.wal);
+ walFsync(tsSdbObj.wal, true);
// browse all items, and process them one by one
taosResetQitems(tsSdbWriteQall);
diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c
index 80909e99aec6d752d35042ca2d761a6e8b923441..0d53fa9617b500d61457889e2228e5e756a244ae 100644
--- a/src/mnode/src/mnodeShow.c
+++ b/src/mnode/src/mnodeShow.c
@@ -110,7 +110,7 @@ static char *mnodeGetShowType(int32_t showType) {
}
static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
- SCMShowMsg *pShowMsg = pMsg->rpcMsg.pCont;
+ SShowMsg *pShowMsg = pMsg->rpcMsg.pCont;
if (pShowMsg->type >= TSDB_MGMT_TABLE_MAX) {
return TSDB_CODE_MND_INVALID_MSG_TYPE;
}
@@ -132,8 +132,8 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_OUT_OF_MEMORY;
}
- int32_t size = sizeof(SCMShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE;
- SCMShowRsp *pShowRsp = rpcMallocCont(size);
+ int32_t size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE;
+ SShowRsp *pShowRsp = rpcMallocCont(size);
if (pShowRsp == NULL) {
mnodeReleaseShowObj(pShow, true);
return TSDB_CODE_MND_OUT_OF_MEMORY;
@@ -146,7 +146,7 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
if (code == TSDB_CODE_SUCCESS) {
pMsg->rpcRsp.rsp = pShowRsp;
- pMsg->rpcRsp.len = sizeof(SCMShowRsp) + sizeof(SSchema) * pShow->numOfColumns;
+ pMsg->rpcRsp.len = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns;
mnodeReleaseShowObj(pShow, false);
return TSDB_CODE_SUCCESS;
} else {
@@ -232,12 +232,17 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
}
static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) {
- SCMHeartBeatRsp *pHBRsp = (SCMHeartBeatRsp *) rpcMallocCont(sizeof(SCMHeartBeatRsp));
- if (pHBRsp == NULL) {
+ SHeartBeatRsp *pRsp = (SHeartBeatRsp *)rpcMallocCont(sizeof(SHeartBeatRsp));
+ if (pRsp == NULL) {
return TSDB_CODE_MND_OUT_OF_MEMORY;
}
- SCMHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont;
+ SHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont;
+ if (taosCheckVersion(pHBMsg->clientVer, version, 3) != TSDB_CODE_SUCCESS) {
+ rpcFreeCont(pRsp);
+ return TSDB_CODE_TSC_INVALID_VERSION; // todo change the error code
+ }
+
SRpcConnInfo connInfo = {0};
rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo);
@@ -251,40 +256,40 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) {
if (pConn == NULL) {
// do not close existing links, otherwise
// mError("failed to create connId, close connect");
- // pHBRsp->killConnection = 1;
+ // pRsp->killConnection = 1;
} else {
- pHBRsp->connId = htonl(pConn->connId);
+ pRsp->connId = htonl(pConn->connId);
mnodeSaveQueryStreamList(pConn, pHBMsg);
if (pConn->killed != 0) {
- pHBRsp->killConnection = 1;
+ pRsp->killConnection = 1;
}
if (pConn->streamId != 0) {
- pHBRsp->streamId = htonl(pConn->streamId);
+ pRsp->streamId = htonl(pConn->streamId);
pConn->streamId = 0;
}
if (pConn->queryId != 0) {
- pHBRsp->queryId = htonl(pConn->queryId);
+ pRsp->queryId = htonl(pConn->queryId);
pConn->queryId = 0;
}
}
- pHBRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum());
- pHBRsp->totalDnodes = htonl(mnodeGetDnodesNum());
- mnodeGetMnodeEpSetForShell(&pHBRsp->epSet);
+ pRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum());
+ pRsp->totalDnodes = htonl(mnodeGetDnodesNum());
+ mnodeGetMnodeEpSetForShell(&pRsp->epSet);
+
+ pMsg->rpcRsp.rsp = pRsp;
+ pMsg->rpcRsp.len = sizeof(SHeartBeatRsp);
- pMsg->rpcRsp.rsp = pHBRsp;
- pMsg->rpcRsp.len = sizeof(SCMHeartBeatRsp);
-
mnodeReleaseConn(pConn);
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
- SCMConnectMsg *pConnectMsg = pMsg->rpcMsg.pCont;
- SCMConnectRsp *pConnectRsp = NULL;
+ SConnectMsg *pConnectMsg = pMsg->rpcMsg.pCont;
+ SConnectRsp *pConnectRsp = NULL;
int32_t code = TSDB_CODE_SUCCESS;
SRpcConnInfo connInfo = {0};
@@ -320,7 +325,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
mnodeDecDbRef(pDb);
}
- pConnectRsp = rpcMallocCont(sizeof(SCMConnectRsp));
+ pConnectRsp = rpcMallocCont(sizeof(SConnectRsp));
if (pConnectRsp == NULL) {
code = TSDB_CODE_MND_OUT_OF_MEMORY;
goto connect_over;
@@ -349,14 +354,14 @@ connect_over:
} else {
mLInfo("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code));
pMsg->rpcRsp.rsp = pConnectRsp;
- pMsg->rpcRsp.len = sizeof(SCMConnectRsp);
+ pMsg->rpcRsp.len = sizeof(SConnectRsp);
}
return code;
}
static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) {
- SCMUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont;
+ SUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont;
int32_t code = TSDB_CODE_SUCCESS;
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pUseDbMsg->db);
@@ -410,7 +415,7 @@ static void mnodeFreeShowObj(void *data) {
sdbFreeIter(pShow->pIter);
mDebug("%p, show is destroyed, data:%p index:%d", pShow, data, pShow->index);
- taosTFree(pShow);
+ tfree(pShow);
}
static void mnodeReleaseShowObj(SShowObj *pShow, bool forceRemove) {
diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c
index 82a062169a591d26d665437273183a4c5e1f27b4..2b5e6455c0ec731d0697f2b8f6b638a1ce139788 100644
--- a/src/mnode/src/mnodeTable.c
+++ b/src/mnode/src/mnodeTable.c
@@ -45,6 +45,10 @@
#include "mnodeRead.h"
#include "mnodePeer.h"
+#define ALTER_CTABLE_RETRY_TIMES 3
+#define CREATE_CTABLE_RETRY_TIMES 10
+#define CREATE_CTABLE_RETRY_SEC 14
+
static void * tsChildTableSdb;
static void * tsSuperTableSdb;
static int32_t tsChildTableUpdateSize;
@@ -52,9 +56,9 @@ static int32_t tsSuperTableUpdateSize;
static void * mnodeGetChildTable(char *tableId);
static void * mnodeGetSuperTable(char *tableId);
static void * mnodeGetSuperTableByUid(uint64_t uid);
-static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable);
-static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable);
-static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable);
+static void mnodeDropAllChildTablesInStable(SSTableObj *pStable);
+static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable);
+static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable);
static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn);
@@ -86,13 +90,13 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg);
static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg);
static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg);
-static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName);
+static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName);
-static void mnodeDestroyChildTable(SChildTableObj *pTable) {
- taosTFree(pTable->info.tableId);
- taosTFree(pTable->schema);
- taosTFree(pTable->sql);
- taosTFree(pTable);
+static void mnodeDestroyChildTable(SCTableObj *pTable) {
+ tfree(pTable->info.tableId);
+ tfree(pTable->schema);
+ tfree(pTable->sql);
+ tfree(pTable);
}
static int32_t mnodeChildTableActionDestroy(SSdbOper *pOper) {
@@ -101,7 +105,7 @@ static int32_t mnodeChildTableActionDestroy(SSdbOper *pOper) {
}
static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) {
- SChildTableObj *pTable = pOper->pObj;
+ SCTableObj *pTable = pOper->pObj;
SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId);
if (pVgroup == NULL) {
@@ -150,7 +154,7 @@ static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) {
}
static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) {
- SChildTableObj *pTable = pOper->pObj;
+ SCTableObj *pTable = pOper->pObj;
if (pTable->vgId == 0) {
return TSDB_CODE_MND_VGROUP_NOT_EXIST;
}
@@ -186,8 +190,8 @@ static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) {
}
static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) {
- SChildTableObj *pNew = pOper->pObj;
- SChildTableObj *pTable = mnodeGetChildTable(pNew->info.tableId);
+ SCTableObj *pNew = pOper->pObj;
+ SCTableObj *pTable = mnodeGetChildTable(pNew->info.tableId);
if (pTable != pNew) {
void *oldTableId = pTable->info.tableId;
void *oldSql = pTable->sql;
@@ -195,7 +199,7 @@ static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) {
void *oldSTable = pTable->superTable;
int32_t oldRefCount = pTable->refCount;
- memcpy(pTable, pNew, sizeof(SChildTableObj));
+ memcpy(pTable, pNew, sizeof(SCTableObj));
pTable->refCount = oldRefCount;
pTable->sql = pNew->sql;
@@ -213,7 +217,7 @@ static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) {
}
static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) {
- SChildTableObj *pTable = pOper->pObj;
+ SCTableObj *pTable = pOper->pObj;
assert(pTable != NULL && pOper->rowData != NULL);
int32_t len = strlen(pTable->info.tableId);
@@ -244,7 +248,7 @@ static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) {
static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) {
assert(pOper->rowData != NULL);
- SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj));
+ SCTableObj *pTable = calloc(1, sizeof(SCTableObj));
if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
int32_t len = strlen(pOper->rowData);
@@ -284,7 +288,7 @@ static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) {
static int32_t mnodeChildTableActionRestored() {
void *pIter = NULL;
- SChildTableObj *pTable = NULL;
+ SCTableObj *pTable = NULL;
while (1) {
pIter = mnodeGetNextChildTable(pIter, &pTable);
@@ -323,7 +327,7 @@ static int32_t mnodeChildTableActionRestored() {
}
if (pTable->info.type == TSDB_CHILD_TABLE) {
- SSuperTableObj *pSuperTable = mnodeGetSuperTableByUid(pTable->suid);
+ SSTableObj *pSuperTable = mnodeGetSuperTableByUid(pTable->suid);
if (pSuperTable == NULL) {
mError("ctable:%s, stable:%" PRIu64 " not exist", pTable->info.tableId, pTable->suid);
pTable->vgId = 0;
@@ -344,14 +348,14 @@ static int32_t mnodeChildTableActionRestored() {
}
static int32_t mnodeInitChildTables() {
- SChildTableObj tObj;
+ SCTableObj tObj;
tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_CTABLE,
.tableName = "ctables",
.hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE,
- .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE,
+ .maxRowSize = sizeof(SCTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_VAR_STRING,
.insertFp = mnodeChildTableActionInsert,
@@ -386,7 +390,7 @@ int64_t mnodeGetChildTableNum() {
return sdbGetNumOfRows(tsChildTableSdb);
}
-static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
+static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable) {
atomic_add_fetch_32(&pStable->numOfTables, 1);
if (pStable->vgHash == NULL) {
@@ -402,7 +406,7 @@ static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCt
}
}
-static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
+static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable) {
atomic_sub_fetch_32(&pStable->numOfTables, 1);
if (pStable->vgHash == NULL) return;
@@ -416,14 +420,14 @@ static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *
mnodeDecVgroupRef(pVgroup);
}
-static void mnodeDestroySuperTable(SSuperTableObj *pStable) {
+static void mnodeDestroySuperTable(SSTableObj *pStable) {
if (pStable->vgHash != NULL) {
taosHashCleanup(pStable->vgHash);
pStable->vgHash = NULL;
}
- taosTFree(pStable->info.tableId);
- taosTFree(pStable->schema);
- taosTFree(pStable);
+ tfree(pStable->info.tableId);
+ tfree(pStable->schema);
+ tfree(pStable);
}
static int32_t mnodeSuperTableActionDestroy(SSdbOper *pOper) {
@@ -432,7 +436,7 @@ static int32_t mnodeSuperTableActionDestroy(SSdbOper *pOper) {
}
static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) {
- SSuperTableObj *pStable = pOper->pObj;
+ SSTableObj *pStable = pOper->pObj;
SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId);
if (pDb != NULL && pDb->status == TSDB_DB_STATUS_READY) {
mnodeAddSuperTableIntoDb(pDb);
@@ -443,11 +447,11 @@ static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) {
}
static int32_t mnodeSuperTableActionDelete(SSdbOper *pOper) {
- SSuperTableObj *pStable = pOper->pObj;
+ SSTableObj *pStable = pOper->pObj;
SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId);
if (pDb != NULL) {
mnodeRemoveSuperTableFromDb(pDb);
- mnodeDropAllChildTablesInStable((SSuperTableObj *)pStable);
+ mnodeDropAllChildTablesInStable((SSTableObj *)pStable);
}
mnodeDecDbRef(pDb);
@@ -455,8 +459,8 @@ static int32_t mnodeSuperTableActionDelete(SSdbOper *pOper) {
}
static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) {
- SSuperTableObj *pNew = pOper->pObj;
- SSuperTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId);
+ SSTableObj *pNew = pOper->pObj;
+ SSTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId);
if (pTable != NULL && pTable != pNew) {
void *oldTableId = pTable->info.tableId;
void *oldSchema = pTable->schema;
@@ -464,7 +468,7 @@ static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) {
int32_t oldRefCount = pTable->refCount;
int32_t oldNumOfTables = pTable->numOfTables;
- memcpy(pTable, pNew, sizeof(SSuperTableObj));
+ memcpy(pTable, pNew, sizeof(SSTableObj));
pTable->vgHash = oldVgHash;
pTable->refCount = oldRefCount;
@@ -480,7 +484,7 @@ static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) {
}
static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) {
- SSuperTableObj *pStable = pOper->pObj;
+ SSTableObj *pStable = pOper->pObj;
assert(pOper->pObj != NULL && pOper->rowData != NULL);
int32_t len = strlen(pStable->info.tableId);
@@ -504,7 +508,7 @@ static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) {
static int32_t mnodeSuperTableActionDecode(SSdbOper *pOper) {
assert(pOper->rowData != NULL);
- SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj));
+ SSTableObj *pStable = (SSTableObj *) calloc(1, sizeof(SSTableObj));
if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
int32_t len = strlen(pOper->rowData);
@@ -537,14 +541,14 @@ static int32_t mnodeSuperTableActionRestored() {
}
static int32_t mnodeInitSuperTables() {
- SSuperTableObj tObj;
+ SSTableObj tObj;
tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
SSdbTableDesc tableDesc = {
.tableId = SDB_TABLE_STABLE,
.tableName = "stables",
.hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE,
- .maxRowSize = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN,
+ .maxRowSize = sizeof(SSTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.keyType = SDB_KEY_VAR_STRING,
.insertFp = mnodeSuperTableActionInsert,
@@ -615,7 +619,7 @@ static void *mnodeGetSuperTable(char *tableId) {
}
static void *mnodeGetSuperTableByUid(uint64_t uid) {
- SSuperTableObj *pStable = NULL;
+ SSTableObj *pStable = NULL;
void *pIter = NULL;
while (1) {
@@ -647,11 +651,11 @@ void *mnodeGetTable(char *tableId) {
return NULL;
}
-void *mnodeGetNextChildTable(void *pIter, SChildTableObj **pTable) {
+void *mnodeGetNextChildTable(void *pIter, SCTableObj **pTable) {
return sdbFetchRow(tsChildTableSdb, pIter, (void **)pTable);
}
-void *mnodeGetNextSuperTable(void *pIter, SSuperTableObj **pTable) {
+void *mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable) {
return sdbFetchRow(tsSuperTableSdb, pIter, (void **)pTable);
}
@@ -765,12 +769,12 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
}
if (pMsg->pTable->type == TSDB_SUPER_TABLE) {
- SSuperTableObj *pSTable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pSTable = (SSTableObj *)pMsg->pTable;
mInfo("app:%p:%p, table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d",
pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pSTable->uid, pSTable->numOfTables, (int32_t)taosHashGetSize(pSTable->vgHash));
return mnodeProcessDropSuperTableMsg(pMsg);
} else {
- SChildTableObj *pCTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pCTable = (SCTableObj *)pMsg->pTable;
mInfo("app:%p:%p, table:%s, start to drop ctable, vgId:%d tid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
pDrop->tableId, pCTable->vgId, pCTable->tid, pCTable->uid);
return mnodeProcessDropChildTableMsg(pMsg);
@@ -778,7 +782,7 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
}
static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
- SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
+ STableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
pInfo->createFlag = htons(pInfo->createFlag);
mDebug("app:%p:%p, table:%s, table meta msg is received from thandle:%p, createFlag:%d", pMsg->rpcMsg.ahandle, pMsg,
pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag);
@@ -816,7 +820,7 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
}
static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pTable = (SSTableObj *)pMsg->pTable;
assert(pTable);
if (code == TSDB_CODE_SUCCESS) {
@@ -835,7 +839,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
- SSuperTableObj * pStable = calloc(1, sizeof(SSuperTableObj));
+ SSTableObj * pStable = calloc(1, sizeof(SSTableObj));
if (pStable == NULL) {
mError("app:%p:%p, table:%s, failed to create, no enough memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
return TSDB_CODE_MND_OUT_OF_MEMORY;
@@ -878,7 +882,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
- .rowSize = sizeof(SSuperTableObj) + schemaSize,
+ .rowSize = sizeof(SSTableObj) + schemaSize,
.pMsg = pMsg,
.writeCb = mnodeCreateSuperTableCb
};
@@ -894,7 +898,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
}
static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pTable = (SSTableObj *)pMsg->pTable;
if (code != TSDB_CODE_SUCCESS) {
mError("app:%p:%p, stable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
} else {
@@ -907,7 +911,7 @@ static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) {
SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash);
while (taosHashIterNext(pIter)) {
@@ -915,8 +919,8 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
SVgObj *pVgroup = mnodeGetVgroup(*pVgId);
if (pVgroup == NULL) break;
- SMDDropSTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropSTableMsg));
- pDrop->contLen = htonl(sizeof(SMDDropSTableMsg));
+ SDropSTableMsg *pDrop = rpcMallocCont(sizeof(SDropSTableMsg));
+ pDrop->contLen = htonl(sizeof(SDropSTableMsg));
pDrop->vgId = htonl(pVgroup->vgId);
pDrop->uid = htobe64(pStable->uid);
mnodeExtractTableName(pStable->info.tableId, pDrop->tableId);
@@ -924,7 +928,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
mInfo("app:%p:%p, stable:%s, send drop stable msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
pVgroup->vgId);
SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pVgroup);
- SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
+ SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
dnodeSendMsgToDnode(&epSet, &rpcMsg);
mnodeDecVgroupRef(pVgroup);
}
@@ -950,7 +954,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
return code;
}
-static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) {
+static int32_t mnodeFindSuperTableTagIndex(SSTableObj *pStable, const char *tagName) {
SSchema *schema = (SSchema *) pStable->schema;
for (int32_t tag = 0; tag < pStable->numOfTags; tag++) {
if (strcasecmp(schema[pStable->numOfColumns + tag].name, tagName) == 0) {
@@ -962,7 +966,7 @@ static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char *
}
static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, stable %s, add tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
tstrerror(code));
@@ -970,7 +974,7 @@ static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
}
static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t ntags) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) {
mError("app:%p:%p, stable:%s, add tag, too many tags", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId);
return TSDB_CODE_MND_TOO_MANY_TAGS;
@@ -1018,14 +1022,14 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t
}
static int32_t mnodeDropSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, stable %s, drop tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
tstrerror(code));
return code;
}
static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
int32_t col = mnodeFindSuperTableTagIndex(pStable, tagName);
if (col < 0) {
mError("app:%p:%p, stable:%s, drop tag, tag:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
@@ -1052,14 +1056,14 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) {
}
static int32_t mnodeModifySuperTableTagNameCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, stable %s, modify tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
tstrerror(code));
return code;
}
static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, char *newTagName) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
int32_t col = mnodeFindSuperTableTagIndex(pStable, oldTagName);
if (col < 0) {
mError("app:%p:%p, stable:%s, failed to modify table tag, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg,
@@ -1095,7 +1099,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c
return sdbUpdateRow(&oper);
}
-static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName) {
+static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName) {
SSchema *schema = (SSchema *) pStable->schema;
for (int32_t col = 0; col < pStable->numOfColumns; col++) {
if (strcasecmp(schema[col].name, colName) == 0) {
@@ -1107,7 +1111,7 @@ static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *col
}
static int32_t mnodeAddSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, stable %s, add column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
tstrerror(code));
return code;
@@ -1115,7 +1119,7 @@ static int32_t mnodeAddSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32_t ncols) {
SDbObj *pDb = pMsg->pDb;
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
if (ncols <= 0) {
mError("app:%p:%p, stable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, ncols);
return TSDB_CODE_MND_APP_ERROR;
@@ -1170,7 +1174,7 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32
}
static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, stable %s, delete column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
tstrerror(code));
return code;
@@ -1178,7 +1182,7 @@ static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) {
SDbObj *pDb = pMsg->pDb;
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
int32_t col = mnodeFindSuperTableColumnIndex(pStable, colName);
if (col <= 0) {
mError("app:%p:%p, stable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
@@ -1215,14 +1219,14 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) {
}
static int32_t mnodeChangeSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, stable %s, change column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
tstrerror(code));
return code;
}
static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) {
- SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
int32_t col = mnodeFindSuperTableColumnIndex(pStable, oldName);
if (col < 0) {
mError("app:%p:%p, stable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg,
@@ -1321,7 +1325,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
int32_t numOfRows = 0;
char * pWrite;
int32_t cols = 0;
- SSuperTableObj *pTable = NULL;
+ SSTableObj *pTable = NULL;
char prefix[64] = {0};
int32_t prefixLen;
@@ -1399,7 +1403,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
void mnodeDropAllSuperTables(SDbObj *pDropDb) {
void * pIter= NULL;
int32_t numOfTables = 0;
- SSuperTableObj *pTable = NULL;
+ SSTableObj *pTable = NULL;
char prefix[64] = {0};
tstrncpy(prefix, pDropDb->name, 64);
@@ -1430,7 +1434,7 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) {
mInfo("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables);
}
-static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) {
+static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSTableObj *pTable) {
int32_t numOfCols = pTable->numOfColumns + pTable->numOfTags;
assert(numOfCols <= TSDB_MAX_COLUMNS);
@@ -1446,7 +1450,7 @@ static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pT
}
static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) {
- SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
+ SSTableObj *pTable = (SSTableObj *)pMsg->pTable;
STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16));
if (pMeta == NULL) {
return TSDB_CODE_MND_OUT_OF_MEMORY;
@@ -1472,32 +1476,32 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) {
}
static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
- SCMSTableVgroupMsg *pInfo = pMsg->rpcMsg.pCont;
+ SSTableVgroupMsg *pInfo = pMsg->rpcMsg.pCont;
int32_t numOfTable = htonl(pInfo->numOfTables);
// reserve space
- int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupMsg) + sizeof(SVgroupsMsg);
+ int32_t contLen = sizeof(SSTableVgroupRspMsg) + 32 * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg);
for (int32_t i = 0; i < numOfTable; ++i) {
- char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN) * i;
- SSuperTableObj *pTable = mnodeGetSuperTable(stableName);
+ char *stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i;
+ SSTableObj *pTable = mnodeGetSuperTable(stableName);
if (pTable != NULL && pTable->vgHash != NULL) {
- contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupMsg) + sizeof(SVgroupsMsg));
- }
-
+ contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg));
+ }
+
mnodeDecTableRef(pTable);
}
- SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen);
+ SSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) {
return TSDB_CODE_MND_OUT_OF_MEMORY;
}
pRsp->numOfTables = 0;
- char *msg = (char *)pRsp + sizeof(SCMSTableVgroupRspMsg);
+ char *msg = (char *)pRsp + sizeof(SSTableVgroupRspMsg);
for (int32_t i = 0; i < numOfTable; ++i) {
- char * stableName = (char *)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i;
- SSuperTableObj *pTable = mnodeGetSuperTable(stableName);
+ char * stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i;
+ SSTableObj *pTable = mnodeGetSuperTable(stableName);
if (pTable == NULL) {
mError("app:%p:%p, stable:%s, not exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, stableName);
mnodeDecTableRef(pTable);
@@ -1548,7 +1552,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
pVgroupMsg->numOfVgroups = htonl(vgSize);
// one table is done, try the next table
- msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SCMVgroupMsg);
+ msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SVgroupMsg);
pRsp->numOfTables++;
}
}
@@ -1569,7 +1573,7 @@ static void mnodeProcessDropSuperTableRsp(SRpcMsg *rpcMsg) {
mInfo("drop stable rsp received, result:%s", tstrerror(rpcMsg->code));
}
-static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) {
+static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SCTableObj *pTable) {
STagData * pTagData = NULL;
int32_t tagDataLen = 0;
int32_t totalCols = 0;
@@ -1643,7 +1647,7 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableO
}
static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
assert(pTable);
mDebug("app:%p:%p, table:%s, created in mnode, vgId:%d sid:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
@@ -1669,7 +1673,7 @@ static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) {
}
static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
assert(pTable);
@@ -1679,12 +1683,12 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
pTable->info.tableId, pMsg->rpcMsg.handle);
pMsg->retry = 0;
- dnodeReprocessMnodeWriteMsg(pMsg);
+ dnodeReprocessMWriteMsg(pMsg);
} else {
mDebug("app:%p:%p, table:%s, created in dnode, thandle:%p", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
pMsg->rpcMsg.handle);
- dnodeSendRpcMnodeWriteRsp(pMsg, TSDB_CODE_SUCCESS);
+ dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS);
}
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
} else {
@@ -1699,7 +1703,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
SVgObj *pVgroup = pMsg->pVgroup;
SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
- SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj));
+ SCTableObj *pTable = calloc(1, sizeof(SCTableObj));
if (pTable == NULL) {
mError("app:%p:%p, table:%s, failed to alloc memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
return TSDB_CODE_MND_OUT_OF_MEMORY;
@@ -1842,7 +1846,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
}
static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
mLInfo("app:%p:%p, ctable:%s, is dropped from sdb", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg));
@@ -1880,7 +1884,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) {
static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) {
if (code != TSDB_CODE_SUCCESS) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
mError("app:%p:%p, ctable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
return code;
}
@@ -1889,7 +1893,7 @@ static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) {
}
static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
if (pMsg->pVgroup == NULL) {
mError("app:%p:%p, table:%s, failed to drop ctable, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg,
@@ -1914,7 +1918,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) {
return code;
}
-static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) {
+static int32_t mnodeFindNormalTableColumnIndex(SCTableObj *pTable, char *colName) {
SSchema *schema = (SSchema *) pTable->schema;
for (int32_t col = 0; col < pTable->numOfColumns; col++) {
if (strcasecmp(schema[col].name, colName) == 0) {
@@ -1926,7 +1930,7 @@ static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *col
}
static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
if (code != TSDB_CODE_SUCCESS) {
mError("app:%p:%p, ctable %s, failed to alter column, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
tstrerror(code));
@@ -1965,7 +1969,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
}
static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32_t ncols) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
SDbObj *pDb = pMsg->pDb;
if (ncols <= 0) {
mError("app:%p:%p, ctable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, ncols);
@@ -2014,7 +2018,7 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3
static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
SDbObj *pDb = pMsg->pDb;
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
int32_t col = mnodeFindNormalTableColumnIndex(pTable, colName);
if (col <= 0) {
mError("app:%p:%p, ctable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
@@ -2046,7 +2050,7 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
}
static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) {
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
int32_t col = mnodeFindNormalTableColumnIndex(pTable, oldName);
if (col < 0) {
mError("app:%p:%p, ctable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg,
@@ -2082,7 +2086,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char
return sdbUpdateRow(&oper);
}
-static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pTable) {
+static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SCTableObj *pTable) {
int32_t numOfCols = pTable->numOfColumns;
for (int32_t i = 0; i < numOfCols; ++i) {
strcpy(pSchema->name, pTable->schema[i].name);
@@ -2097,7 +2101,7 @@ static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *p
static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
SDbObj *pDb = pMsg->pDb;
- SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
pMeta->uid = htobe64(pTable->uid);
pMeta->tid = htonl(pTable->tid);
@@ -2146,7 +2150,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
}
static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
- SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
+ STableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
STagData *pTags = (STagData *)pInfo->tags;
int32_t tagLen = htonl(pTags->dataLen);
if (pTags->name[0] == 0) {
@@ -2156,7 +2160,7 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
}
int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + tagLen;
- SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen);
+ SCMCreateTableMsg *pCreateMsg = calloc(1, contLen);
if (pCreateMsg == NULL) {
mError("app:%p:%p, table:%s, failed to create table while get meta info, no enough memory", pMsg->rpcMsg.ahandle,
pMsg, pInfo->tableId);
@@ -2174,11 +2178,13 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
mDebug("app:%p:%p, table:%s, start to create on demand, tagLen:%d stable:%s",
pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId, tagLen, pTags->name);
- rpcFreeCont(pMsg->rpcMsg.pCont);
+ if (pMsg->rpcMsg.pCont != pMsg->pCont) {
+ tfree(pMsg->rpcMsg.pCont);
+ }
pMsg->rpcMsg.msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE;
pMsg->rpcMsg.pCont = pCreateMsg;
pMsg->rpcMsg.contLen = contLen;
-
+
return TSDB_CODE_MND_ACTION_NEED_REPROCESSED;
}
@@ -2203,7 +2209,7 @@ static int32_t mnodeGetChildTableMeta(SMnodeMsg *pMsg) {
void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) {
void * pIter = NULL;
int32_t numOfTables = 0;
- SChildTableObj *pTable = NULL;
+ SCTableObj *pTable = NULL;
mInfo("vgId:%d, all child tables will be dropped from sdb", pVgroup->vgId);
@@ -2231,7 +2237,7 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) {
void mnodeDropAllChildTables(SDbObj *pDropDb) {
void * pIter = NULL;
int32_t numOfTables = 0;
- SChildTableObj *pTable = NULL;
+ SCTableObj *pTable = NULL;
char prefix[64] = {0};
tstrncpy(prefix, pDropDb->name, 64);
@@ -2261,10 +2267,10 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) {
mInfo("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables);
}
-static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) {
+static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) {
void * pIter = NULL;
int32_t numOfTables = 0;
- SChildTableObj *pTable = NULL;
+ SCTableObj *pTable = NULL;
mInfo("stable:%s uid:%" PRIu64 ", all child tables:%d will be dropped from sdb", pStable->info.tableId, pStable->uid,
pStable->numOfTables);
@@ -2292,11 +2298,11 @@ static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) {
}
#if 0
-static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) {
+static SCTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) {
SVgObj *pVgroup = mnodeGetVgroup(vnode);
if (pVgroup == NULL) return NULL;
- SChildTableObj *pTable = pVgroup->tableList[tid - 1];
+ SCTableObj *pTable = pVgroup->tableList[tid - 1];
mnodeIncTableRef((STableObj *)pTable);
mnodeDecVgroupRef(pVgroup);
@@ -2307,14 +2313,14 @@ static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) {
static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_COM_OPS_NOT_SUPPORT;
#if 0
- SDMConfigTableMsg *pCfg = pMsg->rpcMsg.pCont;
+ SConfigTableMsg *pCfg = pMsg->rpcMsg.pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId);
pCfg->vgId = htonl(pCfg->vgId);
pCfg->sid = htonl(pCfg->sid);
mDebug("app:%p:%p, dnode:%d, vgId:%d sid:%d, receive table config msg", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId,
pCfg->vgId, pCfg->sid);
- SChildTableObj *pTable = mnodeGetTableByPos(pCfg->vgId, pCfg->sid);
+ SCTableObj *pTable = mnodeGetTableByPos(pCfg->vgId, pCfg->sid);
if (pTable == NULL) {
mError("app:%p:%p, dnode:%d, vgId:%d sid:%d, table not found", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId,
pCfg->vgId, pCfg->sid);
@@ -2322,7 +2328,7 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) {
}
SMDCreateTableMsg *pCreate = NULL;
- pCreate = mnodeBuildCreateChildTableMsg(NULL, (SChildTableObj *)pTable);
+ pCreate = mnodeBuildCreateChildTableMsg(NULL, (SCTableObj *)pTable);
mnodeDecTableRef(pTable);
if (pCreate == NULL) return terrno;
@@ -2340,7 +2346,7 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
SMnodeMsg *mnodeMsg = rpcMsg->ahandle;
mnodeMsg->received++;
- SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable;
assert(pTable);
mInfo("app:%p:%p, table:%s, drop table rsp received, vgId:%d sid:%d uid:%" PRIu64 ", thandle:%p result:%s",
@@ -2351,14 +2357,14 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
mError("app:%p:%p, table:%s, failed to drop in dnode, vgId:%d sid:%d uid:%" PRIu64 ", reason:%s",
mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid,
tstrerror(rpcMsg->code));
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
+ dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
return;
}
if (mnodeMsg->pVgroup == NULL) mnodeMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
if (mnodeMsg->pVgroup == NULL) {
mError("app:%p:%p, table:%s, failed to get vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId);
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST);
+ dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST);
return;
}
@@ -2368,7 +2374,7 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
mnodeDropVgroup(mnodeMsg->pVgroup, NULL);
}
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
+ dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
}
/*
@@ -2381,7 +2387,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
SMnodeMsg *mnodeMsg = rpcMsg->ahandle;
mnodeMsg->received++;
- SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable;
assert(pTable);
// If the table is deleted by another thread during creation, stop creating and send drop msg to vnode
@@ -2399,7 +2405,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
mnodeSendDropChildTableMsg(mnodeMsg, false);
rpcMsg->code = TSDB_CODE_SUCCESS;
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
+ dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
return;
}
@@ -2416,25 +2422,32 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mnodeMsg->pTable = NULL;
mnodeDestroyChildTable(pTable);
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, code);
+ dnodeSendRpcMWriteRsp(mnodeMsg, code);
}
} else {
- if (mnodeMsg->retry++ < 10) {
+ mnodeMsg->retry++;
+ int32_t sec = taosGetTimestampSec();
+ if (mnodeMsg->retry < CREATE_CTABLE_RETRY_TIMES && ABS(sec - mnodeMsg->incomingTs) < CREATE_CTABLE_RETRY_SEC) {
mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64
" result:%s thandle:%p",
mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->tid,
pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
- dnodeDelayReprocessMnodeWriteMsg(mnodeMsg);
+ dnodeDelayReprocessMWriteMsg(mnodeMsg);
} else {
- mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 ", result:%s thandle:%p",
+ mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64
+ ", result:%s thandle:%p incomingTs:%d curTs:%d retryTimes:%d",
mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid,
- tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
+ tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle, mnodeMsg->incomingTs, sec, mnodeMsg->retry);
SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable};
sdbDeleteRow(&oper);
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
+ if (rpcMsg->code == TSDB_CODE_APP_NOT_READY) {
+ //Avoid retry again in client
+ rpcMsg->code = TSDB_CODE_MND_VGROUP_NOT_READY;
+ }
+ dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
}
}
}
@@ -2445,31 +2458,31 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) {
SMnodeMsg *mnodeMsg = rpcMsg->ahandle;
mnodeMsg->received++;
- SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable;
+ SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable;
assert(pTable);
if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) {
mDebug("app:%p:%p, ctable:%s, altered in dnode, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
+ dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
} else {
- if (mnodeMsg->retry++ < 3) {
+ if (mnodeMsg->retry++ < ALTER_CTABLE_RETRY_TIMES) {
mDebug("app:%p:%p, table:%s, alter table rsp received, need retry, times:%d result:%s thandle:%p",
mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code),
mnodeMsg->rpcMsg.handle);
- dnodeDelayReprocessMnodeWriteMsg(mnodeMsg);
+ dnodeDelayReprocessMWriteMsg(mnodeMsg);
} else {
mError("app:%p:%p, table:%s, failed to alter in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code);
+ dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
}
}
}
static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
- SCMMultiTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
+ SMultiTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
pInfo->numOfTables = htonl(pInfo->numOfTables);
int32_t totalMallocLen = 4 * 1024 * 1024; // first malloc 4 MB, subsequent reallocation as twice
@@ -2483,7 +2496,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
for (int32_t t = 0; t < pInfo->numOfTables; ++t) {
char * tableId = (char *)(pInfo->tableIds + t * TSDB_TABLE_FNAME_LEN);
- SChildTableObj *pTable = mnodeGetChildTable(tableId);
+ SCTableObj *pTable = mnodeGetChildTable(tableId);
if (pTable == NULL) continue;
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(tableId);
@@ -2607,7 +2620,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
int32_t cols = 0;
int32_t numOfRows = 0;
- SChildTableObj *pTable = NULL;
+ SCTableObj *pTable = NULL;
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
char prefix[64] = {0};
@@ -2705,7 +2718,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
}
static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
- SCMAlterTableMsg *pAlter = pMsg->rpcMsg.pCont;
+ SAlterTableMsg *pAlter = pMsg->rpcMsg.pCont;
mDebug("app:%p:%p, table:%s, alter table msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg,
pAlter->tableId, pMsg->rpcMsg.handle);
@@ -2843,7 +2856,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
}
int32_t numOfRows = 0;
- SChildTableObj *pTable = NULL;
+ SCTableObj *pTable = NULL;
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
char prefix[64] = {0};
diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c
index 779e25254897ff37f1ca884e83469233d7197b8c..cd02d5a935c3ef3cf822d518a9cc3bbd795db46b 100644
--- a/src/mnode/src/mnodeUser.c
+++ b/src/mnode/src/mnodeUser.c
@@ -43,7 +43,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg);
static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg);
static int32_t mnodeUserActionDestroy(SSdbOper *pOper) {
- taosTFree(pOper->pObj);
+ tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
@@ -270,7 +270,7 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) {
code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("user:%s, failed to create by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code));
- taosTFree(pUser);
+ tfree(pUser);
} else {
mLInfo("user:%s, is created by %s", pUser->user, mnodeGetUserFromMsg(pMsg));
}
@@ -414,7 +414,7 @@ static int32_t mnodeProcessCreateUserMsg(SMnodeMsg *pMsg) {
SUserObj *pOperUser = pMsg->pUser;
if (pOperUser->superAuth) {
- SCMCreateUserMsg *pCreate = pMsg->rpcMsg.pCont;
+ SCreateUserMsg *pCreate = pMsg->rpcMsg.pCont;
return mnodeCreateUser(pOperUser->pAcct, pCreate->user, pCreate->pass, pMsg);
} else {
mError("user:%s, no rights to create user", pOperUser->user);
@@ -426,7 +426,7 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) {
int32_t code;
SUserObj *pOperUser = pMsg->pUser;
- SCMAlterUserMsg *pAlter = pMsg->rpcMsg.pCont;
+ SAlterUserMsg *pAlter = pMsg->rpcMsg.pCont;
SUserObj *pUser = mnodeGetUser(pAlter->user);
if (pUser == NULL) {
return TSDB_CODE_MND_INVALID_USER;
@@ -514,7 +514,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) {
int32_t code;
SUserObj *pOperUser = pMsg->pUser;
- SCMDropUserMsg *pDrop = pMsg->rpcMsg.pCont;
+ SDropUserMsg *pDrop = pMsg->rpcMsg.pCont;
SUserObj *pUser = mnodeGetUser(pDrop->user);
if (pUser == NULL) {
return TSDB_CODE_MND_INVALID_USER;
@@ -604,11 +604,11 @@ int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, cha
}
static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg) {
- SDMAuthMsg *pAuthMsg = pMsg->rpcMsg.pCont;
- SDMAuthRsp *pAuthRsp = rpcMallocCont(sizeof(SDMAuthRsp));
+ SAuthMsg *pAuthMsg = pMsg->rpcMsg.pCont;
+ SAuthRsp *pAuthRsp = rpcMallocCont(sizeof(SAuthRsp));
pMsg->rpcRsp.rsp = pAuthRsp;
- pMsg->rpcRsp.len = sizeof(SDMAuthRsp);
+ pMsg->rpcRsp.len = sizeof(SAuthRsp);
return mnodeRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey);
}
diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c
index 5084f1276ae883289939bd74c50a0502f6b3c71d..9e398e94f136de7324f18ee6bef32fdd4a700b33 100644
--- a/src/mnode/src/mnodeVgroup.c
+++ b/src/mnode/src/mnodeVgroup.c
@@ -69,7 +69,7 @@ static void mnodeDestroyVgroup(SVgObj *pVgroup) {
pVgroup->idPool = NULL;
}
- taosTFree(pVgroup);
+ tfree(pVgroup);
}
static int32_t mnodeVgroupActionDestroy(SSdbOper *pOper) {
@@ -529,7 +529,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
(void)sdbUpdateRow(&desc);
- dnodeReprocessMnodeWriteMsg(pMsg);
+ dnodeReprocessMWriteMsg(pMsg);
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
// if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_READY) {
// mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
@@ -537,7 +537,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
// pVgroup->status = TAOS_VG_STATUS_READY;
// SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
// (void)sdbUpdateRow(&desc);
- // dnodeReprocessMnodeWriteMsg(pMsg);
+ // dnodeReprocessMWriteMsg(pMsg);
// return TSDB_CODE_MND_ACTION_IN_PROGRESS;
// } else {
// mError("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d, but vgroup is dropping", pMsg->rpcMsg.ahandle,
@@ -663,13 +663,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
for (int32_t i = 0; i < pShow->maxReplica; ++i) {
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
- snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "dnode%d", i + 1);
+ snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1);
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
- snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dstatus", i + 1);
+ snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1);
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
}
@@ -694,7 +694,7 @@ static bool mnodeFilterVgroups(SVgObj *pVgroup, STableObj *pTable) {
return true;
}
- SChildTableObj *pCTable = (SChildTableObj *)pTable;
+ SCTableObj *pCTable = (SCTableObj *)pTable;
if (pVgroup->vgId == pCTable->vgId) {
return true;
} else {
@@ -791,7 +791,7 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v
return numOfRows;
}
-void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
+void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable) {
int32_t idPoolSize = taosIdPoolMaxSize(pVgroup->idPool);
if (pTable->tid > idPoolSize) {
mnodeAllocVgroupIdPool(pVgroup);
@@ -807,7 +807,7 @@ void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
}
}
-void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
+void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable) {
if (pTable->tid >= 1) {
taosFreeId(pVgroup->idPool, pTable->tid);
pVgroup->numOfTables--;
@@ -818,11 +818,11 @@ void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
}
}
-static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
+static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
SDbObj *pDb = pVgroup->pDb;
if (pDb == NULL) return NULL;
- SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg));
+ SCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SCreateVnodeMsg));
if (pVnode == NULL) return NULL;
strcpy(pVnode->db, pVgroup->dbName);
@@ -830,7 +830,7 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
//TODO: dynamic alloc tables in tsdb
maxTables = MAX(10000, tsMaxTablePerVnode);
- SMDVnodeCfg *pCfg = &pVnode->cfg;
+ SVnodeCfg *pCfg = &pVnode->cfg;
pCfg->vgId = htonl(pVgroup->vgId);
pCfg->cfgVersion = htonl(pDb->cfgVersion);
pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize);
@@ -850,8 +850,9 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
pCfg->replications = (int8_t) pVgroup->numOfVnodes;
pCfg->wals = 3;
pCfg->quorum = pDb->cfg.quorum;
+ pCfg->update = pDb->cfg.update;
- SMDVnodeDesc *pNodes = pVnode->nodes;
+ SVnodeDesc *pNodes = pVnode->nodes;
for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) {
SDnodeObj *pDnode = pVgroup->vnodeGid[j].pDnode;
if (pDnode != NULL) {
@@ -886,11 +887,11 @@ SRpcEpSet mnodeGetEpSetFromIp(char *ep) {
}
static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet) {
- SMDAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup);
+ SAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup);
SRpcMsg rpcMsg = {
.ahandle = NULL,
.pCont = pAlter,
- .contLen = pAlter ? sizeof(SMDAlterVnodeMsg) : 0,
+ .contLen = pAlter ? sizeof(SAlterVnodeMsg) : 0,
.code = 0,
.msgType = TSDB_MSG_TYPE_MD_ALTER_VNODE
};
@@ -909,11 +910,11 @@ void mnodeSendAlterVgroupMsg(SVgObj *pVgroup) {
}
static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet, void *ahandle) {
- SMDCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup);
+ SCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup);
SRpcMsg rpcMsg = {
.ahandle = ahandle,
.pCont = pCreate,
- .contLen = pCreate ? sizeof(SMDCreateVnodeMsg) : 0,
+ .contLen = pCreate ? sizeof(SCreateVnodeMsg) : 0,
.code = 0,
.msgType = TSDB_MSG_TYPE_MD_CREATE_VNODE
};
@@ -969,7 +970,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mnodeMsg->pVgroup = NULL;
mnodeDestroyVgroup(pVgroup);
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, code);
+ dnodeSendRpcMWriteRsp(mnodeMsg, code);
}
} else {
SSdbOper oper = {
@@ -978,12 +979,12 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
.pObj = pVgroup
};
sdbDeleteRow(&oper);
- dnodeSendRpcMnodeWriteRsp(mnodeMsg, mnodeMsg->code);
+ dnodeSendRpcMWriteRsp(mnodeMsg, mnodeMsg->code);
}
}
-static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) {
- SMDDropVnodeMsg *pDrop = rpcMallocCont(sizeof(SMDDropVnodeMsg));
+static SDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) {
+ SDropVnodeMsg *pDrop = rpcMallocCont(sizeof(SDropVnodeMsg));
if (pDrop == NULL) return NULL;
pDrop->vgId = htonl(vgId);
@@ -991,11 +992,11 @@ static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) {
}
void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle) {
- SMDDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId);
+ SDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId);
SRpcMsg rpcMsg = {
.ahandle = ahandle,
.pCont = pDrop,
- .contLen = pDrop ? sizeof(SMDDropVnodeMsg) : 0,
+ .contLen = pDrop ? sizeof(SDropVnodeMsg) : 0,
.code = 0,
.msgType = TSDB_MSG_TYPE_MD_DROP_VNODE
};
@@ -1040,11 +1041,11 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
code = TSDB_CODE_MND_SDB_ERROR;
}
- dnodeReprocessMnodeWriteMsg(mnodeMsg);
+ dnodeReprocessMWriteMsg(mnodeMsg);
}
static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) {
- SDMConfigVnodeMsg *pCfg = pMsg->rpcMsg.pCont;
+ SConfigVnodeMsg *pCfg = pMsg->rpcMsg.pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId);
pCfg->vgId = htonl(pCfg->vgId);
diff --git a/src/os/inc/os.h b/src/os/inc/os.h
index 86e16db8b1446308060945d3a7db2531287c62ec..9383ae48dc5c6151aea0d9e8a2641603b63da144 100644
--- a/src/os/inc/os.h
+++ b/src/os/inc/os.h
@@ -52,9 +52,9 @@ extern "C" {
#include "osWindows.h"
#endif
+#include "osDef.h"
#include "osAtomic.h"
#include "osCommon.h"
-#include "osDef.h"
#include "osDir.h"
#include "osFile.h"
#include "osLz4.h"
diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h
index c1a950fbe6375b14e3cf25277eb933b96e518f10..7bb844831e03dd1e21abd389a61976e14927ebbd 100644
--- a/src/os/inc/osDarwin.h
+++ b/src/os/inc/osDarwin.h
@@ -72,8 +72,6 @@ extern "C" {
#include
#define TAOS_OS_FUNC_FILE_SENDIFLE
- #define taosFSendFile(outfile, infile, offset, count) taosFSendFileImp(outfile, infile, offset, size)
- #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size)
#define TAOS_OS_FUNC_SEMPHONE
#define tsem_t dispatch_semaphore_t
diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h
index dc19c8177c62307ca110486c130f6a6b56047ef4..62e44d8eb0b70fb1526693895847637daa72247a 100644
--- a/src/os/inc/osFile.h
+++ b/src/os/inc/osFile.h
@@ -20,46 +20,52 @@
extern "C" {
#endif
-ssize_t taosTReadImp(int fd, void *buf, size_t count);
-ssize_t taosTWriteImp(int fd, void *buf, size_t count);
+#define tread(fd, buf, count) read(fd, buf, count)
+#define twrite(fd, buf, count) write(fd, buf, count)
+#define tlseek(fd, offset, whence) lseek(fd, offset, whence)
+#define tclose(fd) \
+ { \
+ if (FD_VALID(fd)) { \
+ close(fd); \
+ fd = FD_INITIALIZER; \
+ } \
+ }
-ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size);
-int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count);
+int64_t taosReadImp(int32_t fd, void *buf, int64_t count);
+int64_t taosWriteImp(int32_t fd, void *buf, int64_t count);
+int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence);
+int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath);
-#ifndef TAOS_OS_FUNC_FILE_SENDIFLE
- #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size)
- #define taosFSendFile(outfile, infile, offset, count) taosTSendFileImp(fileno(outfile), fileno(infile), offset, size)
-#endif
+#define taosRead(fd, buf, count) taosReadImp(fd, buf, count)
+#define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count)
+#define taosLSeek(fd, offset, whence) taosLSeekImp(fd, offset, whence)
+#define taosClose(x) tclose(x)
-#define taosTRead(fd, buf, count) taosTReadImp(fd, buf, count)
-#define taosTWrite(fd, buf, count) taosTWriteImp(fd, buf, count)
-#define taosLSeek(fd, offset, whence) lseek(fd, offset, whence)
+// TAOS_OS_FUNC_FILE_SENDIFLE
+int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size);
+int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size);
#ifdef TAOS_RANDOM_FILE_FAIL
- void taosSetRandomFileFailFactor(int factor);
+ void taosSetRandomFileFailFactor(int32_t factor);
void taosSetRandomFileFailOutput(const char *path);
#ifdef TAOS_RANDOM_FILE_FAIL_TEST
- ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
- ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
- off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line);
- #undef taosTRead
- #undef taosTWrite
+ int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line);
+ int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line);
+ int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line);
+ #undef taosRead
+ #undef taosWrite
#undef taosLSeek
- #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__)
- #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+ #define taosRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+ #define taosWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__)
#define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__)
#endif
#endif
-int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath);
-
// TAOS_OS_FUNC_FILE_GETTMPFILEPATH
void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath);
-#ifndef TAOS_OS_FUNC_FILE_FTRUNCATE
- #define taosFtruncate ftruncate
-#endif
-
+// TAOS_OS_FUNC_FILE_FTRUNCATE
+int32_t taosFtruncate(int32_t fd, int64_t length);
#ifdef __cplusplus
}
#endif
diff --git a/src/os/inc/osMemory.h b/src/os/inc/osMemory.h
index 37d9dc9828ec1f51f36542e6c297ec3df8709f7e..0616006650eb2d53cdf7bd70b68c9f60748deac5 100644
--- a/src/os/inc/osMemory.h
+++ b/src/os/inc/osMemory.h
@@ -31,6 +31,7 @@ typedef enum {
void taosSetAllocMode(int mode, const char *path, bool autoDump);
void taosDumpMemoryLeak();
+// used in tsdb module
void * taosTMalloc(size_t size);
void * taosTCalloc(size_t nmemb, size_t size);
void * taosTRealloc(void *ptr, size_t size);
@@ -38,7 +39,14 @@ void taosTZfree(void *ptr);
size_t taosTSizeof(void *ptr);
void taosTMemset(void *ptr, int c);
-#define taosTFree(x) \
+// used in other module
+#define tmalloc(size) malloc(size)
+#define tcalloc(num, size) calloc(num, size)
+#define trealloc(ptr, size) realloc(ptr, size)
+#define tstrdup(str) taosStrdupImp(str)
+#define tstrndup(str, size) taosStrndupImp(str, size)
+#define tgetline(lineptr, n, stream) taosGetlineImp(lineptr, n, stream)
+#define tfree(x) \
do { \
if (x) { \
free((void *)(x)); \
@@ -46,37 +54,30 @@ void taosTMemset(void *ptr, int c);
} \
} while (0);
-#define taosMalloc(size) malloc(size)
-#define taosCalloc(num, size) calloc(num, size)
-#define taosRealloc(ptr, size) realloc(ptr, size)
-#define taosFree(ptr) free(ptr)
-#define taosStrdup(str) taosStrdupImp(str)
-#define taosStrndup(str, size) taosStrndupImp(str, size)
-#define taosGetline(lineptr, n, stream) taosGetlineImp(lineptr, n, stream)
-
#ifdef TAOS_MEM_CHECK
#ifdef TAOS_MEM_CHECK_TEST
- void * taos_malloc(size_t size, const char *file, uint32_t line);
- void * taos_calloc(size_t num, size_t size, const char *file, uint32_t line);
- void * taos_realloc(void *ptr, size_t size, const char *file, uint32_t line);
- void taos_free(void *ptr, const char *file, uint32_t line);
- char * taos_strdup(const char *str, const char *file, uint32_t line);
- char * taos_strndup(const char *str, size_t size, const char *file, uint32_t line);
- ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line);
- #undef taosMalloc
- #undef taosCalloc
- #undef taosRealloc
- #undef taosFree
- #undef taosStrdup
- #undef taosStrndup
- #undef taosGetline
- #define taosMalloc(size) taos_malloc(size, __FILE__, __LINE__)
- #define taosCalloc(num, size) taos_calloc(num, size, __FILE__, __LINE__)
- #define taosRealloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__)
- #define taosFree(ptr) taos_free(ptr, __FILE__, __LINE__)
- //#define taosStrdup(str) taos_strdup(str, __FILE__, __LINE__)
- //#define taosStrndup(str, size) taos_strndup(str, size, __FILE__, __LINE__)
- //#define taosGetline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__)
+ void * taosMallocMem(size_t size, const char *file, uint32_t line);
+ void * taosCallocMem(size_t num, size_t size, const char *file, uint32_t line);
+ void * taosReallocMem(void *ptr, size_t size, const char *file, uint32_t line);
+ void taosFreeMem(void *ptr, const char *file, uint32_t line);
+ char * taosStrdupMem(const char *str, const char *file, uint32_t line);
+ char * taosStrndupMem(const char *str, size_t size, const char *file, uint32_t line);
+ ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line);
+ #undef tmalloc
+ #undef tcalloc
+ #undef trealloc
+ #undef tfree
+ #define tmalloc(size) taosMallocMem(size, __FILE__, __LINE__)
+ #define tcalloc(num, size) taosCallocMem(num, size, __FILE__, __LINE__)
+ #define trealloc(ptr, size) taosReallocMem(ptr, size, __FILE__, __LINE__)
+ #define tfree(ptr) taosFreeMem(ptr, __FILE__, __LINE__)
+
+ // #undef tstrdup
+ // #undef tstrndup
+ // #undef tgetline
+ // #define taosStrdup(str) taos_strdup(str, __FILE__, __LINE__)
+ // #define taosStrndup(str, size) taos_strndup(str, size, __FILE__, __LINE__)
+ // #define tgetline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__)
#endif
#endif
diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h
index 0ab3ff0fcafa43a404533f145a0bb1595bb29061..cbfdedef485f73f0005b2da1e1d9cc3adf9dd377 100644
--- a/src/os/inc/osSocket.h
+++ b/src/os/inc/osSocket.h
@@ -33,21 +33,19 @@ extern "C" {
x = FD_INITIALIZER; \
} \
}
- typedef int SOCKET;
+ typedef int32_t SOCKET;
#endif
#ifndef TAOS_OS_DEF_EPOLL
#define TAOS_EPOLL_WAIT_TIME -1
#endif
-#define taosClose(x) taosCloseSocket(x)
-
#ifdef TAOS_RANDOM_NETWORK_FAIL
#ifdef TAOS_RANDOM_NETWORK_FAIL_TEST
- ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags);
- ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen);
- ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count);
- ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count);
+ int64_t taosSendRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags);
+ int64_t taosSendToRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags, const struct sockaddr *dest_addr, socklen_t addrlen);
+ int64_t taosReadSocketRandomFail(int32_t fd, void *buf, size_t count);
+ int64_t taosWriteSocketRandomFail(int32_t fd, const void *buf, size_t count);
#undef taosSend
#undef taosSendto
#undef taosReadSocket
@@ -60,14 +58,14 @@ extern "C" {
#endif
// TAOS_OS_FUNC_SOCKET
-int taosSetNonblocking(SOCKET sock, int on);
-void taosBlockSIGPIPE();
+int32_t taosSetNonblocking(SOCKET sock, int32_t on);
+void taosBlockSIGPIPE();
// TAOS_OS_FUNC_SOCKET_SETSOCKETOPT
-int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen);
+int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen);
// TAOS_OS_FUNC_SOCKET_INET
-uint32_t taosInetAddr(char *ipAddr);
+uint32_t taosInetAddr(char *ipAddr);
const char *taosInetNtoa(struct in_addr ipInt);
#ifdef __cplusplus
diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h
index dc1da35037dc06c258856a1a143146492eeed0e5..36e30528bf9d84739fd391442145d7e65e1f7df2 100644
--- a/src/os/inc/osWindows.h
+++ b/src/os/inc/osWindows.h
@@ -62,11 +62,8 @@ extern "C" {
#define TAOS_OS_FUNC_FILE_ISDIR
#define TAOS_OS_FUNC_FILE_ISLNK
#define TAOS_OS_FUNC_FILE_SENDIFLE
- #define taosFSendFile(outfile, infile, offset, count) taosFSendFileImp(outfile, infile, offset, size)
- #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size)
#define TAOS_OS_FUNC_FILE_GETTMPFILEPATH
-#define TAOS_OS_FUNC_FILE_FTRUNCATE
- extern int taosFtruncate(int fd, int64_t length);
+#define TAOS_OS_FUNC_FILE_FTRUNCATE
#define TAOS_OS_FUNC_MATH
#define SWAP(a, b, c) \
@@ -139,7 +136,6 @@ typedef int (*__compar_fn_t)(const void *, const void *);
#define in_addr_t unsigned long
#define socklen_t int
#define htobe64 htonll
-#define twrite write
#define getpid _getpid
struct tm *localtime_r(const time_t *timep, struct tm *result);
diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c
index 66bdb5b939d77ab03b30f024466fe3e3ff4e9388..dacf4db74137b8cc58db2b390ba971cf9dfbc012 100644
--- a/src/os/src/darwin/darwinFile.c
+++ b/src/os/src/darwin/darwinFile.c
@@ -19,21 +19,19 @@
#define _SEND_FILE_STEP_ 1000
-int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) {
+int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
fseek(in_file, (int32_t)(*offset), 0);
- int writeLen = 0;
- uint8_t buffer[_SEND_FILE_STEP_] = { 0 };
+ int writeLen = 0;
+ uint8_t buffer[_SEND_FILE_STEP_] = {0};
for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file);
if (rlen <= 0) {
return writeLen;
- }
- else if (rlen < _SEND_FILE_STEP_) {
+ } else if (rlen < _SEND_FILE_STEP_) {
fwrite(buffer, 1, rlen, out_file);
return (int)(writeLen + rlen);
- }
- else {
+ } else {
fwrite(buffer, 1, _SEND_FILE_STEP_, in_file);
writeLen += _SEND_FILE_STEP_;
}
@@ -44,8 +42,7 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou
size_t rlen = fread(buffer, 1, remain, in_file);
if (rlen <= 0) {
return writeLen;
- }
- else {
+ } else {
fwrite(buffer, 1, remain, out_file);
writeLen += remain;
}
@@ -54,7 +51,7 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou
return writeLen;
}
-ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
- uError("not implemented yet");
+int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) {
+ uError("taosSendFile not implemented yet");
return -1;
}
\ No newline at end of file
diff --git a/src/os/src/detail/osFail.c b/src/os/src/detail/osFail.c
index e0eb200851f976e1c9dad47afbc152bfb021e4e5..a99bcd01dbccd0290de1fc38a1220b573ab74b22 100644
--- a/src/os/src/detail/osFail.c
+++ b/src/os/src/detail/osFail.c
@@ -20,7 +20,7 @@
#ifdef TAOS_RANDOM_NETWORK_FAIL
-ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags) {
+int64_t taosSendRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags) {
if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
errno = ECONNRESET;
return -1;
@@ -29,8 +29,8 @@ ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags) {
return send(sockfd, buf, len, flags);
}
-ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr,
- socklen_t addrlen) {
+int64_t taosSendToRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags,
+ const struct sockaddr *dest_addr, socklen_t addrlen) {
if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
errno = ECONNRESET;
return -1;
@@ -39,7 +39,7 @@ ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags,
return sendto(sockfd, buf, len, flags, dest_addr, addrlen);
}
-ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count) {
+int64_t taosReadSocketRandomFail(int32_t fd, void *buf, size_t count) {
if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
errno = ECONNRESET;
return -1;
@@ -48,7 +48,7 @@ ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count) {
return read(fd, buf, count);
}
-ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count) {
+int64_t taosWriteSocketRandomFail(int32_t fd, const void *buf, size_t count) {
if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
errno = EINTR;
return -1;
@@ -61,10 +61,10 @@ ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count) {
#ifdef TAOS_RANDOM_FILE_FAIL
-static int random_file_fail_factor = 20;
+static int32_t random_file_fail_factor = 20;
static FILE *fpRandomFileFailOutput = NULL;
-void taosSetRandomFileFailFactor(int factor) {
+void taosSetRandomFileFailFactor(int32_t factor) {
random_file_fail_factor = factor;
}
@@ -77,7 +77,7 @@ static void close_random_file_fail_output() {
}
}
-static void random_file_fail_output_sig(int sig) {
+static void random_file_fail_output_sig(int32_t sig) {
fprintf(fpRandomFileFailOutput, "signal %d received.\n", sig);
struct sigaction act = {0};
@@ -105,7 +105,7 @@ void taosSetRandomFileFailOutput(const char *path) {
sigaction(SIGILL, &act, NULL);
}
-ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) {
+int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line) {
if (random_file_fail_factor > 0) {
if (rand() % random_file_fail_factor == 0) {
errno = EIO;
@@ -113,10 +113,10 @@ ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file
}
}
- return taosTReadImp(fd, buf, count);
+ return taosReadImp(fd, buf, count);
}
-ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) {
+int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line) {
if (random_file_fail_factor > 0) {
if (rand() % random_file_fail_factor == 0) {
errno = EIO;
@@ -124,10 +124,10 @@ ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *fil
}
}
- return taosTWriteImp(fd, buf, count);
+ return taosWriteImp(fd, buf, count);
}
-off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line) {
+int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line) {
if (random_file_fail_factor > 0) {
if (rand() % random_file_fail_factor == 0) {
errno = EIO;
@@ -135,7 +135,7 @@ off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, ui
}
}
- return lseek(fd, offset, whence);
+ return taosLSeekImp(fd, offset, whence);
}
#endif //TAOS_RANDOM_FILE_FAIL
diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c
index 8f055dd8129f340f267e64cdd905505a7b675a2d..6eb4515f3098f89991640d2fdee2b0aca47c1703 100644
--- a/src/os/src/detail/osFile.c
+++ b/src/os/src/detail/osFile.c
@@ -15,16 +15,22 @@
#define _DEFAULT_SOURCE
#include "os.h"
+#include "tglobal.h"
#ifndef TAOS_OS_FUNC_FILE_GETTMPFILEPATH
+
void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
const char *tdengineTmpFileNamePrefix = "tdengine-";
char tmpPath[PATH_MAX];
- char *tmpDir = "/tmp/";
+ int32_t len = strlen(tsTempDir);
+ memcpy(tmpPath, tsTempDir, len);
+
+ if (tmpPath[len - 1] != '/') {
+ tmpPath[len++] = '/';
+ }
- strcpy(tmpPath, tmpDir);
- strcat(tmpPath, tdengineTmpFileNamePrefix);
+ strcpy(tmpPath + len, tdengineTmpFileNamePrefix);
if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
strcat(tmpPath, fileNamePrefix);
strcat(tmpPath, "-%d-%s");
@@ -34,10 +40,10 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
taosRandStr(rand, tListLen(rand) - 1);
snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
}
+
#endif
-// rename file name
-int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) {
+int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath) {
int32_t ts = taosGetTimestampSec();
char fname[PATH_MAX] = {0}; // max file name length must be less than 255
@@ -46,12 +52,13 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP
if (delimiterPos == NULL) return -1;
int32_t fileNameLen = 0;
- if (suffix)
+ if (suffix) {
fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix);
- else
+ } else {
fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts);
+ }
- size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1);
+ int32_t len = (int32_t)((delimiterPos - fullPath) + fileNameLen + 1);
if (*dstPath == NULL) {
*dstPath = calloc(1, len + 1);
if (*dstPath == NULL) return -1;
@@ -64,9 +71,9 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP
return rename(fullPath, *dstPath);
}
-ssize_t taosTReadImp(int fd, void *buf, size_t count) {
- size_t leftbytes = count;
- ssize_t readbytes;
+int64_t taosReadImp(int32_t fd, void *buf, int64_t count) {
+ int64_t leftbytes = count;
+ int64_t readbytes;
char * tbuf = (char *)buf;
while (leftbytes > 0) {
@@ -78,19 +85,19 @@ ssize_t taosTReadImp(int fd, void *buf, size_t count) {
return -1;
}
} else if (readbytes == 0) {
- return (ssize_t)(count - leftbytes);
+ return (int64_t)(count - leftbytes);
}
leftbytes -= readbytes;
tbuf += readbytes;
}
- return (ssize_t)count;
+ return count;
}
-ssize_t taosTWriteImp(int fd, void *buf, size_t n) {
- size_t nleft = n;
- ssize_t nwritten = 0;
+int64_t taosWriteImp(int32_t fd, void *buf, int64_t n) {
+ int64_t nleft = n;
+ int64_t nwritten = 0;
char * tbuf = (char *)buf;
while (nleft > 0) {
@@ -105,13 +112,18 @@ ssize_t taosTWriteImp(int fd, void *buf, size_t n) {
tbuf += nwritten;
}
- return (ssize_t)n;
+ return n;
+}
+
+int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) {
+ return (int64_t)tlseek(fd, (long)offset, whence);
}
#ifndef TAOS_OS_FUNC_FILE_SENDIFLE
-ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
- size_t leftbytes = size;
- ssize_t sentbytes;
+
+int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) {
+ int64_t leftbytes = size;
+ int64_t sentbytes;
while (leftbytes > 0) {
/*
@@ -126,7 +138,7 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
return -1;
}
} else if (sentbytes == 0) {
- return (ssize_t)(size - leftbytes);
+ return (int64_t)(size - leftbytes);
}
leftbytes -= sentbytes;
@@ -134,4 +146,17 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
return size;
}
+
+int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) {
+ return taosSendFile(fileno(outfile), fileno(infile), offset, size);
+}
+
+#endif
+
+#ifndef TAOS_OS_FUNC_FILE_FTRUNCATE
+
+int32_t taosFtruncate(int32_t fd, int64_t length) {
+ return ftruncate(fd, length);
+}
+
#endif
\ No newline at end of file
diff --git a/src/os/src/detail/osMemory.c b/src/os/src/detail/osMemory.c
index dfd320be89a75a4700a4918f68d56ee66faa3135..53310d179c0090382e009de949e5158146dc282a 100644
--- a/src/os/src/detail/osMemory.c
+++ b/src/os/src/detail/osMemory.c
@@ -28,7 +28,7 @@ static FILE* fpAllocLog = NULL;
extern int32_t taosGetTimestampSec();
static int32_t startTime = INT32_MAX;
-static bool random_alloc_fail(size_t size, const char* file, uint32_t line) {
+static bool taosRandomAllocFail(size_t size, const char* file, uint32_t line) {
if (taosGetTimestampSec() < startTime) {
return false;
}
@@ -48,33 +48,33 @@ static bool random_alloc_fail(size_t size, const char* file, uint32_t line) {
return true;
}
-static void* malloc_random(size_t size, const char* file, uint32_t line) {
- return random_alloc_fail(size, file, line) ? NULL : malloc(size);
+static void* taosRandmoMalloc(size_t size, const char* file, uint32_t line) {
+ return taosRandomAllocFail(size, file, line) ? NULL : malloc(size);
}
-static void* calloc_random(size_t num, size_t size, const char* file, uint32_t line) {
- return random_alloc_fail(num * size, file, line) ? NULL : calloc(num, size);
+static void* taosRandomCalloc(size_t num, size_t size, const char* file, uint32_t line) {
+ return taosRandomAllocFail(num * size, file, line) ? NULL : calloc(num, size);
}
-static void* realloc_random(void* ptr, size_t size, const char* file, uint32_t line) {
- return random_alloc_fail(size, file, line) ? NULL : realloc(ptr, size);
+static void* taosRandomRealloc(void* ptr, size_t size, const char* file, uint32_t line) {
+ return taosRandomAllocFail(size, file, line) ? NULL : realloc(ptr, size);
}
-static char* strdup_random(const char* str, const char* file, uint32_t line) {
+static char* taosRandomStrdup(const char* str, const char* file, uint32_t line) {
size_t len = strlen(str);
- return random_alloc_fail(len + 1, file, line) ? NULL : taosStrdupImp(str);
+ return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrdupImp(str);
}
-static char* strndup_random(const char* str, size_t size, const char* file, uint32_t line) {
+static char* taosRandomStrndup(const char* str, size_t size, const char* file, uint32_t line) {
size_t len = strlen(str);
if (len > size) {
len = size;
}
- return random_alloc_fail(len + 1, file, line) ? NULL : taosStrndupImp(str, len);
+ return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrndupImp(str, len);
}
-static ssize_t getline_random(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
- return random_alloc_fail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream);
+static ssize_t taosRandomGetline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
+ return taosRandomAllocFail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream);
}
////////////////////////////////////////////////////////////////////////////////
@@ -96,7 +96,7 @@ typedef struct SMemBlock {
static SMemBlock *blocks = NULL;
static uintptr_t lock = 0;
-static void add_mem_block(SMemBlock* blk) {
+static void taosAddMemBlock(SMemBlock* blk) {
blk->prev = NULL;
while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0);
blk->next = blocks;
@@ -107,7 +107,7 @@ static void add_mem_block(SMemBlock* blk) {
atomic_store_ptr(&lock, 0);
}
-static void remove_mem_block(SMemBlock* blk) {
+static void taosRemoveMemBlock(SMemBlock* blk) {
while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0);
if (blocks == blk) {
@@ -126,7 +126,7 @@ static void remove_mem_block(SMemBlock* blk) {
blk->next = NULL;
}
-static void free_detect_leak(void* ptr, const char* file, uint32_t line) {
+static void taosFreeDetectLeak(void* ptr, const char* file, uint32_t line) {
if (ptr == NULL) {
return;
}
@@ -140,11 +140,11 @@ static void free_detect_leak(void* ptr, const char* file, uint32_t line) {
return;
}
- remove_mem_block(blk);
+ taosRemoveMemBlock(blk);
free(blk);
}
-static void* malloc_detect_leak(size_t size, const char* file, uint32_t line) {
+static void* taosMallocDetectLeak(size_t size, const char* file, uint32_t line) {
if (size == 0) {
return NULL;
}
@@ -166,28 +166,28 @@ static void* malloc_detect_leak(size_t size, const char* file, uint32_t line) {
blk->line = (uint16_t)line;
blk->magic = MEMBLK_MAGIC;
blk->size = size;
- add_mem_block(blk);
+ taosAddMemBlock(blk);
return blk->data;
}
-static void* calloc_detect_leak(size_t num, size_t size, const char* file, uint32_t line) {
+static void* taosCallocDetectLeak(size_t num, size_t size, const char* file, uint32_t line) {
size *= num;
- void* p = malloc_detect_leak(size, file, line);
+ void* p = taosMallocDetectLeak(size, file, line);
if (p != NULL) {
memset(p, 0, size);
}
return p;
}
-static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint32_t line) {
+static void* taosReallocDetectLeak(void* ptr, size_t size, const char* file, uint32_t line) {
if (size == 0) {
- free_detect_leak(ptr, file, line);
+ taosFreeDetectLeak(ptr, file, line);
return NULL;
}
if (ptr == NULL) {
- return malloc_detect_leak(size, file, line);
+ return taosMallocDetectLeak(size, file, line);
}
SMemBlock* blk = (SMemBlock *)((char*)ptr) - sizeof(SMemBlock);
@@ -198,11 +198,11 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3
return realloc(ptr, size);
}
- remove_mem_block(blk);
+ taosRemoveMemBlock(blk);
void* p = realloc(blk, size + sizeof(SMemBlock));
if (p == NULL) {
- add_mem_block(blk);
+ taosAddMemBlock(blk);
return NULL;
}
@@ -212,13 +212,13 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3
blk = (SMemBlock*)p;
blk->size = size;
- add_mem_block(blk);
+ taosAddMemBlock(blk);
return blk->data;
}
-static char* strdup_detect_leak(const char* str, const char* file, uint32_t line) {
+static char* taosStrdupDetectLeak(const char* str, const char* file, uint32_t line) {
size_t len = strlen(str);
- char *p = malloc_detect_leak(len + 1, file, line);
+ char *p = taosMallocDetectLeak(len + 1, file, line);
if (p != NULL) {
memcpy(p, str, len);
p[len] = 0;
@@ -226,12 +226,12 @@ static char* strdup_detect_leak(const char* str, const char* file, uint32_t line
return p;
}
-static char* strndup_detect_leak(const char* str, size_t size, const char* file, uint32_t line) {
+static char* taosStrndupDetectLeak(const char* str, size_t size, const char* file, uint32_t line) {
size_t len = strlen(str);
if (len > size) {
len = size;
}
- char *p = malloc_detect_leak(len + 1, file, line);
+ char *p = taosMallocDetectLeak(len + 1, file, line);
if (p != NULL) {
memcpy(p, str, len);
p[len] = 0;
@@ -239,13 +239,13 @@ static char* strndup_detect_leak(const char* str, size_t size, const char* file,
return p;
}
-static ssize_t getline_detect_leak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
+static ssize_t taosGetlineDetectLeak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
char* buf = NULL;
size_t bufSize = 0;
ssize_t size = taosGetlineImp(&buf, &bufSize, stream);
if (size != -1) {
if (*n < size + 1) {
- void* p = realloc_detect_leak(*lineptr, size + 1, file, line);
+ void* p = taosReallocDetectLeak(*lineptr, size + 1, file, line);
if (p == NULL) {
free(buf);
return -1;
@@ -260,7 +260,7 @@ static ssize_t getline_detect_leak(char **lineptr, size_t *n, FILE *stream, cons
return size;
}
-static void dump_memory_leak() {
+static void taosDumpMemoryLeakImp() {
const char* hex = "0123456789ABCDEF";
const char* fmt = ":%d: addr=%p, size=%d, content(first 16 bytes)=";
size_t numOfBlk = 0, totalSize = 0;
@@ -299,7 +299,7 @@ static void dump_memory_leak() {
fflush(fpAllocLog);
}
-static void dump_memory_leak_on_sig(int sig) {
+static void taosDumpMemoryLeakOnSig(int sig) {
fprintf(fpAllocLog, "signal %d received.\n", sig);
// restore default signal handler
@@ -307,55 +307,55 @@ static void dump_memory_leak_on_sig(int sig) {
act.sa_handler = SIG_DFL;
sigaction(sig, &act, NULL);
- dump_memory_leak();
+ taosDumpMemoryLeakImp();
}
////////////////////////////////////////////////////////////////////////////////
// interface functions
-void* taos_malloc(size_t size, const char* file, uint32_t line) {
+void* taosMallocMem(size_t size, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return malloc(size);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
- return malloc_random(size, file, line);
+ return taosRandmoMalloc(size, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return malloc_detect_leak(size, file, line);
+ return taosMallocDetectLeak(size, file, line);
}
return malloc(size);
}
-void* taos_calloc(size_t num, size_t size, const char* file, uint32_t line) {
+void* taosCallocMem(size_t num, size_t size, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return calloc(num, size);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
- return calloc_random(num, size, file, line);
+ return taosRandomCalloc(num, size, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return calloc_detect_leak(num, size, file, line);
+ return taosCallocDetectLeak(num, size, file, line);
}
return calloc(num, size);
}
-void* taos_realloc(void* ptr, size_t size, const char* file, uint32_t line) {
+void* taosReallocMem(void* ptr, size_t size, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return realloc(ptr, size);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
- return realloc_random(ptr, size, file, line);
+ return taosRandomRealloc(ptr, size, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return realloc_detect_leak(ptr, size, file, line);
+ return taosReallocDetectLeak(ptr, size, file, line);
}
return realloc(ptr, size);
}
-void taos_free(void* ptr, const char* file, uint32_t line) {
+void taosFreeMem(void* ptr, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return free(ptr);
@@ -364,54 +364,54 @@ void taos_free(void* ptr, const char* file, uint32_t line) {
return free(ptr);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return free_detect_leak(ptr, file, line);
+ return taosFreeDetectLeak(ptr, file, line);
}
return free(ptr);
}
-char* taos_strdup(const char* str, const char* file, uint32_t line) {
+char* taosStrdupMem(const char* str, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return taosStrdupImp(str);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
- return strdup_random(str, file, line);
+ return taosRandomStrdup(str, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return strdup_detect_leak(str, file, line);
+ return taosStrdupDetectLeak(str, file, line);
}
return taosStrdupImp(str);
}
-char* taos_strndup(const char* str, size_t size, const char* file, uint32_t line) {
+char* taosStrndupMem(const char* str, size_t size, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return taosStrndupImp(str, size);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
- return strndup_random(str, size, file, line);
+ return taosRandomStrndup(str, size, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return strndup_detect_leak(str, size, file, line);
+ return taosStrndupDetectLeak(str, size, file, line);
}
return taosStrndupImp(str, size);
}
-ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
+ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return taosGetlineImp(lineptr, n, stream);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
- return getline_random(lineptr, n, stream, file, line);
+ return taosRandomGetline(lineptr, n, stream, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
- return getline_detect_leak(lineptr, n, stream, file, line);
+ return taosGetlineDetectLeak(lineptr, n, stream, file, line);
}
return taosGetlineImp(lineptr, n, stream);
}
-static void close_alloc_log() {
+static void taosCloseAllocLog() {
if (fpAllocLog != NULL) {
if (fpAllocLog != stdout) {
fclose(fpAllocLog);
@@ -432,7 +432,7 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) {
if (path == NULL || path[0] == 0) {
fpAllocLog = stdout;
} else if ((fpAllocLog = fopen(path, "w")) != NULL) {
- atexit(close_alloc_log);
+ atexit(taosCloseAllocLog);
} else {
printf("failed to open memory allocation log file '%s', errno=%d\n", path, errno);
return;
@@ -446,10 +446,10 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) {
}
if (autoDump && mode == TAOS_ALLOC_MODE_DETECT_LEAK) {
- atexit(dump_memory_leak);
+ atexit(taosDumpMemoryLeakImp);
struct sigaction act = {0};
- act.sa_handler = dump_memory_leak_on_sig;
+ act.sa_handler = taosDumpMemoryLeakOnSig;
sigaction(SIGFPE, &act, NULL);
sigaction(SIGSEGV, &act, NULL);
sigaction(SIGILL, &act, NULL);
@@ -457,8 +457,8 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) {
}
void taosDumpMemoryLeak() {
- dump_memory_leak();
- close_alloc_log();
+ taosDumpMemoryLeakImp();
+ taosCloseAllocLog();
}
#else // 'TAOS_MEM_CHECK' not defined
diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c
index 8a51c389e9f2efdcf3eaad97d434269ff3c0061f..c7c9d774271555ae5989aa18b7e00d325a0eddde 100644
--- a/src/os/src/detail/osSocket.c
+++ b/src/os/src/detail/osSocket.c
@@ -19,8 +19,8 @@
#ifndef TAOS_OS_FUNC_SOCKET
-int taosSetNonblocking(SOCKET sock, int on) {
- int flags = 0;
+int32_t taosSetNonblocking(SOCKET sock, int32_t on) {
+ int32_t flags = 0;
if ((flags = fcntl(sock, F_GETFL, 0)) < 0) {
uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno));
return 1;
@@ -43,7 +43,7 @@ void taosBlockSIGPIPE() {
sigset_t signal_mask;
sigemptyset(&signal_mask);
sigaddset(&signal_mask, SIGPIPE);
- int rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
+ int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
if (rc != 0) {
uError("failed to block SIGPIPE");
}
@@ -53,7 +53,7 @@ void taosBlockSIGPIPE() {
#ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT
-int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) {
+int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) {
return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
}
diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c
index 8df671f9c8974ecf77e5bf0943950772e3f9d192..b0ca6139ed9461b12ab2845d0060b51dc388bcab 100644
--- a/src/os/src/detail/osSysinfo.c
+++ b/src/os/src/detail/osSysinfo.c
@@ -61,7 +61,7 @@ bool taosGetProcMemory(float *memoryUsedMB) {
size_t len;
char * line = NULL;
while (!feof(fp)) {
- taosTFree(line);
+ tfree(line);
len = 0;
getline(&line, &len, fp);
if (line == NULL) {
@@ -83,7 +83,7 @@ bool taosGetProcMemory(float *memoryUsedMB) {
sscanf(line, "%s %" PRId64, tmp, &memKB);
*memoryUsedMB = (float)((double)memKB / 1024);
- taosTFree(line);
+ tfree(line);
fclose(fp);
return true;
}
@@ -107,7 +107,7 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
char cpu[10] = {0};
sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle);
- taosTFree(line);
+ tfree(line);
fclose(fp);
return true;
}
@@ -136,7 +136,7 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
}
}
- taosTFree(line);
+ tfree(line);
fclose(fp);
return true;
}
@@ -377,7 +377,7 @@ static bool taosGetCardInfo(int64_t *bytes) {
*bytes += (rbytes + tbytes);
}
- taosTFree(line);
+ tfree(line);
fclose(fp);
return true;
@@ -432,7 +432,7 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) {
int readIndex = 0;
while (!feof(fp)) {
- taosTFree(line);
+ tfree(line);
len = 0;
getline(&line, &len, fp);
if (line == NULL) {
@@ -450,7 +450,7 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) {
if (readIndex >= 2) break;
}
- taosTFree(line);
+ tfree(line);
fclose(fp);
if (readIndex < 2) {
diff --git a/src/os/src/windows/wAtomic.c b/src/os/src/windows/wAtomic.c
index a025cb8f0e67ff65ceb0cfadd420fd10cb99891b..b645893030b852428bff13744050b1f6f8ef1c42 100644
--- a/src/os/src/windows/wAtomic.c
+++ b/src/os/src/windows/wAtomic.c
@@ -44,7 +44,7 @@ long interlocked_add_fetch_32(long volatile* ptr, long val) {
__int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) {
//#ifdef _WIN64
- return _InterlockedExchangeAdd64(ptr, val) + val;
+ return InterlockedExchangeAdd64(ptr, val) + val;
//#else
// return _InterlockedExchangeAdd(ptr, val) + val;
//#endif
diff --git a/src/os/src/windows/wEnv.c b/src/os/src/windows/wEnv.c
index 8110a194904bbbc1166ac57de187de284db34a6b..19351eb7c964a4c2a8a4d1d5d4d1c8ec669908dc 100644
--- a/src/os/src/windows/wEnv.c
+++ b/src/os/src/windows/wEnv.c
@@ -46,5 +46,16 @@ void osInit() {
strcpy(tsDnodeDir, "");
strcpy(tsMnodeDir, "");
strcpy(tsOsName, "Windows");
+
+ const char *tmpDir = getenv("tmp");
+ if (tmpDir != NULL) {
+ tmpDir = getenv("temp");
+ }
+ if (tmpDir != NULL) {
+ strcpy(tsTempDir, tmpDir);
+ } else {
+ strcpy(tsTempDir, "C:\\Windows\\Temp");
+ }
+
taosWinSocketInit();
}
diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c
index 5549c078a5220d3e493ce21e2c19e38fb9611cbb..2204135ae657eb813144dc2bd1ae637d9510842e 100644
--- a/src/os/src/windows/wFile.c
+++ b/src/os/src/windows/wFile.c
@@ -16,17 +16,20 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "tulog.h"
+#include "tglobal.h"
void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
const char* tdengineTmpFileNamePrefix = "tdengine-";
- char tmpPath[PATH_MAX];
+ char tmpPath[PATH_MAX];
- char *tmpDir = getenv("tmp");
- if (tmpDir == NULL) {
- tmpDir = "";
+ int32_t len = (int32_t)strlen(tsTempDir);
+ memcpy(tmpPath, tsTempDir, len);
+
+ if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') {
+ tmpPath[len++] = '\\';
}
-
- strcpy(tmpPath, tmpDir);
+
+ strcpy(tmpPath + len, tdengineTmpFileNamePrefix);
strcat(tmpPath, tdengineTmpFileNamePrefix);
if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
strcat(tmpPath, fileNamePrefix);
@@ -40,19 +43,19 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
#define _SEND_FILE_STEP_ 1000
-int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) {
+int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
fseek(in_file, (int32_t)(*offset), 0);
- int writeLen = 0;
+ int64_t writeLen = 0;
uint8_t buffer[_SEND_FILE_STEP_] = { 0 };
- for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
+ for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file);
if (rlen <= 0) {
return writeLen;
}
else if (rlen < _SEND_FILE_STEP_) {
fwrite(buffer, 1, rlen, out_file);
- return (int)(writeLen + rlen);
+ return (int64_t)(writeLen + rlen);
}
else {
fwrite(buffer, 1, _SEND_FILE_STEP_, in_file);
@@ -60,14 +63,14 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou
}
}
- int remain = count - writeLen;
+ int64_t remain = count - writeLen;
if (remain > 0) {
- size_t rlen = fread(buffer, 1, remain, in_file);
+ size_t rlen = fread(buffer, 1, (size_t) remain, in_file);
if (rlen <= 0) {
return writeLen;
}
else {
- fwrite(buffer, 1, remain, out_file);
+ fwrite(buffer, 1, (size_t) remain, out_file);
writeLen += remain;
}
}
@@ -75,12 +78,12 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou
return writeLen;
}
-ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
- uError("taosTSendFileImp no implemented yet");
+int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) {
+ uError("taosSendFile no implemented yet");
return 0;
}
-int taosFtruncate(int fd, int64_t length) {
+int32_t taosFtruncate(int32_t fd, int64_t length) {
uError("taosFtruncate no implemented yet");
return 0;
}
\ No newline at end of file
diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c
index da9242d6a3bff471c4e749f0faf7c95b6c3a9d8c..3b091b269931e644d9f8c2c01ba3c9cb9ddc520c 100644
--- a/src/os/src/windows/wSocket.c
+++ b/src/os/src/windows/wSocket.c
@@ -34,7 +34,7 @@ void taosWinSocketInit() {
}
}
-int taosSetNonblocking(SOCKET sock, int on) {
+int32_t taosSetNonblocking(SOCKET sock, int32_t on) {
u_long mode;
if (on) {
mode = 1;
@@ -48,7 +48,7 @@ int taosSetNonblocking(SOCKET sock, int on) {
void taosBlockSIGPIPE() {}
-int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) {
+int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) {
if (level == SOL_SOCKET && optname == TCP_KEEPCNT) {
return 0;
}
@@ -72,7 +72,7 @@ int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int op
uint32_t taosInetAddr(char *ipAddr) {
uint32_t value;
- int ret = inet_pton(AF_INET, ipAddr, &value);
+ int32_t ret = inet_pton(AF_INET, ipAddr, &value);
if (ret <= 0) {
return INADDR_NONE;
} else {
diff --git a/src/plugins/http/inc/httpQueue.h b/src/plugins/http/inc/httpQueue.h
index a4590719ff24d48eee875b2f2c4ff2f28a0a31f6..1ffbd5148164ab4d1f74bdac781c8fcf731ad772 100644
--- a/src/plugins/http/inc/httpQueue.h
+++ b/src/plugins/http/inc/httpQueue.h
@@ -22,9 +22,11 @@ extern "C" {
#include
+typedef void (*FHttpResultFp)(void *param, void *result, int32_t code, int32_t rows);
+
bool httpInitResultQueue();
void httpCleanupResultQueue();
-void httpDispatchToResultQueue();
+void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp);
#ifdef __cplusplus
}
diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c
index ec60b984b2b0fcc72367295cef9d956165e678d5..22f464924e2c21cdeccb0c152457fad11a10294a 100644
--- a/src/plugins/http/src/httpContext.c
+++ b/src/plugins/http/src/httpContext.c
@@ -63,7 +63,7 @@ static void httpDestroyContext(void *data) {
pContext->parser = NULL;
}
- taosTFree(pContext);
+ tfree(pContext);
}
bool httpInitContexts() {
diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c
index 43a8ddbd1a0d207addf0576951b2af9b2be16555..1c039abb4d4cd546f6673648217a2410f57ae8e7 100644
--- a/src/plugins/http/src/httpQueue.c
+++ b/src/plugins/http/src/httpQueue.c
@@ -25,6 +25,7 @@
#include "httpResp.h"
#include "httpAuth.h"
#include "httpSession.h"
+#include "httpQueue.h"
typedef struct {
pthread_t thread;
@@ -37,42 +38,45 @@ typedef struct {
} SHttpWorkerPool;
typedef struct {
- void *param;
- void *result;
- int32_t numOfRows;
- void (*fp)(void *param, void *result, int32_t numOfRows);
+ void * param;
+ void * result;
+ int32_t code;
+ int32_t rows;
+ FHttpResultFp fp;
} SHttpResult;
static SHttpWorkerPool tsHttpPool;
static taos_qset tsHttpQset;
static taos_queue tsHttpQueue;
-void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t numOfRows, void (*fp)(void *param, void *result, int32_t numOfRows)) {
+void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp) {
if (tsHttpQueue != NULL) {
- SHttpResult *pMsg = (SHttpResult *)taosAllocateQitem(sizeof(SHttpResult));
+ SHttpResult *pMsg = taosAllocateQitem(sizeof(SHttpResult));
pMsg->param = param;
pMsg->result = result;
- pMsg->numOfRows = numOfRows;
+ pMsg->code = code;
+ pMsg->rows = rows;
pMsg->fp = fp;
taosWriteQitem(tsHttpQueue, TAOS_QTYPE_RPC, pMsg);
} else {
- (*fp)(param, result, numOfRows);
+ (*fp)(param, result, code, rows);
}
}
static void *httpProcessResultQueue(void *param) {
SHttpResult *pMsg;
- int32_t type;
- void *unUsed;
-
+ int32_t type;
+ void * unUsed;
+
while (1) {
if (taosReadQitemFromQset(tsHttpQset, &type, (void **)&pMsg, &unUsed) == 0) {
httpDebug("qset:%p, http queue got no message from qset, exiting", tsHttpQset);
break;
}
- httpTrace("context:%p, res:%p will be processed in result queue", pMsg->param, pMsg->result);
- (*pMsg->fp)(pMsg->param, pMsg->result, pMsg->numOfRows);
+ httpTrace("context:%p, res:%p will be processed in result queue, code:%d rows:%d", pMsg->param, pMsg->result,
+ pMsg->code, pMsg->rows);
+ (*pMsg->fp)(pMsg->param, pMsg->result, pMsg->code, pMsg->rows);
taosFreeQitem(pMsg);
}
diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c
index 70d644146cf5ebe15a42c8f00a2e6e4bd603d081..564f555c403b6f5b3deb5832da6505722104eff8 100644
--- a/src/plugins/http/src/httpSql.c
+++ b/src/plugins/http/src/httpSql.c
@@ -29,9 +29,9 @@
void httpProcessMultiSql(HttpContext *pContext);
-void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows);
+void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows);
-void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int numOfRows) {
+void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
@@ -43,7 +43,7 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n
bool isContinue = false;
- if (numOfRows > 0) {
+ if (code == TSDB_CODE_SUCCESS && numOfRows > 0) {
if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->buildQueryJsonFp) {
isContinue = (encode->buildQueryJsonFp)(pContext, singleCmd, result, numOfRows);
}
@@ -58,9 +58,9 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, stop retrieve, numOfRows:%d, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, numOfRows, sql);
- if (numOfRows < 0) {
+ if (code < 0) {
httpError("context:%p, fd:%d, user:%s, process pos:%d, retrieve failed code:%s, sql:%s", pContext, pContext->fd,
- pContext->user, multiCmds->pos, tstrerror(numOfRows), sql);
+ pContext->user, multiCmds->pos, tstrerror(code), sql);
}
taos_free_result(result);
@@ -73,15 +73,15 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n
}
}
-void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) {
- httpDispatchToResultQueue(param, result, numOfRows, httpProcessMultiSqlRetrieveCallBackImp);
+void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) {
+ int32_t code = taos_errno(result);
+ httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessMultiSqlRetrieveCallBackImp);
}
-void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
+void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
- code = taos_errno(result);
HttpSqlCmds *multiCmds = pContext->multiCmds;
HttpEncodeMethod *encode = pContext->encodeMethod;
@@ -94,7 +94,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
return;
}
- if (code < 0) {
+ if (code != TSDB_CODE_SUCCESS) {
if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code)) {
singleCmd->code = code;
httpDebug("context:%p, fd:%d, user:%s, process pos jump to:%d, last code:%s, last sql:%s", pContext, pContext->fd,
@@ -119,7 +119,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
bool isUpdate = tscIsUpdateQuery(result);
if (isUpdate) {
// not select or show commands
- int affectRows = taos_affected_rows(result);
+ int32_t affectRows = taos_affected_rows(result);
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, affect rows:%d, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, affectRows, sql);
@@ -156,8 +156,10 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
}
}
-void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) {
- httpDispatchToResultQueue(param, result, unUsedCode, httpProcessMultiSqlCallBackImp);
+void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) {
+ int32_t code = taos_errno(result);
+ int32_t affectRows = taos_affected_rows(result);
+ httpDispatchToResultQueue(param, result, code, affectRows, httpProcessMultiSqlCallBackImp);
}
void httpProcessMultiSql(HttpContext *pContext) {
@@ -202,9 +204,9 @@ void httpProcessMultiSqlCmd(HttpContext *pContext) {
httpProcessMultiSql(pContext);
}
-void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows);
+void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows);
-void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int numOfRows) {
+void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
@@ -212,7 +214,7 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int
bool isContinue = false;
- if (numOfRows > 0) {
+ if (code == TSDB_CODE_SUCCESS && numOfRows > 0) {
if (encode->buildQueryJsonFp) {
isContinue = (encode->buildQueryJsonFp)(pContext, &pContext->singleCmd, result, numOfRows);
}
@@ -227,9 +229,9 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int
httpDebug("context:%p, fd:%d, user:%s, stop retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user,
numOfRows);
- if (numOfRows < 0) {
+ if (code < 0) {
httpError("context:%p, fd:%d, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->user,
- tstrerror(numOfRows));
+ tstrerror(code));
}
taos_free_result(result);
@@ -242,30 +244,30 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int
}
}
-void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) {
- httpDispatchToResultQueue(param, result, numOfRows, httpProcessSingleSqlRetrieveCallBackImp);
+void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) {
+ int32_t code = taos_errno(result);
+ httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessSingleSqlRetrieveCallBackImp);
}
-void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCode) {
+void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
- int32_t code = taos_errno(result);
-
HttpEncodeMethod *encode = pContext->encodeMethod;
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
httpError("context:%p, fd:%d, user:%s, query error, code:%s:inprogress, sqlObj:%p", pContext, pContext->fd,
- pContext->user, tstrerror(code), (SSqlObj *)result);
+ pContext->user, tstrerror(code), result);
return;
}
- if (code < 0) {
+ if (code != TSDB_CODE_SUCCESS) {
SSqlObj *pObj = (SSqlObj *)result;
if (code == TSDB_CODE_TSC_INVALID_SQL) {
- httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext,
- pContext->fd, pContext->user, tstrerror(code), pObj, pObj->cmd.payload);
- httpSendTaosdInvalidSqlErrorResp(pContext, pObj->cmd.payload);
+ terrno = code;
+ httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext, pContext->fd,
+ pContext->user, tstrerror(code), pObj, taos_errstr(pObj));
+ httpSendTaosdInvalidSqlErrorResp(pContext, taos_errstr(pObj));
} else {
httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd,
pContext->user, tstrerror(code), pObj);
@@ -278,7 +280,8 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo
bool isUpdate = tscIsUpdateQuery(result);
if (isUpdate) {
// not select or show commands
- int affectRows = taos_affected_rows(result);
+ int32_t affectRows = taos_affected_rows(result);
+ assert(affectRows == affectRowsInput);
httpDebug("context:%p, fd:%d, user:%s, affect rows:%d, stop query, sqlObj:%p", pContext, pContext->fd,
pContext->user, affectRows, result);
@@ -308,8 +311,10 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo
}
}
-void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) {
- httpDispatchToResultQueue(param, result, unUsedCode, httpProcessSingleSqlCallBackImp);
+void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) {
+ int32_t code = taos_errno(result);
+ int32_t affectRows = taos_affected_rows(result);
+ httpDispatchToResultQueue(param, result, code, affectRows, httpProcessSingleSqlCallBackImp);
}
void httpProcessSingleSqlCmd(HttpContext *pContext) {
@@ -373,7 +378,7 @@ void httpExecCmd(HttpContext *pContext) {
}
}
-void httpProcessRequestCb(void *param, TAOS_RES *result, int code) {
+void httpProcessRequestCb(void *param, TAOS_RES *result, int32_t code) {
HttpContext *pContext = param;
taos_free_result(result);
diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c
index 8993b233dd1dde9283679d337845d42f5d649715..3b8858b62e1297f0db5ece60c59a26e376b546f3 100644
--- a/src/plugins/http/src/httpSystem.c
+++ b/src/plugins/http/src/httpSystem.c
@@ -107,7 +107,7 @@ void httpCleanUpSystem() {
httpCleanupResultQueue();
pthread_mutex_destroy(&tsHttpServer.serverMutex);
- taosTFree(tsHttpServer.pThreads);
+ tfree(tsHttpServer.pThreads);
tsHttpServer.pThreads = NULL;
tsHttpServer.status = HTTP_SERVER_CLOSED;
diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c
index 048f839b728fd7ed2a74d59744ebff5d1223cc33..de1e0e233c2b1441a00c226d7305f99a6db41109 100644
--- a/src/plugins/monitor/src/monitorMain.c
+++ b/src/plugins/monitor/src/monitorMain.c
@@ -27,12 +27,12 @@
#include "monitor.h"
#include "taoserror.h"
-#define monitorFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
-#define monitorError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
-#define monitorWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
-#define monitorInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }}
-#define monitorDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
-#define monitorTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
+#define mnFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
+#define mnError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
+#define mnWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
+#define mnInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }}
+#define mnDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
+#define mnTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
#define SQL_LENGTH 1030
#define LOG_LEN_STR 100
@@ -91,12 +91,12 @@ int32_t monitorInitSystem() {
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&tsMonitor.thread, &thAttr, monitorThreadFunc, NULL)) {
- monitorError("failed to create thread to for monitor module, reason:%s", strerror(errno));
+ mnError("failed to create thread to for monitor module, reason:%s", strerror(errno));
return -1;
}
pthread_attr_destroy(&thAttr);
- monitorDebug("monitor thread is launched");
+ mnDebug("monitor thread is launched");
monitorStartSystemFp = monitorStartSystem;
monitorStopSystemFp = monitorStopSystem;
@@ -107,12 +107,12 @@ int32_t monitorStartSystem() {
taos_init();
tsMonitor.start = 1;
monitorExecuteSQLFp = monitorExecuteSQL;
- monitorInfo("monitor module start");
+ mnInfo("monitor module start");
return 0;
}
static void *monitorThreadFunc(void *param) {
- monitorDebug("starting to initialize monitor module ...");
+ mnDebug("starting to initialize monitor module ...");
while (1) {
static int32_t accessTimes = 0;
@@ -121,7 +121,7 @@ static void *monitorThreadFunc(void *param) {
if (tsMonitor.quiting) {
tsMonitor.state = MON_STATE_NOT_INIT;
- monitorInfo("monitor thread will quit, for taosd is quiting");
+ mnInfo("monitor thread will quit, for taosd is quiting");
break;
} else {
taosGetDisk();
@@ -132,7 +132,7 @@ static void *monitorThreadFunc(void *param) {
}
if (dnodeGetDnodeId() <= 0) {
- monitorDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
+ mnDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
continue;
}
@@ -140,10 +140,10 @@ static void *monitorThreadFunc(void *param) {
tsMonitor.state = MON_STATE_NOT_INIT;
tsMonitor.conn = taos_connect(NULL, "monitor", tsInternalPass, "", 0);
if (tsMonitor.conn == NULL) {
- monitorError("failed to connect to database, reason:%s", tstrerror(terrno));
+ mnError("failed to connect to database, reason:%s", tstrerror(terrno));
continue;
} else {
- monitorDebug("connect to database success");
+ mnDebug("connect to database success");
}
}
@@ -155,10 +155,10 @@ static void *monitorThreadFunc(void *param) {
taos_free_result(res);
if (code != 0) {
- monitorError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
+ mnError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
break;
} else {
- monitorDebug("successfully to exec sql:%s", tsMonitor.sql);
+ mnDebug("successfully to exec sql:%s", tsMonitor.sql);
}
}
@@ -174,7 +174,7 @@ static void *monitorThreadFunc(void *param) {
}
}
- monitorInfo("monitor thread is stopped");
+ mnInfo("monitor thread is stopped");
return NULL;
}
@@ -238,7 +238,7 @@ void monitorStopSystem() {
tsMonitor.start = 0;
tsMonitor.state = MON_STATE_NOT_INIT;
monitorExecuteSQLFp = NULL;
- monitorInfo("monitor module stopped");
+ mnInfo("monitor module stopped");
}
void monitorCleanUpSystem() {
@@ -249,7 +249,7 @@ void monitorCleanUpSystem() {
taos_close(tsMonitor.conn);
tsMonitor.conn = NULL;
}
- monitorInfo("monitor module is cleaned up");
+ mnInfo("monitor module is cleaned up");
}
// unit is MB
@@ -257,13 +257,13 @@ static int32_t monitorBuildMemorySql(char *sql) {
float sysMemoryUsedMB = 0;
bool suc = taosGetSysMemory(&sysMemoryUsedMB);
if (!suc) {
- monitorDebug("failed to get sys memory info");
+ mnDebug("failed to get sys memory info");
}
float procMemoryUsedMB = 0;
suc = taosGetProcMemory(&procMemoryUsedMB);
if (!suc) {
- monitorDebug("failed to get proc memory info");
+ mnDebug("failed to get proc memory info");
}
return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB);
@@ -274,7 +274,7 @@ static int32_t monitorBuildCpuSql(char *sql) {
float sysCpuUsage = 0, procCpuUsage = 0;
bool suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage);
if (!suc) {
- monitorDebug("failed to get cpu usage");
+ mnDebug("failed to get cpu usage");
}
if (sysCpuUsage <= procCpuUsage) {
@@ -294,14 +294,14 @@ static int32_t monitorBuildBandSql(char *sql) {
float bandSpeedKb = 0;
bool suc = taosGetBandSpeed(&bandSpeedKb);
if (!suc) {
- monitorDebug("failed to get bandwidth speed");
+ mnDebug("failed to get bandwidth speed");
}
return sprintf(sql, ", %f", bandSpeedKb);
}
static int32_t monitorBuildReqSql(char *sql) {
- SDnodeStatisInfo info = dnodeGetStatisInfo();
+ SStatisInfo info = dnodeGetStatisInfo();
return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum);
}
@@ -309,7 +309,7 @@ static int32_t monitorBuildIoSql(char *sql) {
float readKB = 0, writeKB = 0;
bool suc = taosGetProcIO(&readKB, &writeKB);
if (!suc) {
- monitorDebug("failed to get io info");
+ mnDebug("failed to get io info");
}
return sprintf(sql, ", %f, %f", readKB, writeKB);
@@ -332,19 +332,19 @@ static void monitorSaveSystemInfo() {
taos_free_result(res);
if (code != 0) {
- monitorError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
+ mnError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
} else {
- monitorDebug("successfully to save system info, sql:%s", tsMonitor.sql);
+ mnDebug("successfully to save system info, sql:%s", tsMonitor.sql);
}
}
static void montiorExecSqlCb(void *param, TAOS_RES *result, int32_t code) {
int32_t c = taos_errno(result);
if (c != TSDB_CODE_SUCCESS) {
- monitorError("save %s failed, reason:%s", (char *)param, tstrerror(c));
+ mnError("save %s failed, reason:%s", (char *)param, tstrerror(c));
} else {
int32_t rows = taos_affected_rows(result);
- monitorDebug("save %s succ, rows:%d", (char *)param, rows);
+ mnDebug("save %s succ, rows:%d", (char *)param, rows);
}
taos_free_result(result);
@@ -380,7 +380,7 @@ void monitorSaveAcctLog(SAcctMonitorObj *pMon) {
pMon->totalConns, pMon->maxConns,
pMon->accessState);
- monitorDebug("save account info, sql:%s", sql);
+ mnDebug("save account info, sql:%s", sql);
taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "account info");
}
@@ -401,13 +401,13 @@ void monitorSaveLog(int32_t level, const char *const format, ...) {
len += sprintf(sql + len, "', '%s')", tsLocalEp);
sql[len++] = 0;
- monitorDebug("save log, sql: %s", sql);
+ mnDebug("save log, sql: %s", sql);
taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "log");
}
void monitorExecuteSQL(char *sql) {
if (tsMonitor.state != MON_STATE_INITED) return;
- monitorDebug("execute sql:%s", sql);
+ mnDebug("execute sql:%s", sql);
taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "sql");
}
diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h
index f392644e6736f528fa75f21ee2857c570c0c22c9..22397d0314c3081e8ed23d9671f4ccfcb7abc244 100644
--- a/src/query/inc/qExecutor.h
+++ b/src/query/inc/qExecutor.h
@@ -33,17 +33,26 @@ struct SColumnFilterElem;
typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2);
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
-typedef struct SPosInfo {
- int32_t pageId:20;
- int32_t rowId:12;
-} SPosInfo;
-
typedef struct SGroupResInfo {
int32_t groupId;
int32_t numOfDataPages;
- SPosInfo pos;
+ int32_t pageId;
+ int32_t rowId;
} SGroupResInfo;
+typedef struct SResultRowPool {
+ int32_t elemSize;
+ int32_t blockSize;
+ int32_t numOfElemPerBlock;
+
+ struct {
+ int32_t blockIndex;
+ int32_t pos;
+ } position;
+
+ SArray* pData; // SArray
+} SResultRowPool;
+
typedef struct SSqlGroupbyExpr {
int16_t tableIndex;
SArray* columnInfo; // SArray, group by columns information
@@ -52,13 +61,14 @@ typedef struct SSqlGroupbyExpr {
int16_t orderType; // order by type: asc/desc
} SSqlGroupbyExpr;
-typedef struct SWindowResult {
- SPosInfo pos; // Position of current result in disk-based output buffer
+typedef struct SResultRow {
+ int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer
+ int32_t rowId:15;
+ bool closed:1; // this result status: closed or opened
uint16_t numOfRows; // number of rows of current time window
- bool closed; // this result status: closed or opened
- SResultInfo* resultInfo; // For each result column, there is a resultInfo
+ SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo
union {STimeWindow win; char* key;}; // start key of current time window
-} SWindowResult;
+} SResultRow;
/**
* If the number of generated results is greater than this value,
@@ -72,16 +82,14 @@ typedef struct SResultRec {
} SResultRec;
typedef struct SWindowResInfo {
- SWindowResult* pResult; // result list
- SHashObj* hashList; // hash list for quick access
- int16_t type; // data type for hash key
+ SResultRow** pResult; // result list
+ int16_t type:8; // data type for hash key
+ int32_t size:24; // number of result set
+ int32_t threshold; // threshold to halt query and return the generated results.
int32_t capacity; // max capacity
int32_t curIndex; // current start active index
- int32_t size; // number of result set
int64_t startTime; // start time of the first time window for sliding query
int64_t prevSKey; // previous (not completed) sliding window start key
- int64_t threshold; // threshold to halt query and return the generated results.
- int64_t interval; // time window interval
} SWindowResInfo;
typedef struct SColumnFilterElem {
@@ -97,7 +105,7 @@ typedef struct SSingleColumnFilterInfo {
SColumnFilterElem* pFilters;
} SSingleColumnFilterInfo;
-typedef struct STableQueryInfo { // todo merge with the STableQueryInfo struct
+typedef struct STableQueryInfo {
TSKEY lastKey;
int32_t groupIndex; // group id in table list
int16_t queryRangeSet; // denote if the query range is set, only available for interval query
@@ -125,7 +133,9 @@ typedef struct SQueryCostInfo {
uint32_t discardBlocks;
uint64_t elapsedTime;
uint64_t firstStageMergeTime;
- uint64_t internalSupSize;
+ uint64_t winInfoSize;
+ uint64_t tableInfoSize;
+ uint64_t hashSize;
uint64_t numOfTimeWindows;
} SQueryCostInfo;
@@ -158,11 +168,11 @@ typedef struct SQuery {
typedef struct SQueryRuntimeEnv {
jmp_buf env;
- SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo
+ SResultRow* pResultRow; // todo refactor to merge with SWindowResInfo
SQuery* pQuery;
SQLFunctionCtx* pCtx;
int32_t numOfRowsPerPage;
- int16_t offset[TSDB_MAX_COLUMNS];
+ uint16_t offset[TSDB_MAX_COLUMNS];
uint16_t scanFlag; // denotes reversed scan of data or not
SFillInfo* pFillInfo;
SWindowResInfo windowResInfo;
@@ -178,6 +188,11 @@ typedef struct SQueryRuntimeEnv {
int32_t interBufSize; // intermediate buffer sizse
int32_t prevGroupId; // previous executed group id
SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file
+ SHashObj* pResultRowHashTable; // quick locate the window object for each result
+ char* keyBuf; // window key buffer
+ SResultRowPool* pool; // window result object pool
+
+ int32_t* rowCellInfoOffset;// offset value for each row result cell info
} SQueryRuntimeEnv;
enum {
@@ -185,12 +200,6 @@ enum {
QUERY_RESULT_READY = 2,
};
-typedef struct SMemRef {
- int32_t ref;
- void *mem;
- void *imem;
-} SMemRef;
-
typedef struct SQInfo {
void* signature;
int32_t code; // error code to returned to client
diff --git a/src/query/inc/qFill.h b/src/query/inc/qFill.h
index 6d44fee09557fa8b8d73527677a2c7b238ecb42c..1b5aca77c4d08bc2b51f1147279a7df64476c8e2 100644
--- a/src/query/inc/qFill.h
+++ b/src/query/inc/qFill.h
@@ -49,7 +49,6 @@ typedef struct SFillInfo {
int32_t numOfTags; // number of tags
int32_t numOfCols; // number of columns, including the tags columns
int32_t rowSize; // size of each row
-// char ** pTags; // tags value for current interpolation
SFillTagColInfo* pTags; // tags value for filling gap
SInterval interval;
char * prevValues; // previous row of data, to generate the interpolation results
@@ -71,7 +70,7 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);
-void* taosDestoryFillInfo(SFillInfo *pFillInfo);
+void* taosDestroyFillInfo(SFillInfo *pFillInfo);
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
@@ -83,7 +82,7 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRo
int32_t taosNumOfRemainRows(SFillInfo *pFillInfo);
-int taosDoLinearInterpolation(int32_t type, SPoint *point1, SPoint *point2, SPoint *point);
+int32_t taosGetLinearInterpolationVal(int32_t type, SPoint *point1, SPoint *point2, SPoint *point);
int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity);
diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h
index bc8f9a5e23df72f82bcb3bb5140bec42fe426268..f08e13db6463288fa2b9ec233c904e00d9b7dce8 100644
--- a/src/query/inc/qSqlparser.h
+++ b/src/query/inc/qSqlparser.h
@@ -129,6 +129,7 @@ typedef struct SCreateDBInfo {
int32_t compressionLevel;
SStrToken precision;
bool ignoreExists;
+ int8_t update;
tVariantList *keep;
} SCreateDBInfo;
@@ -223,13 +224,6 @@ typedef struct tSQLExprList {
tSQLExprItem *a; /* One entry for each expression */
} tSQLExprList;
-typedef struct tSQLExprListList {
- int32_t nList; /* Number of expressions on the list */
- int32_t nAlloc; /* Number of entries allocated below */
- tSQLExprList **a; /* one entry for each row */
-} tSQLExprListList;
-
-
/**
*
* @param yyp The parser
diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h
index 46e6f79014f32c9b3052824bdb702556f5c4f060..90bd64336fdeed91deb68b9b490224a7fb29bc80 100644
--- a/src/query/inc/qTsbuf.h
+++ b/src/query/inc/qTsbuf.h
@@ -26,7 +26,7 @@ extern "C" {
#define MEM_BUF_SIZE (1 << 20)
#define TS_COMP_FILE_MAGIC 0x87F5EC4C
-#define TS_COMP_FILE_VNODE_MAX 512
+#define TS_COMP_FILE_GROUP_MAX 512
typedef struct STSList {
char* rawBuf;
@@ -35,17 +35,10 @@ typedef struct STSList {
int32_t len;
} STSList;
-typedef struct STSRawBlock {
- int32_t vnode;
- int64_t tag;
- TSKEY* ts;
- int32_t len;
-} STSRawBlock;
-
typedef struct STSElem {
TSKEY ts;
- tVariant tag;
- int32_t vnode;
+ tVariant* tag;
+ int32_t id;
} STSElem;
typedef struct STSCursor {
@@ -67,26 +60,27 @@ typedef struct STSBlock {
* The size of buffer file should not be greater than 2G,
* and the offset of int32_t type is enough
*/
-typedef struct STSVnodeBlockInfo {
- int32_t vnode; // vnode id
+typedef struct STSGroupBlockInfo {
+ int32_t id; // group id
int32_t offset; // offset set value in file
int32_t numOfBlocks; // number of total blocks
int32_t compLen; // compressed size
-} STSVnodeBlockInfo;
+} STSGroupBlockInfo;
-typedef struct STSVnodeBlockInfoEx {
- STSVnodeBlockInfo info;
+typedef struct STSGroupBlockInfoEx {
+ STSGroupBlockInfo info;
int32_t len; // length before compress
-} STSVnodeBlockInfoEx;
+} STSGroupBlockInfoEx;
typedef struct STSBuf {
FILE* f;
char path[PATH_MAX];
uint32_t fileSize;
- STSVnodeBlockInfoEx* pData;
+ // todo use array
+ STSGroupBlockInfoEx* pData;
uint32_t numOfAlloc;
- uint32_t numOfVnodes;
+ uint32_t numOfGroups;
char* assistBuf;
int32_t bufSize;
@@ -100,30 +94,31 @@ typedef struct STSBuf {
typedef struct STSBufFileHeader {
uint32_t magic; // file magic number
- uint32_t numOfVnode; // number of vnode stored in current file
+ uint32_t numOfGroup; // number of group stored in current file
int32_t tsOrder; // timestamp order in current file
} STSBufFileHeader;
STSBuf* tsBufCreate(bool autoDelete, int32_t order);
STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete);
-STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder);
+STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t id);
void* tsBufDestroy(STSBuf* pTSBuf);
-void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len);
-int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeIdx);
+void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len);
+int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf);
STSBuf* tsBufClone(STSBuf* pTSBuf);
-STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId);
+STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id);
void tsBufFlush(STSBuf* pTSBuf);
void tsBufResetPos(STSBuf* pTSBuf);
STSElem tsBufGetElem(STSBuf* pTSBuf);
+
bool tsBufNextPos(STSBuf* pTSBuf);
-STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag);
+STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag);
STSCursor tsBufGetCursor(STSBuf* pTSBuf);
void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order);
@@ -136,6 +131,16 @@ void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur);
*/
void tsBufDisplay(STSBuf* pTSBuf);
+int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf);
+
+void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id);
+
+int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t id, void* buf, int32_t* len, int32_t* numOfBlocks);
+
+STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag);
+
+bool tsBufIsValidElem(STSElem* pElem);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h
index 5320e5622e70a93c06edb4a1e5a3fe568498ef21..5d649261a6e9fbf8b9eab44aab699ea792f86138 100644
--- a/src/query/inc/qUtil.h
+++ b/src/query/inc/qUtil.h
@@ -15,13 +15,22 @@
#ifndef TDENGINE_QUERYUTIL_H
#define TDENGINE_QUERYUTIL_H
+#define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \
+ do { \
+ assert(sizeof(_uid) == sizeof(uint64_t)); \
+ *(uint64_t *)(_k) = (_uid); \
+ memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \
+ } while (0)
+
+#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t))
+
int32_t getOutputInterResultBufSize(SQuery* pQuery);
-void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes);
-void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, const SWindowResult* src);
+void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow);
+void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src);
+SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index);
-int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size,
- int32_t threshold, int16_t type);
+int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type);
void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo);
void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo);
@@ -33,9 +42,9 @@ void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot);
void closeAllTimeWindow(SWindowResInfo* pWindowResInfo);
void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order);
-static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) {
+static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int32_t slot) {
assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size);
- return &pWindowResInfo->pResult[slot];
+ return pWindowResInfo->pResult[slot];
}
#define curTimeWindowIndex(_winres) ((_winres)->curIndex)
@@ -43,15 +52,15 @@ static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInf
bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);
-int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize);
+int32_t initResultRow(SResultRow *pResultRow);
-static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult,
+static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SResultRow *pResult,
tFilePage* page) {
assert(pResult != NULL && pRuntimeEnv != NULL);
SQuery *pQuery = pRuntimeEnv->pQuery;
- int32_t realRowId = (int32_t)(pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery));
+ int32_t realRowId = (int32_t)(pResult->rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery));
return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage +
pQuery->pSelectExpr[columnIndex].bytes * realRowId;
}
@@ -62,4 +71,14 @@ bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval);
__filter_func_t *getRangeFilterFuncArray(int32_t type);
__filter_func_t *getValueFilterFuncArray(int32_t type);
+size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv);
+
+SResultRowPool* initResultRowPool(size_t size);
+SResultRow* getNewResultRow(SResultRowPool* p);
+int64_t getResultRowPoolMemSize(SResultRowPool* p);
+void* destroyResultRowPool(SResultRowPool* p);
+int32_t getNumOfAllocatedResultRows(SResultRowPool* p);
+int32_t getNumOfUsedResultRows(SResultRowPool* p);
+
+
#endif // TDENGINE_QUERYUTIL_H
diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y
index e5d1185330a47b79fec1becd74a01ed779db8c80..83386e79bbf7b77c0f84e8fa8970235d70b0401e 100644
--- a/src/query/inc/sql.y
+++ b/src/query/inc/sql.y
@@ -239,6 +239,7 @@ wal(Y) ::= WAL INTEGER(X). { Y = X; }
fsync(Y) ::= FSYNC INTEGER(X). { Y = X; }
comp(Y) ::= COMP INTEGER(X). { Y = X; }
prec(Y) ::= PRECISION STRING(X). { Y = X; }
+update(Y) ::= UPDATE INTEGER(X). { Y = X; }
%type db_optr {SCreateDBInfo}
db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);}
@@ -256,6 +257,7 @@ db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z
db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; }
db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; }
+db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); }
%type alter_db_optr {SCreateDBInfo}
alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);}
@@ -267,6 +269,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = s
alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); }
+alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); }
%type typename {TAOS_FIELD}
typename(A) ::= ids(X). {
diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h
index 28b9a60102d41a11fa02fee1ca51cbf8e263bf3b..84ca78d82249b88d68176b0a350a83fb56b8fda9 100644
--- a/src/query/inc/tsqlfunction.h
+++ b/src/query/inc/tsqlfunction.h
@@ -145,15 +145,14 @@ typedef struct SInterpInfoDetail {
int8_t primaryCol;
} SInterpInfoDetail;
-typedef struct SResultInfo {
+typedef struct SResultRowCellInfo {
int8_t hasResult; // result generated, not NULL value
- bool initialized; // output buffer has been initialized
- bool complete; // query has completed
- bool superTableQ; // is super table query
- uint32_t bufLen; // buffer size
- uint64_t numOfRes; // num of output result in current buffer
- void* interResultBuf; // output result buffer
-} SResultInfo;
+ bool initialized; // output buffer has been initialized
+ bool complete; // query has completed
+ uint32_t numOfRes; // num of output result in current buffer
+} SResultRowCellInfo;
+
+#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo)))
struct SQLFunctionCtx;
@@ -175,9 +174,11 @@ typedef struct SQLFunctionCtx {
int16_t inputBytes;
int16_t outputType;
- int16_t outputBytes; // size of results, determined by function and input column data type
- bool hasNull; // null value exist in current block
+ int16_t outputBytes; // size of results, determined by function and input column data type
+ int32_t interBufBytes; // internal buffer size
+ bool hasNull; // null value exist in current block
bool requireNull; // require null in some function
+ bool stableQuery;
int16_t functionId; // function id
void * aInputElemBuf;
char * aOutputBuf; // final result output buffer, point to sdata->data
@@ -189,7 +190,8 @@ typedef struct SQLFunctionCtx {
void * ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
SQLPreAggVal preAggVals;
tVariant tag;
- SResultInfo *resultInfo;
+
+ SResultRowCellInfo *resultInfo;
SExtTagsInfo tagInfo;
} SQLFunctionCtx;
@@ -274,16 +276,16 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha
(_r)->initialized = false; \
} while (0)
-void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf);
+//void setResultInfoBuf(SResultRowCellInfo *pResInfo, char* buf);
-static FORCE_INLINE void initResultInfo(SResultInfo *pResInfo) {
+static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, uint32_t bufLen) {
pResInfo->initialized = true; // the this struct has been initialized flag
pResInfo->complete = false;
pResInfo->hasResult = false;
pResInfo->numOfRes = 0;
- memset(pResInfo->interResultBuf, 0, (size_t)pResInfo->bufLen);
+ memset(GET_ROWCELL_INTERBUF(pResInfo), 0, (size_t)bufLen);
}
#ifdef __cplusplus
diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c
index 893105e44ac4eb82843514cda20928d3e0dcdaf9..f32ed7c39ceeb8f789951cbadefd613c79d709ff 100644
--- a/src/query/src/qAst.c
+++ b/src/query/src/qAst.c
@@ -32,7 +32,6 @@
#include "tstoken.h"
#include "ttokendef.h"
#include "tulog.h"
-#include "tutil.h"
/*
*
@@ -415,9 +414,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
}
if (cond.start != NULL) {
- iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC);
+ iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC);
} else {
- iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->keyInfo.type, TSDB_ORDER_DESC);
+ iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC);
}
if (cond.start != NULL) {
@@ -432,7 +431,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
break;
}
- STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
} else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal
@@ -450,7 +449,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
if (ret == 0 && optr == TSDB_RELATION_GREATER) {
continue;
} else {
- STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
comp = false;
}
@@ -465,22 +464,22 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
continue;
}
- STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
tSkipListDestroyIter(iter);
comp = true;
- iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC);
+ iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC);
while(tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter);
comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0);
- if (comp) {
+ if (comp) {
continue;
}
- STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
@@ -504,7 +503,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
if (ret == 0 && optr == TSDB_RELATION_LESS) {
continue;
} else {
- STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
comp = false; // no need to compare anymore
}
@@ -518,7 +517,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type);
if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) ||
(pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) {
- STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
}
@@ -609,7 +608,7 @@ int32_t intersect(SArray *pLeft, SArray *pRight, SArray *pFinalRes) {
/*
* traverse the result and apply the function to each item to check if the item is qualified or not
*/
-static void tArrayTraverse(tExprNode *pExpr, __result_filter_fn_t fp, SArray *pResult) {
+static UNUSED_FUNC void tArrayTraverse(tExprNode *pExpr, __result_filter_fn_t fp, SArray *pResult) {
assert(pExpr->_node.pLeft->nodeType == TSQL_NODE_COL && pExpr->_node.pRight->nodeType == TSQL_NODE_VALUE && fp != NULL);
// scan the result array list and check for each item in the list
@@ -660,7 +659,7 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *p
* @param pSchema tag schemas
* @param fp filter callback function
*/
-static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) {
+static UNUSED_FUNC void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) {
size_t size = taosArrayGetSize(pResult);
SArray* array = taosArrayInit(size, POINTER_BYTES);
@@ -682,7 +681,7 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki
while (tSkipListIterNext(iter)) {
SSkipListNode *pNode = tSkipListIterGet(iter);
if (filterItem(pExpr, pNode, param)) {
- taosArrayPush(pResult, SL_GET_NODE_DATA(pNode));
+ taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode)));
}
}
tSkipListDestroyIter(iter);
@@ -697,7 +696,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
SSkipListNode *pNode = tSkipListIterGet(iter);
char * pData = SL_GET_NODE_DATA(pNode);
- tstr *name = (tstr*) tsdbGetTableName(*(void**) pData);
+ tstr *name = (tstr*) tsdbGetTableName((void*) pData);
// todo speed up by using hash
if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) {
@@ -711,7 +710,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
}
if (addToResult) {
- STableKeyInfo info = {.pTable = *(void**)pData, .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(res, &info);
}
}
@@ -733,10 +732,6 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY));
param->setupInfoFn(pExpr, param->pExtInfo);
- if (pSkipList == NULL) {
- tArrayTraverse(pExpr, param->nodeFilterFn, result);
- return;
- }
tQueryInfo *pQueryInfo = pExpr->_node.info;
if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) {
@@ -748,49 +743,14 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
return;
}
- // recursive traverse left child branch
+ // The value of hasPK is always 0.
uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK;
+ assert(weight == 0 && pSkipList != NULL && taosArrayGetSize(result) == 0);
- if (weight == 0 ) {
- if (taosArrayGetSize(result) > 0 && pSkipList == NULL) {
- /**
- * Perform the filter operation based on the initial filter result, which is obtained from filtering from index.
- * Since no index presented, the filter operation is done by scan all elements in the result set.
- *
- * if the query is a high selectivity filter, only small portion of meters are retrieved.
- */
- exprTreeTraverseImpl(pExpr, result, param);
- } else {
- /**
- * apply the hierarchical expression to every node in skiplist for find the qualified nodes
- */
- assert(taosArrayGetSize(result) == 0);
- tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param);
- }
-
- return;
- }
-
- if (weight == 2 || (weight == 1 && pExpr->_node.optr == TSDB_RELATION_OR)) {
- SArray* rLeft = taosArrayInit(10, POINTER_BYTES);
- SArray* rRight = taosArrayInit(10, POINTER_BYTES);
-
- tExprTreeTraverse(pLeft, pSkipList, rLeft, param);
- tExprTreeTraverse(pRight, pSkipList, rRight, param);
-
- if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS
- intersect(rLeft, rRight, result);
- } else if (pExpr->_node.optr == TSDB_RELATION_OR) { // or
- merge(rLeft, rRight, result);
- } else {
- assert(false);
- }
-
- taosArrayDestroy(rLeft);
- taosArrayDestroy(rRight);
- return;
- }
+ //apply the hierarchical expression to every node in skiplist for find the qualified nodes
+ tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param);
+ #if 0
/*
* (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here
*
@@ -819,6 +779,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
* So, we do not set the skip list index as a parameter
*/
tExprTreeTraverse(pSecond, NULL, result, param);
+#endif
}
void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c
index 5ad52ef29e6ba0a4b47129db891c0d1318e76c5f..98a4cf2c423dd7b997a3fb59718b93dfae9a3ebf 100644
--- a/src/query/src/qExecutor.c
+++ b/src/query/src/qExecutor.c
@@ -178,13 +178,13 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) {
// todo move to utility
static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group);
-static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult);
-static void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult);
-static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo);
+static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult);
+static void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult);
+static void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow);
static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId);
static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo,
- SDataStatis *pStatis, void *param, int32_t colIndex);
+ SDataStatis *pStatis, void *param, int32_t colIndex, int32_t vgId);
static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo);
@@ -194,6 +194,8 @@ static void buildTagQueryResult(SQInfo *pQInfo);
static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo);
static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo);
+static int32_t checkForQueryBuf(size_t numOfTables);
+static void releaseQueryBuf(size_t numOfTables);
bool doFilterData(SQuery *pQuery, int32_t elemPos) {
for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) {
@@ -253,7 +255,7 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) {
continue;
}
- SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) {
maxOutput = pResInfo->numOfRes;
}
@@ -270,7 +272,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) {
SQuery *pQuery = pRuntimeEnv->pQuery;
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
- SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
int16_t functionId = pRuntimeEnv->pCtx[j].functionId;
if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ ||
@@ -445,11 +447,11 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis
return true;
}
-static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData,
- int16_t bytes, bool masterscan) {
- SQuery *pQuery = pRuntimeEnv->pQuery;
-
- int32_t *p1 = (int32_t *) taosHashGet(pWindowResInfo->hashList, pData, bytes);
+static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData,
+ int16_t bytes, bool masterscan, uint64_t uid) {
+ SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid);
+ int32_t *p1 =
+ (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
if (p1 != NULL) {
pWindowResInfo->curIndex = *p1;
} else {
@@ -459,41 +461,37 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin
// more than the capacity, reallocate the resources
if (pWindowResInfo->size >= pWindowResInfo->capacity) {
- int64_t newCap = 0;
+ int64_t newCapacity = 0;
if (pWindowResInfo->capacity > 10000) {
- newCap = (int64_t)(pWindowResInfo->capacity * 1.25);
+ newCapacity = (int64_t)(pWindowResInfo->capacity * 1.25);
} else {
- newCap = (int64_t)(pWindowResInfo->capacity * 1.5);
+ newCapacity = (int64_t)(pWindowResInfo->capacity * 1.5);
}
- char *t = realloc(pWindowResInfo->pResult, (size_t)(newCap * sizeof(SWindowResult)));
- pRuntimeEnv->summary.internalSupSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult);
- pRuntimeEnv->summary.numOfTimeWindows += (newCap - pWindowResInfo->capacity);
-
+ char *t = realloc(pWindowResInfo->pResult, (size_t)(newCapacity * POINTER_BYTES));
if (t == NULL) {
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
- pWindowResInfo->pResult = (SWindowResult *)t;
+ pWindowResInfo->pResult = (SResultRow **)t;
- int32_t inc = (int32_t)newCap - pWindowResInfo->capacity;
- memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc);
+ int32_t inc = (int32_t)newCapacity - pWindowResInfo->capacity;
+ memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, POINTER_BYTES * inc);
- pRuntimeEnv->summary.internalSupSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc;
-
- for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) {
- int32_t ret = createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize);
- if (ret != TSDB_CODE_SUCCESS) {
- longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
- }
- }
+ pWindowResInfo->capacity = (int32_t)newCapacity;
+ }
- pWindowResInfo->capacity = (int32_t)newCap;
+ SResultRow *pResult = getNewResultRow(pRuntimeEnv->pool);
+ pWindowResInfo->pResult[pWindowResInfo->size] = pResult;
+ int32_t ret = initResultRow(pResult);
+ if (ret != TSDB_CODE_SUCCESS) {
+ longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
// add a new result set for a new group
pWindowResInfo->curIndex = pWindowResInfo->size++;
- taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t));
+ taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes),
+ (char *)&pWindowResInfo->curIndex, sizeof(int32_t));
}
// too many time window in query
@@ -501,7 +499,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
}
- return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex);
+ return getResultRow(pWindowResInfo, pWindowResInfo->curIndex);
}
// get the correct time window according to the handled timestamp
@@ -517,7 +515,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t
}
} else {
int32_t slot = curTimeWindowIndex(pWindowResInfo);
- SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot);
+ SResultRow* pWindowRes = getResultRow(pWindowResInfo, slot);
w = pWindowRes->win;
}
@@ -553,9 +551,9 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t
return w;
}
-static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t sid,
+static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid,
int32_t numOfRowsPerPage) {
- if (pWindowRes->pos.pageId != -1) {
+ if (pWindowRes->pageId != -1) {
return 0;
}
@@ -563,10 +561,10 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
// in the first scan, new space needed for results
int32_t pageId = -1;
- SIDList list = getDataBufPagesIdList(pResultBuf, sid);
+ SIDList list = getDataBufPagesIdList(pResultBuf, tid);
if (taosArrayGetSize(list) == 0) {
- pData = getNewDataBuf(pResultBuf, sid, &pageId);
+ pData = getNewDataBuf(pResultBuf, tid, &pageId);
} else {
SPageInfo* pi = getLastPageInfo(list);
pData = getResBufPage(pResultBuf, pi->pageId);
@@ -576,7 +574,7 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
// release current page first, and prepare the next one
releaseResBufPageInfo(pResultBuf, pi);
- pData = getNewDataBuf(pResultBuf, sid, &pageId);
+ pData = getNewDataBuf(pResultBuf, tid, &pageId);
if (pData != NULL) {
assert(pData->num == 0); // number of elements must be 0 for new allocated buffer
}
@@ -588,24 +586,23 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
}
// set the number of rows in current disk page
- if (pWindowRes->pos.pageId == -1) { // not allocated yet, allocate new buffer
- pWindowRes->pos.pageId = pageId;
- pWindowRes->pos.rowId = (int32_t)(pData->num++);
+ if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer
+ pWindowRes->pageId = pageId;
+ pWindowRes->rowId = (int32_t)(pData->num++);
- assert(pWindowRes->pos.pageId >= 0);
+ assert(pWindowRes->pageId >= 0);
}
return 0;
}
-static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t sid,
+static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, SDataBlockInfo* pBockInfo,
STimeWindow *win, bool masterscan, bool* newWind) {
assert(win->skey <= win->ekey);
SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
- SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey,
- TSDB_KEYSIZE, masterscan);
- if (pWindowRes == NULL) {
+ SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, pBockInfo->uid);
+ if (pResultRow == NULL) {
*newWind = false;
return masterscan? -1:0;
@@ -614,23 +611,22 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes
*newWind = true;
// not assign result buffer yet, add new result buffer
- if (pWindowRes->pos.pageId == -1) {
- int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, sid, pRuntimeEnv->numOfRowsPerPage);
+ if (pResultRow->pageId == -1) {
+ int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, pBockInfo->tid, pRuntimeEnv->numOfRowsPerPage);
if (ret != TSDB_CODE_SUCCESS) {
return -1;
}
}
// set time window for current result
- pWindowRes->win = (*win);
-
- setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes);
+ pResultRow->win = (*win);
+ setWindowResOutputBufInitCtx(pRuntimeEnv, pResultRow);
return TSDB_CODE_SUCCESS;
}
static bool getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) {
assert(slot >= 0 && slot < pWindowResInfo->size);
- return pWindowResInfo->pResult[slot].closed;
+ return pWindowResInfo->pResult[slot]->closed;
}
static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos,
@@ -689,7 +685,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe
int64_t skey = TSKEY_INITIAL_VAL;
for (i = 0; i < pWindowResInfo->size; ++i) {
- SWindowResult *pResult = &pWindowResInfo->pResult[i];
+ SResultRow *pResult = pWindowResInfo->pResult[i];
if (pResult->closed) {
numOfClosed += 1;
continue;
@@ -713,7 +709,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe
pWindowResInfo->curIndex = i;
}
- pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].win.skey;
+ pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey;
// the number of completed slots are larger than the threshold, return current generated results to client.
if (numOfClosed > pWindowResInfo->threshold) {
@@ -938,7 +934,6 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
sas->data = calloc(pQuery->numOfCols, POINTER_BYTES);
if (sas->data == NULL) {
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
@@ -1001,13 +996,13 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport));
if (sasArray == NULL) {
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
+ SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock);
- setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k);
+ setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId);
}
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
@@ -1023,9 +1018,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
bool hasTimeWindow = false;
STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
- if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow) !=
- TSDB_CODE_SUCCESS) {
- taosTFree(sasArray);
+ if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) {
+ tfree(sasArray);
return;
}
@@ -1052,8 +1046,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
// null data, failed to allocate more memory buffer
hasTimeWindow = false;
- if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan,
- &hasTimeWindow) != TSDB_CODE_SUCCESS) {
+ if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) {
break;
}
@@ -1088,13 +1081,13 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
continue;
}
- taosTFree(sasArray[i].data);
+ tfree(sasArray[i].data);
}
- taosTFree(sasArray);
+ tfree(sasArray);
}
-static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes) {
+static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) {
if (isNull(pData, type)) { // ignore the null value
return -1;
}
@@ -1111,13 +1104,14 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
len = varDataLen(pData);
} else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) {
SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
- qError("QInfo:%p group by not supported on double/float/binary/nchar columns, abort", pQInfo);
+ qError("QInfo:%p group by not supported on double/float columns, abort", pQInfo);
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR);
}
- SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true);
- if (pWindowRes == NULL) {
+ uint64_t uid = groupIndex; // uid is always set to be 0.
+ SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid);
+ if (pResultRow == NULL) {
return -1;
}
@@ -1131,23 +1125,21 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
}
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
- pWindowRes->key = malloc(varDataTLen(pData));
- varDataCopy(pWindowRes->key, pData);
+ pResultRow->key = malloc(varDataTLen(pData));
+ varDataCopy(pResultRow->key, pData);
} else {
- pWindowRes->win.skey = v;
- pWindowRes->win.ekey = v;
+ pResultRow->win.skey = v;
+ pResultRow->win.ekey = v;
}
- assert(pRuntimeEnv->windowResInfo.interval == 0);
-
- if (pWindowRes->pos.pageId == -1) {
- int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage);
+ if (pResultRow->pageId == -1) {
+ int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage);
if (ret != 0) {
return -1;
}
}
- setWindowResOutputBuf(pRuntimeEnv, pWindowRes);
+ setResultOutputBuf(pRuntimeEnv, pResultRow);
initCtxOutputBuf(pRuntimeEnv);
return TSDB_CODE_SUCCESS;
}
@@ -1200,7 +1192,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) {
SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
// compare tag first
- if (tVariantCompare(&pCtx[0].tag, &elem.tag) != 0) {
+ if (tVariantCompare(&pCtx[0].tag, elem.tag) != 0) {
return TS_JOIN_TAG_NOT_EQUALS;
}
@@ -1230,7 +1222,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) {
}
static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId) {
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
SQuery* pQuery = pRuntimeEnv->pQuery;
// in case of timestamp column, always generated results.
@@ -1274,7 +1266,6 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport));
if (sasArray == NULL) {
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
@@ -1286,9 +1277,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
groupbyColumnData = getGroupbyColumnData(pQuery, &type, &bytes, pDataBlock);
}
+ SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock);
- setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k);
+ setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId);
}
// set the input column data
@@ -1303,7 +1295,6 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
// from top to bottom in desc
// from bottom to top in asc order
if (pRuntimeEnv->pTSBuf != NULL) {
- SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pRuntimeEnv);
qDebug("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->rows,
pQuery->order.order, pRuntimeEnv->pTSBuf->cur.order);
}
@@ -1335,7 +1326,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
bool hasTimeWindow = false;
- int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow);
+ int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow);
if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code
continue;
}
@@ -1363,7 +1354,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
// null data, failed to allocate more memory buffer
hasTimeWindow = false;
- if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) {
+ if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) {
break;
}
@@ -1379,7 +1370,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
if (groupbyColumnValue) {
char *val = groupbyColumnData + bytes * offset;
- int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes);
+ int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes, item->groupIndex);
if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code
continue;
}
@@ -1409,13 +1400,17 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
item->lastKey = (QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey) + step;
}
+ if (pRuntimeEnv->pTSBuf != NULL) {
+ item->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf);
+ }
+
// todo refactor: extract method
for(int32_t i = 0; i < pQuery->numOfOutput; ++i) {
if (pQuery->pSelectExpr[i].base.functionId != TSDB_FUNC_ARITHM) {
continue;
}
- taosTFree(sasArray[i].data);
+ tfree(sasArray[i].data);
}
free(sasArray);
@@ -1469,7 +1464,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
}
void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo,
- SDataStatis *pStatis, void *param, int32_t colIndex) {
+ SDataStatis *pStatis, void *param, int32_t colIndex, int32_t vgId) {
int32_t functionId = pQuery->pSelectExpr[colIndex].base.functionId;
int32_t colId = pQuery->pSelectExpr[colIndex].base.colInfo.colId;
@@ -1514,7 +1509,8 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY
* top/bottom values emerge, so does diff function
*/
if (functionId == TSDB_FUNC_TWA) {
- STwaInfo *pTWAInfo = GET_RES_INFO(pCtx)->interResultBuf;
+ SResultRowCellInfo* pInfo = GET_RES_INFO(pCtx);
+ STwaInfo *pTWAInfo = (STwaInfo*) GET_ROWCELL_INTERBUF(pInfo);
pTWAInfo->SKey = pQuery->window.skey;
pTWAInfo->EKey = pQuery->window.ekey;
}
@@ -1528,7 +1524,9 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY
pCtx->preAggVals.statis.max = pBlockInfo->window.ekey;
}
} else if (functionId == TSDB_FUNC_INTERP) {
- SInterpInfoDetail *pInterpInfo = GET_RES_INFO(pCtx)->interResultBuf;
+ SResultRowCellInfo* pInfo = GET_RES_INFO(pCtx);
+
+ SInterpInfoDetail *pInterpInfo = (SInterpInfoDetail *)GET_ROWCELL_INTERBUF(pInfo);
pInterpInfo->type = (int8_t)pQuery->fillType;
pInterpInfo->ts = pQuery->window.skey;
pInterpInfo->primaryCol = (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX);
@@ -1542,6 +1540,9 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY
}
}
}
+ } else if (functionId == TSDB_FUNC_TS_COMP) {
+ pCtx->param[0].i64Key = vgId;
+ pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT;
}
#if defined(_DEBUG_VIEW)
@@ -1595,33 +1596,22 @@ static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx
p->tagInfo.numOfTagCols = num;
p->tagInfo.tagsLen = tagLen;
} else {
- taosTFree(pTagCtx);
+ tfree(pTagCtx);
}
}
return TSDB_CODE_SUCCESS;
}
-static FORCE_INLINE void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery, char* buf) {
- char* p = buf;
- for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- int32_t size = pQuery->pSelectExpr[i].interBytes;
- setResultInfoBuf(&pResultInfo[i], size, isStableQuery, p);
-
- p += size;
- }
-}
-
static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order) {
qDebug("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv));
SQuery *pQuery = pRuntimeEnv->pQuery;
- size_t size = pRuntimeEnv->interBufSize + pQuery->numOfOutput * sizeof(SResultInfo);
-
- pRuntimeEnv->resultInfo = calloc(1, size);
pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx));
+ pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t));
+ pRuntimeEnv->pResultRow = getNewResultRow(pRuntimeEnv->pool);
- if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) {
+ if (pRuntimeEnv->pResultRow == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) {
goto _clean;
}
@@ -1658,7 +1648,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
pCtx->inputType = pQuery->colList[index].type;
}
-
assert(isValidDataType(pCtx->inputType));
pCtx->ptsOutputBuf = NULL;
@@ -1667,6 +1656,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
pCtx->order = pQuery->order.order;
pCtx->functionId = pSqlFuncMsg->functionId;
+ pCtx->stableQuery = pRuntimeEnv->stableQuery;
+ pCtx->interBufBytes = pQuery->pSelectExpr[i].interBytes;
pCtx->numOfParams = pSqlFuncMsg->numOfParams;
for (int32_t j = 0; j < pCtx->numOfParams; ++j) {
@@ -1696,16 +1687,14 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
if (i > 0) {
pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes;
+ pRuntimeEnv->rowCellInfoOffset[i] = pRuntimeEnv->rowCellInfoOffset[i - 1] + sizeof(SResultRowCellInfo) + pQuery->pSelectExpr[i - 1].interBytes;
}
- }
-
- char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultInfo) * pQuery->numOfOutput;
- // set the intermediate result output buffer
- setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf);
+ }
// if it is group by normal column, do not set output buffer, the output buffer is pResult
- if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery) {
+ // fixed output query/multi-output query for normal table
+ if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) {
resetCtxOutputBuf(pRuntimeEnv);
}
@@ -1717,12 +1706,24 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
return TSDB_CODE_SUCCESS;
_clean:
- taosTFree(pRuntimeEnv->resultInfo);
- taosTFree(pRuntimeEnv->pCtx);
+ tfree(pRuntimeEnv->pCtx);
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
+static void doFreeQueryHandle(SQInfo* pQInfo) {
+ SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
+
+ tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
+ tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
+
+ pRuntimeEnv->pQueryHandle = NULL;
+ pRuntimeEnv->pSecQueryHandle = NULL;
+
+ SMemRef* pMemRef = &pQInfo->memRef;
+ assert(pMemRef->ref == 0 && pMemRef->imem == NULL && pMemRef->mem == NULL);
+}
+
static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
if (pRuntimeEnv->pQuery == NULL) {
return;
@@ -1743,20 +1744,25 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
}
tVariantDestroy(&pCtx->tag);
- taosTFree(pCtx->tagInfo.pTagCtxList);
+ tfree(pCtx->tagInfo.pTagCtxList);
}
- taosTFree(pRuntimeEnv->resultInfo);
- taosTFree(pRuntimeEnv->pCtx);
+ tfree(pRuntimeEnv->pCtx);
}
- pRuntimeEnv->pFillInfo = taosDestoryFillInfo(pRuntimeEnv->pFillInfo);
+ pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo);
destroyResultBuf(pRuntimeEnv->pResultBuf);
- tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
- tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
+ doFreeQueryHandle(pQInfo);
pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf);
+ tfree(pRuntimeEnv->keyBuf);
+ tfree(pRuntimeEnv->rowCellInfoOffset);
+
+ taosHashCleanup(pRuntimeEnv->pResultRowHashTable);
+ pRuntimeEnv->pResultRowHashTable = NULL;
+
+ pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool);
}
#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED)
@@ -1847,8 +1853,11 @@ static bool needReverseScan(SQuery *pQuery) {
}
if (functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) {
+ // the scan order to acquire the last result of the specified column
int32_t order = (int32_t)pQuery->pSelectExpr[i].base.arg->argValue.i64;
- return order != pQuery->order.order;
+ if (order != pQuery->order.order) {
+ return true;
+ }
}
}
@@ -2252,7 +2261,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo * pW
TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey;
STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery);
- if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo->tid, &win, masterScan, &hasTimeWindow) !=
+ if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, masterScan, &hasTimeWindow) !=
TSDB_CODE_SUCCESS) {
// todo handle error in set result for timewindow
}
@@ -2621,21 +2630,28 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) {
pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
assert(pFuncMsg->numOfParams == 1);
- int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64;
- SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId);
+ int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64;
+ SColumnInfo *pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId);
doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes);
- qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%"PRId64, pQInfo, pExprInfo->base.arg->argValue.i64,
- pRuntimeEnv->pCtx[0].tag.i64Key)
+
+ int16_t tagType = pRuntimeEnv->pCtx[0].tag.nType;
+ if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) {
+ qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%s", pQInfo,
+ pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.pz);
+ } else {
+ qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, pQInfo,
+ pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.i64Key);
+ }
}
}
}
-static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowResult *pWindowRes, bool mergeFlag) {
+static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SResultRow *pWindowRes, bool mergeFlag) {
SQuery * pQuery = pRuntimeEnv->pQuery;
SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
- tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
+ tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId);
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
@@ -2805,15 +2821,15 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param)
}
SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo;
- SWindowResult * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos);
- tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pos.pageId);
+ SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos);
+ tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId);
char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1);
TSKEY leftTimestamp = GET_INT64_VAL(b1);
SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo;
- SWindowResult * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos);
- tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pos.pageId);
+ SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos);
+ tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId);
char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2);
TSKEY rightTimestamp = GET_INT64_VAL(b2);
@@ -2850,7 +2866,7 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) {
}
SGroupResInfo* info = &pQInfo->groupResInfo;
- if (pQInfo->groupIndex == numOfGroups && info->pos.pageId == info->numOfDataPages) {
+ if (pQInfo->groupIndex == numOfGroups && info->pageId == info->numOfDataPages) {
SET_STABLE_QUERY_OVER(pQInfo);
}
@@ -2866,10 +2882,10 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo;
// all results have been return to client, try next group
- if (pGroupResInfo->pos.pageId == pGroupResInfo->numOfDataPages) {
+ if (pGroupResInfo->pageId == pGroupResInfo->numOfDataPages) {
pGroupResInfo->numOfDataPages = 0;
- pGroupResInfo->pos.pageId = 0;
- pGroupResInfo->pos.rowId = 0;
+ pGroupResInfo->pageId = 0;
+ pGroupResInfo->rowId = 0;
// current results of group has been sent to client, try next group
if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) {
@@ -2897,22 +2913,22 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
assert(size == pGroupResInfo->numOfDataPages);
bool done = false;
- for (int32_t j = pGroupResInfo->pos.pageId; j < size; ++j) {
+ for (int32_t j = pGroupResInfo->pageId; j < size; ++j) {
SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j);
tFilePage* pData = getResBufPage(pResultBuf, pi->pageId);
- assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->pos.rowId < pData->num);
- int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->pos.rowId);
+ assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->rowId < pData->num);
+ int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->rowId);
if (numOfRes > pQuery->rec.capacity - offset) {
numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset);
- pGroupResInfo->pos.rowId += numOfCopiedRows;
+ pGroupResInfo->rowId += numOfCopiedRows;
done = true;
} else {
numOfCopiedRows = (int32_t)pData->num;
- pGroupResInfo->pos.pageId += 1;
- pGroupResInfo->pos.rowId = 0;
+ pGroupResInfo->pageId += 1;
+ pGroupResInfo->rowId = 0;
}
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
@@ -2933,7 +2949,9 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
pQuery->rec.rows += offset;
}
-int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) {
+int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) {
+ SQuery* pQuery = pRuntimeEnv->pQuery;
+
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
int32_t functionId = pQuery->pSelectExpr[j].base.functionId;
@@ -2945,7 +2963,7 @@ int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) {
continue;
}
- SResultInfo *pResultInfo = &pWindowRes->resultInfo[j];
+ SResultRowCellInfo *pResultInfo = getResultCell(pRuntimeEnv, pResultRow, j);
assert(pResultInfo != NULL);
if (pResultInfo->numOfRes > 0) {
@@ -2967,8 +2985,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
STableQueryInfo **pTableList = malloc(POINTER_BYTES * size);
if (pTableList == NULL || posList == NULL) {
- taosTFree(posList);
- taosTFree(pTableList);
+ tfree(posList);
+ tfree(pTableList);
qError("QInfo:%p failed alloc memory", pQInfo);
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
@@ -2992,19 +3010,19 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
// there is no data in current group
if (numOfTables == 0) {
- taosTFree(posList);
- taosTFree(pTableList);
+ tfree(posList);
+ tfree(pTableList);
return 0;
} else if (numOfTables == 1) { // no need to merge results since only one table in each group
- taosTFree(posList);
- taosTFree(pTableList);
+ tfree(posList);
+ tfree(pTableList);
SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo;
pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList);
pGroupResInfo->groupId = tid;
- pGroupResInfo->pos.pageId = 0;
- pGroupResInfo->pos.rowId = 0;
+ pGroupResInfo->pageId = 0;
+ pGroupResInfo->rowId = 0;
return pGroupResInfo->numOfDataPages;
}
@@ -3014,18 +3032,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
SLoserTreeInfo *pTree = NULL;
tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn);
- SResultInfo *pResultInfo = calloc(pQuery->numOfOutput, sizeof(SResultInfo));
- if (pResultInfo == NULL) {
- longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
- }
-
- char* buf = calloc(1, pRuntimeEnv->interBufSize);
- if (buf == NULL) {
- longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
- }
-
- setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf);
- resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo);
+ SResultRow* pRow = getNewResultRow(pRuntimeEnv->pool);
+ resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow);
pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex);
@@ -3037,26 +3045,23 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
if (IS_QUERY_KILLED(pQInfo)) {
qDebug("QInfo:%p it is already killed, abort", pQInfo);
- taosTFree(pTableList);
- taosTFree(posList);
- taosTFree(pTree);
- taosTFree(pResultInfo);
- taosTFree(buf);
-
+ tfree(pTableList);
+ tfree(posList);
+ tfree(pTree);
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
int32_t pos = pTree->pNode[0].index;
SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo;
- SWindowResult *pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]);
- tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
+ SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.position[pos]);
+ tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId);
char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page);
TSKEY ts = GET_INT64_VAL(b);
assert(ts == pWindowRes->win.skey);
- int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes);
+ int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes);
if (num <= 0) {
cs.position[pos] += 1;
@@ -3077,7 +3082,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
return -1;
}
- resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo);
+ resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow);
}
doMerge(pRuntimeEnv, ts, pWindowRes, false);
@@ -3087,7 +3092,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
lastTimestamp = ts;
// move to the next element of current entry
- int32_t currentPageId = pWindowRes->pos.pageId;
+ int32_t currentPageId = pWindowRes->pageId;
cs.position[pos] += 1;
if (cs.position[pos] >= pWindowResInfo->size) {
@@ -3099,8 +3104,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
}
} else {
// current page is not needed anymore
- SWindowResult *pNextWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]);
- if (pNextWindowRes->pos.pageId != currentPageId) {
+ SResultRow *pNextWindowRes = getResultRow(pWindowResInfo, cs.position[pos]);
+ if (pNextWindowRes->pageId != currentPageId) {
releaseResBufPage(pRuntimeEnv->pResultBuf, page);
}
}
@@ -3113,11 +3118,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) {
qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo);
- taosTFree(pTree);
- taosTFree(pTableList);
- taosTFree(posList);
- taosTFree(pResultInfo);
-
+ tfree(pTree);
+ tfree(pTableList);
+ tfree(posList);
return -1;
}
}
@@ -3130,12 +3133,12 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt);
- taosTFree(pTableList);
- taosTFree(posList);
- taosTFree(pTree);
+ tfree(pTableList);
+ tfree(posList);
+ tfree(pTree);
- taosTFree(pResultInfo);
- taosTFree(buf);
+// tfree(pResultInfo);
+// tfree(buf);
return pQInfo->groupResInfo.numOfDataPages;
}
@@ -3178,12 +3181,14 @@ int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupR
return TSDB_CODE_SUCCESS;
}
-void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo) {
+void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow) {
+ SQuery* pQuery = pRuntimeEnv->pQuery;
+
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes;
pCtx[k].size = 1;
pCtx[k].startOffset = 0;
- pCtx[k].resultInfo = &pResultInfo[k];
+ pCtx[k].resultInfo = getResultCell(pRuntimeEnv, pRow, k);
pQuery->sdata[k]->num = 0;
}
@@ -3220,8 +3225,8 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *
pTableQueryInfo->windowResInfo.curIndex = pTableQueryInfo->windowResInfo.size - 1;
}
-static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindowResInfo, int32_t order) {
- SQuery* pQuery = pQInfo->runtimeEnv.pQuery;
+static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t order) {
+ SQuery* pQuery = pRuntimeEnv->pQuery;
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
bool closed = getTimeWindowResStatus(pWindowResInfo, i);
@@ -3229,17 +3234,18 @@ static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindow
continue;
}
- SWindowResult *buf = getWindowResult(pWindowResInfo, i);
+ SResultRow *pRow = getResultRow(pWindowResInfo, i);
// open/close the specified query for each group result
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
int32_t functId = pQuery->pSelectExpr[j].base.functionId;
+ SResultRowCellInfo* pInfo = getResultCell(pRuntimeEnv, pRow, j);
if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) ||
((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) {
- buf->resultInfo[j].complete = false;
+ pInfo->complete = false;
} else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) {
- buf->resultInfo[j].complete = true;
+ pInfo->complete = true;
}
}
}
@@ -3253,7 +3259,7 @@ void disableFuncInReverseScan(SQInfo *pQInfo) {
// group by normal columns and interval query on normal table
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) {
- disableFuncInReverseScanImpl(pQInfo, pWindowResInfo, order);
+ disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order);
} else { // for simple result of table query,
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor
int32_t functId = pQuery->pSelectExpr[j].base.functionId;
@@ -3303,26 +3309,16 @@ void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) {
}
}
-int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize) {
- int32_t numOfCols = pQuery->numOfOutput;
-
- size_t size = numOfCols * sizeof(SResultInfo) + interBufSize;
- pResultRow->resultInfo = calloc(1, size);
- if (pResultRow->resultInfo == NULL) {
- return TSDB_CODE_QRY_OUT_OF_MEMORY;
- }
-
- pResultRow->pos = (SPosInfo) {-1, -1};
-
- char* buf = (char*) pResultRow->resultInfo + numOfCols * sizeof(SResultInfo);
-
- // set the intermediate result output buffer
- setWindowResultInfo(pResultRow->resultInfo, pQuery, isSTableQuery, buf);
+int32_t initResultRow(SResultRow *pResultRow) {
+ pResultRow->pCellInfo = (SResultRowCellInfo*)((char*)pResultRow + sizeof(SResultRow));
+ pResultRow->pageId = -1;
+ pResultRow->rowId = -1;
return TSDB_CODE_SUCCESS;
}
void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) {
SQuery *pQuery = pRuntimeEnv->pQuery;
+ SResultRow* pRow = pRuntimeEnv->pResultRow;
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
@@ -3332,8 +3328,9 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) {
* set the output buffer information and intermediate buffer
* not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc.
*/
- RESET_RESULT_INFO(&pRuntimeEnv->resultInfo[i]);
- pCtx->resultInfo = &pRuntimeEnv->resultInfo[i];
+ SResultRowCellInfo* pCellInfo = getResultCell(pRuntimeEnv, pRow, i);
+ RESET_RESULT_INFO(pCellInfo);
+ pCtx->resultInfo = pCellInfo;
// set the timestamp output buffer for top/bottom/diff query
int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
@@ -3382,7 +3379,7 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) {
int32_t functionId = pQuery->pSelectExpr[j].base.functionId;
pRuntimeEnv->pCtx[j].currentStage = 0;
- SResultInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
+ SResultRowCellInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
if (pResInfo->initialized) {
continue;
}
@@ -3451,12 +3448,12 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) {
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
- SWindowResult *pResult = getWindowResult(pWindowResInfo, i);
+ SResultRow *pResult = getResultRow(pWindowResInfo, i);
if (!pResult->closed) {
continue;
}
- setWindowResOutputBuf(pRuntimeEnv, pResult);
+ setResultOutputBuf(pRuntimeEnv, pResult);
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
int16_t functId = pQuery->pSelectExpr[j].base.functionId;
@@ -3465,7 +3462,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) {
}
aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]);
- SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
toContinue |= (!pResInfo->complete);
}
@@ -3478,7 +3475,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) {
}
aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]);
- SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]);
toContinue |= (!pResInfo->complete);
}
@@ -3551,7 +3548,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI
tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
}
- pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+ pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
if (pRuntimeEnv->pSecQueryHandle == NULL) {
longjmp(pRuntimeEnv->env, terrno);
}
@@ -3635,7 +3632,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
}
restoreTimeWindow(&pQInfo->tableGroupInfo, &cond);
- pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+ pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
if (pRuntimeEnv->pSecQueryHandle == NULL) {
longjmp(pRuntimeEnv->env, terrno);
}
@@ -3649,7 +3646,6 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
// check if query is killed or not
if (IS_QUERY_KILLED(pQInfo)) {
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
}
@@ -3678,12 +3674,12 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) {
}
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
- SWindowResult *buf = &pWindowResInfo->pResult[i];
+ SResultRow *buf = pWindowResInfo->pResult[i];
if (!isWindowResClosed(pWindowResInfo, i)) {
continue;
}
- setWindowResOutputBuf(pRuntimeEnv, buf);
+ setResultOutputBuf(pRuntimeEnv, buf);
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
aAggs[pQuery->pSelectExpr[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]);
@@ -3730,7 +3726,7 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void
if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) {
int32_t initialSize = 16;
int32_t initialThreshold = 100;
- int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, initialSize, initialThreshold, TSDB_DATA_TYPE_INT);
+ int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, initialSize, initialThreshold, TSDB_DATA_TYPE_INT);
if (code != TSDB_CODE_SUCCESS) {
return NULL;
}
@@ -3745,6 +3741,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) {
return;
}
+ tVariantDestroy(&pTableQueryInfo->tag);
cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo);
}
@@ -3769,9 +3766,10 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) {
return;
}
- SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex,
- sizeof(groupIndex), true);
- if (pWindowRes == NULL) {
+ uint64_t uid = 0; // uid is always set to be 0
+ SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex,
+ sizeof(groupIndex), true, uid);
+ if (pResultRow == NULL) {
return;
}
@@ -3779,8 +3777,8 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) {
* not assign result buffer yet, add new result buffer
* all group belong to one result set, and each group result has different group id so set the id to be one
*/
- if (pWindowRes->pos.pageId == -1) {
- if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) !=
+ if (pResultRow->pageId == -1) {
+ if (addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) !=
TSDB_CODE_SUCCESS) {
return;
}
@@ -3788,15 +3786,15 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) {
// record the current active group id
pRuntimeEnv->prevGroupId = groupIndex;
- setWindowResOutputBuf(pRuntimeEnv, pWindowRes);
+ setResultOutputBuf(pRuntimeEnv, pResultRow);
initCtxOutputBuf(pRuntimeEnv);
}
-void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) {
+void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) {
SQuery *pQuery = pRuntimeEnv->pQuery;
// Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
- tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
+ tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId);
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
@@ -3808,27 +3806,23 @@ void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult
}
/*
- * set the output buffer information and intermediate buffer
+ * set the output buffer information and intermediate buffer,
* not all queries require the interResultBuf, such as COUNT
*/
- pCtx->resultInfo = &pResult->resultInfo[i];
-
- // set super table query flag
- SResultInfo *pResInfo = GET_RES_INFO(pCtx);
- pResInfo->superTableQ = pRuntimeEnv->stableQuery;
+ pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i);
}
}
-void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) {
+void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) {
SQuery *pQuery = pRuntimeEnv->pQuery;
// Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
- tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
+ tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId);
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
- pCtx->resultInfo = &pResult->resultInfo[i];
+ pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i);
if (pCtx->resultInfo->initialized && pCtx->resultInfo->complete) {
continue;
}
@@ -3841,12 +3835,6 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *
pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf;
}
- /*
- * set the output buffer information and intermediate buffer
- * not all queries require the interResultBuf, such as COUNT
- */
- pCtx->resultInfo->superTableQ = pRuntimeEnv->stableQuery; // set super table query flag
-
if (!pCtx->resultInfo->initialized) {
aAggs[functionId].init(pCtx);
}
@@ -3860,14 +3848,40 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ
// both the master and supplement scan needs to set the correct ts comp start position
if (pRuntimeEnv->pTSBuf != NULL) {
+ tVariant* pTag = &pRuntimeEnv->pCtx[0].tag;
+
if (pTableQueryInfo->cur.vgroupIndex == -1) {
- tVariantAssign(&pTableQueryInfo->tag, &pRuntimeEnv->pCtx[0].tag);
- tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pTableQueryInfo->tag);
+ tVariantAssign(&pTableQueryInfo->tag, pTag);
+
+ STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pTableQueryInfo->tag);
+
+ // failed to find data with the specified tag value and vnodeId
+ if (!tsBufIsValidElem(&elem)) {
+ if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
+ qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz);
+ } else {
+ qError("QInfo:%p failed to find tag:%" PRId64 " in ts_comp", pQInfo, pTag->i64Key);
+ }
+
+ return false;
+ }
// keep the cursor info of current meter
- pTableQueryInfo->cur = pRuntimeEnv->pTSBuf->cur;
+ pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf);
+ if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
+ qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex);
+ } else {
+ qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex);
+ }
+
} else {
tsBufSetCursor(pRuntimeEnv->pTSBuf, &pTableQueryInfo->cur);
+
+ if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
+ qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex);
+ } else {
+ qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex);
+ }
}
}
@@ -3961,7 +3975,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_
qDebug("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo);
int32_t totalSet = numOfClosedTimeWindow(pResultInfo);
- SWindowResult* result = pResultInfo->pResult;
+ SResultRow** result = pResultInfo->pResult;
if (orderType == TSDB_ORDER_ASC) {
startIdx = pQInfo->groupIndex;
@@ -3974,14 +3988,14 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_
SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo;
for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) {
- if (result[i].numOfRows == 0) {
+ if (result[i]->numOfRows == 0) {
pQInfo->groupIndex += 1;
- pGroupResInfo->pos.rowId = 0;
+ pGroupResInfo->rowId = 0;
continue;
}
- int32_t numOfRowsToCopy = result[i].numOfRows - pGroupResInfo->pos.rowId;
- int32_t oldOffset = pGroupResInfo->pos.rowId;
+ int32_t numOfRowsToCopy = result[i]->numOfRows - pGroupResInfo->rowId;
+ int32_t oldOffset = pGroupResInfo->rowId;
/*
* current output space is not enough to accommodate all data of this page, only partial results
@@ -3989,19 +4003,19 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_
*/
if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) {
numOfRowsToCopy = (int32_t) pQuery->rec.capacity - numOfResult;
- pGroupResInfo->pos.rowId += numOfRowsToCopy;
+ pGroupResInfo->rowId += numOfRowsToCopy;
} else {
- pGroupResInfo->pos.rowId = 0;
+ pGroupResInfo->rowId = 0;
pQInfo->groupIndex += 1;
}
- tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i].pos.pageId);
+ tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i]->pageId);
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
int32_t size = pRuntimeEnv->pCtx[j].outputBytes;
char *out = pQuery->sdata[j]->data + numOfResult * size;
- char *in = getPosInResultPage(pRuntimeEnv, j, &result[i], page);
+ char *in = getPosInResultPage(pRuntimeEnv, j, result[i], page);
memcpy(out, in + oldOffset * size, size * numOfRowsToCopy);
}
@@ -4048,7 +4062,7 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) {
}
for (int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) {
- SWindowResult *pResult = &pRuntimeEnv->windowResInfo.pResult[i];
+ SResultRow *pResult = pRuntimeEnv->windowResInfo.pResult[i];
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
int32_t functionId = pRuntimeEnv->pCtx[j].functionId;
@@ -4056,7 +4070,8 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) {
continue;
}
- pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes));
+ SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j);
+ pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes));
}
}
}
@@ -4202,16 +4217,24 @@ static void queryCostStatis(SQInfo *pQInfo) {
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
SQueryCostInfo *pSummary = &pRuntimeEnv->summary;
+ uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable);
+ hashSize += taosHashGetMemSize(pQInfo->tableqinfoGroupInfo.map);
+ pSummary->hashSize = hashSize;
+
// add the merge time
pSummary->elapsedTime += pSummary->firstStageMergeTime;
+ SResultRowPool* p = pQInfo->runtimeEnv.pool;
+ pSummary->winInfoSize = getResultRowPoolMemSize(p);
+ pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p);
+
qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, "
"load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64,
pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis,
pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows);
- qDebug("QInfo:%p :cost summary: internal size:%"PRId64"B, numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize,
- pSummary->numOfTimeWindows);
+ qDebug("QInfo:%p :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo, pSummary->winInfoSize/1024.0,
+ pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0);
}
static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) {
@@ -4266,7 +4289,6 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
while (tsdbNextDataBlock(pQueryHandle)) {
if (IS_QUERY_KILLED(GET_QINFO_ADDR(pRuntimeEnv))) {
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
@@ -4452,7 +4474,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery)
terrno = TSDB_CODE_SUCCESS;
if (isFirstLastRowQuery(pQuery)) {
- pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+ pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
// update the query time window
pQuery->window = cond.twindow;
@@ -4474,9 +4496,9 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery)
}
}
} else if (isPointInterpoQuery(pQuery)) {
- pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+ pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
} else {
- pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+ pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
}
return terrno;
@@ -4570,7 +4592,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
}
}
- code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 8, threshold, type);
+ code = initWindowResInfo(&pRuntimeEnv->windowResInfo, 8, threshold, type);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@@ -4590,7 +4612,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
type = TSDB_DATA_TYPE_TIMESTAMP;
}
- code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, numOfResultRows, 4096, type);
+ code = initWindowResInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, 1024, type);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@@ -4617,7 +4639,7 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) {
SQuery *pQuery = pRuntimeEnv->pQuery;
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
- SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]);
+ SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]);
if (pResInfo != NULL) {
pResInfo->complete = false;
}
@@ -4755,7 +4777,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
pRuntimeEnv->pQueryHandle = NULL;
}
- pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
+ pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef);
taosArrayDestroy(tx);
taosArrayDestroy(g1);
if (pRuntimeEnv->pQueryHandle == NULL) {
@@ -4763,15 +4785,62 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
}
if (pRuntimeEnv->pTSBuf != NULL) {
+ tVariant* pTag = &pRuntimeEnv->pCtx[0].tag;
+
if (pRuntimeEnv->cur.vgroupIndex == -1) {
- STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pRuntimeEnv->pCtx[0].tag);
+ STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag);
+ // failed to find data with the specified tag value and vnodeId
+ if (!tsBufIsValidElem(&elem)) {
+ if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
+ qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz);
+ } else {
+ qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key);
+ }
- // failed to find data with the specified tag value
- if (elem.vnode < 0) {
return false;
+ } else {
+ STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf);
+
+ if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
+ qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz,
+ cur.blockIndex, cur.tsIndex);
+ } else {
+ qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key,
+ cur.blockIndex, cur.tsIndex);
+ }
}
} else {
- tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur);
+ STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf);
+ if (tVariantCompare(elem.tag, &pRuntimeEnv->pCtx[0].tag) != 0) {
+
+ STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag);
+ // failed to find data with the specified tag value and vnodeId
+ if (!tsBufIsValidElem(&elem1)) {
+ if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
+ qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz);
+ } else {
+ qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key);
+ }
+
+ return false;
+ } else {
+ STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf);
+ if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
+ qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex);
+ } else {
+ qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex);
+ }
+ }
+
+ } else {
+ tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur);
+ STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf);
+ if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
+ qDebug("QInfo:%p continue scan ts_comp file, tag:%s blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex);
+ } else {
+ qDebug("QInfo:%p continue scan ts_comp file, tag:%"PRId64" blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex);
+ }
+ }
}
}
@@ -4793,7 +4862,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo);
- if (isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery)) {
+ if (isPointInterpoQuery(pQuery)) {
resetCtxOutputBuf(pRuntimeEnv);
assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0);
@@ -4823,11 +4892,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
pRuntimeEnv->pQueryHandle = NULL;
}
- if (isFirstLastRowQuery(pQuery)) {
- assert(0); // last_row query switch to other routine to handle
- } else {
- pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo);
- }
+ pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef);
taosArrayDestroy(tx);
taosArrayDestroy(g1);
@@ -4841,10 +4906,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
assert(taosArrayGetSize(s) >= 1);
setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb);
- if (isFirstLastRowQuery(pQuery)) {
- assert(taosArrayGetSize(s) == 1);
- }
-
taosArrayDestroy(s);
// here we simply set the first table as current table
@@ -4897,7 +4958,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
}
// no need to update the lastkey for each table
- pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
+ pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef);
taosArrayDestroy(g1);
taosArrayDestroy(tx);
@@ -4923,11 +4984,12 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
}
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
- pWindowResInfo->pResult[i].closed = true; // enable return all results for group by normal columns
+ pWindowResInfo->pResult[i]->closed = true; // enable return all results for group by normal columns
- SWindowResult *pResult = &pWindowResInfo->pResult[i];
+ SResultRow *pResult = pWindowResInfo->pResult[i];
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
- pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes));
+ SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j);
+ pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes));
}
}
@@ -5027,6 +5089,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
break;
}
+ if (pRuntimeEnv->pTSBuf != NULL) {
+ pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur;
+ }
+
} else {
// all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter
if (pQuery->rec.rows == 0) {
@@ -5079,7 +5145,7 @@ static void doSaveContext(SQInfo *pQInfo) {
SWITCH_ORDER(pQuery->order.order);
if (pRuntimeEnv->pTSBuf != NULL) {
- pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order;
+ SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order);
}
STsdbQueryCond cond = {
@@ -5101,7 +5167,7 @@ static void doSaveContext(SQInfo *pQInfo) {
setupQueryRangeForReverseScan(pQInfo);
pRuntimeEnv->prevGroupId = INT32_MIN;
- pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+ pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
if (pRuntimeEnv->pSecQueryHandle == NULL) {
longjmp(pRuntimeEnv->env, terrno);
}
@@ -5172,7 +5238,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
// query error occurred or query is killed, abort current execution
if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) {
qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code));
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
@@ -5194,7 +5259,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) {
qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code));
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
+ //TODO finalizeQueryResult may cause SEGSEV, since the memory may not allocated yet, add a cleanup function instead
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
@@ -5234,7 +5299,6 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
finalizeQueryResult(pRuntimeEnv);
if (IS_QUERY_KILLED(pQInfo)) {
- finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
@@ -5816,13 +5880,13 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
return TSDB_CODE_SUCCESS;
_cleanup:
- taosTFree(*pExpr);
+ tfree(*pExpr);
taosArrayDestroy(*pTableIdList);
*pTableIdList = NULL;
- taosTFree(*tbnameCond);
- taosTFree(*groupbyCols);
- taosTFree(*tagCols);
- taosTFree(*tagCond);
+ tfree(*tbnameCond);
+ tfree(*groupbyCols);
+ tfree(*tagCols);
+ tfree(*tagCond);
return code;
}
@@ -5873,7 +5937,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg);
if (code != TSDB_CODE_SUCCESS) {
- taosTFree(pExprs);
+ tfree(pExprs);
return code;
}
@@ -5912,7 +5976,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
int32_t param = (int32_t)pExprs[i].base.arg[0].argValue.i64;
if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].type, &pExprs[i].bytes,
&pExprs[i].interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) {
- taosTFree(pExprs);
+ tfree(pExprs);
return TSDB_CODE_QRY_INVALID_MSG;
}
@@ -6186,8 +6250,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
calResultBufSize(pQuery);
for (int32_t col = 0; col < pQuery->numOfOutput; ++col) {
- assert(pExprs[col].interBytes >= pExprs[col].bytes);
-
// allocate additional memory for interResults that are usually larger then final results
size_t size = (size_t)((pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interBytes + sizeof(tFilePage));
pQuery->sdata[col] = (tFilePage *)calloc(1, size);
@@ -6213,12 +6275,18 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
pQInfo->tableqinfoGroupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES);
pQInfo->tableqinfoGroupInfo.numOfTables = pTableGroupInfo->numOfTables;
pQInfo->tableqinfoGroupInfo.map = taosHashInit(pTableGroupInfo->numOfTables,
- taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
+ taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
}
int tableIndex = 0;
pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery);
+ pQInfo->runtimeEnv.summary.tableInfoSize += (pTableGroupInfo->numOfTables * sizeof(STableQueryInfo));
+
+ pQInfo->runtimeEnv.pResultRowHashTable = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+ pQInfo->runtimeEnv.keyBuf = malloc(TSDB_MAX_BYTES_PER_ROW);
+ pQInfo->runtimeEnv.pool = initResultRowPool(getWindowResultSize(&pQInfo->runtimeEnv));
+
pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo));
if (pQInfo->pBuf == NULL) {
goto _cleanup;
@@ -6253,7 +6321,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
for(int32_t j = 0; j < s; ++j) {
STableKeyInfo* info = taosArrayGet(pa, j);
- void* buf = (char*)pQInfo->pBuf + index * sizeof(STableQueryInfo);
+ void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo);
window.skey = info->lastKey;
STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf);
@@ -6284,7 +6352,7 @@ _cleanup_query:
free(pGroupbyExpr);
}
- taosTFree(pTagCols);
+ tfree(pTagCols);
for (int32_t i = 0; i < numOfOutput; ++i) {
SExprInfo* pExprInfo = &pExprs[i];
if (pExprInfo->pExpr != NULL) {
@@ -6292,7 +6360,7 @@ _cleanup_query:
}
}
- taosTFree(pExprs);
+ tfree(pExprs);
_cleanup:
freeQInfo(pQInfo);
@@ -6318,12 +6386,13 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
STSBuf *pTSBuf = NULL;
- if (pQueryMsg->tsLen > 0) { // open new file to save the result
+ if (pQueryMsg->tsLen > 0) { // open new file to save the result
char *tsBlock = (char *) pQueryMsg + pQueryMsg->tsOffset;
- pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder);
+ pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder, vgId);
tsBufResetPos(pTSBuf);
bool ret = tsBufNextPos(pTSBuf);
+
UNUSED(ret);
}
@@ -6402,25 +6471,27 @@ static void freeQInfo(SQInfo *pQInfo) {
qDebug("QInfo:%p start to free QInfo", pQInfo);
+ releaseQueryBuf(pQInfo->tableqinfoGroupInfo.numOfTables);
+
teardownQueryRuntimeEnv(&pQInfo->runtimeEnv);
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
if (pQuery != NULL) {
if (pQuery->sdata != NULL) {
for (int32_t col = 0; col < pQuery->numOfOutput; ++col) {
- taosTFree(pQuery->sdata[col]);
+ tfree(pQuery->sdata[col]);
}
- taosTFree(pQuery->sdata);
+ tfree(pQuery->sdata);
}
if (pQuery->fillVal != NULL) {
- taosTFree(pQuery->fillVal);
+ tfree(pQuery->fillVal);
}
for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) {
SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i];
if (pColFilter->numOfFilters > 0) {
- taosTFree(pColFilter->pFilters);
+ tfree(pColFilter->pFilters);
}
}
@@ -6433,31 +6504,31 @@ static void freeQInfo(SQInfo *pQInfo) {
}
}
- taosTFree(pQuery->pSelectExpr);
+ tfree(pQuery->pSelectExpr);
}
if (pQuery->pGroupbyExpr != NULL) {
taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo);
- taosTFree(pQuery->pGroupbyExpr);
+ tfree(pQuery->pGroupbyExpr);
}
- taosTFree(pQuery->tagColList);
- taosTFree(pQuery->pFilterInfo);
+ tfree(pQuery->tagColList);
+ tfree(pQuery->pFilterInfo);
if (pQuery->colList != NULL) {
for (int32_t i = 0; i < pQuery->numOfCols; i++) {
SColumnInfo *column = pQuery->colList + i;
freeColumnFilterInfo(column->filters, column->numOfFilters);
}
- taosTFree(pQuery->colList);
+ tfree(pQuery->colList);
}
- taosTFree(pQuery);
+ tfree(pQuery);
}
doDestroyTableQueryInfo(&pQInfo->tableqinfoGroupInfo);
- taosTFree(pQInfo->pBuf);
+ tfree(pQInfo->pBuf);
tsdbDestroyTableGroup(&pQInfo->tableGroupInfo);
taosArrayDestroy(pQInfo->arrTableIdInfo);
@@ -6465,7 +6536,7 @@ static void freeQInfo(SQInfo *pQInfo) {
qDebug("QInfo:%p QInfo is freed", pQInfo);
- taosTFree(pQInfo);
+ tfree(pQInfo);
}
static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) {
@@ -6636,6 +6707,11 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
assert(0);
}
+ code = checkForQueryBuf(tableGroupInfo.numOfTables);
+ if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort
+ goto _over;
+ }
+
(*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &tableGroupInfo, pTagColumnInfo, isSTableQuery);
pExprs = NULL;
pGroupbyExpr = NULL;
@@ -6777,7 +6853,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex
int32_t code = TSDB_CODE_SUCCESS;
-#if 0
+#if _NON_BLOCKING_RETRIEVE
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
pthread_mutex_lock(&pQInfo->lock);
@@ -6849,6 +6925,8 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
pQInfo->dataReady = QUERY_RESULT_NOT_READY;
if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
+ // here current thread hold the refcount, so it is safe to free tsdbQueryHandle.
+ doFreeQueryHandle(pQInfo);
*continueExec = false;
(*pRsp)->completed = 1; // notify no more result to client
} else {
@@ -7037,6 +7115,48 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
setQueryStatus(pQuery, QUERY_COMPLETED);
}
+static int64_t getQuerySupportBufSize(size_t numOfTables) {
+ size_t s1 = sizeof(STableQueryInfo);
+ size_t s2 = sizeof(SHashNode);
+
+// size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb
+ return (int64_t)((s1 + s2) * 1.5 * numOfTables);
+}
+
+int32_t checkForQueryBuf(size_t numOfTables) {
+ int64_t t = getQuerySupportBufSize(numOfTables);
+ if (tsQueryBufferSize < 0) {
+ return TSDB_CODE_SUCCESS;
+ } else if (tsQueryBufferSize > 0) {
+
+ while(1) {
+ int64_t s = tsQueryBufferSize;
+ int64_t remain = s - t;
+ if (remain >= 0) {
+ if (atomic_val_compare_exchange_64(&tsQueryBufferSize, s, remain) == s) {
+ return TSDB_CODE_SUCCESS;
+ }
+ } else {
+ return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER;
+ }
+ }
+ }
+
+ // disable query processing if the value of tsQueryBufferSize is zero.
+ return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER;
+}
+
+void releaseQueryBuf(size_t numOfTables) {
+ if (tsQueryBufferSize <= 0) {
+ return;
+ }
+
+ int64_t t = getQuerySupportBufSize(numOfTables);
+
+ // restore value is not enough buffer available
+ atomic_add_fetch_64(&tsQueryBufferSize, t);
+}
+
void* qGetResultRetrieveMsg(qinfo_t qinfo) {
SQInfo* pQInfo = (SQInfo*) qinfo;
assert(pQInfo != NULL);
@@ -7111,9 +7231,9 @@ void qCleanupQueryMgmt(void* pQMgmt) {
taosCacheCleanup(pqinfoPool);
pthread_mutex_destroy(&pQueryMgmt->lock);
- taosTFree(pQueryMgmt);
+ tfree(pQueryMgmt);
- qDebug("vgId:%d queryMgmt cleanup completed", vgId);
+ qDebug("vgId:%d, queryMgmt cleanup completed", vgId);
}
void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) {
diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c
index fc9c60b39b0cfa4b591cc77c1efcdac4e6647ce9..fa3fe285a857cc0d146e25f52710d0ebc0a69193 100644
--- a/src/query/src/qExtbuffer.c
+++ b/src/query/src/qExtbuffer.c
@@ -64,7 +64,7 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) {
// release flush out info link
SExtFileInfo *pFileMeta = &pMemBuffer->fileMeta;
if (pFileMeta->flushoutData.nAllocSize != 0 && pFileMeta->flushoutData.pFlushoutInfo != NULL) {
- taosTFree(pFileMeta->flushoutData.pFlushoutInfo);
+ tfree(pFileMeta->flushoutData.pFlushoutInfo);
}
// release all in-memory buffer pages
@@ -72,7 +72,7 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) {
while (pFilePages != NULL) {
tFilePagesItem *pTmp = pFilePages;
pFilePages = pFilePages->pNext;
- taosTFree(pTmp);
+ tfree(pTmp);
}
// close temp file
@@ -87,8 +87,8 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) {
destroyColumnModel(pMemBuffer->pColumnModel);
- taosTFree(pMemBuffer->path);
- taosTFree(pMemBuffer);
+ tfree(pMemBuffer->path);
+ tfree(pMemBuffer);
return NULL;
}
@@ -275,7 +275,7 @@ int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) {
tFilePagesItem *ptmp = first;
first = first->pNext;
- taosTFree(ptmp); // release all data in memory buffer
+ tfree(ptmp); // release all data in memory buffer
}
fflush(pMemBuffer->file); // flush to disk
@@ -300,7 +300,7 @@ void tExtMemBufferClear(tExtMemBuffer *pMemBuffer) {
while (first != NULL) {
tFilePagesItem *ptmp = first;
first = first->pNext;
- taosTFree(ptmp);
+ tfree(ptmp);
}
pMemBuffer->fileMeta.numOfElemsInFile = 0;
@@ -344,8 +344,6 @@ static FORCE_INLINE int32_t primaryKeyComparator(int64_t f1, int64_t f2, int32_t
return 0;
}
- assert(colIdx == 0);
-
if (tsOrder == TSDB_ORDER_DESC) { // primary column desc order
return (f1 < f2) ? 1 : -1;
} else { // asc
@@ -804,7 +802,7 @@ void destroyColumnModel(SColumnModel *pModel) {
return;
}
- taosTFree(pModel);
+ tfree(pModel);
}
static void printBinaryData(char *data, int32_t len) {
@@ -1089,5 +1087,5 @@ void tOrderDescDestroy(tOrderDescriptor *pDesc) {
}
destroyColumnModel(pDesc->pColumnModel);
- taosTFree(pDesc);
+ tfree(pDesc);
}
diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c
index f186726c0120f2e2e34580fec0da00c2083e5da9..33d627c83973cf40b378573e6aa9ddbf2be1a258 100644
--- a/src/query/src/qFill.c
+++ b/src/query/src/qFill.c
@@ -91,23 +91,23 @@ void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
pFillInfo->numOfTotal = 0;
}
-void* taosDestoryFillInfo(SFillInfo* pFillInfo) {
+void* taosDestroyFillInfo(SFillInfo* pFillInfo) {
if (pFillInfo == NULL) {
return NULL;
}
- taosTFree(pFillInfo->prevValues);
- taosTFree(pFillInfo->nextValues);
- taosTFree(pFillInfo->pTags);
+ tfree(pFillInfo->prevValues);
+ tfree(pFillInfo->nextValues);
+ tfree(pFillInfo->pTags);
for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
- taosTFree(pFillInfo->pData[i]);
+ tfree(pFillInfo->pData[i]);
}
- taosTFree(pFillInfo->pData);
- taosTFree(pFillInfo->pFillCol);
+ tfree(pFillInfo->pData);
+ tfree(pFillInfo->pFillCol);
- taosTFree(pFillInfo);
+ tfree(pFillInfo);
return NULL;
}
@@ -170,7 +170,7 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows
int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
TSKEY ekey1 = ekey;
- if (pFillInfo->order != TSDB_ORDER_ASC) {
+ if (!FILL_IS_ASC_FILL(pFillInfo)) {
pFillInfo->endKey = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
}
@@ -215,7 +215,7 @@ static double linearInterpolationImpl(double v1, double v2, double k1, double k2
return v1 + (v2 - v1) * (k - k1) / (k2 - k1);
}
-int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) {
+int32_t taosGetLinearInterpolationVal(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) {
switch (type) {
case TSDB_DATA_TYPE_INT: {
*(int32_t*)point->val = (int32_t)linearInterpolationImpl(*(int32_t*)point1->val, *(int32_t*)point2->val, (double)point1->key,
@@ -343,7 +343,7 @@ static void doFillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t* nu
point1 = (SPoint){.key = *(TSKEY*)(prevValues), .val = prevValues + pCol->col.offset};
point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->rowIdx * bytes};
point = (SPoint){.key = pFillInfo->start, .val = val1};
- taosDoLinearInterpolation(type, &point1, &point2, &point);
+ taosGetLinearInterpolationVal(type, &point1, &point2, &point);
}
setTagsValue(pFillInfo, data, *num);
@@ -496,7 +496,7 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu
pFillInfo->numOfRows = 0;
/* the raw data block is exhausted, next value does not exists */
- taosTFree(*nextValues);
+ tfree(*nextValues);
}
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c
index 703ee2c52129a067d1070cfbe567087ec48a7a6e..35e5906d1f797c2f53f3906bf93c6797ac766c3e 100644
--- a/src/query/src/qHistogram.c
+++ b/src/query/src/qHistogram.c
@@ -168,7 +168,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
(*pHisto)->numOfEntries += 1;
}
} else { /* insert a new slot */
- if ((*pHisto)->numOfElems > 1 && idx < (*pHisto)->numOfEntries) {
+ if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) {
if (idx > 0) {
assert((*pHisto)->elems[idx - 1].val <= val);
}
@@ -661,4 +661,4 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2
free(pHistoBins);
return pResHistogram;
-}
\ No newline at end of file
+}
diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c
index 7e8128f200658af282bab89bd73dc2c27ddb25c1..02a7012b0ef40d9b8635cb0bc502e791379581f7 100644
--- a/src/query/src/qParserImpl.c
+++ b/src/query/src/qParserImpl.c
@@ -571,7 +571,7 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) {
doDestroyQuerySql(pQuerySql);
}
- taosTFree(pClause->pClause);
+ tfree(pClause->pClause);
}
SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pStableName,
@@ -641,12 +641,12 @@ void SQLInfoDestroy(SSqlInfo *pInfo) {
tFieldListDestroy(pCreateTableInfo->colInfo.pTagColumns);
tVariantListDestroy(pCreateTableInfo->usingInfo.pTagVals);
- taosTFree(pInfo->pCreateTableInfo);
+ tfree(pInfo->pCreateTableInfo);
} else if (pInfo->type == TSDB_SQL_ALTER_TABLE) {
tVariantListDestroy(pInfo->pAlterInfo->varList);
tFieldListDestroy(pInfo->pAlterInfo->pAddColumns);
- taosTFree(pInfo->pAlterInfo);
+ tfree(pInfo->pAlterInfo);
} else {
if (pInfo->pDCLInfo != NULL && pInfo->pDCLInfo->nAlloc > 0) {
free(pInfo->pDCLInfo->a);
@@ -656,7 +656,7 @@ void SQLInfoDestroy(SSqlInfo *pInfo) {
tVariantListDestroy(pInfo->pDCLInfo->dbOpt.keep);
}
- taosTFree(pInfo->pDCLInfo);
+ tfree(pInfo->pDCLInfo);
}
}
@@ -872,5 +872,6 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) {
pDBInfo->quorum = -1;
pDBInfo->keep = NULL;
+ pDBInfo->update = -1;
memset(&pDBInfo->precision, 0, sizeof(SStrToken));
}
diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c
index ab9ffb7bcb458129b7f170e7020cee904d2dfda6..3bdc0d477f777faa71a8e5d2264021932dddeea4 100644
--- a/src/query/src/qPercentile.c
+++ b/src/query/src/qPercentile.c
@@ -361,8 +361,8 @@ void tMemBucketDestroy(tMemBucket *pBucket) {
}
destroyResultBuf(pBucket->pBuffer);
- taosTFree(pBucket->pSlots);
- taosTFree(pBucket);
+ tfree(pBucket->pSlots);
+ tfree(pBucket);
}
void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) {
@@ -680,7 +680,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
}
double val = (1 - fraction) * td + fraction * nd;
- taosTFree(buffer);
+ tfree(buffer);
return val;
} else { // incur a second round bucket split
diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c
index b3e97459d36c68d48c04447b159473bd179a5dbb..2645cff678b7fe91f62dda0f3fe65f3a968f9c11 100644
--- a/src/query/src/qResultbuf.c
+++ b/src/query/src/qResultbuf.c
@@ -267,7 +267,7 @@ static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) {
assert(d->pn == pn);
d->pn = NULL;
- taosTFree(pn);
+ tfree(pn);
bufPage = flushPageToDisk(pResultBuf, d);
}
@@ -407,18 +407,18 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
}
if (pResultBuf->file != NULL) {
- qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, inmem size:%dbytes, file size:%"PRId64" bytes",
- pResultBuf->handle, pResultBuf->totalBufSize, listNEles(pResultBuf->lruList) * pResultBuf->pageSize,
- pResultBuf->fileSize);
+ qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f",
+ pResultBuf->handle, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0,
+ pResultBuf->fileSize/1024.0);
fclose(pResultBuf->file);
} else {
- qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle,
- pResultBuf->totalBufSize);
+ qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, no file created", pResultBuf->handle,
+ pResultBuf->totalBufSize/1024.0);
}
unlink(pResultBuf->path);
- taosTFree(pResultBuf->path);
+ tfree(pResultBuf->path);
SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet);
while(taosHashIterNext(iter)) {
@@ -426,8 +426,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
size_t n = taosArrayGetSize(*p);
for(int32_t i = 0; i < n; ++i) {
SPageInfo* pi = taosArrayGetP(*p, i);
- taosTFree(pi->pData);
- taosTFree(pi);
+ tfree(pi->pData);
+ tfree(pi);
}
taosArrayDestroy(*p);
@@ -440,8 +440,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
taosHashCleanup(pResultBuf->groupSet);
taosHashCleanup(pResultBuf->all);
- taosTFree(pResultBuf->assistBuf);
- taosTFree(pResultBuf);
+ tfree(pResultBuf->assistBuf);
+ tfree(pResultBuf);
}
SPageInfo* getLastPageInfo(SIDList pList) {
diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c
index 0c9f92786fdf837e2453fe70f594c10e7a093421..98545c8ef3930587b6f7bd2c5b8d40ded0066d52 100644
--- a/src/query/src/qTokenizer.c
+++ b/src/query/src/qTokenizer.c
@@ -155,6 +155,7 @@ static SKeyword keywordTable[] = {
{"INSERT", TK_INSERT},
{"INTO", TK_INTO},
{"VALUES", TK_VALUES},
+ {"UPDATE", TK_UPDATE},
{"RESET", TK_RESET},
{"QUERY", TK_QUERY},
{"ADD", TK_ADD},
@@ -251,16 +252,16 @@ static const char isIdChar[] = {
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
};
-static void* KeywordHashTable = NULL;
+static void* keywordHashTable = NULL;
static void doInitKeywordsTable(void) {
int numOfEntries = tListLen(keywordTable);
- KeywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false);
+ keywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false);
for (int32_t i = 0; i < numOfEntries; i++) {
keywordTable[i].len = (uint8_t)strlen(keywordTable[i].name);
void* ptr = &keywordTable[i];
- taosHashPut(KeywordHashTable, keywordTable[i].name, keywordTable[i].len, (void*)&ptr, POINTER_BYTES);
+ taosHashPut(keywordHashTable, keywordTable[i].name, keywordTable[i].len, (void*)&ptr, POINTER_BYTES);
}
}
@@ -282,7 +283,7 @@ int tSQLKeywordCode(const char* z, int n) {
}
}
- SKeyword** pKey = (SKeyword**)taosHashGet(KeywordHashTable, key, n);
+ SKeyword** pKey = (SKeyword**)taosHashGet(keywordHashTable, key, n);
return (pKey != NULL)? (*pKey)->type:TK_ID;
}
@@ -660,5 +661,8 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn
bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); }
void taosCleanupKeywordsTable() {
- taosHashCleanup(KeywordHashTable);
-}
\ No newline at end of file
+ void* m = keywordHashTable;
+ if (m != NULL && atomic_val_compare_exchange_ptr(&keywordHashTable, m, 0) == m) {
+ taosHashCleanup(m);
+ }
+}
diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c
index b264f6cdc9d815a12cc5a3ab0e5c09c0d670bcdb..8a5a87baab5abbfd04f1c97dbe91a7226606263e 100644
--- a/src/query/src/qTsbuf.c
+++ b/src/query/src/qTsbuf.c
@@ -4,7 +4,7 @@
#include "tutil.h"
static int32_t getDataStartOffset();
-static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo);
+static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo);
static STSBuf* allocResForTSBuf(STSBuf* pTSBuf);
static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader);
@@ -32,7 +32,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
}
// update the header info
- STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = TSDB_ORDER_ASC};
+ STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = TSDB_ORDER_ASC};
STSBufUpdateHeader(pTSBuf, &header);
tsBufResetPos(pTSBuf);
@@ -75,9 +75,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
return NULL;
}
- if (header.numOfVnode > pTSBuf->numOfAlloc) {
- pTSBuf->numOfAlloc = header.numOfVnode;
- STSVnodeBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * pTSBuf->numOfAlloc);
+ if (header.numOfGroup > pTSBuf->numOfAlloc) {
+ pTSBuf->numOfAlloc = header.numOfGroup;
+ STSGroupBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc);
if (tmp == NULL) {
tsBufDestroy(pTSBuf);
return NULL;
@@ -86,7 +86,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
pTSBuf->pData = tmp;
}
- pTSBuf->numOfVnodes = header.numOfVnode;
+ pTSBuf->numOfGroups = header.numOfGroup;
// check the ts order
pTSBuf->tsOrder = header.tsOrder;
@@ -96,9 +96,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
return NULL;
}
- size_t infoSize = sizeof(STSVnodeBlockInfo) * pTSBuf->numOfVnodes;
+ size_t infoSize = sizeof(STSGroupBlockInfo) * pTSBuf->numOfGroups;
- STSVnodeBlockInfo* buf = (STSVnodeBlockInfo*)calloc(1, infoSize);
+ STSGroupBlockInfo* buf = (STSGroupBlockInfo*)calloc(1, infoSize);
if (buf == NULL) {
tsBufDestroy(pTSBuf);
return NULL;
@@ -109,9 +109,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
UNUSED(sz);
// the length value for each vnode is not kept in file, so does not set the length value
- for (int32_t i = 0; i < pTSBuf->numOfVnodes; ++i) {
- STSVnodeBlockInfoEx* pBlockList = &pTSBuf->pData[i];
- memcpy(&pBlockList->info, &buf[i], sizeof(STSVnodeBlockInfo));
+ for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) {
+ STSGroupBlockInfoEx* pBlockList = &pTSBuf->pData[i];
+ memcpy(&pBlockList->info, &buf[i], sizeof(STSGroupBlockInfo));
}
free(buf);
@@ -131,8 +131,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
pTSBuf->cur.order = TSDB_ORDER_ASC;
pTSBuf->autoDelete = autoDelete;
-// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfVnode:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f),
-// pTSBuf->fileSize, pTSBuf->numOfVnodes, pTSBuf->autoDelete);
+// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f),
+// pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete);
return pTSBuf;
}
@@ -142,11 +142,11 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
return NULL;
}
- taosTFree(pTSBuf->assistBuf);
- taosTFree(pTSBuf->tsData.rawBuf);
+ tfree(pTSBuf->assistBuf);
+ tfree(pTSBuf->tsData.rawBuf);
- taosTFree(pTSBuf->pData);
- taosTFree(pTSBuf->block.payload);
+ tfree(pTSBuf->pData);
+ tfree(pTSBuf->block.payload);
fclose(pTSBuf->f);
@@ -156,58 +156,59 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
} else {
// tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path);
}
-
+
+ tVariantDestroy(&pTSBuf->block.tag);
free(pTSBuf);
return NULL;
}
-static STSVnodeBlockInfoEx* tsBufGetLastVnodeInfo(STSBuf* pTSBuf) {
- int32_t last = pTSBuf->numOfVnodes - 1;
+static STSGroupBlockInfoEx* tsBufGetLastGroupInfo(STSBuf* pTSBuf) {
+ int32_t last = pTSBuf->numOfGroups - 1;
assert(last >= 0);
return &pTSBuf->pData[last];
}
-static STSVnodeBlockInfoEx* addOneVnodeInfo(STSBuf* pTSBuf, int32_t vnodeId) {
- if (pTSBuf->numOfAlloc <= pTSBuf->numOfVnodes) {
+static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) {
+ if (pTSBuf->numOfAlloc <= pTSBuf->numOfGroups) {
uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5);
assert((int32_t)newSize > pTSBuf->numOfAlloc);
- STSVnodeBlockInfoEx* tmp = (STSVnodeBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * newSize);
+ STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize);
if (tmp == NULL) {
return NULL;
}
pTSBuf->pData = tmp;
pTSBuf->numOfAlloc = newSize;
- memset(&pTSBuf->pData[pTSBuf->numOfVnodes], 0, sizeof(STSVnodeBlockInfoEx) * (newSize - pTSBuf->numOfVnodes));
+ memset(&pTSBuf->pData[pTSBuf->numOfGroups], 0, sizeof(STSGroupBlockInfoEx) * (newSize - pTSBuf->numOfGroups));
}
- if (pTSBuf->numOfVnodes > 0) {
- STSVnodeBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastVnodeInfo(pTSBuf);
+ if (pTSBuf->numOfGroups > 0) {
+ STSGroupBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf);
// update prev vnode length info in file
- TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, &pPrevBlockInfoEx->info);
+ TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pPrevBlockInfoEx->info);
}
// set initial value for vnode block
- STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfVnodes].info;
- pBlockInfo->vnode = vnodeId;
+ STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfGroups].info;
+ pBlockInfo->id = id;
pBlockInfo->offset = pTSBuf->fileSize;
assert(pBlockInfo->offset >= getDataStartOffset());
// update vnode info in file
- TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes, pBlockInfo);
+ TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups, pBlockInfo);
// add one vnode info
- pTSBuf->numOfVnodes += 1;
+ pTSBuf->numOfGroups += 1;
// update the header info
STSBufFileHeader header = {
- .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder};
+ .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder};
STSBufUpdateHeader(pTSBuf, &header);
- return tsBufGetLastVnodeInfo(pTSBuf);
+ return tsBufGetLastGroupInfo(pTSBuf);
}
static void shrinkBuffer(STSList* ptsData) {
@@ -218,6 +219,15 @@ static void shrinkBuffer(STSList* ptsData) {
}
}
+static int32_t getTagAreaLength(tVariant* pa) {
+ int32_t t = sizeof(pa->nLen) * 2 + sizeof(pa->nType);
+ if (pa->nType != TSDB_DATA_TYPE_NULL) {
+ t += pa->nLen;
+ }
+
+ return t;
+}
+
static void writeDataToDisk(STSBuf* pTSBuf) {
if (pTSBuf->tsData.len == 0) {
return;
@@ -243,28 +253,36 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
*/
int32_t metaLen = 0;
metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f);
- metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+ int32_t trueLen = pBlock->tag.nLen;
if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) {
+ metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
- metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f);
+ metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+ metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
+ } else {
+ trueLen = 0;
+ metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
}
fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
-
+
+ metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+ assert(metaLen == getTagAreaLength(&pBlock->tag));
+
int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen;
pTSBuf->fileSize += blockSize;
pTSBuf->tsData.len = 0;
- STSVnodeBlockInfoEx* pVnodeBlockInfoEx = tsBufGetLastVnodeInfo(pTSBuf);
+ STSGroupBlockInfoEx* pGroupBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf);
- pVnodeBlockInfoEx->info.compLen += blockSize;
- pVnodeBlockInfoEx->info.numOfBlocks += 1;
+ pGroupBlockInfoEx->info.compLen += blockSize;
+ pGroupBlockInfoEx->info.numOfBlocks += 1;
shrinkBuffer(&pTSBuf->tsData);
}
@@ -284,23 +302,28 @@ static void expandBuffer(STSList* ptsData, int32_t inputSize) {
STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
STSBlock* pBlock = &pTSBuf->block;
-
+
// clear the memory buffer
- void* tmp = pBlock->payload;
- memset(pBlock, 0, sizeof(STSBlock));
- pBlock->payload = tmp;
-
+ pBlock->compLen = 0;
+ pBlock->padding = 0;
+ pBlock->numOfElem = 0;
+
+ int32_t offset = -1;
+
if (order == TSDB_ORDER_DESC) {
/*
* set the right position for the reversed traverse, the reversed traverse is started from
* the end of each comp data block
*/
- int32_t ret = fseek(pTSBuf->f, -(int32_t)(sizeof(pBlock->padding)), SEEK_CUR);
- size_t sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
+ int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen));
+ int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR);
+ size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f);
+ sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
UNUSED(sz);
-
+
pBlock->compLen = pBlock->padding;
- int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
+
+ offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
ret = fseek(pTSBuf->f, -offset, SEEK_CUR);
UNUSED(ret);
}
@@ -319,7 +342,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
- sz = fread(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f);
+ sz = fread(&pBlock->tag.i64Key, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
}
sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
@@ -327,8 +350,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
UNUSED(sz);
sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
- UNUSED(sz);
-
+
if (decomp) {
pTSBuf->tsData.len =
tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf,
@@ -337,11 +359,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
// read the comp length at the length of comp block
sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
+ assert(pBlock->padding == pBlock->compLen);
+
+ int32_t n = 0;
+ sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
+ if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) {
+ assert(n == 0);
+ } else {
+ assert(n == pBlock->tag.nLen);
+ }
+
UNUSED(sz);
// for backwards traverse, set the start position at the end of previous block
if (order == TSDB_ORDER_DESC) {
- int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR);
UNUSED(r);
}
@@ -382,20 +413,20 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) {
return TSDB_CODE_SUCCESS;
}
-void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len) {
- STSVnodeBlockInfoEx* pBlockInfo = NULL;
+void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len) {
+ STSGroupBlockInfoEx* pBlockInfo = NULL;
STSList* ptsData = &pTSBuf->tsData;
- if (pTSBuf->numOfVnodes == 0 || tsBufGetLastVnodeInfo(pTSBuf)->info.vnode != vnodeId) {
+ if (pTSBuf->numOfGroups == 0 || tsBufGetLastGroupInfo(pTSBuf)->info.id != id) {
writeDataToDisk(pTSBuf);
shrinkBuffer(ptsData);
- pBlockInfo = addOneVnodeInfo(pTSBuf, vnodeId);
+ pBlockInfo = addOneGroupInfo(pTSBuf, id);
} else {
- pBlockInfo = tsBufGetLastVnodeInfo(pTSBuf);
+ pBlockInfo = tsBufGetLastGroupInfo(pTSBuf);
}
- assert(pBlockInfo->info.vnode == vnodeId);
+ assert(pBlockInfo->info.id == id);
if ((tVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) {
// new arrived data with different tags value, save current value into disk first
@@ -403,7 +434,7 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pDa
} else {
expandBuffer(ptsData, len);
}
-
+
tVariantAssign(&pTSBuf->block.tag, tag);
memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len);
@@ -433,23 +464,23 @@ void tsBufFlush(STSBuf* pTSBuf) {
writeDataToDisk(pTSBuf);
shrinkBuffer(&pTSBuf->tsData);
- STSVnodeBlockInfoEx* pBlockInfoEx = tsBufGetLastVnodeInfo(pTSBuf);
+ STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf);
// update prev vnode length info in file
- TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, &pBlockInfoEx->info);
+ TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pBlockInfoEx->info);
// save the ts order into header
STSBufFileHeader header = {
- .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder};
+ .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder};
STSBufUpdateHeader(pTSBuf, &header);
fsync(fileno(pTSBuf->f));
}
-static int32_t tsBufFindVnodeIndexFromId(STSVnodeBlockInfoEx* pVnodeInfoEx, int32_t numOfVnodes, int32_t vnodeId) {
+static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t numOfGroups, int32_t id) {
int32_t j = -1;
- for (int32_t i = 0; i < numOfVnodes; ++i) {
- if (pVnodeInfoEx[i].info.vnode == vnodeId) {
+ for (int32_t i = 0; i < numOfGroups; ++i) {
+ if (pGroupInfoEx[i].info.id == id) {
j = i;
break;
}
@@ -459,7 +490,7 @@ static int32_t tsBufFindVnodeIndexFromId(STSVnodeBlockInfoEx* pVnodeInfoEx, int3
}
// todo opt performance by cache blocks info
-static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int32_t blockIndex) {
+static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) {
if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
return -1;
}
@@ -478,7 +509,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int
if (pTSBuf->cur.order == TSDB_ORDER_DESC) {
STSBlock* pBlock = &pTSBuf->block;
int32_t compBlockSize =
- pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag);
+ pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR);
UNUSED(ret);
}
@@ -486,7 +517,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int
return 0;
}
-static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, tVariant* tag) {
+static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, tVariant* tag) {
bool decomp = false;
int64_t offset = 0;
@@ -506,21 +537,21 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo
}
if (tVariantCompare(&pTSBuf->block.tag, tag) == 0) {
- return i;
+ return (pTSBuf->cur.order == TSDB_ORDER_ASC)? i: (pBlockInfo->numOfBlocks - (i + 1));
}
}
return -1;
}
-static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex) {
- STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[vnodeIndex].info;
+static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex) {
+ STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info;
if (pBlockInfo->numOfBlocks <= blockIndex) {
assert(false);
}
STSCursor* pCur = &pTSBuf->cur;
- if (pCur->vgroupIndex == vnodeIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) ||
+ if (pCur->vgroupIndex == groupIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) ||
(pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) {
int32_t i = 0;
bool decomp = false;
@@ -555,14 +586,27 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex
assert((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len));
- pCur->vgroupIndex = vnodeIndex;
+ pCur->vgroupIndex = groupIndex;
pCur->blockIndex = blockIndex;
pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1;
}
-STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) {
- int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId);
+static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockInfo* pVInfo) {
+ if (offset < 0 || offset >= getDataStartOffset()) {
+ return -1;
+ }
+
+ if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) {
+ return -1;
+ }
+
+ fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f);
+ return 0;
+}
+
+STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) {
+ int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id);
if (j == -1) {
return NULL;
}
@@ -571,7 +615,7 @@ STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) {
}
int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
- if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfVnode == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
+ if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
return -1;
}
@@ -587,7 +631,7 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
}
bool tsBufNextPos(STSBuf* pTSBuf) {
- if (pTSBuf == NULL || pTSBuf->numOfVnodes == 0) {
+ if (pTSBuf == NULL || pTSBuf->numOfGroups == 0) {
return false;
}
@@ -606,16 +650,16 @@ bool tsBufNextPos(STSBuf* pTSBuf) {
}
} else { // get the last timestamp record in the last block of the last vnode
- assert(pTSBuf->numOfVnodes > 0);
+ assert(pTSBuf->numOfGroups > 0);
- int32_t vnodeIndex = pTSBuf->numOfVnodes - 1;
- pCur->vgroupIndex = vnodeIndex;
+ int32_t groupIndex = pTSBuf->numOfGroups - 1;
+ pCur->vgroupIndex = groupIndex;
- int32_t vnodeId = pTSBuf->pData[pCur->vgroupIndex].info.vnode;
- STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId);
+ int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id;
+ STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id);
int32_t blockIndex = pBlockInfo->numOfBlocks - 1;
- tsBufGetBlock(pTSBuf, vnodeIndex, blockIndex);
+ tsBufGetBlock(pTSBuf, groupIndex, blockIndex);
pCur->tsIndex = pTSBuf->block.numOfElem - 1;
if (pTSBuf->block.numOfElem == 0) {
@@ -634,12 +678,12 @@ bool tsBufNextPos(STSBuf* pTSBuf) {
if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) ||
(pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) {
- int32_t vnodeId = pTSBuf->pData[pCur->vgroupIndex].info.vnode;
+ int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id;
- STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId);
+ STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id);
if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) ||
(pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) {
- if ((pCur->vgroupIndex >= pTSBuf->numOfVnodes - 1 && pCur->order == TSDB_ORDER_ASC) ||
+ if ((pCur->vgroupIndex >= pTSBuf->numOfGroups - 1 && pCur->order == TSDB_ORDER_ASC) ||
(pCur->vgroupIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) {
pCur->vgroupIndex = -1;
return false;
@@ -649,7 +693,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) {
return false;
}
- int32_t blockIndex = pCur->order == TSDB_ORDER_ASC ? 0 : pBlockInfo->numOfBlocks - 1;
+ int32_t blockIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : (pBlockInfo->numOfBlocks - 1);
tsBufGetBlock(pTSBuf, pCur->vgroupIndex + step, blockIndex);
break;
@@ -675,8 +719,7 @@ void tsBufResetPos(STSBuf* pTSBuf) {
}
STSElem tsBufGetElem(STSBuf* pTSBuf) {
- STSElem elem1 = {.vnode = -1};
-
+ STSElem elem1 = {.id = -1};
if (pTSBuf == NULL) {
return elem1;
}
@@ -688,9 +731,9 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) {
STSBlock* pBlock = &pTSBuf->block;
- elem1.vnode = pTSBuf->pData[pCur->vgroupIndex].info.vnode;
+ elem1.id = pTSBuf->pData[pCur->vgroupIndex].info.id;
elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE);
- tVariantAssign(&elem1.tag, &pBlock->tag);
+ elem1.tag = &pBlock->tag;
return elem1;
}
@@ -699,35 +742,34 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) {
* current only support ts comp data from two vnode merge
* @param pDestBuf
* @param pSrcBuf
- * @param vnodeId
+ * @param id
* @return
*/
-int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) {
- if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfVnodes <= 0) {
+int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
+ if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfGroups <= 0) {
return 0;
}
- if (pDestBuf->numOfVnodes + pSrcBuf->numOfVnodes > TS_COMP_FILE_VNODE_MAX) {
+ if (pDestBuf->numOfGroups + pSrcBuf->numOfGroups > TS_COMP_FILE_GROUP_MAX) {
return -1;
}
// src can only have one vnode index
- if (pSrcBuf->numOfVnodes > 1) {
- return -1;
- }
-
+ assert(pSrcBuf->numOfGroups == 1);
+
// there are data in buffer, flush to disk first
tsBufFlush(pDestBuf);
// compared with the last vnode id
- if (vnodeId != tsBufGetLastVnodeInfo(pDestBuf)->info.vnode) {
- int32_t oldSize = pDestBuf->numOfVnodes;
- int32_t newSize = oldSize + pSrcBuf->numOfVnodes;
+ int32_t id = tsBufGetLastGroupInfo((STSBuf*) pSrcBuf)->info.id;
+ if (id != tsBufGetLastGroupInfo(pDestBuf)->info.id) {
+ int32_t oldSize = pDestBuf->numOfGroups;
+ int32_t newSize = oldSize + pSrcBuf->numOfGroups;
if (pDestBuf->numOfAlloc < newSize) {
pDestBuf->numOfAlloc = newSize;
- STSVnodeBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSVnodeBlockInfoEx) * newSize);
+ STSGroupBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize);
if (tmp == NULL) {
return -1;
}
@@ -736,23 +778,23 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) {
}
// directly copy the vnode index information
- memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfVnodes * sizeof(STSVnodeBlockInfoEx));
+ memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfGroups * sizeof(STSGroupBlockInfoEx));
// set the new offset value
- for (int32_t i = 0; i < pSrcBuf->numOfVnodes; ++i) {
- STSVnodeBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize];
+ for (int32_t i = 0; i < pSrcBuf->numOfGroups; ++i) {
+ STSGroupBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize];
pBlockInfoEx->info.offset = (pSrcBuf->pData[i].info.offset - getDataStartOffset()) + pDestBuf->fileSize;
- pBlockInfoEx->info.vnode = vnodeId;
+ pBlockInfoEx->info.id = id;
}
- pDestBuf->numOfVnodes = newSize;
+ pDestBuf->numOfGroups = newSize;
} else {
- STSVnodeBlockInfoEx* pBlockInfoEx = tsBufGetLastVnodeInfo(pDestBuf);
+ STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pDestBuf);
pBlockInfoEx->len += pSrcBuf->pData[0].len;
pBlockInfoEx->info.numOfBlocks += pSrcBuf->pData[0].info.numOfBlocks;
pBlockInfoEx->info.compLen += pSrcBuf->pData[0].info.compLen;
- pBlockInfoEx->info.vnode = vnodeId;
+ pBlockInfoEx->info.id = id;
}
int32_t r = fseek(pDestBuf->f, 0, SEEK_END);
@@ -761,7 +803,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) {
int64_t offset = getDataStartOffset();
int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset;
- ssize_t rc = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size);
+ int64_t rc = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size);
if (rc == -1) {
// tscError("failed to merge tsBuf from:%s to %s, reason:%s\n", pSrcBuf->path, pDestBuf->path, strerror(errno));
@@ -785,23 +827,23 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) {
assert(pDestBuf->fileSize == oldSize + size);
-// tscDebug("tsBuf merge success, %p, path:%s, fd:%d, file size:%d, numOfVnode:%d, autoDelete:%d", pDestBuf,
-// pDestBuf->path, fileno(pDestBuf->f), pDestBuf->fileSize, pDestBuf->numOfVnodes, pDestBuf->autoDelete);
+// tscDebug("tsBuf merge success, %p, path:%s, fd:%d, file size:%d, numOfGroups:%d, autoDelete:%d", pDestBuf,
+// pDestBuf->path, fileno(pDestBuf->f), pDestBuf->fileSize, pDestBuf->numOfGroups, pDestBuf->autoDelete);
return 0;
}
-STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order) {
+STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order, int32_t id) {
STSBuf* pTSBuf = tsBufCreate(true, order);
- STSVnodeBlockInfo* pBlockInfo = &(addOneVnodeInfo(pTSBuf, 0)->info);
+ STSGroupBlockInfo* pBlockInfo = &(addOneGroupInfo(pTSBuf, 0)->info);
pBlockInfo->numOfBlocks = numOfBlocks;
pBlockInfo->compLen = len;
pBlockInfo->offset = getDataStartOffset();
- pBlockInfo->vnode = 0;
+ pBlockInfo->id = id;
// update prev vnode length info in file
- TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, pBlockInfo);
+ TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo);
int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET);
UNUSED(ret);
@@ -813,7 +855,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);
STSBufFileHeader header = {
- .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder};
+ .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder};
STSBufUpdateHeader(pTSBuf, &header);
fsync(fileno(pTSBuf->f));
@@ -821,14 +863,14 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
return pTSBuf;
}
-STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) {
- STSElem elem = {.vnode = -1};
+STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag) {
+ STSElem elem = {.id = -1};
if (pTSBuf == NULL) {
return elem;
}
- int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId);
+ int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id);
if (j == -1) {
return elem;
}
@@ -837,7 +879,7 @@ STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) {
// tsBufDisplay(pTSBuf);
STSCursor* pCur = &pTSBuf->cur;
- STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[j].info;
+ STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[j].info;
int32_t blockIndex = tsBufFindBlockByTag(pTSBuf, pBlockInfo, tag);
if (blockIndex < 0) {
@@ -893,7 +935,7 @@ STSBuf* tsBufClone(STSBuf* pTSBuf) {
void tsBufDisplay(STSBuf* pTSBuf) {
printf("-------start of ts comp file-------\n");
- printf("number of vnode:%d\n", pTSBuf->numOfVnodes);
+ printf("number of vnode:%d\n", pTSBuf->numOfGroups);
int32_t old = pTSBuf->cur.order;
pTSBuf->cur.order = TSDB_ORDER_ASC;
@@ -902,8 +944,8 @@ void tsBufDisplay(STSBuf* pTSBuf) {
while (tsBufNextPos(pTSBuf)) {
STSElem elem = tsBufGetElem(pTSBuf);
- if (elem.tag.nType == TSDB_DATA_TYPE_BIGINT) {
- printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag.i64Key, elem.ts);
+ if (elem.tag->nType == TSDB_DATA_TYPE_BIGINT) {
+ printf("%d-%" PRId64 "-%" PRId64 "\n", elem.id, elem.tag->i64Key, elem.ts);
}
}
@@ -912,33 +954,20 @@ void tsBufDisplay(STSBuf* pTSBuf) {
}
static int32_t getDataStartOffset() {
- return sizeof(STSBufFileHeader) + TS_COMP_FILE_VNODE_MAX * sizeof(STSVnodeBlockInfo);
-}
-
-static int32_t doUpdateVnodeInfo(STSBuf* pTSBuf, int64_t offset, STSVnodeBlockInfo* pVInfo) {
- if (offset < 0 || offset >= getDataStartOffset()) {
- return -1;
- }
-
- if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) {
- return -1;
- }
-
- fwrite(pVInfo, sizeof(STSVnodeBlockInfo), 1, pTSBuf->f);
- return 0;
+ return sizeof(STSBufFileHeader) + TS_COMP_FILE_GROUP_MAX * sizeof(STSGroupBlockInfo);
}
// update prev vnode length info in file
-static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo) {
- int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSVnodeBlockInfo);
- doUpdateVnodeInfo(pTSBuf, offset, pBlockInfo);
+static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo) {
+ int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSGroupBlockInfo);
+ doUpdateGroupInfo(pTSBuf, offset, pBlockInfo);
}
static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) {
- const int32_t INITIAL_VNODEINFO_SIZE = 4;
+ const int32_t INITIAL_GROUPINFO_SIZE = 4;
- pTSBuf->numOfAlloc = INITIAL_VNODEINFO_SIZE;
- pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSVnodeBlockInfoEx));
+ pTSBuf->numOfAlloc = INITIAL_GROUPINFO_SIZE;
+ pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx));
if (pTSBuf->pData == NULL) {
tsBufDestroy(pTSBuf);
return NULL;
@@ -969,3 +998,72 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) {
pTSBuf->fileSize += getDataStartOffset();
return pTSBuf;
}
+
+int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf) {
+ if (pTSBuf == NULL) {
+ return 0;
+ }
+
+ return pTSBuf->numOfGroups;
+}
+
+void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) {
+ int32_t size = tsBufGetNumOfGroup(pTSBuf);
+ if (num != NULL) {
+ *num = size;
+ }
+
+ *id = NULL;
+ if (size == 0) {
+ return;
+ }
+
+ (*id) = malloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t));
+
+ for(int32_t i = 0; i < size; ++i) {
+ (*id)[i] = pTSBuf->pData[i].info.id;
+ }
+}
+
+int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, int32_t* len, int32_t* numOfBlocks) {
+ assert(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups);
+ STSGroupBlockInfo *pBlockInfo = &pTSBuf->pData[groupIndex].info;
+
+ *len = 0;
+ *numOfBlocks = 0;
+
+ if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
+ int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
+// qError("%p: fseek failed: %s", pSql, tstrerror(code));
+ return code;
+ }
+
+ size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f);
+ if (s != pBlockInfo->compLen) {
+ int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
+// tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
+ return code;
+ }
+
+ *len = pBlockInfo->compLen;
+ *numOfBlocks = pBlockInfo->numOfBlocks;
+
+ return TSDB_CODE_SUCCESS;
+}
+
+STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag) {
+ STSElem el = {.id = -1};
+
+ for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) {
+ el = tsBufGetElemStartPos(pTSBuf, pTSBuf->pData[i].info.id, pTag);
+ if (el.id == pTSBuf->pData[i].info.id) {
+ return el;
+ }
+ }
+
+ return el;
+}
+
+bool tsBufIsValidElem(STSElem* pElem) {
+ return pElem->id >= 0;
+}
diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c
index 2bd92c74a456c16fe946288968d25d797907a390..f057dc1a497f63b6a91aa69c83a531230c2ae8d4 100644
--- a/src/query/src/qUtil.c
+++ b/src/query/src/qUtil.c
@@ -14,8 +14,9 @@
*/
#include "os.h"
-#include "hash.h"
#include "taosmsg.h"
+#include "hash.h"
+
#include "qExecutor.h"
#include "qUtil.h"
@@ -26,75 +27,37 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) {
size += pQuery->pSelectExpr[i].interBytes;
}
- assert(size > 0);
+ assert(size >= 0);
return size;
}
-int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t size,
- int32_t threshold, int16_t type) {
+int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int32_t threshold, int16_t type) {
pWindowResInfo->capacity = size;
pWindowResInfo->threshold = threshold;
pWindowResInfo->type = type;
- _hash_fn_t fn = taosGetDefaultHashFunction(type);
- pWindowResInfo->hashList = taosHashInit(threshold, fn, true, false);
- if (pWindowResInfo->hashList == NULL) {
- return TSDB_CODE_QRY_OUT_OF_MEMORY;
- }
-
pWindowResInfo->curIndex = -1;
pWindowResInfo->size = 0;
pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
- SQueryCostInfo* pSummary = &pRuntimeEnv->summary;
-
- // use the pointer arraylist
- pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult));
+ pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, POINTER_BYTES);
if (pWindowResInfo->pResult == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
- pWindowResInfo->interval = pRuntimeEnv->pQuery->interval.interval;
-
- pSummary->internalSupSize += sizeof(SWindowResult) * threshold;
- pSummary->internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity;
- pSummary->numOfTimeWindows = threshold;
-
- for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
- int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize);
- if (code != TSDB_CODE_SUCCESS) {
- return code;
- }
- }
-
return TSDB_CODE_SUCCESS;
}
-void destroyTimeWindowRes(SWindowResult *pWindowRes) {
- if (pWindowRes == NULL) {
- return;
- }
-
- free(pWindowRes->resultInfo);
-}
-
void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) {
if (pWindowResInfo == NULL) {
return;
}
if (pWindowResInfo->capacity == 0) {
- assert(pWindowResInfo->hashList == NULL && pWindowResInfo->pResult == NULL);
+ assert(/*pWindowResInfo->hashList == NULL && */pWindowResInfo->pResult == NULL);
return;
}
- if (pWindowResInfo->pResult != NULL) {
- for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
- destroyTimeWindowRes(&pWindowResInfo->pResult[i]);
- }
- }
-
- taosHashCleanup(pWindowResInfo->hashList);
- taosTFree(pWindowResInfo->pResult);
+ tfree(pWindowResInfo->pResult);
}
void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo) {
@@ -103,17 +66,13 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR
}
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
- SWindowResult *pWindowRes = &pWindowResInfo->pResult[i];
- clearTimeWindowResBuf(pRuntimeEnv, pWindowRes);
+ SResultRow *pWindowRes = pWindowResInfo->pResult[i];
+ clearResultRow(pRuntimeEnv, pWindowRes);
}
pWindowResInfo->curIndex = -1;
- taosHashCleanup(pWindowResInfo->hashList);
pWindowResInfo->size = 0;
- _hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type);
- pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, true, false);
-
pWindowResInfo->startTime = TSKEY_INITIAL_VAL;
pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
}
@@ -127,13 +86,13 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo);
assert(num >= 0 && num <= numOfClosed);
- int16_t type = pWindowResInfo->type;
-
- char *key = NULL;
- int16_t bytes = -1;
+ int16_t type = pWindowResInfo->type;
+ STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); // uid is always set to be 0.
+ char *key = NULL;
+ int16_t bytes = -1;
for (int32_t i = 0; i < num; ++i) {
- SWindowResult *pResult = &pWindowResInfo->pResult[i];
+ SResultRow *pResult = pWindowResInfo->pResult[i];
if (pResult->closed) { // remove the window slot from hash table
// todo refactor
@@ -145,7 +104,8 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
}
- taosHashRemove(pWindowResInfo->hashList, (const char *)key, bytes);
+ SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
+ taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
} else {
break;
}
@@ -155,19 +115,19 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
// clear all the closed windows from the window list
for (int32_t k = 0; k < remain; ++k) {
- copyTimeWindowResBuf(pRuntimeEnv, &pWindowResInfo->pResult[k], &pWindowResInfo->pResult[num + k]);
+ copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k]);
}
// move the unclosed window in the front of the window list
for (int32_t k = remain; k < pWindowResInfo->size; ++k) {
- SWindowResult *pWindowRes = &pWindowResInfo->pResult[k];
- clearTimeWindowResBuf(pRuntimeEnv, pWindowRes);
+ SResultRow *pWindowRes = pWindowResInfo->pResult[k];
+ clearResultRow(pRuntimeEnv, pWindowRes);
}
pWindowResInfo->size = remain;
for (int32_t k = 0; k < pWindowResInfo->size; ++k) {
- SWindowResult *pResult = &pWindowResInfo->pResult[k];
+ SResultRow *pResult = pWindowResInfo->pResult[k];
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
key = varDataVal(pResult->key);
@@ -177,12 +137,15 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
}
- int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)key, bytes);
+ SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
+ int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
assert(p != NULL);
int32_t v = (*p - num);
assert(v >= 0 && v <= pWindowResInfo->size);
- taosHashPut(pWindowResInfo->hashList, (char *)key, bytes, (char *)&v, sizeof(int32_t));
+
+ SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
+ taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t));
}
pWindowResInfo->curIndex = -1;
@@ -200,7 +163,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) {
int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) {
int32_t i = 0;
- while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].closed) {
+ while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) {
++i;
}
@@ -211,11 +174,11 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) {
assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size);
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
- if (pWindowResInfo->pResult[i].closed) {
+ if (pWindowResInfo->pResult[i]->closed) {
continue;
}
- pWindowResInfo->pResult[i].closed = true;
+ pWindowResInfo->pResult[i]->closed = true;
}
}
@@ -231,19 +194,19 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_
}
// get the result order
- int32_t resultOrder = (pWindowResInfo->pResult[0].win.skey < pWindowResInfo->pResult[1].win.skey)? 1:-1;
+ int32_t resultOrder = (pWindowResInfo->pResult[0]->win.skey < pWindowResInfo->pResult[1]->win.skey)? 1:-1;
if (order != resultOrder) {
return;
}
int32_t i = 0;
if (order == QUERY_ASC_FORWARD_STEP) {
- TSKEY ekey = pWindowResInfo->pResult[i].win.ekey;
+ TSKEY ekey = pWindowResInfo->pResult[i]->win.ekey;
while (i < pWindowResInfo->size && (ekey < lastKey)) {
++i;
}
} else if (order == QUERY_DESC_FORWARD_STEP) {
- while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].win.skey > lastKey)) {
+ while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i]->win.skey > lastKey)) {
++i;
}
}
@@ -254,32 +217,36 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_
}
bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) {
- return (getWindowResult(pWindowResInfo, slot)->closed == true);
+ return (getResultRow(pWindowResInfo, slot)->closed == true);
}
void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) {
- getWindowResult(pWindowResInfo, slot)->closed = true;
+ getResultRow(pWindowResInfo, slot)->closed = true;
}
-void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) {
+void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) {
if (pWindowRes == NULL) {
return;
}
- tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
+ // the result does not put into the SDiskbasedResultBuf, ignore it.
+ if (pWindowRes->pageId >= 0) {
+ tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId);
- for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) {
- SResultInfo *pResultInfo = &pWindowRes->resultInfo[i];
-
- char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
- size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
- memset(s, 0, size);
-
- RESET_RESULT_INFO(pResultInfo);
+ for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) {
+ SResultRowCellInfo *pResultInfo = &pWindowRes->pCellInfo[i];
+
+ char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
+ size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
+ memset(s, 0, size);
+
+ RESET_RESULT_INFO(pResultInfo);
+ }
}
-
+
pWindowRes->numOfRows = 0;
- pWindowRes->pos = (SPosInfo){-1, -1};
+ pWindowRes->pageId = -1;
+ pWindowRes->rowId = -1;
pWindowRes->closed = false;
pWindowRes->win = TSWINDOW_INITIALIZER;
}
@@ -289,7 +256,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow
* since the attribute of "Pos" is bound to each window result when the window result is created in the
* disk-based result buffer.
*/
-void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) {
+void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src) {
dst->numOfRows = src->numOfRows;
dst->win = src->win;
dst->closed = src->closed;
@@ -297,25 +264,105 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con
int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput;
for (int32_t i = 0; i < nOutputCols; ++i) {
- SResultInfo *pDst = &dst->resultInfo[i];
- SResultInfo *pSrc = &src->resultInfo[i];
+ SResultRowCellInfo *pDst = getResultCell(pRuntimeEnv, dst, i);
+ SResultRowCellInfo *pSrc = getResultCell(pRuntimeEnv, src, i);
- char *buf = pDst->interResultBuf;
- memcpy(pDst, pSrc, sizeof(SResultInfo));
- pDst->interResultBuf = buf; // restore the allocated buffer
+// char *buf = pDst->interResultBuf;
+ memcpy(pDst, pSrc, sizeof(SResultRowCellInfo) + pRuntimeEnv->pCtx[i].interBufBytes);
+// pDst->interResultBuf = buf; // restore the allocated buffer
// copy the result info struct
- memcpy(pDst->interResultBuf, pSrc->interResultBuf, pDst->bufLen);
+// memcpy(pDst->interResultBuf, pSrc->interResultBuf, pRuntimeEnv->pCtx[i].interBufBytes);
// copy the output buffer data from src to dst, the position info keep unchanged
- tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pos.pageId);
+ tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pageId);
char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage);
- tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pos.pageId);
- char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src, srcpage);
+ tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pageId);
+ char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SResultRow *)src, srcpage);
size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
memcpy(dstBuf, srcBuf, s);
}
}
+SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index) {
+ assert(index >= 0 && index < pRuntimeEnv->pQuery->numOfOutput);
+ return (SResultRowCellInfo*)((char*) pRow->pCellInfo + pRuntimeEnv->rowCellInfoOffset[index]);
+}
+
+size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv) {
+ return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pRuntimeEnv->interBufSize + sizeof(SResultRow);
+}
+
+SResultRowPool* initResultRowPool(size_t size) {
+ SResultRowPool* p = calloc(1, sizeof(SResultRowPool));
+ if (p == NULL) {
+ return NULL;
+ }
+
+ p->numOfElemPerBlock = 128;
+
+ p->elemSize = (int32_t) size;
+ p->blockSize = p->numOfElemPerBlock * p->elemSize;
+ p->position.pos = 0;
+
+ p->pData = taosArrayInit(8, POINTER_BYTES);
+ return p;
+}
+
+SResultRow* getNewResultRow(SResultRowPool* p) {
+ if (p == NULL) {
+ return NULL;
+ }
+
+ void* ptr = NULL;
+ if (p->position.pos == 0) {
+ ptr = calloc(1, p->blockSize);
+ taosArrayPush(p->pData, &ptr);
+
+ } else {
+ size_t last = taosArrayGetSize(p->pData);
+
+ void** pBlock = taosArrayGet(p->pData, last - 1);
+ ptr = ((char*) (*pBlock)) + p->elemSize * p->position.pos;
+ }
+
+ p->position.pos = (p->position.pos + 1)%p->numOfElemPerBlock;
+ initResultRow(ptr);
+
+ return ptr;
+}
+
+int64_t getResultRowPoolMemSize(SResultRowPool* p) {
+ if (p == NULL) {
+ return 0;
+ }
+
+ return taosArrayGetSize(p->pData) * p->blockSize;
+}
+
+int32_t getNumOfAllocatedResultRows(SResultRowPool* p) {
+ return (int32_t) taosArrayGetSize(p->pData) * p->numOfElemPerBlock;
+}
+
+int32_t getNumOfUsedResultRows(SResultRowPool* p) {
+ return getNumOfAllocatedResultRows(p) - p->numOfElemPerBlock + p->position.pos;
+}
+
+void* destroyResultRowPool(SResultRowPool* p) {
+ if (p == NULL) {
+ return NULL;
+ }
+
+ size_t size = taosArrayGetSize(p->pData);
+ for(int32_t i = 0; i < size; ++i) {
+ void** ptr = taosArrayGet(p->pData, i);
+ tfree(*ptr);
+ }
+
+ taosArrayDestroy(p->pData);
+
+ tfree(p);
+ return NULL;
+}
diff --git a/src/query/src/sql.c b/src/query/src/sql.c
index a18efdeb744ba039c685d5fc1067c95e3ca15d86..93bb9b2ee75af6bc46f49ab7535fbf081fe6d8cd 100644
--- a/src/query/src/sql.c
+++ b/src/query/src/sql.c
@@ -97,27 +97,27 @@
#endif
/************* Begin control #defines *****************************************/
#define YYCODETYPE unsigned short int
-#define YYNOCODE 272
+#define YYNOCODE 274
#define YYACTIONTYPE unsigned short int
#define ParseTOKENTYPE SStrToken
typedef union {
int yyinit;
ParseTOKENTYPE yy0;
- SSubclauseInfo* yy25;
- tSQLExpr* yy66;
- SCreateAcctSQL yy73;
- int yy82;
- SQuerySQL* yy150;
- SCreateDBInfo yy158;
- TAOS_FIELD yy181;
- SLimitVal yy188;
- tSQLExprList* yy224;
- int64_t yy271;
- tVariant yy312;
- SIntervalVal yy314;
- SCreateTableSQL* yy374;
- tFieldList* yy449;
- tVariantList* yy494;
+ int yy46;
+ tSQLExpr* yy64;
+ tVariant yy134;
+ SCreateAcctSQL yy149;
+ int64_t yy207;
+ SLimitVal yy216;
+ TAOS_FIELD yy223;
+ SSubclauseInfo* yy231;
+ SCreateDBInfo yy268;
+ tSQLExprList* yy290;
+ SQuerySQL* yy414;
+ SCreateTableSQL* yy470;
+ tVariantList* yy498;
+ tFieldList* yy523;
+ SIntervalVal yy532;
} YYMINORTYPE;
#ifndef YYSTACKDEPTH
#define YYSTACKDEPTH 100
@@ -127,17 +127,17 @@ typedef union {
#define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo
#define ParseARG_STORE yypParser->pInfo = pInfo
#define YYFALLBACK 1
-#define YYNSTATE 252
-#define YYNRULE 230
-#define YYNTOKEN 206
-#define YY_MAX_SHIFT 251
-#define YY_MIN_SHIFTREDUCE 416
-#define YY_MAX_SHIFTREDUCE 645
-#define YY_ERROR_ACTION 646
-#define YY_ACCEPT_ACTION 647
-#define YY_NO_ACTION 648
-#define YY_MIN_REDUCE 649
-#define YY_MAX_REDUCE 878
+#define YYNSTATE 253
+#define YYNRULE 233
+#define YYNTOKEN 207
+#define YY_MAX_SHIFT 252
+#define YY_MIN_SHIFTREDUCE 420
+#define YY_MAX_SHIFTREDUCE 652
+#define YY_ERROR_ACTION 653
+#define YY_ACCEPT_ACTION 654
+#define YY_NO_ACTION 655
+#define YY_MIN_REDUCE 656
+#define YY_MAX_REDUCE 888
/************* End control #defines *******************************************/
/* Define the yytestcase() macro to be a no-op if is not already defined
@@ -203,223 +203,224 @@ typedef union {
** yy_default[] Default action for each state.
**
*********** Begin parsing tables **********************************************/
-#define YY_ACTTAB_COUNT (566)
+#define YY_ACTTAB_COUNT (571)
static const YYACTIONTYPE yy_action[] = {
- /* 0 */ 751, 459, 11, 749, 750, 647, 251, 459, 752, 460,
- /* 10 */ 754, 755, 753, 35, 36, 460, 37, 38, 159, 249,
- /* 20 */ 170, 29, 141, 459, 206, 41, 39, 43, 40, 140,
- /* 30 */ 145, 460, 865, 34, 33, 862, 141, 32, 31, 30,
- /* 40 */ 35, 36, 781, 37, 38, 165, 866, 170, 29, 141,
- /* 50 */ 62, 206, 41, 39, 43, 40, 191, 525, 164, 866,
- /* 60 */ 34, 33, 27, 21, 32, 31, 30, 417, 418, 419,
- /* 70 */ 420, 421, 422, 423, 424, 425, 426, 427, 428, 250,
- /* 80 */ 35, 36, 181, 37, 38, 227, 226, 170, 29, 781,
- /* 90 */ 176, 206, 41, 39, 43, 40, 174, 162, 767, 792,
- /* 100 */ 34, 33, 56, 160, 32, 31, 30, 21, 36, 8,
- /* 110 */ 37, 38, 63, 118, 170, 29, 770, 108, 206, 41,
- /* 120 */ 39, 43, 40, 32, 31, 30, 599, 34, 33, 78,
- /* 130 */ 875, 32, 31, 30, 238, 37, 38, 108, 238, 170,
- /* 140 */ 29, 184, 766, 206, 41, 39, 43, 40, 188, 187,
- /* 150 */ 789, 177, 34, 33, 224, 223, 32, 31, 30, 16,
- /* 160 */ 218, 244, 243, 217, 216, 215, 242, 214, 241, 240,
- /* 170 */ 239, 213, 747, 818, 735, 736, 737, 738, 739, 740,
- /* 180 */ 741, 742, 743, 744, 745, 746, 169, 612, 103, 12,
- /* 190 */ 603, 17, 606, 819, 609, 201, 169, 612, 26, 108,
- /* 200 */ 603, 108, 606, 861, 609, 153, 169, 612, 173, 567,
- /* 210 */ 603, 154, 606, 105, 609, 90, 89, 148, 166, 167,
- /* 220 */ 34, 33, 205, 102, 32, 31, 30, 770, 166, 167,
- /* 230 */ 26, 21, 557, 41, 39, 43, 40, 549, 166, 167,
- /* 240 */ 194, 34, 33, 17, 193, 32, 31, 30, 860, 16,
- /* 250 */ 26, 244, 243, 203, 21, 60, 242, 61, 241, 240,
- /* 260 */ 239, 248, 247, 96, 175, 229, 767, 76, 80, 245,
- /* 270 */ 190, 554, 21, 85, 88, 79, 18, 156, 121, 122,
- /* 280 */ 605, 82, 608, 42, 70, 66, 69, 225, 770, 767,
- /* 290 */ 135, 133, 601, 42, 611, 768, 93, 92, 91, 690,
- /* 300 */ 168, 207, 131, 42, 611, 230, 545, 767, 546, 610,
- /* 310 */ 699, 157, 691, 131, 611, 131, 604, 541, 607, 610,
- /* 320 */ 538, 571, 539, 47, 540, 46, 580, 581, 602, 610,
- /* 330 */ 572, 631, 613, 50, 14, 13, 13, 531, 543, 3,
- /* 340 */ 544, 46, 48, 530, 75, 74, 811, 22, 178, 179,
- /* 350 */ 51, 211, 10, 9, 829, 22, 87, 86, 101, 99,
- /* 360 */ 158, 143, 144, 146, 147, 151, 152, 150, 139, 149,
- /* 370 */ 769, 142, 828, 171, 825, 824, 172, 791, 761, 796,
- /* 380 */ 228, 783, 798, 104, 810, 119, 120, 701, 117, 212,
- /* 390 */ 615, 137, 24, 221, 698, 26, 222, 192, 874, 72,
- /* 400 */ 873, 871, 123, 719, 25, 100, 23, 138, 566, 688,
- /* 410 */ 81, 686, 83, 84, 684, 195, 780, 683, 161, 542,
- /* 420 */ 180, 199, 132, 681, 680, 679, 52, 49, 678, 677,
- /* 430 */ 109, 134, 44, 675, 204, 673, 671, 669, 667, 202,
- /* 440 */ 200, 198, 196, 28, 136, 220, 57, 58, 812, 77,
- /* 450 */ 231, 232, 233, 234, 235, 236, 237, 246, 209, 645,
- /* 460 */ 53, 182, 183, 644, 110, 64, 67, 155, 186, 185,
- /* 470 */ 682, 643, 94, 636, 676, 189, 126, 125, 720, 124,
- /* 480 */ 127, 128, 130, 129, 95, 668, 1, 551, 193, 765,
- /* 490 */ 2, 55, 113, 111, 114, 112, 115, 116, 59, 568,
- /* 500 */ 163, 106, 197, 5, 573, 107, 6, 65, 614, 19,
- /* 510 */ 4, 20, 15, 208, 616, 7, 210, 500, 496, 494,
- /* 520 */ 493, 492, 489, 463, 219, 68, 45, 71, 73, 22,
- /* 530 */ 527, 526, 524, 54, 484, 482, 474, 480, 476, 478,
- /* 540 */ 472, 470, 499, 498, 497, 495, 491, 490, 46, 461,
- /* 550 */ 432, 430, 649, 648, 648, 648, 648, 648, 648, 648,
- /* 560 */ 648, 648, 648, 648, 97, 98,
+ /* 0 */ 108, 463, 141, 11, 654, 252, 802, 463, 140, 464,
+ /* 10 */ 162, 165, 876, 35, 36, 464, 37, 38, 159, 250,
+ /* 20 */ 170, 29, 141, 463, 206, 41, 39, 43, 40, 173,
+ /* 30 */ 780, 464, 875, 34, 33, 145, 141, 32, 31, 30,
+ /* 40 */ 35, 36, 791, 37, 38, 164, 876, 170, 29, 780,
+ /* 50 */ 21, 206, 41, 39, 43, 40, 191, 829, 799, 201,
+ /* 60 */ 34, 33, 21, 21, 32, 31, 30, 421, 422, 423,
+ /* 70 */ 424, 425, 426, 427, 428, 429, 430, 431, 432, 251,
+ /* 80 */ 35, 36, 181, 37, 38, 532, 776, 170, 29, 238,
+ /* 90 */ 246, 206, 41, 39, 43, 40, 174, 175, 777, 777,
+ /* 100 */ 34, 33, 872, 56, 32, 31, 30, 176, 871, 36,
+ /* 110 */ 780, 37, 38, 227, 226, 170, 29, 791, 17, 206,
+ /* 120 */ 41, 39, 43, 40, 108, 26, 870, 606, 34, 33,
+ /* 130 */ 78, 160, 32, 31, 30, 238, 157, 16, 218, 245,
+ /* 140 */ 244, 217, 216, 215, 243, 214, 242, 241, 240, 213,
+ /* 150 */ 239, 755, 103, 743, 744, 745, 746, 747, 748, 749,
+ /* 160 */ 750, 751, 752, 753, 754, 756, 37, 38, 229, 177,
+ /* 170 */ 170, 29, 224, 223, 206, 41, 39, 43, 40, 203,
+ /* 180 */ 62, 60, 8, 34, 33, 63, 118, 32, 31, 30,
+ /* 190 */ 169, 619, 27, 12, 610, 184, 613, 158, 616, 778,
+ /* 200 */ 169, 619, 188, 187, 610, 194, 613, 108, 616, 153,
+ /* 210 */ 169, 619, 561, 108, 610, 154, 613, 18, 616, 90,
+ /* 220 */ 89, 148, 166, 167, 34, 33, 205, 143, 32, 31,
+ /* 230 */ 30, 697, 166, 167, 131, 144, 564, 41, 39, 43,
+ /* 240 */ 40, 706, 166, 167, 131, 34, 33, 146, 17, 32,
+ /* 250 */ 31, 30, 32, 31, 30, 26, 16, 207, 245, 244,
+ /* 260 */ 21, 587, 588, 243, 828, 242, 241, 240, 698, 239,
+ /* 270 */ 61, 131, 76, 80, 147, 190, 102, 151, 85, 88,
+ /* 280 */ 79, 760, 156, 26, 758, 759, 82, 21, 42, 761,
+ /* 290 */ 556, 763, 764, 762, 225, 765, 777, 193, 42, 618,
+ /* 300 */ 249, 248, 96, 574, 121, 122, 608, 105, 42, 618,
+ /* 310 */ 70, 66, 69, 578, 617, 168, 579, 46, 152, 618,
+ /* 320 */ 14, 230, 548, 777, 617, 545, 638, 546, 150, 547,
+ /* 330 */ 13, 135, 133, 612, 617, 615, 139, 93, 92, 91,
+ /* 340 */ 620, 611, 609, 614, 13, 47, 538, 622, 50, 552,
+ /* 350 */ 46, 553, 537, 178, 179, 3, 22, 211, 75, 74,
+ /* 360 */ 149, 22, 10, 9, 48, 51, 142, 550, 885, 551,
+ /* 370 */ 87, 86, 101, 99, 779, 839, 838, 171, 835, 834,
+ /* 380 */ 172, 801, 771, 228, 806, 793, 808, 104, 821, 119,
+ /* 390 */ 820, 117, 120, 708, 212, 137, 24, 221, 705, 222,
+ /* 400 */ 26, 192, 100, 884, 72, 883, 881, 123, 726, 25,
+ /* 410 */ 573, 23, 138, 695, 49, 81, 693, 83, 84, 691,
+ /* 420 */ 790, 690, 195, 161, 199, 549, 57, 52, 180, 132,
+ /* 430 */ 688, 687, 686, 685, 684, 134, 682, 109, 680, 678,
+ /* 440 */ 44, 676, 674, 136, 204, 202, 58, 822, 200, 198,
+ /* 450 */ 196, 220, 77, 28, 231, 232, 233, 235, 652, 234,
+ /* 460 */ 236, 237, 247, 209, 183, 53, 651, 182, 185, 186,
+ /* 470 */ 64, 67, 155, 650, 643, 189, 193, 689, 558, 94,
+ /* 480 */ 683, 675, 126, 125, 727, 129, 124, 127, 128, 95,
+ /* 490 */ 130, 1, 114, 110, 111, 775, 2, 55, 59, 116,
+ /* 500 */ 112, 113, 115, 575, 106, 163, 197, 5, 580, 107,
+ /* 510 */ 6, 65, 621, 19, 4, 20, 15, 208, 623, 7,
+ /* 520 */ 210, 504, 500, 498, 497, 496, 493, 467, 219, 68,
+ /* 530 */ 45, 71, 73, 22, 534, 533, 531, 488, 54, 486,
+ /* 540 */ 478, 484, 480, 482, 476, 474, 505, 503, 502, 501,
+ /* 550 */ 499, 495, 494, 46, 465, 436, 434, 656, 655, 655,
+ /* 560 */ 655, 655, 655, 655, 655, 655, 655, 655, 655, 97,
+ /* 570 */ 98,
};
static const YYCODETYPE yy_lookahead[] = {
- /* 0 */ 226, 1, 260, 229, 230, 207, 208, 1, 234, 9,
- /* 10 */ 236, 237, 238, 13, 14, 9, 16, 17, 209, 210,
- /* 20 */ 20, 21, 260, 1, 24, 25, 26, 27, 28, 260,
- /* 30 */ 260, 9, 270, 33, 34, 260, 260, 37, 38, 39,
- /* 40 */ 13, 14, 244, 16, 17, 269, 270, 20, 21, 260,
- /* 50 */ 247, 24, 25, 26, 27, 28, 258, 5, 269, 270,
- /* 60 */ 33, 34, 259, 210, 37, 38, 39, 45, 46, 47,
+ /* 0 */ 211, 1, 262, 262, 208, 209, 211, 1, 262, 9,
+ /* 10 */ 228, 271, 272, 13, 14, 9, 16, 17, 210, 211,
+ /* 20 */ 20, 21, 262, 1, 24, 25, 26, 27, 28, 228,
+ /* 30 */ 248, 9, 272, 33, 34, 262, 262, 37, 38, 39,
+ /* 40 */ 13, 14, 246, 16, 17, 271, 272, 20, 21, 248,
+ /* 50 */ 211, 24, 25, 26, 27, 28, 260, 268, 263, 270,
+ /* 60 */ 33, 34, 211, 211, 37, 38, 39, 45, 46, 47,
/* 70 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57,
- /* 80 */ 13, 14, 60, 16, 17, 33, 34, 20, 21, 244,
- /* 90 */ 66, 24, 25, 26, 27, 28, 243, 227, 245, 210,
- /* 100 */ 33, 34, 102, 258, 37, 38, 39, 210, 14, 98,
- /* 110 */ 16, 17, 101, 102, 20, 21, 246, 210, 24, 25,
- /* 120 */ 26, 27, 28, 37, 38, 39, 99, 33, 34, 73,
- /* 130 */ 246, 37, 38, 39, 78, 16, 17, 210, 78, 20,
- /* 140 */ 21, 126, 245, 24, 25, 26, 27, 28, 133, 134,
- /* 150 */ 261, 127, 33, 34, 130, 131, 37, 38, 39, 85,
- /* 160 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95,
- /* 170 */ 96, 97, 226, 266, 228, 229, 230, 231, 232, 233,
- /* 180 */ 234, 235, 236, 237, 238, 239, 1, 2, 210, 44,
- /* 190 */ 5, 98, 7, 266, 9, 268, 1, 2, 105, 210,
- /* 200 */ 5, 210, 7, 260, 9, 60, 1, 2, 227, 99,
- /* 210 */ 5, 66, 7, 103, 9, 70, 71, 72, 33, 34,
- /* 220 */ 33, 34, 37, 98, 37, 38, 39, 246, 33, 34,
- /* 230 */ 105, 210, 37, 25, 26, 27, 28, 99, 33, 34,
- /* 240 */ 262, 33, 34, 98, 106, 37, 38, 39, 260, 85,
- /* 250 */ 105, 87, 88, 264, 210, 266, 92, 266, 94, 95,
- /* 260 */ 96, 63, 64, 65, 243, 210, 245, 61, 62, 227,
- /* 270 */ 125, 103, 210, 67, 68, 69, 108, 132, 61, 62,
- /* 280 */ 5, 75, 7, 98, 67, 68, 69, 243, 246, 245,
- /* 290 */ 61, 62, 1, 98, 109, 240, 67, 68, 69, 214,
- /* 300 */ 59, 15, 217, 98, 109, 243, 5, 245, 7, 124,
- /* 310 */ 214, 260, 214, 217, 109, 217, 5, 2, 7, 124,
- /* 320 */ 5, 99, 7, 103, 9, 103, 115, 116, 37, 124,
- /* 330 */ 99, 99, 99, 103, 103, 103, 103, 99, 5, 98,
- /* 340 */ 7, 103, 122, 99, 128, 129, 267, 103, 33, 34,
- /* 350 */ 120, 99, 128, 129, 241, 103, 73, 74, 61, 62,
- /* 360 */ 260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
- /* 370 */ 246, 260, 241, 241, 241, 241, 241, 210, 242, 210,
- /* 380 */ 241, 244, 210, 210, 267, 210, 210, 210, 248, 210,
- /* 390 */ 104, 210, 210, 210, 210, 105, 210, 244, 210, 210,
- /* 400 */ 210, 210, 210, 210, 210, 59, 210, 210, 109, 210,
- /* 410 */ 210, 210, 210, 210, 210, 263, 257, 210, 263, 104,
- /* 420 */ 210, 263, 210, 210, 210, 210, 119, 121, 210, 210,
- /* 430 */ 256, 210, 118, 210, 113, 210, 210, 210, 210, 117,
- /* 440 */ 112, 111, 110, 123, 210, 76, 211, 211, 211, 84,
- /* 450 */ 83, 49, 80, 82, 53, 81, 79, 76, 211, 5,
- /* 460 */ 211, 135, 5, 5, 255, 215, 215, 211, 5, 135,
- /* 470 */ 211, 5, 212, 86, 211, 126, 219, 223, 225, 224,
- /* 480 */ 222, 220, 218, 221, 212, 211, 216, 99, 106, 244,
- /* 490 */ 213, 107, 252, 254, 251, 253, 250, 249, 103, 99,
- /* 500 */ 1, 98, 98, 114, 99, 98, 114, 73, 99, 103,
- /* 510 */ 98, 103, 98, 100, 104, 98, 100, 9, 5, 5,
- /* 520 */ 5, 5, 5, 77, 15, 73, 16, 129, 129, 103,
- /* 530 */ 5, 5, 99, 98, 5, 5, 5, 5, 5, 5,
- /* 540 */ 5, 5, 5, 5, 5, 5, 5, 5, 103, 77,
- /* 550 */ 59, 58, 0, 271, 271, 271, 271, 271, 271, 271,
- /* 560 */ 271, 271, 271, 271, 21, 21, 271, 271, 271, 271,
- /* 570 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 580 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 590 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 600 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 610 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 620 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 630 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 640 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 650 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 660 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 670 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 680 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 690 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 700 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 710 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 720 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 730 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 740 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 750 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 760 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271,
- /* 770 */ 271, 271,
+ /* 80 */ 13, 14, 60, 16, 17, 5, 247, 20, 21, 78,
+ /* 90 */ 228, 24, 25, 26, 27, 28, 245, 245, 247, 247,
+ /* 100 */ 33, 34, 262, 103, 37, 38, 39, 66, 262, 14,
+ /* 110 */ 248, 16, 17, 33, 34, 20, 21, 246, 99, 24,
+ /* 120 */ 25, 26, 27, 28, 211, 106, 262, 100, 33, 34,
+ /* 130 */ 73, 260, 37, 38, 39, 78, 262, 85, 86, 87,
+ /* 140 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 97,
+ /* 150 */ 98, 227, 211, 229, 230, 231, 232, 233, 234, 235,
+ /* 160 */ 236, 237, 238, 239, 240, 241, 16, 17, 211, 128,
+ /* 170 */ 20, 21, 131, 132, 24, 25, 26, 27, 28, 266,
+ /* 180 */ 249, 268, 99, 33, 34, 102, 103, 37, 38, 39,
+ /* 190 */ 1, 2, 261, 44, 5, 127, 7, 262, 9, 242,
+ /* 200 */ 1, 2, 134, 135, 5, 264, 7, 211, 9, 60,
+ /* 210 */ 1, 2, 104, 211, 5, 66, 7, 109, 9, 70,
+ /* 220 */ 71, 72, 33, 34, 33, 34, 37, 262, 37, 38,
+ /* 230 */ 39, 215, 33, 34, 218, 262, 37, 25, 26, 27,
+ /* 240 */ 28, 215, 33, 34, 218, 33, 34, 262, 99, 37,
+ /* 250 */ 38, 39, 37, 38, 39, 106, 85, 15, 87, 88,
+ /* 260 */ 211, 116, 117, 92, 268, 94, 95, 96, 215, 98,
+ /* 270 */ 268, 218, 61, 62, 262, 126, 99, 262, 67, 68,
+ /* 280 */ 69, 227, 133, 106, 230, 231, 75, 211, 99, 235,
+ /* 290 */ 100, 237, 238, 239, 245, 241, 247, 107, 99, 110,
+ /* 300 */ 63, 64, 65, 100, 61, 62, 1, 104, 99, 110,
+ /* 310 */ 67, 68, 69, 100, 125, 59, 100, 104, 262, 110,
+ /* 320 */ 104, 245, 2, 247, 125, 5, 100, 7, 262, 9,
+ /* 330 */ 104, 61, 62, 5, 125, 7, 262, 67, 68, 69,
+ /* 340 */ 100, 5, 37, 7, 104, 104, 100, 105, 104, 5,
+ /* 350 */ 104, 7, 100, 33, 34, 99, 104, 100, 129, 130,
+ /* 360 */ 262, 104, 129, 130, 123, 121, 262, 5, 248, 7,
+ /* 370 */ 73, 74, 61, 62, 248, 243, 243, 243, 243, 243,
+ /* 380 */ 243, 211, 244, 243, 211, 246, 211, 211, 269, 211,
+ /* 390 */ 269, 250, 211, 211, 211, 211, 211, 211, 211, 211,
+ /* 400 */ 106, 246, 59, 211, 211, 211, 211, 211, 211, 211,
+ /* 410 */ 110, 211, 211, 211, 122, 211, 211, 211, 211, 211,
+ /* 420 */ 259, 211, 265, 265, 265, 105, 212, 120, 211, 211,
+ /* 430 */ 211, 211, 211, 211, 211, 211, 211, 258, 211, 211,
+ /* 440 */ 119, 211, 211, 211, 114, 118, 212, 212, 113, 112,
+ /* 450 */ 111, 76, 84, 124, 83, 49, 80, 53, 5, 82,
+ /* 460 */ 81, 79, 76, 212, 5, 212, 5, 136, 136, 5,
+ /* 470 */ 216, 216, 212, 5, 86, 127, 107, 212, 100, 213,
+ /* 480 */ 212, 212, 220, 224, 226, 222, 225, 223, 221, 213,
+ /* 490 */ 219, 217, 253, 257, 256, 246, 214, 108, 104, 251,
+ /* 500 */ 255, 254, 252, 100, 99, 1, 99, 115, 100, 99,
+ /* 510 */ 115, 73, 100, 104, 99, 104, 99, 101, 105, 99,
+ /* 520 */ 101, 9, 5, 5, 5, 5, 5, 77, 15, 73,
+ /* 530 */ 16, 130, 130, 104, 5, 5, 100, 5, 99, 5,
+ /* 540 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
+ /* 550 */ 5, 5, 5, 104, 77, 59, 58, 0, 273, 273,
+ /* 560 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 21,
+ /* 570 */ 21, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 580 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 590 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 600 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 610 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 620 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 630 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 640 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 650 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 660 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 670 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 680 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 690 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 700 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 710 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 720 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 730 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 740 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 750 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 760 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273,
+ /* 770 */ 273, 273, 273, 273, 273, 273, 273, 273,
};
-#define YY_SHIFT_COUNT (251)
+#define YY_SHIFT_COUNT (252)
#define YY_SHIFT_MIN (0)
-#define YY_SHIFT_MAX (552)
+#define YY_SHIFT_MAX (557)
static const unsigned short int yy_shift_ofst[] = {
- /* 0 */ 145, 74, 164, 185, 205, 6, 6, 6, 6, 6,
- /* 10 */ 6, 0, 22, 205, 315, 315, 315, 93, 6, 6,
- /* 20 */ 6, 6, 6, 56, 60, 60, 566, 195, 205, 205,
- /* 30 */ 205, 205, 205, 205, 205, 205, 205, 205, 205, 205,
- /* 40 */ 205, 205, 205, 205, 205, 315, 315, 52, 52, 52,
- /* 50 */ 52, 52, 52, 11, 52, 125, 6, 6, 6, 6,
- /* 60 */ 211, 211, 168, 6, 6, 6, 6, 6, 6, 6,
+ /* 0 */ 149, 52, 171, 189, 209, 6, 6, 6, 6, 6,
+ /* 10 */ 6, 0, 22, 209, 320, 320, 320, 19, 6, 6,
+ /* 20 */ 6, 6, 6, 57, 11, 11, 571, 199, 209, 209,
+ /* 30 */ 209, 209, 209, 209, 209, 209, 209, 209, 209, 209,
+ /* 40 */ 209, 209, 209, 209, 209, 320, 320, 80, 80, 80,
+ /* 50 */ 80, 80, 80, 83, 80, 177, 6, 6, 6, 6,
+ /* 60 */ 145, 145, 108, 6, 6, 6, 6, 6, 6, 6,
/* 70 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
/* 80 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
/* 90 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
- /* 100 */ 6, 6, 290, 346, 346, 299, 299, 299, 346, 307,
- /* 110 */ 306, 314, 321, 322, 328, 330, 332, 320, 290, 346,
- /* 120 */ 346, 369, 369, 346, 365, 367, 402, 372, 371, 401,
- /* 130 */ 374, 377, 346, 381, 346, 381, 346, 566, 566, 27,
- /* 140 */ 67, 67, 67, 94, 119, 208, 208, 208, 206, 187,
- /* 150 */ 187, 187, 187, 217, 229, 24, 15, 86, 86, 198,
- /* 160 */ 138, 110, 222, 231, 232, 233, 275, 311, 291, 241,
- /* 170 */ 286, 220, 230, 238, 244, 252, 216, 224, 301, 333,
- /* 180 */ 283, 297, 454, 326, 457, 458, 334, 463, 466, 387,
- /* 190 */ 349, 382, 388, 384, 395, 400, 403, 499, 404, 405,
- /* 200 */ 407, 406, 389, 408, 392, 409, 412, 410, 414, 413,
- /* 210 */ 417, 416, 434, 508, 513, 514, 515, 516, 517, 446,
- /* 220 */ 509, 452, 510, 398, 399, 426, 525, 526, 433, 435,
- /* 230 */ 426, 529, 530, 531, 532, 533, 534, 535, 536, 537,
- /* 240 */ 538, 539, 540, 541, 542, 445, 472, 543, 544, 491,
- /* 250 */ 493, 552,
+ /* 100 */ 6, 6, 294, 343, 343, 300, 300, 300, 343, 307,
+ /* 110 */ 292, 321, 330, 327, 335, 337, 339, 329, 294, 343,
+ /* 120 */ 343, 375, 375, 343, 368, 371, 406, 376, 377, 404,
+ /* 130 */ 379, 382, 343, 386, 343, 386, 343, 571, 571, 27,
+ /* 140 */ 67, 67, 67, 95, 150, 212, 212, 212, 211, 191,
+ /* 150 */ 191, 191, 191, 243, 270, 41, 68, 215, 215, 237,
+ /* 160 */ 190, 203, 213, 216, 226, 240, 328, 336, 305, 256,
+ /* 170 */ 242, 241, 244, 246, 252, 257, 229, 233, 344, 362,
+ /* 180 */ 297, 311, 453, 331, 459, 461, 332, 464, 468, 388,
+ /* 190 */ 348, 369, 378, 389, 394, 403, 405, 504, 407, 408,
+ /* 200 */ 410, 409, 392, 411, 395, 412, 415, 413, 417, 416,
+ /* 210 */ 420, 419, 438, 512, 517, 518, 519, 520, 521, 450,
+ /* 220 */ 513, 456, 514, 401, 402, 429, 529, 530, 436, 439,
+ /* 230 */ 429, 532, 534, 535, 536, 537, 538, 539, 540, 541,
+ /* 240 */ 542, 543, 544, 545, 546, 547, 449, 477, 548, 549,
+ /* 250 */ 496, 498, 557,
};
#define YY_REDUCE_COUNT (138)
-#define YY_REDUCE_MIN (-258)
-#define YY_REDUCE_MAX (277)
+#define YY_REDUCE_MIN (-260)
+#define YY_REDUCE_MAX (282)
static const short yy_reduce_ofst[] = {
- /* 0 */ -202, -54, -226, -224, -211, -73, -11, -147, 21, 44,
- /* 10 */ 62, -111, -191, -238, -130, -19, 42, -155, -22, -93,
- /* 20 */ -9, 55, -103, 85, 96, 98, -197, -258, -231, -230,
- /* 30 */ -225, -57, -12, 51, 100, 101, 102, 103, 104, 105,
- /* 40 */ 106, 107, 108, 109, 111, -116, 124, 113, 131, 132,
- /* 50 */ 133, 134, 135, 136, 139, 137, 167, 169, 172, 173,
- /* 60 */ 79, 117, 140, 175, 176, 177, 179, 181, 182, 183,
- /* 70 */ 184, 186, 188, 189, 190, 191, 192, 193, 194, 196,
- /* 80 */ 197, 199, 200, 201, 202, 203, 204, 207, 210, 212,
- /* 90 */ 213, 214, 215, 218, 219, 221, 223, 225, 226, 227,
- /* 100 */ 228, 234, 153, 235, 236, 152, 155, 158, 237, 159,
- /* 110 */ 174, 209, 239, 242, 240, 243, 246, 248, 245, 247,
- /* 120 */ 249, 250, 251, 256, 253, 255, 254, 257, 258, 261,
- /* 130 */ 262, 264, 259, 260, 263, 272, 274, 270, 277,
+ /* 0 */ -204, -76, 54, -260, -226, -211, -87, -149, -148, 49,
+ /* 10 */ 76, -205, -192, -240, -218, -199, -138, -129, -59, -4,
+ /* 20 */ 2, -43, -161, 16, 26, 53, -69, -259, -254, -227,
+ /* 30 */ -160, -154, -136, -126, -65, -35, -27, -15, 12, 15,
+ /* 40 */ 56, 66, 74, 98, 104, 120, 126, 132, 133, 134,
+ /* 50 */ 135, 136, 137, 138, 140, 139, 170, 173, 175, 176,
+ /* 60 */ 119, 121, 141, 178, 181, 182, 183, 184, 185, 186,
+ /* 70 */ 187, 188, 192, 193, 194, 195, 196, 197, 198, 200,
+ /* 80 */ 201, 202, 204, 205, 206, 207, 208, 210, 217, 218,
+ /* 90 */ 219, 220, 221, 222, 223, 224, 225, 227, 228, 230,
+ /* 100 */ 231, 232, 155, 214, 234, 157, 158, 159, 235, 161,
+ /* 110 */ 179, 236, 238, 245, 247, 239, 250, 248, 249, 251,
+ /* 120 */ 253, 254, 255, 260, 258, 261, 259, 262, 264, 267,
+ /* 130 */ 263, 271, 265, 266, 268, 276, 269, 274, 282,
};
static const YYACTIONTYPE yy_default[] = {
- /* 0 */ 646, 700, 689, 868, 868, 646, 646, 646, 646, 646,
- /* 10 */ 646, 793, 664, 868, 646, 646, 646, 646, 646, 646,
- /* 20 */ 646, 646, 646, 702, 702, 702, 788, 646, 646, 646,
- /* 30 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
- /* 40 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
- /* 50 */ 646, 646, 646, 646, 646, 646, 646, 795, 797, 646,
- /* 60 */ 815, 815, 786, 646, 646, 646, 646, 646, 646, 646,
- /* 70 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
- /* 80 */ 646, 687, 646, 685, 646, 646, 646, 646, 646, 646,
- /* 90 */ 646, 646, 646, 646, 646, 646, 674, 646, 646, 646,
- /* 100 */ 646, 646, 646, 666, 666, 646, 646, 646, 666, 822,
- /* 110 */ 826, 820, 808, 816, 807, 803, 802, 830, 646, 666,
- /* 120 */ 666, 697, 697, 666, 718, 716, 714, 706, 712, 708,
- /* 130 */ 710, 704, 666, 695, 666, 695, 666, 734, 748, 646,
- /* 140 */ 831, 867, 821, 857, 856, 863, 855, 854, 646, 850,
- /* 150 */ 851, 853, 852, 646, 646, 646, 646, 859, 858, 646,
- /* 160 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 833,
- /* 170 */ 646, 827, 823, 646, 646, 646, 646, 646, 646, 646,
- /* 180 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 646,
- /* 190 */ 646, 785, 646, 646, 794, 646, 646, 646, 646, 646,
- /* 200 */ 646, 817, 646, 809, 646, 646, 646, 646, 646, 646,
- /* 210 */ 646, 762, 646, 646, 646, 646, 646, 646, 646, 646,
- /* 220 */ 646, 646, 646, 646, 646, 872, 646, 646, 646, 756,
- /* 230 */ 870, 646, 646, 646, 646, 646, 646, 646, 646, 646,
- /* 240 */ 646, 646, 646, 646, 646, 721, 646, 672, 670, 646,
- /* 250 */ 662, 646,
+ /* 0 */ 653, 707, 696, 878, 878, 653, 653, 653, 653, 653,
+ /* 10 */ 653, 803, 671, 878, 653, 653, 653, 653, 653, 653,
+ /* 20 */ 653, 653, 653, 709, 709, 709, 798, 653, 653, 653,
+ /* 30 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653,
+ /* 40 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653,
+ /* 50 */ 653, 653, 653, 653, 653, 653, 653, 805, 807, 653,
+ /* 60 */ 825, 825, 796, 653, 653, 653, 653, 653, 653, 653,
+ /* 70 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653,
+ /* 80 */ 653, 694, 653, 692, 653, 653, 653, 653, 653, 653,
+ /* 90 */ 653, 653, 653, 653, 653, 653, 681, 653, 653, 653,
+ /* 100 */ 653, 653, 653, 673, 673, 653, 653, 653, 673, 832,
+ /* 110 */ 836, 830, 818, 826, 817, 813, 812, 840, 653, 673,
+ /* 120 */ 673, 704, 704, 673, 725, 723, 721, 713, 719, 715,
+ /* 130 */ 717, 711, 673, 702, 673, 702, 673, 742, 757, 653,
+ /* 140 */ 841, 877, 831, 867, 866, 873, 865, 864, 653, 860,
+ /* 150 */ 861, 863, 862, 653, 653, 653, 653, 869, 868, 653,
+ /* 160 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 843,
+ /* 170 */ 653, 837, 833, 653, 653, 653, 653, 653, 653, 653,
+ /* 180 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653,
+ /* 190 */ 653, 795, 653, 653, 804, 653, 653, 653, 653, 653,
+ /* 200 */ 653, 827, 653, 819, 653, 653, 653, 653, 653, 653,
+ /* 210 */ 653, 772, 653, 653, 653, 653, 653, 653, 653, 653,
+ /* 220 */ 653, 653, 653, 653, 653, 882, 653, 653, 653, 766,
+ /* 230 */ 880, 653, 653, 653, 653, 653, 653, 653, 653, 653,
+ /* 240 */ 653, 653, 653, 653, 653, 653, 728, 653, 679, 677,
+ /* 250 */ 653, 669, 653,
};
/********** End of lemon-generated parsing tables *****************************/
@@ -537,6 +538,7 @@ static const YYCODETYPE yyFallback[] = {
0, /* FSYNC => nothing */
0, /* COMP => nothing */
0, /* PRECISION => nothing */
+ 0, /* UPDATE => nothing */
0, /* LP => nothing */
0, /* RP => nothing */
0, /* TAGS => nothing */
@@ -829,179 +831,181 @@ static const char *const yyTokenName[] = {
/* 95 */ "FSYNC",
/* 96 */ "COMP",
/* 97 */ "PRECISION",
- /* 98 */ "LP",
- /* 99 */ "RP",
- /* 100 */ "TAGS",
- /* 101 */ "USING",
- /* 102 */ "AS",
- /* 103 */ "COMMA",
- /* 104 */ "NULL",
- /* 105 */ "SELECT",
- /* 106 */ "UNION",
- /* 107 */ "ALL",
- /* 108 */ "FROM",
- /* 109 */ "VARIABLE",
- /* 110 */ "INTERVAL",
- /* 111 */ "FILL",
- /* 112 */ "SLIDING",
- /* 113 */ "ORDER",
- /* 114 */ "BY",
- /* 115 */ "ASC",
- /* 116 */ "DESC",
- /* 117 */ "GROUP",
- /* 118 */ "HAVING",
- /* 119 */ "LIMIT",
- /* 120 */ "OFFSET",
- /* 121 */ "SLIMIT",
- /* 122 */ "SOFFSET",
- /* 123 */ "WHERE",
- /* 124 */ "NOW",
- /* 125 */ "RESET",
- /* 126 */ "QUERY",
- /* 127 */ "ADD",
- /* 128 */ "COLUMN",
- /* 129 */ "TAG",
- /* 130 */ "CHANGE",
- /* 131 */ "SET",
- /* 132 */ "KILL",
- /* 133 */ "CONNECTION",
- /* 134 */ "STREAM",
- /* 135 */ "COLON",
- /* 136 */ "ABORT",
- /* 137 */ "AFTER",
- /* 138 */ "ATTACH",
- /* 139 */ "BEFORE",
- /* 140 */ "BEGIN",
- /* 141 */ "CASCADE",
- /* 142 */ "CLUSTER",
- /* 143 */ "CONFLICT",
- /* 144 */ "COPY",
- /* 145 */ "DEFERRED",
- /* 146 */ "DELIMITERS",
- /* 147 */ "DETACH",
- /* 148 */ "EACH",
- /* 149 */ "END",
- /* 150 */ "EXPLAIN",
- /* 151 */ "FAIL",
- /* 152 */ "FOR",
- /* 153 */ "IGNORE",
- /* 154 */ "IMMEDIATE",
- /* 155 */ "INITIALLY",
- /* 156 */ "INSTEAD",
- /* 157 */ "MATCH",
- /* 158 */ "KEY",
- /* 159 */ "OF",
- /* 160 */ "RAISE",
- /* 161 */ "REPLACE",
- /* 162 */ "RESTRICT",
- /* 163 */ "ROW",
- /* 164 */ "STATEMENT",
- /* 165 */ "TRIGGER",
- /* 166 */ "VIEW",
- /* 167 */ "COUNT",
- /* 168 */ "SUM",
- /* 169 */ "AVG",
- /* 170 */ "MIN",
- /* 171 */ "MAX",
- /* 172 */ "FIRST",
- /* 173 */ "LAST",
- /* 174 */ "TOP",
- /* 175 */ "BOTTOM",
- /* 176 */ "STDDEV",
- /* 177 */ "PERCENTILE",
- /* 178 */ "APERCENTILE",
- /* 179 */ "LEASTSQUARES",
- /* 180 */ "HISTOGRAM",
- /* 181 */ "DIFF",
- /* 182 */ "SPREAD",
- /* 183 */ "TWA",
- /* 184 */ "INTERP",
- /* 185 */ "LAST_ROW",
- /* 186 */ "RATE",
- /* 187 */ "IRATE",
- /* 188 */ "SUM_RATE",
- /* 189 */ "SUM_IRATE",
- /* 190 */ "AVG_RATE",
- /* 191 */ "AVG_IRATE",
- /* 192 */ "TBID",
- /* 193 */ "SEMI",
- /* 194 */ "NONE",
- /* 195 */ "PREV",
- /* 196 */ "LINEAR",
- /* 197 */ "IMPORT",
- /* 198 */ "METRIC",
- /* 199 */ "TBNAME",
- /* 200 */ "JOIN",
- /* 201 */ "METRICS",
- /* 202 */ "STABLE",
- /* 203 */ "INSERT",
- /* 204 */ "INTO",
- /* 205 */ "VALUES",
- /* 206 */ "error",
- /* 207 */ "program",
- /* 208 */ "cmd",
- /* 209 */ "dbPrefix",
- /* 210 */ "ids",
- /* 211 */ "cpxName",
- /* 212 */ "ifexists",
- /* 213 */ "alter_db_optr",
- /* 214 */ "acct_optr",
- /* 215 */ "ifnotexists",
- /* 216 */ "db_optr",
- /* 217 */ "pps",
- /* 218 */ "tseries",
- /* 219 */ "dbs",
- /* 220 */ "streams",
- /* 221 */ "storage",
- /* 222 */ "qtime",
- /* 223 */ "users",
- /* 224 */ "conns",
- /* 225 */ "state",
- /* 226 */ "keep",
- /* 227 */ "tagitemlist",
- /* 228 */ "cache",
- /* 229 */ "replica",
- /* 230 */ "quorum",
- /* 231 */ "days",
- /* 232 */ "minrows",
- /* 233 */ "maxrows",
- /* 234 */ "blocks",
- /* 235 */ "ctime",
- /* 236 */ "wal",
- /* 237 */ "fsync",
- /* 238 */ "comp",
- /* 239 */ "prec",
- /* 240 */ "typename",
- /* 241 */ "signed",
- /* 242 */ "create_table_args",
- /* 243 */ "columnlist",
- /* 244 */ "select",
- /* 245 */ "column",
- /* 246 */ "tagitem",
- /* 247 */ "selcollist",
- /* 248 */ "from",
- /* 249 */ "where_opt",
- /* 250 */ "interval_opt",
- /* 251 */ "fill_opt",
- /* 252 */ "sliding_opt",
- /* 253 */ "groupby_opt",
- /* 254 */ "orderby_opt",
- /* 255 */ "having_opt",
- /* 256 */ "slimit_opt",
- /* 257 */ "limit_opt",
- /* 258 */ "union",
- /* 259 */ "sclp",
- /* 260 */ "expr",
- /* 261 */ "as",
- /* 262 */ "tablelist",
- /* 263 */ "tmvar",
- /* 264 */ "sortlist",
- /* 265 */ "sortitem",
- /* 266 */ "item",
- /* 267 */ "sortorder",
- /* 268 */ "grouplist",
- /* 269 */ "exprlist",
- /* 270 */ "expritem",
+ /* 98 */ "UPDATE",
+ /* 99 */ "LP",
+ /* 100 */ "RP",
+ /* 101 */ "TAGS",
+ /* 102 */ "USING",
+ /* 103 */ "AS",
+ /* 104 */ "COMMA",
+ /* 105 */ "NULL",
+ /* 106 */ "SELECT",
+ /* 107 */ "UNION",
+ /* 108 */ "ALL",
+ /* 109 */ "FROM",
+ /* 110 */ "VARIABLE",
+ /* 111 */ "INTERVAL",
+ /* 112 */ "FILL",
+ /* 113 */ "SLIDING",
+ /* 114 */ "ORDER",
+ /* 115 */ "BY",
+ /* 116 */ "ASC",
+ /* 117 */ "DESC",
+ /* 118 */ "GROUP",
+ /* 119 */ "HAVING",
+ /* 120 */ "LIMIT",
+ /* 121 */ "OFFSET",
+ /* 122 */ "SLIMIT",
+ /* 123 */ "SOFFSET",
+ /* 124 */ "WHERE",
+ /* 125 */ "NOW",
+ /* 126 */ "RESET",
+ /* 127 */ "QUERY",
+ /* 128 */ "ADD",
+ /* 129 */ "COLUMN",
+ /* 130 */ "TAG",
+ /* 131 */ "CHANGE",
+ /* 132 */ "SET",
+ /* 133 */ "KILL",
+ /* 134 */ "CONNECTION",
+ /* 135 */ "STREAM",
+ /* 136 */ "COLON",
+ /* 137 */ "ABORT",
+ /* 138 */ "AFTER",
+ /* 139 */ "ATTACH",
+ /* 140 */ "BEFORE",
+ /* 141 */ "BEGIN",
+ /* 142 */ "CASCADE",
+ /* 143 */ "CLUSTER",
+ /* 144 */ "CONFLICT",
+ /* 145 */ "COPY",
+ /* 146 */ "DEFERRED",
+ /* 147 */ "DELIMITERS",
+ /* 148 */ "DETACH",
+ /* 149 */ "EACH",
+ /* 150 */ "END",
+ /* 151 */ "EXPLAIN",
+ /* 152 */ "FAIL",
+ /* 153 */ "FOR",
+ /* 154 */ "IGNORE",
+ /* 155 */ "IMMEDIATE",
+ /* 156 */ "INITIALLY",
+ /* 157 */ "INSTEAD",
+ /* 158 */ "MATCH",
+ /* 159 */ "KEY",
+ /* 160 */ "OF",
+ /* 161 */ "RAISE",
+ /* 162 */ "REPLACE",
+ /* 163 */ "RESTRICT",
+ /* 164 */ "ROW",
+ /* 165 */ "STATEMENT",
+ /* 166 */ "TRIGGER",
+ /* 167 */ "VIEW",
+ /* 168 */ "COUNT",
+ /* 169 */ "SUM",
+ /* 170 */ "AVG",
+ /* 171 */ "MIN",
+ /* 172 */ "MAX",
+ /* 173 */ "FIRST",
+ /* 174 */ "LAST",
+ /* 175 */ "TOP",
+ /* 176 */ "BOTTOM",
+ /* 177 */ "STDDEV",
+ /* 178 */ "PERCENTILE",
+ /* 179 */ "APERCENTILE",
+ /* 180 */ "LEASTSQUARES",
+ /* 181 */ "HISTOGRAM",
+ /* 182 */ "DIFF",
+ /* 183 */ "SPREAD",
+ /* 184 */ "TWA",
+ /* 185 */ "INTERP",
+ /* 186 */ "LAST_ROW",
+ /* 187 */ "RATE",
+ /* 188 */ "IRATE",
+ /* 189 */ "SUM_RATE",
+ /* 190 */ "SUM_IRATE",
+ /* 191 */ "AVG_RATE",
+ /* 192 */ "AVG_IRATE",
+ /* 193 */ "TBID",
+ /* 194 */ "SEMI",
+ /* 195 */ "NONE",
+ /* 196 */ "PREV",
+ /* 197 */ "LINEAR",
+ /* 198 */ "IMPORT",
+ /* 199 */ "METRIC",
+ /* 200 */ "TBNAME",
+ /* 201 */ "JOIN",
+ /* 202 */ "METRICS",
+ /* 203 */ "STABLE",
+ /* 204 */ "INSERT",
+ /* 205 */ "INTO",
+ /* 206 */ "VALUES",
+ /* 207 */ "error",
+ /* 208 */ "program",
+ /* 209 */ "cmd",
+ /* 210 */ "dbPrefix",
+ /* 211 */ "ids",
+ /* 212 */ "cpxName",
+ /* 213 */ "ifexists",
+ /* 214 */ "alter_db_optr",
+ /* 215 */ "acct_optr",
+ /* 216 */ "ifnotexists",
+ /* 217 */ "db_optr",
+ /* 218 */ "pps",
+ /* 219 */ "tseries",
+ /* 220 */ "dbs",
+ /* 221 */ "streams",
+ /* 222 */ "storage",
+ /* 223 */ "qtime",
+ /* 224 */ "users",
+ /* 225 */ "conns",
+ /* 226 */ "state",
+ /* 227 */ "keep",
+ /* 228 */ "tagitemlist",
+ /* 229 */ "cache",
+ /* 230 */ "replica",
+ /* 231 */ "quorum",
+ /* 232 */ "days",
+ /* 233 */ "minrows",
+ /* 234 */ "maxrows",
+ /* 235 */ "blocks",
+ /* 236 */ "ctime",
+ /* 237 */ "wal",
+ /* 238 */ "fsync",
+ /* 239 */ "comp",
+ /* 240 */ "prec",
+ /* 241 */ "update",
+ /* 242 */ "typename",
+ /* 243 */ "signed",
+ /* 244 */ "create_table_args",
+ /* 245 */ "columnlist",
+ /* 246 */ "select",
+ /* 247 */ "column",
+ /* 248 */ "tagitem",
+ /* 249 */ "selcollist",
+ /* 250 */ "from",
+ /* 251 */ "where_opt",
+ /* 252 */ "interval_opt",
+ /* 253 */ "fill_opt",
+ /* 254 */ "sliding_opt",
+ /* 255 */ "groupby_opt",
+ /* 256 */ "orderby_opt",
+ /* 257 */ "having_opt",
+ /* 258 */ "slimit_opt",
+ /* 259 */ "limit_opt",
+ /* 260 */ "union",
+ /* 261 */ "sclp",
+ /* 262 */ "expr",
+ /* 263 */ "as",
+ /* 264 */ "tablelist",
+ /* 265 */ "tmvar",
+ /* 266 */ "sortlist",
+ /* 267 */ "sortitem",
+ /* 268 */ "item",
+ /* 269 */ "sortorder",
+ /* 270 */ "grouplist",
+ /* 271 */ "exprlist",
+ /* 272 */ "expritem",
};
#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */
@@ -1094,151 +1098,154 @@ static const char *const yyRuleName[] = {
/* 82 */ "fsync ::= FSYNC INTEGER",
/* 83 */ "comp ::= COMP INTEGER",
/* 84 */ "prec ::= PRECISION STRING",
- /* 85 */ "db_optr ::=",
- /* 86 */ "db_optr ::= db_optr cache",
- /* 87 */ "db_optr ::= db_optr replica",
- /* 88 */ "db_optr ::= db_optr quorum",
- /* 89 */ "db_optr ::= db_optr days",
- /* 90 */ "db_optr ::= db_optr minrows",
- /* 91 */ "db_optr ::= db_optr maxrows",
- /* 92 */ "db_optr ::= db_optr blocks",
- /* 93 */ "db_optr ::= db_optr ctime",
- /* 94 */ "db_optr ::= db_optr wal",
- /* 95 */ "db_optr ::= db_optr fsync",
- /* 96 */ "db_optr ::= db_optr comp",
- /* 97 */ "db_optr ::= db_optr prec",
- /* 98 */ "db_optr ::= db_optr keep",
- /* 99 */ "alter_db_optr ::=",
- /* 100 */ "alter_db_optr ::= alter_db_optr replica",
- /* 101 */ "alter_db_optr ::= alter_db_optr quorum",
- /* 102 */ "alter_db_optr ::= alter_db_optr keep",
- /* 103 */ "alter_db_optr ::= alter_db_optr blocks",
- /* 104 */ "alter_db_optr ::= alter_db_optr comp",
- /* 105 */ "alter_db_optr ::= alter_db_optr wal",
- /* 106 */ "alter_db_optr ::= alter_db_optr fsync",
- /* 107 */ "typename ::= ids",
- /* 108 */ "typename ::= ids LP signed RP",
- /* 109 */ "signed ::= INTEGER",
- /* 110 */ "signed ::= PLUS INTEGER",
- /* 111 */ "signed ::= MINUS INTEGER",
- /* 112 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args",
- /* 113 */ "create_table_args ::= LP columnlist RP",
- /* 114 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP",
- /* 115 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP",
- /* 116 */ "create_table_args ::= AS select",
- /* 117 */ "columnlist ::= columnlist COMMA column",
- /* 118 */ "columnlist ::= column",
- /* 119 */ "column ::= ids typename",
- /* 120 */ "tagitemlist ::= tagitemlist COMMA tagitem",
- /* 121 */ "tagitemlist ::= tagitem",
- /* 122 */ "tagitem ::= INTEGER",
- /* 123 */ "tagitem ::= FLOAT",
- /* 124 */ "tagitem ::= STRING",
- /* 125 */ "tagitem ::= BOOL",
- /* 126 */ "tagitem ::= NULL",
- /* 127 */ "tagitem ::= MINUS INTEGER",
- /* 128 */ "tagitem ::= MINUS FLOAT",
- /* 129 */ "tagitem ::= PLUS INTEGER",
- /* 130 */ "tagitem ::= PLUS FLOAT",
- /* 131 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt",
- /* 132 */ "union ::= select",
- /* 133 */ "union ::= LP union RP",
- /* 134 */ "union ::= union UNION ALL select",
- /* 135 */ "union ::= union UNION ALL LP select RP",
- /* 136 */ "cmd ::= union",
- /* 137 */ "select ::= SELECT selcollist",
- /* 138 */ "sclp ::= selcollist COMMA",
- /* 139 */ "sclp ::=",
- /* 140 */ "selcollist ::= sclp expr as",
- /* 141 */ "selcollist ::= sclp STAR",
- /* 142 */ "as ::= AS ids",
- /* 143 */ "as ::= ids",
- /* 144 */ "as ::=",
- /* 145 */ "from ::= FROM tablelist",
- /* 146 */ "tablelist ::= ids cpxName",
- /* 147 */ "tablelist ::= ids cpxName ids",
- /* 148 */ "tablelist ::= tablelist COMMA ids cpxName",
- /* 149 */ "tablelist ::= tablelist COMMA ids cpxName ids",
- /* 150 */ "tmvar ::= VARIABLE",
- /* 151 */ "interval_opt ::= INTERVAL LP tmvar RP",
- /* 152 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP",
- /* 153 */ "interval_opt ::=",
- /* 154 */ "fill_opt ::=",
- /* 155 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP",
- /* 156 */ "fill_opt ::= FILL LP ID RP",
- /* 157 */ "sliding_opt ::= SLIDING LP tmvar RP",
- /* 158 */ "sliding_opt ::=",
- /* 159 */ "orderby_opt ::=",
- /* 160 */ "orderby_opt ::= ORDER BY sortlist",
- /* 161 */ "sortlist ::= sortlist COMMA item sortorder",
- /* 162 */ "sortlist ::= item sortorder",
- /* 163 */ "item ::= ids cpxName",
- /* 164 */ "sortorder ::= ASC",
- /* 165 */ "sortorder ::= DESC",
- /* 166 */ "sortorder ::=",
- /* 167 */ "groupby_opt ::=",
- /* 168 */ "groupby_opt ::= GROUP BY grouplist",
- /* 169 */ "grouplist ::= grouplist COMMA item",
- /* 170 */ "grouplist ::= item",
- /* 171 */ "having_opt ::=",
- /* 172 */ "having_opt ::= HAVING expr",
- /* 173 */ "limit_opt ::=",
- /* 174 */ "limit_opt ::= LIMIT signed",
- /* 175 */ "limit_opt ::= LIMIT signed OFFSET signed",
- /* 176 */ "limit_opt ::= LIMIT signed COMMA signed",
- /* 177 */ "slimit_opt ::=",
- /* 178 */ "slimit_opt ::= SLIMIT signed",
- /* 179 */ "slimit_opt ::= SLIMIT signed SOFFSET signed",
- /* 180 */ "slimit_opt ::= SLIMIT signed COMMA signed",
- /* 181 */ "where_opt ::=",
- /* 182 */ "where_opt ::= WHERE expr",
- /* 183 */ "expr ::= LP expr RP",
- /* 184 */ "expr ::= ID",
- /* 185 */ "expr ::= ID DOT ID",
- /* 186 */ "expr ::= ID DOT STAR",
- /* 187 */ "expr ::= INTEGER",
- /* 188 */ "expr ::= MINUS INTEGER",
- /* 189 */ "expr ::= PLUS INTEGER",
- /* 190 */ "expr ::= FLOAT",
- /* 191 */ "expr ::= MINUS FLOAT",
- /* 192 */ "expr ::= PLUS FLOAT",
- /* 193 */ "expr ::= STRING",
- /* 194 */ "expr ::= NOW",
- /* 195 */ "expr ::= VARIABLE",
- /* 196 */ "expr ::= BOOL",
- /* 197 */ "expr ::= ID LP exprlist RP",
- /* 198 */ "expr ::= ID LP STAR RP",
- /* 199 */ "expr ::= expr IS NULL",
- /* 200 */ "expr ::= expr IS NOT NULL",
- /* 201 */ "expr ::= expr LT expr",
- /* 202 */ "expr ::= expr GT expr",
- /* 203 */ "expr ::= expr LE expr",
- /* 204 */ "expr ::= expr GE expr",
- /* 205 */ "expr ::= expr NE expr",
- /* 206 */ "expr ::= expr EQ expr",
- /* 207 */ "expr ::= expr AND expr",
- /* 208 */ "expr ::= expr OR expr",
- /* 209 */ "expr ::= expr PLUS expr",
- /* 210 */ "expr ::= expr MINUS expr",
- /* 211 */ "expr ::= expr STAR expr",
- /* 212 */ "expr ::= expr SLASH expr",
- /* 213 */ "expr ::= expr REM expr",
- /* 214 */ "expr ::= expr LIKE expr",
- /* 215 */ "expr ::= expr IN LP exprlist RP",
- /* 216 */ "exprlist ::= exprlist COMMA expritem",
- /* 217 */ "exprlist ::= expritem",
- /* 218 */ "expritem ::= expr",
- /* 219 */ "expritem ::=",
- /* 220 */ "cmd ::= RESET QUERY CACHE",
- /* 221 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist",
- /* 222 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids",
- /* 223 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist",
- /* 224 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids",
- /* 225 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids",
- /* 226 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem",
- /* 227 */ "cmd ::= KILL CONNECTION INTEGER",
- /* 228 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER",
- /* 229 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER",
+ /* 85 */ "update ::= UPDATE INTEGER",
+ /* 86 */ "db_optr ::=",
+ /* 87 */ "db_optr ::= db_optr cache",
+ /* 88 */ "db_optr ::= db_optr replica",
+ /* 89 */ "db_optr ::= db_optr quorum",
+ /* 90 */ "db_optr ::= db_optr days",
+ /* 91 */ "db_optr ::= db_optr minrows",
+ /* 92 */ "db_optr ::= db_optr maxrows",
+ /* 93 */ "db_optr ::= db_optr blocks",
+ /* 94 */ "db_optr ::= db_optr ctime",
+ /* 95 */ "db_optr ::= db_optr wal",
+ /* 96 */ "db_optr ::= db_optr fsync",
+ /* 97 */ "db_optr ::= db_optr comp",
+ /* 98 */ "db_optr ::= db_optr prec",
+ /* 99 */ "db_optr ::= db_optr keep",
+ /* 100 */ "db_optr ::= db_optr update",
+ /* 101 */ "alter_db_optr ::=",
+ /* 102 */ "alter_db_optr ::= alter_db_optr replica",
+ /* 103 */ "alter_db_optr ::= alter_db_optr quorum",
+ /* 104 */ "alter_db_optr ::= alter_db_optr keep",
+ /* 105 */ "alter_db_optr ::= alter_db_optr blocks",
+ /* 106 */ "alter_db_optr ::= alter_db_optr comp",
+ /* 107 */ "alter_db_optr ::= alter_db_optr wal",
+ /* 108 */ "alter_db_optr ::= alter_db_optr fsync",
+ /* 109 */ "alter_db_optr ::= alter_db_optr update",
+ /* 110 */ "typename ::= ids",
+ /* 111 */ "typename ::= ids LP signed RP",
+ /* 112 */ "signed ::= INTEGER",
+ /* 113 */ "signed ::= PLUS INTEGER",
+ /* 114 */ "signed ::= MINUS INTEGER",
+ /* 115 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args",
+ /* 116 */ "create_table_args ::= LP columnlist RP",
+ /* 117 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP",
+ /* 118 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP",
+ /* 119 */ "create_table_args ::= AS select",
+ /* 120 */ "columnlist ::= columnlist COMMA column",
+ /* 121 */ "columnlist ::= column",
+ /* 122 */ "column ::= ids typename",
+ /* 123 */ "tagitemlist ::= tagitemlist COMMA tagitem",
+ /* 124 */ "tagitemlist ::= tagitem",
+ /* 125 */ "tagitem ::= INTEGER",
+ /* 126 */ "tagitem ::= FLOAT",
+ /* 127 */ "tagitem ::= STRING",
+ /* 128 */ "tagitem ::= BOOL",
+ /* 129 */ "tagitem ::= NULL",
+ /* 130 */ "tagitem ::= MINUS INTEGER",
+ /* 131 */ "tagitem ::= MINUS FLOAT",
+ /* 132 */ "tagitem ::= PLUS INTEGER",
+ /* 133 */ "tagitem ::= PLUS FLOAT",
+ /* 134 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt",
+ /* 135 */ "union ::= select",
+ /* 136 */ "union ::= LP union RP",
+ /* 137 */ "union ::= union UNION ALL select",
+ /* 138 */ "union ::= union UNION ALL LP select RP",
+ /* 139 */ "cmd ::= union",
+ /* 140 */ "select ::= SELECT selcollist",
+ /* 141 */ "sclp ::= selcollist COMMA",
+ /* 142 */ "sclp ::=",
+ /* 143 */ "selcollist ::= sclp expr as",
+ /* 144 */ "selcollist ::= sclp STAR",
+ /* 145 */ "as ::= AS ids",
+ /* 146 */ "as ::= ids",
+ /* 147 */ "as ::=",
+ /* 148 */ "from ::= FROM tablelist",
+ /* 149 */ "tablelist ::= ids cpxName",
+ /* 150 */ "tablelist ::= ids cpxName ids",
+ /* 151 */ "tablelist ::= tablelist COMMA ids cpxName",
+ /* 152 */ "tablelist ::= tablelist COMMA ids cpxName ids",
+ /* 153 */ "tmvar ::= VARIABLE",
+ /* 154 */ "interval_opt ::= INTERVAL LP tmvar RP",
+ /* 155 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP",
+ /* 156 */ "interval_opt ::=",
+ /* 157 */ "fill_opt ::=",
+ /* 158 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP",
+ /* 159 */ "fill_opt ::= FILL LP ID RP",
+ /* 160 */ "sliding_opt ::= SLIDING LP tmvar RP",
+ /* 161 */ "sliding_opt ::=",
+ /* 162 */ "orderby_opt ::=",
+ /* 163 */ "orderby_opt ::= ORDER BY sortlist",
+ /* 164 */ "sortlist ::= sortlist COMMA item sortorder",
+ /* 165 */ "sortlist ::= item sortorder",
+ /* 166 */ "item ::= ids cpxName",
+ /* 167 */ "sortorder ::= ASC",
+ /* 168 */ "sortorder ::= DESC",
+ /* 169 */ "sortorder ::=",
+ /* 170 */ "groupby_opt ::=",
+ /* 171 */ "groupby_opt ::= GROUP BY grouplist",
+ /* 172 */ "grouplist ::= grouplist COMMA item",
+ /* 173 */ "grouplist ::= item",
+ /* 174 */ "having_opt ::=",
+ /* 175 */ "having_opt ::= HAVING expr",
+ /* 176 */ "limit_opt ::=",
+ /* 177 */ "limit_opt ::= LIMIT signed",
+ /* 178 */ "limit_opt ::= LIMIT signed OFFSET signed",
+ /* 179 */ "limit_opt ::= LIMIT signed COMMA signed",
+ /* 180 */ "slimit_opt ::=",
+ /* 181 */ "slimit_opt ::= SLIMIT signed",
+ /* 182 */ "slimit_opt ::= SLIMIT signed SOFFSET signed",
+ /* 183 */ "slimit_opt ::= SLIMIT signed COMMA signed",
+ /* 184 */ "where_opt ::=",
+ /* 185 */ "where_opt ::= WHERE expr",
+ /* 186 */ "expr ::= LP expr RP",
+ /* 187 */ "expr ::= ID",
+ /* 188 */ "expr ::= ID DOT ID",
+ /* 189 */ "expr ::= ID DOT STAR",
+ /* 190 */ "expr ::= INTEGER",
+ /* 191 */ "expr ::= MINUS INTEGER",
+ /* 192 */ "expr ::= PLUS INTEGER",
+ /* 193 */ "expr ::= FLOAT",
+ /* 194 */ "expr ::= MINUS FLOAT",
+ /* 195 */ "expr ::= PLUS FLOAT",
+ /* 196 */ "expr ::= STRING",
+ /* 197 */ "expr ::= NOW",
+ /* 198 */ "expr ::= VARIABLE",
+ /* 199 */ "expr ::= BOOL",
+ /* 200 */ "expr ::= ID LP exprlist RP",
+ /* 201 */ "expr ::= ID LP STAR RP",
+ /* 202 */ "expr ::= expr IS NULL",
+ /* 203 */ "expr ::= expr IS NOT NULL",
+ /* 204 */ "expr ::= expr LT expr",
+ /* 205 */ "expr ::= expr GT expr",
+ /* 206 */ "expr ::= expr LE expr",
+ /* 207 */ "expr ::= expr GE expr",
+ /* 208 */ "expr ::= expr NE expr",
+ /* 209 */ "expr ::= expr EQ expr",
+ /* 210 */ "expr ::= expr AND expr",
+ /* 211 */ "expr ::= expr OR expr",
+ /* 212 */ "expr ::= expr PLUS expr",
+ /* 213 */ "expr ::= expr MINUS expr",
+ /* 214 */ "expr ::= expr STAR expr",
+ /* 215 */ "expr ::= expr SLASH expr",
+ /* 216 */ "expr ::= expr REM expr",
+ /* 217 */ "expr ::= expr LIKE expr",
+ /* 218 */ "expr ::= expr IN LP exprlist RP",
+ /* 219 */ "exprlist ::= exprlist COMMA expritem",
+ /* 220 */ "exprlist ::= expritem",
+ /* 221 */ "expritem ::= expr",
+ /* 222 */ "expritem ::=",
+ /* 223 */ "cmd ::= RESET QUERY CACHE",
+ /* 224 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist",
+ /* 225 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids",
+ /* 226 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist",
+ /* 227 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids",
+ /* 228 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids",
+ /* 229 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem",
+ /* 230 */ "cmd ::= KILL CONNECTION INTEGER",
+ /* 231 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER",
+ /* 232 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER",
};
#endif /* NDEBUG */
@@ -1359,50 +1366,50 @@ static void yy_destructor(
** inside the C code.
*/
/********* Begin destructor definitions ***************************************/
- case 226: /* keep */
- case 227: /* tagitemlist */
- case 251: /* fill_opt */
- case 253: /* groupby_opt */
- case 254: /* orderby_opt */
- case 264: /* sortlist */
- case 268: /* grouplist */
+ case 227: /* keep */
+ case 228: /* tagitemlist */
+ case 253: /* fill_opt */
+ case 255: /* groupby_opt */
+ case 256: /* orderby_opt */
+ case 266: /* sortlist */
+ case 270: /* grouplist */
{
-tVariantListDestroy((yypminor->yy494));
+tVariantListDestroy((yypminor->yy498));
}
break;
- case 243: /* columnlist */
+ case 245: /* columnlist */
{
-tFieldListDestroy((yypminor->yy449));
+tFieldListDestroy((yypminor->yy523));
}
break;
- case 244: /* select */
+ case 246: /* select */
{
-doDestroyQuerySql((yypminor->yy150));
+doDestroyQuerySql((yypminor->yy414));
}
break;
- case 247: /* selcollist */
- case 259: /* sclp */
- case 269: /* exprlist */
+ case 249: /* selcollist */
+ case 261: /* sclp */
+ case 271: /* exprlist */
{
-tSQLExprListDestroy((yypminor->yy224));
+tSQLExprListDestroy((yypminor->yy290));
}
break;
- case 249: /* where_opt */
- case 255: /* having_opt */
- case 260: /* expr */
- case 270: /* expritem */
+ case 251: /* where_opt */
+ case 257: /* having_opt */
+ case 262: /* expr */
+ case 272: /* expritem */
{
-tSQLExprDestroy((yypminor->yy66));
+tSQLExprDestroy((yypminor->yy64));
}
break;
- case 258: /* union */
+ case 260: /* union */
{
-destroyAllSelectClause((yypminor->yy25));
+destroyAllSelectClause((yypminor->yy231));
}
break;
- case 265: /* sortitem */
+ case 267: /* sortitem */
{
-tVariantDestroy(&(yypminor->yy312));
+tVariantDestroy(&(yypminor->yy134));
}
break;
/********* End destructor definitions *****************************************/
@@ -1696,236 +1703,239 @@ static const struct {
YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */
signed char nrhs; /* Negative of the number of RHS symbols in the rule */
} yyRuleInfo[] = {
- { 207, -1 }, /* (0) program ::= cmd */
- { 208, -2 }, /* (1) cmd ::= SHOW DATABASES */
- { 208, -2 }, /* (2) cmd ::= SHOW MNODES */
- { 208, -2 }, /* (3) cmd ::= SHOW DNODES */
- { 208, -2 }, /* (4) cmd ::= SHOW ACCOUNTS */
- { 208, -2 }, /* (5) cmd ::= SHOW USERS */
- { 208, -2 }, /* (6) cmd ::= SHOW MODULES */
- { 208, -2 }, /* (7) cmd ::= SHOW QUERIES */
- { 208, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */
- { 208, -2 }, /* (9) cmd ::= SHOW STREAMS */
- { 208, -2 }, /* (10) cmd ::= SHOW VARIABLES */
- { 208, -2 }, /* (11) cmd ::= SHOW SCORES */
- { 208, -2 }, /* (12) cmd ::= SHOW GRANTS */
- { 208, -2 }, /* (13) cmd ::= SHOW VNODES */
- { 208, -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */
- { 209, 0 }, /* (15) dbPrefix ::= */
- { 209, -2 }, /* (16) dbPrefix ::= ids DOT */
- { 211, 0 }, /* (17) cpxName ::= */
- { 211, -2 }, /* (18) cpxName ::= DOT ids */
- { 208, -5 }, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */
- { 208, -4 }, /* (20) cmd ::= SHOW CREATE DATABASE ids */
- { 208, -3 }, /* (21) cmd ::= SHOW dbPrefix TABLES */
- { 208, -5 }, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */
- { 208, -3 }, /* (23) cmd ::= SHOW dbPrefix STABLES */
- { 208, -5 }, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */
- { 208, -3 }, /* (25) cmd ::= SHOW dbPrefix VGROUPS */
- { 208, -4 }, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */
- { 208, -5 }, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */
- { 208, -4 }, /* (28) cmd ::= DROP DATABASE ifexists ids */
- { 208, -3 }, /* (29) cmd ::= DROP DNODE ids */
- { 208, -3 }, /* (30) cmd ::= DROP USER ids */
- { 208, -3 }, /* (31) cmd ::= DROP ACCOUNT ids */
- { 208, -2 }, /* (32) cmd ::= USE ids */
- { 208, -3 }, /* (33) cmd ::= DESCRIBE ids cpxName */
- { 208, -5 }, /* (34) cmd ::= ALTER USER ids PASS ids */
- { 208, -5 }, /* (35) cmd ::= ALTER USER ids PRIVILEGE ids */
- { 208, -4 }, /* (36) cmd ::= ALTER DNODE ids ids */
- { 208, -5 }, /* (37) cmd ::= ALTER DNODE ids ids ids */
- { 208, -3 }, /* (38) cmd ::= ALTER LOCAL ids */
- { 208, -4 }, /* (39) cmd ::= ALTER LOCAL ids ids */
- { 208, -4 }, /* (40) cmd ::= ALTER DATABASE ids alter_db_optr */
- { 208, -4 }, /* (41) cmd ::= ALTER ACCOUNT ids acct_optr */
- { 208, -6 }, /* (42) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */
- { 210, -1 }, /* (43) ids ::= ID */
- { 210, -1 }, /* (44) ids ::= STRING */
- { 212, -2 }, /* (45) ifexists ::= IF EXISTS */
- { 212, 0 }, /* (46) ifexists ::= */
- { 215, -3 }, /* (47) ifnotexists ::= IF NOT EXISTS */
- { 215, 0 }, /* (48) ifnotexists ::= */
- { 208, -3 }, /* (49) cmd ::= CREATE DNODE ids */
- { 208, -6 }, /* (50) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */
- { 208, -5 }, /* (51) cmd ::= CREATE DATABASE ifnotexists ids db_optr */
- { 208, -5 }, /* (52) cmd ::= CREATE USER ids PASS ids */
- { 217, 0 }, /* (53) pps ::= */
- { 217, -2 }, /* (54) pps ::= PPS INTEGER */
- { 218, 0 }, /* (55) tseries ::= */
- { 218, -2 }, /* (56) tseries ::= TSERIES INTEGER */
- { 219, 0 }, /* (57) dbs ::= */
- { 219, -2 }, /* (58) dbs ::= DBS INTEGER */
- { 220, 0 }, /* (59) streams ::= */
- { 220, -2 }, /* (60) streams ::= STREAMS INTEGER */
- { 221, 0 }, /* (61) storage ::= */
- { 221, -2 }, /* (62) storage ::= STORAGE INTEGER */
- { 222, 0 }, /* (63) qtime ::= */
- { 222, -2 }, /* (64) qtime ::= QTIME INTEGER */
- { 223, 0 }, /* (65) users ::= */
- { 223, -2 }, /* (66) users ::= USERS INTEGER */
- { 224, 0 }, /* (67) conns ::= */
- { 224, -2 }, /* (68) conns ::= CONNS INTEGER */
- { 225, 0 }, /* (69) state ::= */
- { 225, -2 }, /* (70) state ::= STATE ids */
- { 214, -9 }, /* (71) acct_optr ::= pps tseries storage streams qtime dbs users conns state */
- { 226, -2 }, /* (72) keep ::= KEEP tagitemlist */
- { 228, -2 }, /* (73) cache ::= CACHE INTEGER */
- { 229, -2 }, /* (74) replica ::= REPLICA INTEGER */
- { 230, -2 }, /* (75) quorum ::= QUORUM INTEGER */
- { 231, -2 }, /* (76) days ::= DAYS INTEGER */
- { 232, -2 }, /* (77) minrows ::= MINROWS INTEGER */
- { 233, -2 }, /* (78) maxrows ::= MAXROWS INTEGER */
- { 234, -2 }, /* (79) blocks ::= BLOCKS INTEGER */
- { 235, -2 }, /* (80) ctime ::= CTIME INTEGER */
- { 236, -2 }, /* (81) wal ::= WAL INTEGER */
- { 237, -2 }, /* (82) fsync ::= FSYNC INTEGER */
- { 238, -2 }, /* (83) comp ::= COMP INTEGER */
- { 239, -2 }, /* (84) prec ::= PRECISION STRING */
- { 216, 0 }, /* (85) db_optr ::= */
- { 216, -2 }, /* (86) db_optr ::= db_optr cache */
- { 216, -2 }, /* (87) db_optr ::= db_optr replica */
- { 216, -2 }, /* (88) db_optr ::= db_optr quorum */
- { 216, -2 }, /* (89) db_optr ::= db_optr days */
- { 216, -2 }, /* (90) db_optr ::= db_optr minrows */
- { 216, -2 }, /* (91) db_optr ::= db_optr maxrows */
- { 216, -2 }, /* (92) db_optr ::= db_optr blocks */
- { 216, -2 }, /* (93) db_optr ::= db_optr ctime */
- { 216, -2 }, /* (94) db_optr ::= db_optr wal */
- { 216, -2 }, /* (95) db_optr ::= db_optr fsync */
- { 216, -2 }, /* (96) db_optr ::= db_optr comp */
- { 216, -2 }, /* (97) db_optr ::= db_optr prec */
- { 216, -2 }, /* (98) db_optr ::= db_optr keep */
- { 213, 0 }, /* (99) alter_db_optr ::= */
- { 213, -2 }, /* (100) alter_db_optr ::= alter_db_optr replica */
- { 213, -2 }, /* (101) alter_db_optr ::= alter_db_optr quorum */
- { 213, -2 }, /* (102) alter_db_optr ::= alter_db_optr keep */
- { 213, -2 }, /* (103) alter_db_optr ::= alter_db_optr blocks */
- { 213, -2 }, /* (104) alter_db_optr ::= alter_db_optr comp */
- { 213, -2 }, /* (105) alter_db_optr ::= alter_db_optr wal */
- { 213, -2 }, /* (106) alter_db_optr ::= alter_db_optr fsync */
- { 240, -1 }, /* (107) typename ::= ids */
- { 240, -4 }, /* (108) typename ::= ids LP signed RP */
- { 241, -1 }, /* (109) signed ::= INTEGER */
- { 241, -2 }, /* (110) signed ::= PLUS INTEGER */
- { 241, -2 }, /* (111) signed ::= MINUS INTEGER */
- { 208, -6 }, /* (112) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
- { 242, -3 }, /* (113) create_table_args ::= LP columnlist RP */
- { 242, -7 }, /* (114) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
- { 242, -7 }, /* (115) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
- { 242, -2 }, /* (116) create_table_args ::= AS select */
- { 243, -3 }, /* (117) columnlist ::= columnlist COMMA column */
- { 243, -1 }, /* (118) columnlist ::= column */
- { 245, -2 }, /* (119) column ::= ids typename */
- { 227, -3 }, /* (120) tagitemlist ::= tagitemlist COMMA tagitem */
- { 227, -1 }, /* (121) tagitemlist ::= tagitem */
- { 246, -1 }, /* (122) tagitem ::= INTEGER */
- { 246, -1 }, /* (123) tagitem ::= FLOAT */
- { 246, -1 }, /* (124) tagitem ::= STRING */
- { 246, -1 }, /* (125) tagitem ::= BOOL */
- { 246, -1 }, /* (126) tagitem ::= NULL */
- { 246, -2 }, /* (127) tagitem ::= MINUS INTEGER */
- { 246, -2 }, /* (128) tagitem ::= MINUS FLOAT */
- { 246, -2 }, /* (129) tagitem ::= PLUS INTEGER */
- { 246, -2 }, /* (130) tagitem ::= PLUS FLOAT */
- { 244, -12 }, /* (131) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
- { 258, -1 }, /* (132) union ::= select */
- { 258, -3 }, /* (133) union ::= LP union RP */
- { 258, -4 }, /* (134) union ::= union UNION ALL select */
- { 258, -6 }, /* (135) union ::= union UNION ALL LP select RP */
- { 208, -1 }, /* (136) cmd ::= union */
- { 244, -2 }, /* (137) select ::= SELECT selcollist */
- { 259, -2 }, /* (138) sclp ::= selcollist COMMA */
- { 259, 0 }, /* (139) sclp ::= */
- { 247, -3 }, /* (140) selcollist ::= sclp expr as */
- { 247, -2 }, /* (141) selcollist ::= sclp STAR */
- { 261, -2 }, /* (142) as ::= AS ids */
- { 261, -1 }, /* (143) as ::= ids */
- { 261, 0 }, /* (144) as ::= */
- { 248, -2 }, /* (145) from ::= FROM tablelist */
- { 262, -2 }, /* (146) tablelist ::= ids cpxName */
- { 262, -3 }, /* (147) tablelist ::= ids cpxName ids */
- { 262, -4 }, /* (148) tablelist ::= tablelist COMMA ids cpxName */
- { 262, -5 }, /* (149) tablelist ::= tablelist COMMA ids cpxName ids */
- { 263, -1 }, /* (150) tmvar ::= VARIABLE */
- { 250, -4 }, /* (151) interval_opt ::= INTERVAL LP tmvar RP */
- { 250, -6 }, /* (152) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */
- { 250, 0 }, /* (153) interval_opt ::= */
- { 251, 0 }, /* (154) fill_opt ::= */
- { 251, -6 }, /* (155) fill_opt ::= FILL LP ID COMMA tagitemlist RP */
- { 251, -4 }, /* (156) fill_opt ::= FILL LP ID RP */
- { 252, -4 }, /* (157) sliding_opt ::= SLIDING LP tmvar RP */
- { 252, 0 }, /* (158) sliding_opt ::= */
- { 254, 0 }, /* (159) orderby_opt ::= */
- { 254, -3 }, /* (160) orderby_opt ::= ORDER BY sortlist */
- { 264, -4 }, /* (161) sortlist ::= sortlist COMMA item sortorder */
- { 264, -2 }, /* (162) sortlist ::= item sortorder */
- { 266, -2 }, /* (163) item ::= ids cpxName */
- { 267, -1 }, /* (164) sortorder ::= ASC */
- { 267, -1 }, /* (165) sortorder ::= DESC */
- { 267, 0 }, /* (166) sortorder ::= */
- { 253, 0 }, /* (167) groupby_opt ::= */
- { 253, -3 }, /* (168) groupby_opt ::= GROUP BY grouplist */
- { 268, -3 }, /* (169) grouplist ::= grouplist COMMA item */
- { 268, -1 }, /* (170) grouplist ::= item */
- { 255, 0 }, /* (171) having_opt ::= */
- { 255, -2 }, /* (172) having_opt ::= HAVING expr */
- { 257, 0 }, /* (173) limit_opt ::= */
- { 257, -2 }, /* (174) limit_opt ::= LIMIT signed */
- { 257, -4 }, /* (175) limit_opt ::= LIMIT signed OFFSET signed */
- { 257, -4 }, /* (176) limit_opt ::= LIMIT signed COMMA signed */
- { 256, 0 }, /* (177) slimit_opt ::= */
- { 256, -2 }, /* (178) slimit_opt ::= SLIMIT signed */
- { 256, -4 }, /* (179) slimit_opt ::= SLIMIT signed SOFFSET signed */
- { 256, -4 }, /* (180) slimit_opt ::= SLIMIT signed COMMA signed */
- { 249, 0 }, /* (181) where_opt ::= */
- { 249, -2 }, /* (182) where_opt ::= WHERE expr */
- { 260, -3 }, /* (183) expr ::= LP expr RP */
- { 260, -1 }, /* (184) expr ::= ID */
- { 260, -3 }, /* (185) expr ::= ID DOT ID */
- { 260, -3 }, /* (186) expr ::= ID DOT STAR */
- { 260, -1 }, /* (187) expr ::= INTEGER */
- { 260, -2 }, /* (188) expr ::= MINUS INTEGER */
- { 260, -2 }, /* (189) expr ::= PLUS INTEGER */
- { 260, -1 }, /* (190) expr ::= FLOAT */
- { 260, -2 }, /* (191) expr ::= MINUS FLOAT */
- { 260, -2 }, /* (192) expr ::= PLUS FLOAT */
- { 260, -1 }, /* (193) expr ::= STRING */
- { 260, -1 }, /* (194) expr ::= NOW */
- { 260, -1 }, /* (195) expr ::= VARIABLE */
- { 260, -1 }, /* (196) expr ::= BOOL */
- { 260, -4 }, /* (197) expr ::= ID LP exprlist RP */
- { 260, -4 }, /* (198) expr ::= ID LP STAR RP */
- { 260, -3 }, /* (199) expr ::= expr IS NULL */
- { 260, -4 }, /* (200) expr ::= expr IS NOT NULL */
- { 260, -3 }, /* (201) expr ::= expr LT expr */
- { 260, -3 }, /* (202) expr ::= expr GT expr */
- { 260, -3 }, /* (203) expr ::= expr LE expr */
- { 260, -3 }, /* (204) expr ::= expr GE expr */
- { 260, -3 }, /* (205) expr ::= expr NE expr */
- { 260, -3 }, /* (206) expr ::= expr EQ expr */
- { 260, -3 }, /* (207) expr ::= expr AND expr */
- { 260, -3 }, /* (208) expr ::= expr OR expr */
- { 260, -3 }, /* (209) expr ::= expr PLUS expr */
- { 260, -3 }, /* (210) expr ::= expr MINUS expr */
- { 260, -3 }, /* (211) expr ::= expr STAR expr */
- { 260, -3 }, /* (212) expr ::= expr SLASH expr */
- { 260, -3 }, /* (213) expr ::= expr REM expr */
- { 260, -3 }, /* (214) expr ::= expr LIKE expr */
- { 260, -5 }, /* (215) expr ::= expr IN LP exprlist RP */
- { 269, -3 }, /* (216) exprlist ::= exprlist COMMA expritem */
- { 269, -1 }, /* (217) exprlist ::= expritem */
- { 270, -1 }, /* (218) expritem ::= expr */
- { 270, 0 }, /* (219) expritem ::= */
- { 208, -3 }, /* (220) cmd ::= RESET QUERY CACHE */
- { 208, -7 }, /* (221) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
- { 208, -7 }, /* (222) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
- { 208, -7 }, /* (223) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
- { 208, -7 }, /* (224) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
- { 208, -8 }, /* (225) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
- { 208, -9 }, /* (226) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
- { 208, -3 }, /* (227) cmd ::= KILL CONNECTION INTEGER */
- { 208, -5 }, /* (228) cmd ::= KILL STREAM INTEGER COLON INTEGER */
- { 208, -5 }, /* (229) cmd ::= KILL QUERY INTEGER COLON INTEGER */
+ { 208, -1 }, /* (0) program ::= cmd */
+ { 209, -2 }, /* (1) cmd ::= SHOW DATABASES */
+ { 209, -2 }, /* (2) cmd ::= SHOW MNODES */
+ { 209, -2 }, /* (3) cmd ::= SHOW DNODES */
+ { 209, -2 }, /* (4) cmd ::= SHOW ACCOUNTS */
+ { 209, -2 }, /* (5) cmd ::= SHOW USERS */
+ { 209, -2 }, /* (6) cmd ::= SHOW MODULES */
+ { 209, -2 }, /* (7) cmd ::= SHOW QUERIES */
+ { 209, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */
+ { 209, -2 }, /* (9) cmd ::= SHOW STREAMS */
+ { 209, -2 }, /* (10) cmd ::= SHOW VARIABLES */
+ { 209, -2 }, /* (11) cmd ::= SHOW SCORES */
+ { 209, -2 }, /* (12) cmd ::= SHOW GRANTS */
+ { 209, -2 }, /* (13) cmd ::= SHOW VNODES */
+ { 209, -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */
+ { 210, 0 }, /* (15) dbPrefix ::= */
+ { 210, -2 }, /* (16) dbPrefix ::= ids DOT */
+ { 212, 0 }, /* (17) cpxName ::= */
+ { 212, -2 }, /* (18) cpxName ::= DOT ids */
+ { 209, -5 }, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */
+ { 209, -4 }, /* (20) cmd ::= SHOW CREATE DATABASE ids */
+ { 209, -3 }, /* (21) cmd ::= SHOW dbPrefix TABLES */
+ { 209, -5 }, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */
+ { 209, -3 }, /* (23) cmd ::= SHOW dbPrefix STABLES */
+ { 209, -5 }, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */
+ { 209, -3 }, /* (25) cmd ::= SHOW dbPrefix VGROUPS */
+ { 209, -4 }, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */
+ { 209, -5 }, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */
+ { 209, -4 }, /* (28) cmd ::= DROP DATABASE ifexists ids */
+ { 209, -3 }, /* (29) cmd ::= DROP DNODE ids */
+ { 209, -3 }, /* (30) cmd ::= DROP USER ids */
+ { 209, -3 }, /* (31) cmd ::= DROP ACCOUNT ids */
+ { 209, -2 }, /* (32) cmd ::= USE ids */
+ { 209, -3 }, /* (33) cmd ::= DESCRIBE ids cpxName */
+ { 209, -5 }, /* (34) cmd ::= ALTER USER ids PASS ids */
+ { 209, -5 }, /* (35) cmd ::= ALTER USER ids PRIVILEGE ids */
+ { 209, -4 }, /* (36) cmd ::= ALTER DNODE ids ids */
+ { 209, -5 }, /* (37) cmd ::= ALTER DNODE ids ids ids */
+ { 209, -3 }, /* (38) cmd ::= ALTER LOCAL ids */
+ { 209, -4 }, /* (39) cmd ::= ALTER LOCAL ids ids */
+ { 209, -4 }, /* (40) cmd ::= ALTER DATABASE ids alter_db_optr */
+ { 209, -4 }, /* (41) cmd ::= ALTER ACCOUNT ids acct_optr */
+ { 209, -6 }, /* (42) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */
+ { 211, -1 }, /* (43) ids ::= ID */
+ { 211, -1 }, /* (44) ids ::= STRING */
+ { 213, -2 }, /* (45) ifexists ::= IF EXISTS */
+ { 213, 0 }, /* (46) ifexists ::= */
+ { 216, -3 }, /* (47) ifnotexists ::= IF NOT EXISTS */
+ { 216, 0 }, /* (48) ifnotexists ::= */
+ { 209, -3 }, /* (49) cmd ::= CREATE DNODE ids */
+ { 209, -6 }, /* (50) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */
+ { 209, -5 }, /* (51) cmd ::= CREATE DATABASE ifnotexists ids db_optr */
+ { 209, -5 }, /* (52) cmd ::= CREATE USER ids PASS ids */
+ { 218, 0 }, /* (53) pps ::= */
+ { 218, -2 }, /* (54) pps ::= PPS INTEGER */
+ { 219, 0 }, /* (55) tseries ::= */
+ { 219, -2 }, /* (56) tseries ::= TSERIES INTEGER */
+ { 220, 0 }, /* (57) dbs ::= */
+ { 220, -2 }, /* (58) dbs ::= DBS INTEGER */
+ { 221, 0 }, /* (59) streams ::= */
+ { 221, -2 }, /* (60) streams ::= STREAMS INTEGER */
+ { 222, 0 }, /* (61) storage ::= */
+ { 222, -2 }, /* (62) storage ::= STORAGE INTEGER */
+ { 223, 0 }, /* (63) qtime ::= */
+ { 223, -2 }, /* (64) qtime ::= QTIME INTEGER */
+ { 224, 0 }, /* (65) users ::= */
+ { 224, -2 }, /* (66) users ::= USERS INTEGER */
+ { 225, 0 }, /* (67) conns ::= */
+ { 225, -2 }, /* (68) conns ::= CONNS INTEGER */
+ { 226, 0 }, /* (69) state ::= */
+ { 226, -2 }, /* (70) state ::= STATE ids */
+ { 215, -9 }, /* (71) acct_optr ::= pps tseries storage streams qtime dbs users conns state */
+ { 227, -2 }, /* (72) keep ::= KEEP tagitemlist */
+ { 229, -2 }, /* (73) cache ::= CACHE INTEGER */
+ { 230, -2 }, /* (74) replica ::= REPLICA INTEGER */
+ { 231, -2 }, /* (75) quorum ::= QUORUM INTEGER */
+ { 232, -2 }, /* (76) days ::= DAYS INTEGER */
+ { 233, -2 }, /* (77) minrows ::= MINROWS INTEGER */
+ { 234, -2 }, /* (78) maxrows ::= MAXROWS INTEGER */
+ { 235, -2 }, /* (79) blocks ::= BLOCKS INTEGER */
+ { 236, -2 }, /* (80) ctime ::= CTIME INTEGER */
+ { 237, -2 }, /* (81) wal ::= WAL INTEGER */
+ { 238, -2 }, /* (82) fsync ::= FSYNC INTEGER */
+ { 239, -2 }, /* (83) comp ::= COMP INTEGER */
+ { 240, -2 }, /* (84) prec ::= PRECISION STRING */
+ { 241, -2 }, /* (85) update ::= UPDATE INTEGER */
+ { 217, 0 }, /* (86) db_optr ::= */
+ { 217, -2 }, /* (87) db_optr ::= db_optr cache */
+ { 217, -2 }, /* (88) db_optr ::= db_optr replica */
+ { 217, -2 }, /* (89) db_optr ::= db_optr quorum */
+ { 217, -2 }, /* (90) db_optr ::= db_optr days */
+ { 217, -2 }, /* (91) db_optr ::= db_optr minrows */
+ { 217, -2 }, /* (92) db_optr ::= db_optr maxrows */
+ { 217, -2 }, /* (93) db_optr ::= db_optr blocks */
+ { 217, -2 }, /* (94) db_optr ::= db_optr ctime */
+ { 217, -2 }, /* (95) db_optr ::= db_optr wal */
+ { 217, -2 }, /* (96) db_optr ::= db_optr fsync */
+ { 217, -2 }, /* (97) db_optr ::= db_optr comp */
+ { 217, -2 }, /* (98) db_optr ::= db_optr prec */
+ { 217, -2 }, /* (99) db_optr ::= db_optr keep */
+ { 217, -2 }, /* (100) db_optr ::= db_optr update */
+ { 214, 0 }, /* (101) alter_db_optr ::= */
+ { 214, -2 }, /* (102) alter_db_optr ::= alter_db_optr replica */
+ { 214, -2 }, /* (103) alter_db_optr ::= alter_db_optr quorum */
+ { 214, -2 }, /* (104) alter_db_optr ::= alter_db_optr keep */
+ { 214, -2 }, /* (105) alter_db_optr ::= alter_db_optr blocks */
+ { 214, -2 }, /* (106) alter_db_optr ::= alter_db_optr comp */
+ { 214, -2 }, /* (107) alter_db_optr ::= alter_db_optr wal */
+ { 214, -2 }, /* (108) alter_db_optr ::= alter_db_optr fsync */
+ { 214, -2 }, /* (109) alter_db_optr ::= alter_db_optr update */
+ { 242, -1 }, /* (110) typename ::= ids */
+ { 242, -4 }, /* (111) typename ::= ids LP signed RP */
+ { 243, -1 }, /* (112) signed ::= INTEGER */
+ { 243, -2 }, /* (113) signed ::= PLUS INTEGER */
+ { 243, -2 }, /* (114) signed ::= MINUS INTEGER */
+ { 209, -6 }, /* (115) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
+ { 244, -3 }, /* (116) create_table_args ::= LP columnlist RP */
+ { 244, -7 }, /* (117) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
+ { 244, -7 }, /* (118) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
+ { 244, -2 }, /* (119) create_table_args ::= AS select */
+ { 245, -3 }, /* (120) columnlist ::= columnlist COMMA column */
+ { 245, -1 }, /* (121) columnlist ::= column */
+ { 247, -2 }, /* (122) column ::= ids typename */
+ { 228, -3 }, /* (123) tagitemlist ::= tagitemlist COMMA tagitem */
+ { 228, -1 }, /* (124) tagitemlist ::= tagitem */
+ { 248, -1 }, /* (125) tagitem ::= INTEGER */
+ { 248, -1 }, /* (126) tagitem ::= FLOAT */
+ { 248, -1 }, /* (127) tagitem ::= STRING */
+ { 248, -1 }, /* (128) tagitem ::= BOOL */
+ { 248, -1 }, /* (129) tagitem ::= NULL */
+ { 248, -2 }, /* (130) tagitem ::= MINUS INTEGER */
+ { 248, -2 }, /* (131) tagitem ::= MINUS FLOAT */
+ { 248, -2 }, /* (132) tagitem ::= PLUS INTEGER */
+ { 248, -2 }, /* (133) tagitem ::= PLUS FLOAT */
+ { 246, -12 }, /* (134) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
+ { 260, -1 }, /* (135) union ::= select */
+ { 260, -3 }, /* (136) union ::= LP union RP */
+ { 260, -4 }, /* (137) union ::= union UNION ALL select */
+ { 260, -6 }, /* (138) union ::= union UNION ALL LP select RP */
+ { 209, -1 }, /* (139) cmd ::= union */
+ { 246, -2 }, /* (140) select ::= SELECT selcollist */
+ { 261, -2 }, /* (141) sclp ::= selcollist COMMA */
+ { 261, 0 }, /* (142) sclp ::= */
+ { 249, -3 }, /* (143) selcollist ::= sclp expr as */
+ { 249, -2 }, /* (144) selcollist ::= sclp STAR */
+ { 263, -2 }, /* (145) as ::= AS ids */
+ { 263, -1 }, /* (146) as ::= ids */
+ { 263, 0 }, /* (147) as ::= */
+ { 250, -2 }, /* (148) from ::= FROM tablelist */
+ { 264, -2 }, /* (149) tablelist ::= ids cpxName */
+ { 264, -3 }, /* (150) tablelist ::= ids cpxName ids */
+ { 264, -4 }, /* (151) tablelist ::= tablelist COMMA ids cpxName */
+ { 264, -5 }, /* (152) tablelist ::= tablelist COMMA ids cpxName ids */
+ { 265, -1 }, /* (153) tmvar ::= VARIABLE */
+ { 252, -4 }, /* (154) interval_opt ::= INTERVAL LP tmvar RP */
+ { 252, -6 }, /* (155) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */
+ { 252, 0 }, /* (156) interval_opt ::= */
+ { 253, 0 }, /* (157) fill_opt ::= */
+ { 253, -6 }, /* (158) fill_opt ::= FILL LP ID COMMA tagitemlist RP */
+ { 253, -4 }, /* (159) fill_opt ::= FILL LP ID RP */
+ { 254, -4 }, /* (160) sliding_opt ::= SLIDING LP tmvar RP */
+ { 254, 0 }, /* (161) sliding_opt ::= */
+ { 256, 0 }, /* (162) orderby_opt ::= */
+ { 256, -3 }, /* (163) orderby_opt ::= ORDER BY sortlist */
+ { 266, -4 }, /* (164) sortlist ::= sortlist COMMA item sortorder */
+ { 266, -2 }, /* (165) sortlist ::= item sortorder */
+ { 268, -2 }, /* (166) item ::= ids cpxName */
+ { 269, -1 }, /* (167) sortorder ::= ASC */
+ { 269, -1 }, /* (168) sortorder ::= DESC */
+ { 269, 0 }, /* (169) sortorder ::= */
+ { 255, 0 }, /* (170) groupby_opt ::= */
+ { 255, -3 }, /* (171) groupby_opt ::= GROUP BY grouplist */
+ { 270, -3 }, /* (172) grouplist ::= grouplist COMMA item */
+ { 270, -1 }, /* (173) grouplist ::= item */
+ { 257, 0 }, /* (174) having_opt ::= */
+ { 257, -2 }, /* (175) having_opt ::= HAVING expr */
+ { 259, 0 }, /* (176) limit_opt ::= */
+ { 259, -2 }, /* (177) limit_opt ::= LIMIT signed */
+ { 259, -4 }, /* (178) limit_opt ::= LIMIT signed OFFSET signed */
+ { 259, -4 }, /* (179) limit_opt ::= LIMIT signed COMMA signed */
+ { 258, 0 }, /* (180) slimit_opt ::= */
+ { 258, -2 }, /* (181) slimit_opt ::= SLIMIT signed */
+ { 258, -4 }, /* (182) slimit_opt ::= SLIMIT signed SOFFSET signed */
+ { 258, -4 }, /* (183) slimit_opt ::= SLIMIT signed COMMA signed */
+ { 251, 0 }, /* (184) where_opt ::= */
+ { 251, -2 }, /* (185) where_opt ::= WHERE expr */
+ { 262, -3 }, /* (186) expr ::= LP expr RP */
+ { 262, -1 }, /* (187) expr ::= ID */
+ { 262, -3 }, /* (188) expr ::= ID DOT ID */
+ { 262, -3 }, /* (189) expr ::= ID DOT STAR */
+ { 262, -1 }, /* (190) expr ::= INTEGER */
+ { 262, -2 }, /* (191) expr ::= MINUS INTEGER */
+ { 262, -2 }, /* (192) expr ::= PLUS INTEGER */
+ { 262, -1 }, /* (193) expr ::= FLOAT */
+ { 262, -2 }, /* (194) expr ::= MINUS FLOAT */
+ { 262, -2 }, /* (195) expr ::= PLUS FLOAT */
+ { 262, -1 }, /* (196) expr ::= STRING */
+ { 262, -1 }, /* (197) expr ::= NOW */
+ { 262, -1 }, /* (198) expr ::= VARIABLE */
+ { 262, -1 }, /* (199) expr ::= BOOL */
+ { 262, -4 }, /* (200) expr ::= ID LP exprlist RP */
+ { 262, -4 }, /* (201) expr ::= ID LP STAR RP */
+ { 262, -3 }, /* (202) expr ::= expr IS NULL */
+ { 262, -4 }, /* (203) expr ::= expr IS NOT NULL */
+ { 262, -3 }, /* (204) expr ::= expr LT expr */
+ { 262, -3 }, /* (205) expr ::= expr GT expr */
+ { 262, -3 }, /* (206) expr ::= expr LE expr */
+ { 262, -3 }, /* (207) expr ::= expr GE expr */
+ { 262, -3 }, /* (208) expr ::= expr NE expr */
+ { 262, -3 }, /* (209) expr ::= expr EQ expr */
+ { 262, -3 }, /* (210) expr ::= expr AND expr */
+ { 262, -3 }, /* (211) expr ::= expr OR expr */
+ { 262, -3 }, /* (212) expr ::= expr PLUS expr */
+ { 262, -3 }, /* (213) expr ::= expr MINUS expr */
+ { 262, -3 }, /* (214) expr ::= expr STAR expr */
+ { 262, -3 }, /* (215) expr ::= expr SLASH expr */
+ { 262, -3 }, /* (216) expr ::= expr REM expr */
+ { 262, -3 }, /* (217) expr ::= expr LIKE expr */
+ { 262, -5 }, /* (218) expr ::= expr IN LP exprlist RP */
+ { 271, -3 }, /* (219) exprlist ::= exprlist COMMA expritem */
+ { 271, -1 }, /* (220) exprlist ::= expritem */
+ { 272, -1 }, /* (221) expritem ::= expr */
+ { 272, 0 }, /* (222) expritem ::= */
+ { 209, -3 }, /* (223) cmd ::= RESET QUERY CACHE */
+ { 209, -7 }, /* (224) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
+ { 209, -7 }, /* (225) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
+ { 209, -7 }, /* (226) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
+ { 209, -7 }, /* (227) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
+ { 209, -8 }, /* (228) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
+ { 209, -9 }, /* (229) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
+ { 209, -3 }, /* (230) cmd ::= KILL CONNECTION INTEGER */
+ { 209, -5 }, /* (231) cmd ::= KILL STREAM INTEGER COLON INTEGER */
+ { 209, -5 }, /* (232) cmd ::= KILL QUERY INTEGER COLON INTEGER */
};
static void yy_accept(yyParser*); /* Forward Declaration */
@@ -2156,13 +2166,13 @@ static void yy_reduce(
{ setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); }
break;
case 40: /* cmd ::= ALTER DATABASE ids alter_db_optr */
-{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy158, &t);}
+{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);}
break;
case 41: /* cmd ::= ALTER ACCOUNT ids acct_optr */
-{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy73);}
+{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);}
break;
case 42: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */
-{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy73);}
+{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);}
break;
case 43: /* ids ::= ID */
case 44: /* ids ::= STRING */ yytestcase(yyruleno==44);
@@ -2183,10 +2193,10 @@ static void yy_reduce(
{ setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);}
break;
case 50: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */
-{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy73);}
+{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);}
break;
case 51: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */
-{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy158, &yymsp[-2].minor.yy0);}
+{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &yymsp[-2].minor.yy0);}
break;
case 52: /* cmd ::= CREATE USER ids PASS ids */
{ setCreateUserSQL(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);}
@@ -2215,20 +2225,20 @@ static void yy_reduce(
break;
case 71: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */
{
- yylhsminor.yy73.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1;
- yylhsminor.yy73.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1;
- yylhsminor.yy73.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1;
- yylhsminor.yy73.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1;
- yylhsminor.yy73.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1;
- yylhsminor.yy73.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1;
- yylhsminor.yy73.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1;
- yylhsminor.yy73.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1;
- yylhsminor.yy73.stat = yymsp[0].minor.yy0;
-}
- yymsp[-8].minor.yy73 = yylhsminor.yy73;
+ yylhsminor.yy149.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1;
+ yylhsminor.yy149.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1;
+ yylhsminor.yy149.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1;
+ yylhsminor.yy149.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1;
+ yylhsminor.yy149.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1;
+ yylhsminor.yy149.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1;
+ yylhsminor.yy149.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1;
+ yylhsminor.yy149.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1;
+ yylhsminor.yy149.stat = yymsp[0].minor.yy0;
+}
+ yymsp[-8].minor.yy149 = yylhsminor.yy149;
break;
case 72: /* keep ::= KEEP tagitemlist */
-{ yymsp[-1].minor.yy494 = yymsp[0].minor.yy494; }
+{ yymsp[-1].minor.yy498 = yymsp[0].minor.yy498; }
break;
case 73: /* cache ::= CACHE INTEGER */
case 74: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==74);
@@ -2242,540 +2252,546 @@ static void yy_reduce(
case 82: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==82);
case 83: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==83);
case 84: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==84);
+ case 85: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==85);
{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; }
break;
- case 85: /* db_optr ::= */
-{setDefaultCreateDbOption(&yymsp[1].minor.yy158);}
- break;
- case 86: /* db_optr ::= db_optr cache */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 87: /* db_optr ::= db_optr replica */
- case 100: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==100);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 88: /* db_optr ::= db_optr quorum */
- case 101: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==101);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 89: /* db_optr ::= db_optr days */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 90: /* db_optr ::= db_optr minrows */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 91: /* db_optr ::= db_optr maxrows */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 92: /* db_optr ::= db_optr blocks */
- case 103: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==103);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 93: /* db_optr ::= db_optr ctime */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 94: /* db_optr ::= db_optr wal */
- case 105: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==105);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 95: /* db_optr ::= db_optr fsync */
- case 106: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==106);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 96: /* db_optr ::= db_optr comp */
- case 104: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==104);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 97: /* db_optr ::= db_optr prec */
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.precision = yymsp[0].minor.yy0; }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 98: /* db_optr ::= db_optr keep */
- case 102: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==102);
-{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.keep = yymsp[0].minor.yy494; }
- yymsp[-1].minor.yy158 = yylhsminor.yy158;
- break;
- case 99: /* alter_db_optr ::= */
-{ setDefaultCreateDbOption(&yymsp[1].minor.yy158);}
- break;
- case 107: /* typename ::= ids */
+ case 86: /* db_optr ::= */
+{setDefaultCreateDbOption(&yymsp[1].minor.yy268);}
+ break;
+ case 87: /* db_optr ::= db_optr cache */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 88: /* db_optr ::= db_optr replica */
+ case 102: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==102);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 89: /* db_optr ::= db_optr quorum */
+ case 103: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==103);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 90: /* db_optr ::= db_optr days */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 91: /* db_optr ::= db_optr minrows */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 92: /* db_optr ::= db_optr maxrows */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 93: /* db_optr ::= db_optr blocks */
+ case 105: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==105);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 94: /* db_optr ::= db_optr ctime */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 95: /* db_optr ::= db_optr wal */
+ case 107: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==107);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 96: /* db_optr ::= db_optr fsync */
+ case 108: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==108);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 97: /* db_optr ::= db_optr comp */
+ case 106: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==106);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 98: /* db_optr ::= db_optr prec */
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.precision = yymsp[0].minor.yy0; }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 99: /* db_optr ::= db_optr keep */
+ case 104: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==104);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.keep = yymsp[0].minor.yy498; }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 100: /* db_optr ::= db_optr update */
+ case 109: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==109);
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[-1].minor.yy268 = yylhsminor.yy268;
+ break;
+ case 101: /* alter_db_optr ::= */
+{ setDefaultCreateDbOption(&yymsp[1].minor.yy268);}
+ break;
+ case 110: /* typename ::= ids */
{
yymsp[0].minor.yy0.type = 0;
- tSQLSetColumnType (&yylhsminor.yy181, &yymsp[0].minor.yy0);
+ tSQLSetColumnType (&yylhsminor.yy223, &yymsp[0].minor.yy0);
}
- yymsp[0].minor.yy181 = yylhsminor.yy181;
+ yymsp[0].minor.yy223 = yylhsminor.yy223;
break;
- case 108: /* typename ::= ids LP signed RP */
+ case 111: /* typename ::= ids LP signed RP */
{
- if (yymsp[-1].minor.yy271 <= 0) {
+ if (yymsp[-1].minor.yy207 <= 0) {
yymsp[-3].minor.yy0.type = 0;
- tSQLSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0);
+ tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0);
} else {
- yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy271; // negative value of name length
- tSQLSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0);
+ yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy207; // negative value of name length
+ tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0);
}
}
- yymsp[-3].minor.yy181 = yylhsminor.yy181;
+ yymsp[-3].minor.yy223 = yylhsminor.yy223;
break;
- case 109: /* signed ::= INTEGER */
-{ yylhsminor.yy271 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
- yymsp[0].minor.yy271 = yylhsminor.yy271;
+ case 112: /* signed ::= INTEGER */
+{ yylhsminor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ yymsp[0].minor.yy207 = yylhsminor.yy207;
break;
- case 110: /* signed ::= PLUS INTEGER */
-{ yymsp[-1].minor.yy271 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+ case 113: /* signed ::= PLUS INTEGER */
+{ yymsp[-1].minor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
break;
- case 111: /* signed ::= MINUS INTEGER */
-{ yymsp[-1].minor.yy271 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);}
+ case 114: /* signed ::= MINUS INTEGER */
+{ yymsp[-1].minor.yy207 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);}
break;
- case 112: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
+ case 115: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */
{
yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n;
setCreatedTableName(pInfo, &yymsp[-2].minor.yy0, &yymsp[-3].minor.yy0);
}
break;
- case 113: /* create_table_args ::= LP columnlist RP */
+ case 116: /* create_table_args ::= LP columnlist RP */
{
- yymsp[-2].minor.yy374 = tSetCreateSQLElems(yymsp[-1].minor.yy449, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE);
- setSQLInfo(pInfo, yymsp[-2].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+ yymsp[-2].minor.yy470 = tSetCreateSQLElems(yymsp[-1].minor.yy523, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE);
+ setSQLInfo(pInfo, yymsp[-2].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
}
break;
- case 114: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
+ case 117: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
{
- yymsp[-6].minor.yy374 = tSetCreateSQLElems(yymsp[-5].minor.yy449, yymsp[-1].minor.yy449, NULL, NULL, NULL, TSQL_CREATE_STABLE);
- setSQLInfo(pInfo, yymsp[-6].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+ yymsp[-6].minor.yy470 = tSetCreateSQLElems(yymsp[-5].minor.yy523, yymsp[-1].minor.yy523, NULL, NULL, NULL, TSQL_CREATE_STABLE);
+ setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
}
break;
- case 115: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
+ case 118: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
{
yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n;
- yymsp[-6].minor.yy374 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy494, NULL, TSQL_CREATE_TABLE_FROM_STABLE);
- setSQLInfo(pInfo, yymsp[-6].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+ yymsp[-6].minor.yy470 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy498, NULL, TSQL_CREATE_TABLE_FROM_STABLE);
+ setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
}
break;
- case 116: /* create_table_args ::= AS select */
+ case 119: /* create_table_args ::= AS select */
{
- yymsp[-1].minor.yy374 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy150, TSQL_CREATE_STREAM);
- setSQLInfo(pInfo, yymsp[-1].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE);
+ yymsp[-1].minor.yy470 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy414, TSQL_CREATE_STREAM);
+ setSQLInfo(pInfo, yymsp[-1].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
}
break;
- case 117: /* columnlist ::= columnlist COMMA column */
-{yylhsminor.yy449 = tFieldListAppend(yymsp[-2].minor.yy449, &yymsp[0].minor.yy181); }
- yymsp[-2].minor.yy449 = yylhsminor.yy449;
+ case 120: /* columnlist ::= columnlist COMMA column */
+{yylhsminor.yy523 = tFieldListAppend(yymsp[-2].minor.yy523, &yymsp[0].minor.yy223); }
+ yymsp[-2].minor.yy523 = yylhsminor.yy523;
break;
- case 118: /* columnlist ::= column */
-{yylhsminor.yy449 = tFieldListAppend(NULL, &yymsp[0].minor.yy181);}
- yymsp[0].minor.yy449 = yylhsminor.yy449;
+ case 121: /* columnlist ::= column */
+{yylhsminor.yy523 = tFieldListAppend(NULL, &yymsp[0].minor.yy223);}
+ yymsp[0].minor.yy523 = yylhsminor.yy523;
break;
- case 119: /* column ::= ids typename */
+ case 122: /* column ::= ids typename */
{
- tSQLSetColumnInfo(&yylhsminor.yy181, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy181);
+ tSQLSetColumnInfo(&yylhsminor.yy223, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy223);
}
- yymsp[-1].minor.yy181 = yylhsminor.yy181;
+ yymsp[-1].minor.yy223 = yylhsminor.yy223;
break;
- case 120: /* tagitemlist ::= tagitemlist COMMA tagitem */
-{ yylhsminor.yy494 = tVariantListAppend(yymsp[-2].minor.yy494, &yymsp[0].minor.yy312, -1); }
- yymsp[-2].minor.yy494 = yylhsminor.yy494;
+ case 123: /* tagitemlist ::= tagitemlist COMMA tagitem */
+{ yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1); }
+ yymsp[-2].minor.yy498 = yylhsminor.yy498;
break;
- case 121: /* tagitemlist ::= tagitem */
-{ yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[0].minor.yy312, -1); }
- yymsp[0].minor.yy494 = yylhsminor.yy494;
+ case 124: /* tagitemlist ::= tagitem */
+{ yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); }
+ yymsp[0].minor.yy498 = yylhsminor.yy498;
break;
- case 122: /* tagitem ::= INTEGER */
- case 123: /* tagitem ::= FLOAT */ yytestcase(yyruleno==123);
- case 124: /* tagitem ::= STRING */ yytestcase(yyruleno==124);
- case 125: /* tagitem ::= BOOL */ yytestcase(yyruleno==125);
-{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy312, &yymsp[0].minor.yy0); }
- yymsp[0].minor.yy312 = yylhsminor.yy312;
+ case 125: /* tagitem ::= INTEGER */
+ case 126: /* tagitem ::= FLOAT */ yytestcase(yyruleno==126);
+ case 127: /* tagitem ::= STRING */ yytestcase(yyruleno==127);
+ case 128: /* tagitem ::= BOOL */ yytestcase(yyruleno==128);
+{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); }
+ yymsp[0].minor.yy134 = yylhsminor.yy134;
break;
- case 126: /* tagitem ::= NULL */
-{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy312, &yymsp[0].minor.yy0); }
- yymsp[0].minor.yy312 = yylhsminor.yy312;
+ case 129: /* tagitem ::= NULL */
+{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); }
+ yymsp[0].minor.yy134 = yylhsminor.yy134;
break;
- case 127: /* tagitem ::= MINUS INTEGER */
- case 128: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==128);
- case 129: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==129);
- case 130: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==130);
+ case 130: /* tagitem ::= MINUS INTEGER */
+ case 131: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==131);
+ case 132: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==132);
+ case 133: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==133);
{
yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type;
toTSDBType(yymsp[-1].minor.yy0.type);
- tVariantCreate(&yylhsminor.yy312, &yymsp[-1].minor.yy0);
+ tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0);
}
- yymsp[-1].minor.yy312 = yylhsminor.yy312;
+ yymsp[-1].minor.yy134 = yylhsminor.yy134;
break;
- case 131: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
+ case 134: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
{
- yylhsminor.yy150 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy224, yymsp[-9].minor.yy494, yymsp[-8].minor.yy66, yymsp[-4].minor.yy494, yymsp[-3].minor.yy494, &yymsp[-7].minor.yy314, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy494, &yymsp[0].minor.yy188, &yymsp[-1].minor.yy188);
+ yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy290, yymsp[-9].minor.yy498, yymsp[-8].minor.yy64, yymsp[-4].minor.yy498, yymsp[-3].minor.yy498, &yymsp[-7].minor.yy532, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy498, &yymsp[0].minor.yy216, &yymsp[-1].minor.yy216);
}
- yymsp[-11].minor.yy150 = yylhsminor.yy150;
+ yymsp[-11].minor.yy414 = yylhsminor.yy414;
break;
- case 132: /* union ::= select */
-{ yylhsminor.yy25 = setSubclause(NULL, yymsp[0].minor.yy150); }
- yymsp[0].minor.yy25 = yylhsminor.yy25;
+ case 135: /* union ::= select */
+{ yylhsminor.yy231 = setSubclause(NULL, yymsp[0].minor.yy414); }
+ yymsp[0].minor.yy231 = yylhsminor.yy231;
break;
- case 133: /* union ::= LP union RP */
-{ yymsp[-2].minor.yy25 = yymsp[-1].minor.yy25; }
+ case 136: /* union ::= LP union RP */
+{ yymsp[-2].minor.yy231 = yymsp[-1].minor.yy231; }
break;
- case 134: /* union ::= union UNION ALL select */
-{ yylhsminor.yy25 = appendSelectClause(yymsp[-3].minor.yy25, yymsp[0].minor.yy150); }
- yymsp[-3].minor.yy25 = yylhsminor.yy25;
+ case 137: /* union ::= union UNION ALL select */
+{ yylhsminor.yy231 = appendSelectClause(yymsp[-3].minor.yy231, yymsp[0].minor.yy414); }
+ yymsp[-3].minor.yy231 = yylhsminor.yy231;
break;
- case 135: /* union ::= union UNION ALL LP select RP */
-{ yylhsminor.yy25 = appendSelectClause(yymsp[-5].minor.yy25, yymsp[-1].minor.yy150); }
- yymsp[-5].minor.yy25 = yylhsminor.yy25;
+ case 138: /* union ::= union UNION ALL LP select RP */
+{ yylhsminor.yy231 = appendSelectClause(yymsp[-5].minor.yy231, yymsp[-1].minor.yy414); }
+ yymsp[-5].minor.yy231 = yylhsminor.yy231;
break;
- case 136: /* cmd ::= union */
-{ setSQLInfo(pInfo, yymsp[0].minor.yy25, NULL, TSDB_SQL_SELECT); }
+ case 139: /* cmd ::= union */
+{ setSQLInfo(pInfo, yymsp[0].minor.yy231, NULL, TSDB_SQL_SELECT); }
break;
- case 137: /* select ::= SELECT selcollist */
+ case 140: /* select ::= SELECT selcollist */
{
- yylhsminor.yy150 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy224, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+ yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy290, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
}
- yymsp[-1].minor.yy150 = yylhsminor.yy150;
+ yymsp[-1].minor.yy414 = yylhsminor.yy414;
break;
- case 138: /* sclp ::= selcollist COMMA */
-{yylhsminor.yy224 = yymsp[-1].minor.yy224;}
- yymsp[-1].minor.yy224 = yylhsminor.yy224;
+ case 141: /* sclp ::= selcollist COMMA */
+{yylhsminor.yy290 = yymsp[-1].minor.yy290;}
+ yymsp[-1].minor.yy290 = yylhsminor.yy290;
break;
- case 139: /* sclp ::= */
-{yymsp[1].minor.yy224 = 0;}
+ case 142: /* sclp ::= */
+{yymsp[1].minor.yy290 = 0;}
break;
- case 140: /* selcollist ::= sclp expr as */
+ case 143: /* selcollist ::= sclp expr as */
{
- yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224, yymsp[-1].minor.yy66, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0);
+ yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290, yymsp[-1].minor.yy64, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0);
}
- yymsp[-2].minor.yy224 = yylhsminor.yy224;
+ yymsp[-2].minor.yy290 = yylhsminor.yy290;
break;
- case 141: /* selcollist ::= sclp STAR */
+ case 144: /* selcollist ::= sclp STAR */
{
tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL);
- yylhsminor.yy224 = tSQLExprListAppend(yymsp[-1].minor.yy224, pNode, 0);
+ yylhsminor.yy290 = tSQLExprListAppend(yymsp[-1].minor.yy290, pNode, 0);
}
- yymsp[-1].minor.yy224 = yylhsminor.yy224;
+ yymsp[-1].minor.yy290 = yylhsminor.yy290;
break;
- case 142: /* as ::= AS ids */
+ case 145: /* as ::= AS ids */
{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; }
break;
- case 143: /* as ::= ids */
+ case 146: /* as ::= ids */
{ yylhsminor.yy0 = yymsp[0].minor.yy0; }
yymsp[0].minor.yy0 = yylhsminor.yy0;
break;
- case 144: /* as ::= */
+ case 147: /* as ::= */
{ yymsp[1].minor.yy0.n = 0; }
break;
- case 145: /* from ::= FROM tablelist */
-{yymsp[-1].minor.yy494 = yymsp[0].minor.yy494;}
+ case 148: /* from ::= FROM tablelist */
+{yymsp[-1].minor.yy498 = yymsp[0].minor.yy498;}
break;
- case 146: /* tablelist ::= ids cpxName */
+ case 149: /* tablelist ::= ids cpxName */
{
toTSDBType(yymsp[-1].minor.yy0.type);
yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
- yylhsminor.yy494 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
- yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[-1].minor.yy0, -1); // table alias name
+ yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
+ yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[-1].minor.yy0, -1); // table alias name
}
- yymsp[-1].minor.yy494 = yylhsminor.yy494;
+ yymsp[-1].minor.yy498 = yylhsminor.yy498;
break;
- case 147: /* tablelist ::= ids cpxName ids */
+ case 150: /* tablelist ::= ids cpxName ids */
{
toTSDBType(yymsp[-2].minor.yy0.type);
toTSDBType(yymsp[0].minor.yy0.type);
yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n;
- yylhsminor.yy494 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
- yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[0].minor.yy0, -1);
+ yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
+ yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[0].minor.yy0, -1);
}
- yymsp[-2].minor.yy494 = yylhsminor.yy494;
+ yymsp[-2].minor.yy498 = yylhsminor.yy498;
break;
- case 148: /* tablelist ::= tablelist COMMA ids cpxName */
+ case 151: /* tablelist ::= tablelist COMMA ids cpxName */
{
toTSDBType(yymsp[-1].minor.yy0.type);
yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
- yylhsminor.yy494 = tVariantListAppendToken(yymsp[-3].minor.yy494, &yymsp[-1].minor.yy0, -1);
- yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[-1].minor.yy0, -1);
+ yylhsminor.yy498 = tVariantListAppendToken(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy0, -1);
+ yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[-1].minor.yy0, -1);
}
- yymsp[-3].minor.yy494 = yylhsminor.yy494;
+ yymsp[-3].minor.yy498 = yylhsminor.yy498;
break;
- case 149: /* tablelist ::= tablelist COMMA ids cpxName ids */
+ case 152: /* tablelist ::= tablelist COMMA ids cpxName ids */
{
toTSDBType(yymsp[-2].minor.yy0.type);
toTSDBType(yymsp[0].minor.yy0.type);
yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n;
- yylhsminor.yy494 = tVariantListAppendToken(yymsp[-4].minor.yy494, &yymsp[-2].minor.yy0, -1);
- yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[0].minor.yy0, -1);
+ yylhsminor.yy498 = tVariantListAppendToken(yymsp[-4].minor.yy498, &yymsp[-2].minor.yy0, -1);
+ yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[0].minor.yy0, -1);
}
- yymsp[-4].minor.yy494 = yylhsminor.yy494;
+ yymsp[-4].minor.yy498 = yylhsminor.yy498;
break;
- case 150: /* tmvar ::= VARIABLE */
+ case 153: /* tmvar ::= VARIABLE */
{yylhsminor.yy0 = yymsp[0].minor.yy0;}
yymsp[0].minor.yy0 = yylhsminor.yy0;
break;
- case 151: /* interval_opt ::= INTERVAL LP tmvar RP */
-{yymsp[-3].minor.yy314.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy314.offset.n = 0; yymsp[-3].minor.yy314.offset.z = NULL; yymsp[-3].minor.yy314.offset.type = 0;}
+ case 154: /* interval_opt ::= INTERVAL LP tmvar RP */
+{yymsp[-3].minor.yy532.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy532.offset.n = 0; yymsp[-3].minor.yy532.offset.z = NULL; yymsp[-3].minor.yy532.offset.type = 0;}
break;
- case 152: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */
-{yymsp[-5].minor.yy314.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy314.offset = yymsp[-1].minor.yy0;}
+ case 155: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */
+{yymsp[-5].minor.yy532.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy532.offset = yymsp[-1].minor.yy0;}
break;
- case 153: /* interval_opt ::= */
-{memset(&yymsp[1].minor.yy314, 0, sizeof(yymsp[1].minor.yy314));}
+ case 156: /* interval_opt ::= */
+{memset(&yymsp[1].minor.yy532, 0, sizeof(yymsp[1].minor.yy532));}
break;
- case 154: /* fill_opt ::= */
-{yymsp[1].minor.yy494 = 0; }
+ case 157: /* fill_opt ::= */
+{yymsp[1].minor.yy498 = 0; }
break;
- case 155: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */
+ case 158: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */
{
tVariant A = {0};
toTSDBType(yymsp[-3].minor.yy0.type);
tVariantCreate(&A, &yymsp[-3].minor.yy0);
- tVariantListInsert(yymsp[-1].minor.yy494, &A, -1, 0);
- yymsp[-5].minor.yy494 = yymsp[-1].minor.yy494;
+ tVariantListInsert(yymsp[-1].minor.yy498, &A, -1, 0);
+ yymsp[-5].minor.yy498 = yymsp[-1].minor.yy498;
}
break;
- case 156: /* fill_opt ::= FILL LP ID RP */
+ case 159: /* fill_opt ::= FILL LP ID RP */
{
toTSDBType(yymsp[-1].minor.yy0.type);
- yymsp[-3].minor.yy494 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
+ yymsp[-3].minor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
}
break;
- case 157: /* sliding_opt ::= SLIDING LP tmvar RP */
+ case 160: /* sliding_opt ::= SLIDING LP tmvar RP */
{yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; }
break;
- case 158: /* sliding_opt ::= */
+ case 161: /* sliding_opt ::= */
{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; }
break;
- case 159: /* orderby_opt ::= */
- case 167: /* groupby_opt ::= */ yytestcase(yyruleno==167);
-{yymsp[1].minor.yy494 = 0;}
+ case 162: /* orderby_opt ::= */
+ case 170: /* groupby_opt ::= */ yytestcase(yyruleno==170);
+{yymsp[1].minor.yy498 = 0;}
break;
- case 160: /* orderby_opt ::= ORDER BY sortlist */
- case 168: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==168);
-{yymsp[-2].minor.yy494 = yymsp[0].minor.yy494;}
+ case 163: /* orderby_opt ::= ORDER BY sortlist */
+ case 171: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==171);
+{yymsp[-2].minor.yy498 = yymsp[0].minor.yy498;}
break;
- case 161: /* sortlist ::= sortlist COMMA item sortorder */
+ case 164: /* sortlist ::= sortlist COMMA item sortorder */
{
- yylhsminor.yy494 = tVariantListAppend(yymsp[-3].minor.yy494, &yymsp[-1].minor.yy312, yymsp[0].minor.yy82);
+ yylhsminor.yy498 = tVariantListAppend(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
}
- yymsp[-3].minor.yy494 = yylhsminor.yy494;
+ yymsp[-3].minor.yy498 = yylhsminor.yy498;
break;
- case 162: /* sortlist ::= item sortorder */
+ case 165: /* sortlist ::= item sortorder */
{
- yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[-1].minor.yy312, yymsp[0].minor.yy82);
+ yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
}
- yymsp[-1].minor.yy494 = yylhsminor.yy494;
+ yymsp[-1].minor.yy498 = yylhsminor.yy498;
break;
- case 163: /* item ::= ids cpxName */
+ case 166: /* item ::= ids cpxName */
{
toTSDBType(yymsp[-1].minor.yy0.type);
yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
- tVariantCreate(&yylhsminor.yy312, &yymsp[-1].minor.yy0);
+ tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0);
}
- yymsp[-1].minor.yy312 = yylhsminor.yy312;
+ yymsp[-1].minor.yy134 = yylhsminor.yy134;
break;
- case 164: /* sortorder ::= ASC */
-{yymsp[0].minor.yy82 = TSDB_ORDER_ASC; }
+ case 167: /* sortorder ::= ASC */
+{yymsp[0].minor.yy46 = TSDB_ORDER_ASC; }
break;
- case 165: /* sortorder ::= DESC */
-{yymsp[0].minor.yy82 = TSDB_ORDER_DESC;}
+ case 168: /* sortorder ::= DESC */
+{yymsp[0].minor.yy46 = TSDB_ORDER_DESC;}
break;
- case 166: /* sortorder ::= */
-{yymsp[1].minor.yy82 = TSDB_ORDER_ASC;}
+ case 169: /* sortorder ::= */
+{yymsp[1].minor.yy46 = TSDB_ORDER_ASC;}
break;
- case 169: /* grouplist ::= grouplist COMMA item */
+ case 172: /* grouplist ::= grouplist COMMA item */
{
- yylhsminor.yy494 = tVariantListAppend(yymsp[-2].minor.yy494, &yymsp[0].minor.yy312, -1);
+ yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1);
}
- yymsp[-2].minor.yy494 = yylhsminor.yy494;
+ yymsp[-2].minor.yy498 = yylhsminor.yy498;
break;
- case 170: /* grouplist ::= item */
+ case 173: /* grouplist ::= item */
{
- yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[0].minor.yy312, -1);
+ yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1);
}
- yymsp[0].minor.yy494 = yylhsminor.yy494;
+ yymsp[0].minor.yy498 = yylhsminor.yy498;
break;
- case 171: /* having_opt ::= */
- case 181: /* where_opt ::= */ yytestcase(yyruleno==181);
- case 219: /* expritem ::= */ yytestcase(yyruleno==219);
-{yymsp[1].minor.yy66 = 0;}
+ case 174: /* having_opt ::= */
+ case 184: /* where_opt ::= */ yytestcase(yyruleno==184);
+ case 222: /* expritem ::= */ yytestcase(yyruleno==222);
+{yymsp[1].minor.yy64 = 0;}
break;
- case 172: /* having_opt ::= HAVING expr */
- case 182: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==182);
-{yymsp[-1].minor.yy66 = yymsp[0].minor.yy66;}
+ case 175: /* having_opt ::= HAVING expr */
+ case 185: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==185);
+{yymsp[-1].minor.yy64 = yymsp[0].minor.yy64;}
break;
- case 173: /* limit_opt ::= */
- case 177: /* slimit_opt ::= */ yytestcase(yyruleno==177);
-{yymsp[1].minor.yy188.limit = -1; yymsp[1].minor.yy188.offset = 0;}
+ case 176: /* limit_opt ::= */
+ case 180: /* slimit_opt ::= */ yytestcase(yyruleno==180);
+{yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;}
break;
- case 174: /* limit_opt ::= LIMIT signed */
- case 178: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==178);
-{yymsp[-1].minor.yy188.limit = yymsp[0].minor.yy271; yymsp[-1].minor.yy188.offset = 0;}
+ case 177: /* limit_opt ::= LIMIT signed */
+ case 181: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==181);
+{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;}
break;
- case 175: /* limit_opt ::= LIMIT signed OFFSET signed */
- case 179: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==179);
-{yymsp[-3].minor.yy188.limit = yymsp[-2].minor.yy271; yymsp[-3].minor.yy188.offset = yymsp[0].minor.yy271;}
+ case 178: /* limit_opt ::= LIMIT signed OFFSET signed */
+ case 182: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==182);
+{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
break;
- case 176: /* limit_opt ::= LIMIT signed COMMA signed */
- case 180: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==180);
-{yymsp[-3].minor.yy188.limit = yymsp[0].minor.yy271; yymsp[-3].minor.yy188.offset = yymsp[-2].minor.yy271;}
+ case 179: /* limit_opt ::= LIMIT signed COMMA signed */
+ case 183: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==183);
+{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
break;
- case 183: /* expr ::= LP expr RP */
-{yymsp[-2].minor.yy66 = yymsp[-1].minor.yy66; }
+ case 186: /* expr ::= LP expr RP */
+{yymsp[-2].minor.yy64 = yymsp[-1].minor.yy64; }
break;
- case 184: /* expr ::= ID */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 187: /* expr ::= ID */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 185: /* expr ::= ID DOT ID */
-{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 188: /* expr ::= ID DOT ID */
+{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 186: /* expr ::= ID DOT STAR */
-{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 189: /* expr ::= ID DOT STAR */
+{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 187: /* expr ::= INTEGER */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 190: /* expr ::= INTEGER */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 188: /* expr ::= MINUS INTEGER */
- case 189: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==189);
-{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);}
- yymsp[-1].minor.yy66 = yylhsminor.yy66;
+ case 191: /* expr ::= MINUS INTEGER */
+ case 192: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==192);
+{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);}
+ yymsp[-1].minor.yy64 = yylhsminor.yy64;
break;
- case 190: /* expr ::= FLOAT */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 193: /* expr ::= FLOAT */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 191: /* expr ::= MINUS FLOAT */
- case 192: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==192);
-{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);}
- yymsp[-1].minor.yy66 = yylhsminor.yy66;
+ case 194: /* expr ::= MINUS FLOAT */
+ case 195: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==195);
+{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);}
+ yymsp[-1].minor.yy64 = yylhsminor.yy64;
break;
- case 193: /* expr ::= STRING */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 196: /* expr ::= STRING */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 194: /* expr ::= NOW */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); }
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 197: /* expr ::= NOW */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); }
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 195: /* expr ::= VARIABLE */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 198: /* expr ::= VARIABLE */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 196: /* expr ::= BOOL */
-{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 199: /* expr ::= BOOL */
+{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 197: /* expr ::= ID LP exprlist RP */
-{ yylhsminor.yy66 = tSQLExprCreateFunction(yymsp[-1].minor.yy224, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
- yymsp[-3].minor.yy66 = yylhsminor.yy66;
+ case 200: /* expr ::= ID LP exprlist RP */
+{ yylhsminor.yy64 = tSQLExprCreateFunction(yymsp[-1].minor.yy290, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
+ yymsp[-3].minor.yy64 = yylhsminor.yy64;
break;
- case 198: /* expr ::= ID LP STAR RP */
-{ yylhsminor.yy66 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
- yymsp[-3].minor.yy66 = yylhsminor.yy66;
+ case 201: /* expr ::= ID LP STAR RP */
+{ yylhsminor.yy64 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
+ yymsp[-3].minor.yy64 = yylhsminor.yy64;
break;
- case 199: /* expr ::= expr IS NULL */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, NULL, TK_ISNULL);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 202: /* expr ::= expr IS NULL */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, NULL, TK_ISNULL);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 200: /* expr ::= expr IS NOT NULL */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-3].minor.yy66, NULL, TK_NOTNULL);}
- yymsp[-3].minor.yy66 = yylhsminor.yy66;
+ case 203: /* expr ::= expr IS NOT NULL */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-3].minor.yy64, NULL, TK_NOTNULL);}
+ yymsp[-3].minor.yy64 = yylhsminor.yy64;
break;
- case 201: /* expr ::= expr LT expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LT);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 204: /* expr ::= expr LT expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LT);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 202: /* expr ::= expr GT expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_GT);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 205: /* expr ::= expr GT expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GT);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 203: /* expr ::= expr LE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LE);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 206: /* expr ::= expr LE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LE);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 204: /* expr ::= expr GE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_GE);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 207: /* expr ::= expr GE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GE);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 205: /* expr ::= expr NE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_NE);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 208: /* expr ::= expr NE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_NE);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 206: /* expr ::= expr EQ expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_EQ);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 209: /* expr ::= expr EQ expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_EQ);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 207: /* expr ::= expr AND expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_AND);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 210: /* expr ::= expr AND expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_AND);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 208: /* expr ::= expr OR expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_OR); }
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 211: /* expr ::= expr OR expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_OR); }
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 209: /* expr ::= expr PLUS expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_PLUS); }
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 212: /* expr ::= expr PLUS expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_PLUS); }
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 210: /* expr ::= expr MINUS expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_MINUS); }
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 213: /* expr ::= expr MINUS expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_MINUS); }
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 211: /* expr ::= expr STAR expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_STAR); }
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 214: /* expr ::= expr STAR expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_STAR); }
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 212: /* expr ::= expr SLASH expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_DIVIDE);}
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 215: /* expr ::= expr SLASH expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_DIVIDE);}
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 213: /* expr ::= expr REM expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_REM); }
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 216: /* expr ::= expr REM expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_REM); }
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 214: /* expr ::= expr LIKE expr */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LIKE); }
- yymsp[-2].minor.yy66 = yylhsminor.yy66;
+ case 217: /* expr ::= expr LIKE expr */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LIKE); }
+ yymsp[-2].minor.yy64 = yylhsminor.yy64;
break;
- case 215: /* expr ::= expr IN LP exprlist RP */
-{yylhsminor.yy66 = tSQLExprCreate(yymsp[-4].minor.yy66, (tSQLExpr*)yymsp[-1].minor.yy224, TK_IN); }
- yymsp[-4].minor.yy66 = yylhsminor.yy66;
+ case 218: /* expr ::= expr IN LP exprlist RP */
+{yylhsminor.yy64 = tSQLExprCreate(yymsp[-4].minor.yy64, (tSQLExpr*)yymsp[-1].minor.yy290, TK_IN); }
+ yymsp[-4].minor.yy64 = yylhsminor.yy64;
break;
- case 216: /* exprlist ::= exprlist COMMA expritem */
-{yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224,yymsp[0].minor.yy66,0);}
- yymsp[-2].minor.yy224 = yylhsminor.yy224;
+ case 219: /* exprlist ::= exprlist COMMA expritem */
+{yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290,yymsp[0].minor.yy64,0);}
+ yymsp[-2].minor.yy290 = yylhsminor.yy290;
break;
- case 217: /* exprlist ::= expritem */
-{yylhsminor.yy224 = tSQLExprListAppend(0,yymsp[0].minor.yy66,0);}
- yymsp[0].minor.yy224 = yylhsminor.yy224;
+ case 220: /* exprlist ::= expritem */
+{yylhsminor.yy290 = tSQLExprListAppend(0,yymsp[0].minor.yy64,0);}
+ yymsp[0].minor.yy290 = yylhsminor.yy290;
break;
- case 218: /* expritem ::= expr */
-{yylhsminor.yy66 = yymsp[0].minor.yy66;}
- yymsp[0].minor.yy66 = yylhsminor.yy66;
+ case 221: /* expritem ::= expr */
+{yylhsminor.yy64 = yymsp[0].minor.yy64;}
+ yymsp[0].minor.yy64 = yylhsminor.yy64;
break;
- case 220: /* cmd ::= RESET QUERY CACHE */
+ case 223: /* cmd ::= RESET QUERY CACHE */
{ setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);}
break;
- case 221: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
+ case 224: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
{
yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
- SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_COLUMN);
+ SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
- case 222: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
+ case 225: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */
{
yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
@@ -2786,14 +2802,14 @@ static void yy_reduce(
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
- case 223: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
+ case 226: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
{
yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
- SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN);
+ SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
- case 224: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
+ case 227: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */
{
yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
@@ -2804,7 +2820,7 @@ static void yy_reduce(
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
- case 225: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
+ case 228: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */
{
yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n;
@@ -2818,25 +2834,25 @@ static void yy_reduce(
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
- case 226: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
+ case 229: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */
{
yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n;
toTSDBType(yymsp[-2].minor.yy0.type);
tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
- A = tVariantListAppend(A, &yymsp[0].minor.yy312, -1);
+ A = tVariantListAppend(A, &yymsp[0].minor.yy134, -1);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
- case 227: /* cmd ::= KILL CONNECTION INTEGER */
+ case 230: /* cmd ::= KILL CONNECTION INTEGER */
{setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);}
break;
- case 228: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */
+ case 231: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */
{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);}
break;
- case 229: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */
+ case 232: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */
{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);}
break;
default:
@@ -3130,4 +3146,4 @@ void Parse(
}
#endif
return;
-}
+}
\ No newline at end of file
diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp
index b78c5314f243874e348748a6434d224592489528..bada3194bd90e30de465dceaef07742cf6e8ac07 100644
--- a/src/query/tests/tsBufTest.cpp
+++ b/src/query/tests/tsBufTest.cpp
@@ -42,7 +42,7 @@ void simpleTest() {
EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num);
EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0);
- EXPECT_EQ(pTSBuf->numOfVnodes, 1);
+ EXPECT_EQ(pTSBuf->numOfGroups, 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
@@ -69,7 +69,7 @@ void largeTSTest() {
// the data has been flush to disk, no data in cache
EXPECT_EQ(pTSBuf->tsData.len, 0);
EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0);
- EXPECT_EQ(pTSBuf->numOfVnodes, 1);
+ EXPECT_EQ(pTSBuf->numOfGroups, 1);
EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC);
tsBufFlush(pTSBuf);
@@ -105,7 +105,7 @@ void multiTagsTest() {
EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1);
- EXPECT_EQ(pTSBuf->numOfVnodes, 1);
+ EXPECT_EQ(pTSBuf->numOfGroups, 1);
tsBufFlush(pTSBuf);
EXPECT_EQ(pTSBuf->tsData.len, 0);
@@ -139,7 +139,7 @@ void multiVnodeTagsTest() {
start += step * num;
}
- EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
+ EXPECT_EQ(pTSBuf->numOfGroups, j + 1);
}
EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC);
@@ -184,7 +184,7 @@ void loadDataTest() {
start += step * num;
}
- EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
+ EXPECT_EQ(pTSBuf->numOfGroups, j + 1);
}
EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC);
@@ -203,7 +203,7 @@ void loadDataTest() {
// create from exists file
STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false);
EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder);
- EXPECT_EQ(pNewBuf->numOfVnodes, numOfVnode);
+ EXPECT_EQ(pNewBuf->numOfGroups, numOfVnode);
EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize);
EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset);
@@ -269,7 +269,7 @@ void TSTraverse() {
start += step * num;
}
- EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
+ EXPECT_EQ(pTSBuf->numOfGroups, j + 1);
}
tsBufResetPos(pTSBuf);
@@ -304,7 +304,7 @@ void TSTraverse() {
int32_t totalOutput = 10;
while (1) {
STSElem elem = tsBufGetElem(pTSBuf);
- printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts);
+ printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts);
if (!tsBufNextPos(pTSBuf)) {
break;
@@ -352,7 +352,7 @@ void TSTraverse() {
totalOutput = 10;
while (1) {
STSElem elem = tsBufGetElem(pTSBuf);
- printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts);
+ printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts);
if (!tsBufNextPos(pTSBuf)) {
break;
@@ -416,8 +416,8 @@ void mergeDiffVnodeBufferTest() {
int64_t* list = createTsList(num, start, step);
t.i64Key = i;
- tsBufAppend(pTSBuf1, 0, &t, (const char*)list, num * sizeof(int64_t));
- tsBufAppend(pTSBuf2, 0, &t, (const char*)list, num * sizeof(int64_t));
+ tsBufAppend(pTSBuf1, 1, &t, (const char*)list, num * sizeof(int64_t));
+ tsBufAppend(pTSBuf2, 9, &t, (const char*)list, num * sizeof(int64_t));
free(list);
@@ -426,8 +426,8 @@ void mergeDiffVnodeBufferTest() {
tsBufFlush(pTSBuf2);
- tsBufMerge(pTSBuf1, pTSBuf2, 9);
- EXPECT_EQ(pTSBuf1->numOfVnodes, 2);
+ tsBufMerge(pTSBuf1, pTSBuf2);
+ EXPECT_EQ(pTSBuf1->numOfGroups, 2);
EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);
tsBufDisplay(pTSBuf1);
@@ -459,8 +459,6 @@ void mergeIdenticalVnodeBufferTest() {
start += step * num;
}
-
-
for (int32_t i = numOfTags; i < numOfTags * 2; ++i) {
int64_t* list = createTsList(num, start, step);
@@ -473,16 +471,23 @@ void mergeIdenticalVnodeBufferTest() {
tsBufFlush(pTSBuf2);
- tsBufMerge(pTSBuf1, pTSBuf2, 12);
- EXPECT_EQ(pTSBuf1->numOfVnodes, 1);
+ tsBufMerge(pTSBuf1, pTSBuf2);
+ EXPECT_EQ(pTSBuf1->numOfGroups, 2);
EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);
tsBufResetPos(pTSBuf1);
+
+ int32_t count = 0;
while (tsBufNextPos(pTSBuf1)) {
STSElem elem = tsBufGetElem(pTSBuf1);
- EXPECT_EQ(elem.vnode, 12);
- printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts);
+ if (count++ < numOfTags * num) {
+ EXPECT_EQ(elem.id, 12);
+ } else {
+ EXPECT_EQ(elem.id, 77);
+ }
+
+ printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts);
}
tsBufDestroy(pTSBuf1);
diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c
index 46b0d4e3bb9428f98c1022556f2f57c6ec0e9a14..09d8f3bff1faad5596d85e931adea7f83670a48a 100644
--- a/src/rpc/src/rpcCache.c
+++ b/src/rpc/src/rpcCache.c
@@ -101,9 +101,9 @@ void rpcCloseConnCache(void *handle) {
if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool);
- taosTFree(pCache->connHashList);
- taosTFree(pCache->count);
- taosTFree(pCache->lockedBy);
+ tfree(pCache->connHashList);
+ tfree(pCache->count);
+ tfree(pCache->lockedBy);
pthread_mutex_unlock(&pCache->mutex);
diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c
index 6e9088d9fb82a80bb28857d563ad51f19594e6b7..f963eeb68dd4ffcecf4a945c81b7658924670b71 100644
--- a/src/rpc/src/rpcMain.c
+++ b/src/rpc/src/rpcMain.c
@@ -20,6 +20,7 @@
#include "ttimer.h"
#include "tutil.h"
#include "lz4.h"
+#include "tref.h"
#include "taoserror.h"
#include "tsocket.h"
#include "tglobal.h"
@@ -72,7 +73,6 @@ typedef struct {
SRpcInfo *pRpc; // associated SRpcInfo
SRpcEpSet epSet; // ip list provided by app
void *ahandle; // handle provided by app
- void *signature; // for validation
struct SRpcConn *pConn; // pConn allocated
char msgType; // message type
uint8_t *pCont; // content provided by app
@@ -82,6 +82,7 @@ typedef struct {
int8_t oldInUse; // server EP inUse passed by app
int8_t redirect; // flag to indicate redirect
int8_t connType; // connection type
+ int64_t rid; // refId returned by taosAddRef
SRpcMsg *pRsp; // for synchronous API
tsem_t *pSem; // for synchronous API
SRpcEpSet *pSet; // for synchronous API
@@ -132,6 +133,10 @@ int tsRpcMaxRetry;
int tsRpcHeadSize;
int tsRpcOverhead;
+static int tsRpcRefId = -1;
+static int32_t tsRpcNum = 0;
+static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT;
+
// server:0 client:1 tcp:2 udp:0
#define RPC_CONN_UDPS 0
#define RPC_CONN_UDPC 1
@@ -211,14 +216,30 @@ static void rpcUnlockConn(SRpcConn *pConn);
static void rpcAddRef(SRpcInfo *pRpc);
static void rpcDecRef(SRpcInfo *pRpc);
-void *rpcOpen(const SRpcInit *pInit) {
- SRpcInfo *pRpc;
+static void rpcFree(void *p) {
+ tTrace("free mem: %p", p);
+ free(p);
+}
+void rpcInit(void) {
tsProgressTimer = tsRpcTimer/2;
tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer;
tsRpcHeadSize = RPC_MSG_OVERHEAD;
tsRpcOverhead = sizeof(SRpcReqContext);
+ tsRpcRefId = taosOpenRef(200, rpcFree);
+}
+
+void rpcCleanup(void) {
+ taosCloseRef(tsRpcRefId);
+ tsRpcRefId = -1;
+}
+
+void *rpcOpen(const SRpcInit *pInit) {
+ SRpcInfo *pRpc;
+
+ pthread_once(&tsRpcInit, rpcInit);
+
pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo));
if (pRpc == NULL) return NULL;
@@ -237,6 +258,8 @@ void *rpcOpen(const SRpcInit *pInit) {
pRpc->afp = pInit->afp;
pRpc->refCount = 1;
+ atomic_add_fetch_32(&tsRpcNum, 1);
+
size_t size = sizeof(SRpcConn) * pRpc->sessions;
pRpc->connList = (SRpcConn *)calloc(1, size);
if (pRpc->connList == NULL) {
@@ -323,7 +346,7 @@ void *rpcMallocCont(int contLen) {
tError("failed to malloc msg, size:%d", size);
return NULL;
} else {
- tTrace("malloc mem: %p", start);
+ tTrace("malloc mem:%p size:%d", start, size);
}
return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
@@ -356,14 +379,13 @@ void *rpcReallocCont(void *ptr, int contLen) {
return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
}
-void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
+int64_t rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
SRpcInfo *pRpc = (SRpcInfo *)shandle;
SRpcReqContext *pContext;
int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen);
pContext = (SRpcReqContext *) ((char*)pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext));
pContext->ahandle = pMsg->ahandle;
- pContext->signature = pContext;
pContext->pRpc = (SRpcInfo *)shandle;
pContext->epSet = *pEpSet;
pContext->contLen = contLen;
@@ -386,9 +408,11 @@ void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
// set the handle to pContext, so app can cancel the request
if (pMsg->handle) *((void **)pMsg->handle) = pContext;
+ pContext->rid = taosAddRef(tsRpcRefId, pContext);
+
rpcSendReqToServer(pRpc, pContext);
- return;
+ return pContext->rid;
}
void rpcSendResponse(const SRpcMsg *pRsp) {
@@ -533,17 +557,14 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) {
return code;
}
-void rpcCancelRequest(void *handle) {
- SRpcReqContext *pContext = handle;
+void rpcCancelRequest(int64_t rid) {
- // signature is used to check if pContext is freed.
- // pContext may have been released just before app calls the rpcCancelRequest
- if (pContext == NULL || pContext->signature != pContext) return;
+ SRpcReqContext *pContext = taosAcquireRef(tsRpcRefId, rid);
+ if (pContext == NULL) return;
- if (pContext->pConn) {
- tDebug("%s, app tries to cancel request", pContext->pConn->info);
- rpcCloseConn(pContext->pConn);
- }
+ rpcCloseConn(pContext->pConn);
+
+ taosReleaseRef(tsRpcRefId, rid);
}
static void rpcFreeMsg(void *msg) {
@@ -612,7 +633,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
// if there is an outgoing message, free it
if (pConn->outType && pConn->pReqMsg) {
if (pConn->pContext) pConn->pContext->pConn = NULL;
- rpcFreeMsg(pConn->pReqMsg);
+ taosRemoveRef(tsRpcRefId, pConn->pContext->rid);
}
}
@@ -636,6 +657,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
static void rpcCloseConn(void *thandle) {
SRpcConn *pConn = (SRpcConn *)thandle;
+ if (pConn == NULL) return;
rpcLockConn(pConn);
@@ -1007,6 +1029,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) {
if (pConn->outType) {
SRpcReqContext *pContext = pConn->pContext;
pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+ pContext->pConn = NULL;
pConn->pReqMsg = NULL;
taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl);
}
@@ -1057,6 +1080,13 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
}
} else { // msg is passed to app only parsing is ok
+
+ if (pHead->msgType == TSDB_MSG_TYPE_NETWORK_TEST) {
+ rpcSendQuickRsp(pConn, TSDB_CODE_SUCCESS);
+ rpcFreeMsg(pRecv->msg);
+ return pConn;
+ }
+
rpcProcessIncomingMsg(pConn, pHead, pContext);
}
}
@@ -1068,7 +1098,6 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
SRpcInfo *pRpc = pContext->pRpc;
- pContext->signature = NULL;
pContext->pConn = NULL;
if (pContext->pRsp) {
// for synchronous API
@@ -1085,7 +1114,7 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
}
// free the request message
- rpcFreeCont(pContext->pCont);
+ taosRemoveRef(tsRpcRefId, pContext->rid);
}
static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) {
@@ -1110,6 +1139,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte
// it's a response
rpcMsg.handle = pContext;
rpcMsg.ahandle = pContext->ahandle;
+ pContext->pConn = NULL;
// for UDP, port may be changed by server, the port in epSet shall be used for cache
if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) {
@@ -1345,6 +1375,7 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) {
tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort);
if (pConn->pContext) {
pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+ pConn->pContext->pConn = NULL;
pConn->pReqMsg = NULL;
taosTmrStart(rpcProcessConnError, 0, pConn->pContext, pRpc->tmrCtrl);
rpcReleaseConn(pConn);
@@ -1453,7 +1484,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) {
pNewHead->msgLen = rpcMsgLenFromCont(origLen);
rpcFreeMsg(pHead); // free the compressed message buffer
pHead = pNewHead;
- tTrace("decomp malloc mem: %p", temp);
+ tTrace("decomp malloc mem:%p", temp);
} else {
tError("failed to allocate memory to decompress msg, contLen:%d", contLen);
}
@@ -1589,10 +1620,12 @@ static void rpcDecRef(SRpcInfo *pRpc)
taosTmrCleanUp(pRpc->tmrCtrl);
taosIdPoolCleanUp(pRpc->idPool);
- taosTFree(pRpc->connList);
+ tfree(pRpc->connList);
pthread_mutex_destroy(&pRpc->mutex);
tDebug("%s rpc resources are released", pRpc->label);
- taosTFree(pRpc);
+ tfree(pRpc);
+
+ atomic_sub_fetch_32(&tsRpcNum, 1);
}
}
diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c
index bc8d360d39509ce9a2fdbe6a9dd883c5c5c99190..7b32d3416d784b4cbc6e319a73664a28dd578d4d 100644
--- a/src/rpc/src/rpcTcp.c
+++ b/src/rpc/src/rpcTcp.c
@@ -236,8 +236,8 @@ void taosCleanUpTcpServer(void *handle) {
tDebug("%s TCP server is cleaned up", pServerObj->label);
- taosTFree(pServerObj->pThreadObj);
- taosTFree(pServerObj);
+ tfree(pServerObj->pThreadObj);
+ tfree(pServerObj);
}
static void *taosAcceptTcpConnection(void *arg) {
@@ -437,12 +437,13 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
}
msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
- buffer = malloc(msgLen + tsRpcOverhead);
+ int32_t size = msgLen + tsRpcOverhead;
+ buffer = malloc(size);
if (NULL == buffer) {
tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
return -1;
} else {
- tTrace("TCP malloc mem: %p", buffer);
+ tTrace("TCP malloc mem:%p size:%d", buffer, size);
}
msg = buffer + tsRpcOverhead;
@@ -534,7 +535,7 @@ static void *taosProcessTcpData(void *param) {
pthread_mutex_destroy(&(pThreadObj->mutex));
tDebug("%s TCP thread exits ...", pThreadObj->label);
- taosTFree(pThreadObj);
+ tfree(pThreadObj);
return NULL;
}
@@ -555,7 +556,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) {
event.events = EPOLLIN | EPOLLRDHUP;
event.data.ptr = pFdObj;
if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
- taosTFree(pFdObj);
+ tfree(pFdObj);
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
@@ -608,5 +609,5 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d",
pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds);
- taosTFree(pFdObj);
+ tfree(pFdObj);
}
diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c
index 6f653046615f162c516b5eebf08995d30c6214d7..22301fcecc83fb1f4c9a29132b0311e05b2382e6 100644
--- a/src/rpc/src/rpcUdp.c
+++ b/src/rpc/src/rpcUdp.c
@@ -147,7 +147,7 @@ void taosStopUdpConnection(void *handle) {
if (taosCheckPthreadValid(pConn->thread)) {
pthread_join(pConn->thread, NULL);
}
- taosTFree(pConn->buffer);
+ tfree(pConn->buffer);
// tTrace("%s UDP thread is closed, index:%d", pConn->label, i);
}
@@ -166,7 +166,7 @@ void taosCleanUpUdpConnection(void *handle) {
}
tDebug("%s UDP is cleaned up", pSet->label);
- taosTFree(pSet);
+ tfree(pSet);
}
void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) {
@@ -209,12 +209,13 @@ static void *taosRecvUdpData(void *param) {
continue;
}
- char *tmsg = malloc(dataLen + tsRpcOverhead);
+ int32_t size = dataLen + tsRpcOverhead;
+ char *tmsg = malloc(size);
if (NULL == tmsg) {
tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen);
continue;
} else {
- tTrace("UDP malloc mem: %p", tmsg);
+ tTrace("UDP malloc mem:%p size:%d", tmsg, size);
}
tmsg += tsRpcOverhead; // overhead for SRpcReqContext
diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h
index f6818106462cb9c9d694d59c7cb9012cd1a54c8b..93c6efc20a8d79a7d6436680f01cee87dadda853 100644
--- a/src/sync/inc/syncInt.h
+++ b/src/sync/inc/syncInt.h
@@ -35,6 +35,8 @@ extern "C" {
#define TAOS_SMSG_SYNC_MUST 6
#define TAOS_SMSG_STATUS 7
+#define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16)
+
#define nodeRole pNode->peerInfo[pNode->selfIndex]->role
#define nodeVersion pNode->peerInfo[pNode->selfIndex]->version
#define nodeSStatus pNode->peerInfo[pNode->selfIndex]->sstatus
@@ -89,11 +91,11 @@ typedef struct {
#pragma pack(pop)
typedef struct {
- char *buffer;
- int bufferSize;
- char *offset;
- int forwards;
- int code;
+ char * buffer;
+ int32_t bufferSize;
+ char * offset;
+ int32_t forwards;
+ int32_t code;
} SRecvBuffer;
typedef struct {
@@ -107,10 +109,10 @@ typedef struct {
} SFwdInfo;
typedef struct {
- int first;
- int last;
- int fwds; // number of forwards
- SFwdInfo fwdInfo[];
+ int32_t first;
+ int32_t last;
+ int32_t fwds; // number of forwards
+ SFwdInfo fwdInfo[];
} SSyncFwds;
typedef struct SsyncPeer {
@@ -123,15 +125,15 @@ typedef struct SsyncPeer {
int8_t sstatus; // sync status
uint64_t version;
uint64_t sversion; // track the peer version in retrieve process
- int syncFd;
- int peerFd; // forward FD
- int numOfRetrieves; // number of retrieves tried
- int fileChanged; // a flag to indicate file is changed during retrieving process
+ int32_t syncFd;
+ int32_t peerFd; // forward FD
+ int32_t numOfRetrieves; // number of retrieves tried
+ int32_t fileChanged; // a flag to indicate file is changed during retrieving process
void * timer;
void * pConn;
- int notifyFd;
- int watchNum;
- int * watchFd;
+ int32_t notifyFd;
+ int32_t watchNum;
+ int32_t *watchFd;
int8_t refCount; // reference count
struct SSyncNode *pSyncNode;
} SSyncPeer;
@@ -141,6 +143,7 @@ typedef struct SSyncNode {
int8_t replica;
int8_t quorum;
uint32_t vgId;
+ int64_t rid;
void *ahandle;
int8_t selfIndex;
SSyncPeer *peerInfo[TAOS_SYNC_MAX_REPLICA+1]; // extra one for arbitrator
@@ -160,16 +163,16 @@ typedef struct SSyncNode {
} SSyncNode;
// sync module global
-extern int tsSyncNum;
-extern char tsNodeFqdn[TSDB_FQDN_LEN];
+extern int32_t tsSyncNum;
+extern char tsNodeFqdn[TSDB_FQDN_LEN];
void *syncRetrieveData(void *param);
void *syncRestoreData(void *param);
-int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead);
-void syncRestartConnection(SSyncPeer *pPeer);
-void syncBroadcastStatus(SSyncNode *pNode);
-void syncAddPeerRef(SSyncPeer *pPeer);
-int syncDecPeerRef(SSyncPeer *pPeer);
+int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead);
+void syncRestartConnection(SSyncPeer *pPeer);
+void syncBroadcastStatus(SSyncNode *pNode);
+void syncAddPeerRef(SSyncPeer *pPeer);
+int32_t syncDecPeerRef(SSyncPeer *pPeer);
#ifdef __cplusplus
}
diff --git a/src/sync/inc/taosTcpPool.h b/src/sync/inc/taosTcpPool.h
index 261d190ad3b7cbe4fbf46a061a1432c217262d24..41043b0cd4c886616d5cecd2739eae684052c395 100644
--- a/src/sync/inc/taosTcpPool.h
+++ b/src/sync/inc/taosTcpPool.h
@@ -20,23 +20,23 @@
extern "C" {
#endif
-typedef void* ttpool_h;
-typedef void* tthread_h;
+typedef void *ttpool_h;
+typedef void *tthread_h;
typedef struct {
- int numOfThreads;
+ int32_t numOfThreads;
uint32_t serverIp;
int16_t port;
- int bufferSize;
- void (*processBrokenLink)(void *ahandle);
- int (*processIncomingMsg)(void *ahandle, void *buffer);
- void (*processIncomingConn)(int fd, uint32_t ip);
+ int32_t bufferSize;
+ void (*processBrokenLink)(void *ahandle);
+ int32_t (*processIncomingMsg)(void *ahandle, void *buffer);
+ void (*processIncomingConn)(int32_t fd, uint32_t ip);
} SPoolInfo;
-ttpool_h taosOpenTcpThreadPool(SPoolInfo *pInfo);
-void taosCloseTcpThreadPool(ttpool_h);
-void *taosAllocateTcpConn(void *, void *ahandle, int connFd);
-void taosFreeTcpConn(void *);
+ttpool_h taosOpenTcpThreadPool(SPoolInfo *pInfo);
+void taosCloseTcpThreadPool(ttpool_h);
+void * taosAllocateTcpConn(void *, void *ahandle, int32_t connFd);
+void taosFreeTcpConn(void *);
#ifdef __cplusplus
}
diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c
index ef635e6efc1ca5f071c64dbe00920c3987837494..5c92801cc389df6745377d81b4110ccf8fbc8441 100644
--- a/src/sync/src/syncMain.c
+++ b/src/sync/src/syncMain.c
@@ -20,6 +20,7 @@
#include "tlog.h"
#include "tutil.h"
#include "ttimer.h"
+#include "tref.h"
#include "tsocket.h"
#include "tglobal.h"
#include "taoserror.h"
@@ -30,37 +31,38 @@
#include "syncInt.h"
// global configurable
-int tsMaxSyncNum = 2;
-int tsSyncTcpThreads = 2;
-int tsMaxWatchFiles = 500;
-int tsMaxFwdInfo = 200;
-int tsSyncTimer = 1;
+int32_t tsMaxSyncNum = 2;
+int32_t tsSyncTcpThreads = 2;
+int32_t tsMaxWatchFiles = 500;
+int32_t tsMaxFwdInfo = 200;
+int32_t tsSyncTimer = 1;
// module global, not configurable
-int tsSyncNum; // number of sync in process in whole system
-char tsNodeFqdn[TSDB_FQDN_LEN];
+int32_t tsSyncNum; // number of sync in process in whole system
+char tsNodeFqdn[TSDB_FQDN_LEN];
static ttpool_h tsTcpPool;
-static void * syncTmrCtrl = NULL;
-static void * vgIdHash;
+static void * tsSyncTmrCtrl = NULL;
+static void * tsVgIdHash;
+static int32_t tsSyncRefId = -1;
// local functions
-static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
-static void syncRecoverFromMaster(SSyncPeer *pPeer);
-static void syncCheckPeerConnection(void *param, void *tmrId);
-static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack);
-static void syncProcessBrokenLink(void *param);
-static int syncProcessPeerMsg(void *param, void *buffer);
-static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp);
-static void syncRemovePeer(SSyncPeer *pPeer);
-static void syncAddArbitrator(SSyncNode *pNode);
-static void syncAddNodeRef(SSyncNode *pNode);
-static void syncDecNodeRef(SSyncNode *pNode);
-static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode);
-static void syncMonitorFwdInfos(void *param, void *tmrId);
-static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code);
-static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle);
-static void syncRestartPeer(SSyncPeer *pPeer);
+static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
+static void syncRecoverFromMaster(SSyncPeer *pPeer);
+static void syncCheckPeerConnection(void *param, void *tmrId);
+static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack);
+static void syncProcessBrokenLink(void *param);
+static int32_t syncProcessPeerMsg(void *param, void *buffer);
+static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp);
+static void syncRemovePeer(SSyncPeer *pPeer);
+static void syncAddArbitrator(SSyncNode *pNode);
+static void syncFreeNode(void *);
+static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode);
+static void syncMonitorFwdInfos(void *param, void *tmrId);
+static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code);
+static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle);
+static void syncRestartPeer(SSyncPeer *pPeer);
+static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtyp);
static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo);
char* syncRole[] = {
@@ -77,7 +79,7 @@ int32_t syncInit() {
info.numOfThreads = tsSyncTcpThreads;
info.serverIp = 0;
info.port = tsSyncPort;
- info.bufferSize = 640000;
+ info.bufferSize = SYNC_MAX_SIZE;
info.processBrokenLink = syncProcessBrokenLink;
info.processIncomingMsg = syncProcessPeerMsg;
info.processIncomingConn = syncProcessIncommingConnection;
@@ -88,21 +90,27 @@ int32_t syncInit() {
return -1;
}
- syncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC");
- if (syncTmrCtrl == NULL) {
+ tsSyncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC");
+ if (tsSyncTmrCtrl == NULL) {
sError("failed to init tmrCtrl");
taosCloseTcpThreadPool(tsTcpPool);
tsTcpPool = NULL;
return -1;
}
- vgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
- if (vgIdHash == NULL) {
- sError("failed to init vgIdHash");
- taosTmrCleanUp(syncTmrCtrl);
+ tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
+ if (tsVgIdHash == NULL) {
+ sError("failed to init tsVgIdHash");
+ taosTmrCleanUp(tsSyncTmrCtrl);
taosCloseTcpThreadPool(tsTcpPool);
tsTcpPool = NULL;
- syncTmrCtrl = NULL;
+ tsSyncTmrCtrl = NULL;
+ return -1;
+ }
+
+ tsSyncRefId = taosOpenRef(200, syncFreeNode);
+ if (tsSyncRefId < 0) {
+ syncCleanUp();
return -1;
}
@@ -118,27 +126,30 @@ void syncCleanUp() {
tsTcpPool = NULL;
}
- if (syncTmrCtrl) {
- taosTmrCleanUp(syncTmrCtrl);
- syncTmrCtrl = NULL;
+ if (tsSyncTmrCtrl) {
+ taosTmrCleanUp(tsSyncTmrCtrl);
+ tsSyncTmrCtrl = NULL;
}
- if (vgIdHash) {
- taosHashCleanup(vgIdHash);
- vgIdHash = NULL;
+ if (tsVgIdHash) {
+ taosHashCleanup(tsVgIdHash);
+ tsVgIdHash = NULL;
}
+ taosCloseRef(tsSyncRefId);
+ tsSyncRefId = -1;
+
sInfo("sync module is cleaned up");
}
-void *syncStart(const SSyncInfo *pInfo) {
+int64_t syncStart(const SSyncInfo *pInfo) {
const SSyncCfg *pCfg = &pInfo->syncCfg;
SSyncNode *pNode = (SSyncNode *)calloc(sizeof(SSyncNode), 1);
if (pNode == NULL) {
sError("no memory to allocate syncNode");
terrno = TAOS_SYSTEM_ERROR(errno);
- return NULL;
+ return -1;
}
tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path));
@@ -159,7 +170,13 @@ void *syncStart(const SSyncInfo *pInfo) {
pNode->quorum = pCfg->quorum;
if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
- for (int i = 0; i < pCfg->replica; ++i) {
+ pNode->rid = taosAddRef(tsSyncRefId, pNode);
+ if (pNode->rid < 0) {
+ syncFreeNode(pNode);
+ return -1;
+ }
+
+ for (int32_t i = 0; i < pCfg->replica; ++i) {
const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i;
pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo);
if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) {
@@ -167,13 +184,11 @@ void *syncStart(const SSyncInfo *pInfo) {
}
}
- syncAddNodeRef(pNode);
-
if (pNode->selfIndex < 0) {
sInfo("vgId:%d, this node is not configured", pNode->vgId);
terrno = TSDB_CODE_SYN_INVALID_CONFIG;
- syncStop(pNode);
- return NULL;
+ syncStop(pNode->rid);
+ return -1;
}
nodeVersion = pInfo->version; // set the initial version
@@ -185,40 +200,41 @@ void *syncStart(const SSyncInfo *pInfo) {
if (pNode->pSyncFwds == NULL) {
sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId);
terrno = TAOS_SYSTEM_ERROR(errno);
- syncStop(pNode);
- return NULL;
+ syncStop(pNode->rid);
+ return -1;
}
- pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
+ pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl);
if (pNode->pFwdTimer == NULL) {
sError("vgId:%d, failed to allocate timer", pNode->vgId);
- syncStop(pNode);
- return NULL;
+ syncStop(pNode->rid);
+ return -1;
}
syncAddArbitrator(pNode);
- taosHashPut(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *));
+ taosHashPut(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *));
if (pNode->notifyRole) {
(*pNode->notifyRole)(pNode->ahandle, nodeRole);
}
- return pNode;
+ return pNode->rid;
}
-void syncStop(void *param) {
- SSyncNode *pNode = param;
+void syncStop(int64_t rid) {
SSyncPeer *pPeer;
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return;
+
sInfo("vgId:%d, cleanup sync", pNode->vgId);
pthread_mutex_lock(&(pNode->mutex));
- if (vgIdHash) taosHashRemove(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t));
+ if (tsVgIdHash) taosHashRemove(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t));
if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer);
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
pPeer = pNode->peerInfo[i];
if (pPeer) syncRemovePeer(pPeer);
}
@@ -228,14 +244,16 @@ void syncStop(void *param) {
pthread_mutex_unlock(&(pNode->mutex));
- syncDecNodeRef(pNode);
+ taosReleaseRef(tsSyncRefId, rid);
+ taosRemoveRef(tsSyncRefId, rid);
}
-int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
- SSyncNode *pNode = param;
- int i, j;
+int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
+ int32_t i, j;
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG;
+
sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica,
pNode->replica);
@@ -298,105 +316,58 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
syncRole[nodeRole]);
syncBroadcastStatus(pNode);
+ taosReleaseRef(tsSyncRefId, rid);
+
return 0;
}
-int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) {
- SSyncNode *pNode = param;
- SSyncPeer *pPeer;
- SSyncHead *pSyncHead;
- SWalHead * pWalHead = data;
- int fwdLen;
- int code = 0;
-
- if (pNode == NULL) return 0;
-
- if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) {
- sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId,
- pWalHead->version, nodeVersion);
- for (int i = 0; i < pNode->replica; ++i) {
- pPeer = pNode->peerInfo[i];
- syncRestartConnection(pPeer);
- }
- return TSDB_CODE_SYN_INVALID_VERSION;
- }
+int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype) {
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+ if (pNode == NULL) return 0;
- // always update version
- nodeVersion = pWalHead->version;
- sDebug("vgId:%d, replica:%d nodeRole:%s qtype:%d ver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole],
- qtype, pWalHead->version);
+ int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype);
- if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0;
-
- // only pkt from RPC or CQ can be forwarded
- if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0;
-
- // a hacker way to improve the performance
- pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead));
- pSyncHead->type = TAOS_SMSG_FORWARD;
- pSyncHead->pversion = 0;
- pSyncHead->len = sizeof(SWalHead) + pWalHead->len;
- fwdLen = pSyncHead->len + sizeof(SSyncHead); // include the WAL and SYNC head
-
- pthread_mutex_lock(&(pNode->mutex));
-
- for (int i = 0; i < pNode->replica; ++i) {
- pPeer = pNode->peerInfo[i];
- if (pPeer == NULL || pPeer->peerFd < 0) continue;
- if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue;
-
- if (pNode->quorum > 1 && code == 0) {
- syncSaveFwdInfo(pNode, pWalHead->version, mhandle);
- code = 1;
- }
-
- int retLen = write(pPeer->peerFd, pSyncHead, fwdLen);
- if (retLen == fwdLen) {
- sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len);
- } else {
- sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen);
- syncRestartConnection(pPeer);
- }
- }
-
- pthread_mutex_unlock(&(pNode->mutex));
+ taosReleaseRef(tsSyncRefId, rid);
return code;
}
-void syncConfirmForward(void *param, uint64_t version, int32_t code) {
- SSyncNode *pNode = param;
+void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return;
- if (pNode->quorum <= 1) return;
SSyncPeer *pPeer = pNode->pMaster;
- if (pPeer == NULL) return;
-
- char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0};
+ if (pPeer && pNode->quorum > 1) {
+ char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0};
- SSyncHead *pHead = (SSyncHead *)msg;
- pHead->type = TAOS_SMSG_FORWARD_RSP;
- pHead->len = sizeof(SFwdRsp);
+ SSyncHead *pHead = (SSyncHead *)msg;
+ pHead->type = TAOS_SMSG_FORWARD_RSP;
+ pHead->len = sizeof(SFwdRsp);
- SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead));
- pFwdRsp->version = version;
- pFwdRsp->code = code;
+ SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead));
+ pFwdRsp->version = version;
+ pFwdRsp->code = code;
- int msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp);
- int retLen = write(pPeer->peerFd, msg, msgLen);
+ int32_t msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp);
+ int32_t retLen = write(pPeer->peerFd, msg, msgLen);
- if (retLen == msgLen) {
- sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version);
- } else {
- sDebug("%s, failed to send forward ack, restart", pPeer->id);
- syncRestartConnection(pPeer);
+ if (retLen == msgLen) {
+ sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version);
+ } else {
+ sDebug("%s, failed to send forward ack, restart", pPeer->id);
+ syncRestartConnection(pPeer);
+ }
}
+
+ taosReleaseRef(tsSyncRefId, rid);
}
-void syncRecover(void *param) {
- SSyncNode *pNode = param;
+void syncRecover(int64_t rid) {
SSyncPeer *pPeer;
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+ if (pNode == NULL) return;
+
// to do: add a few lines to check if recover is OK
// if take this node to unsync state, the whole system may not work
@@ -406,7 +377,7 @@ void syncRecover(void *param) {
pthread_mutex_lock(&(pNode->mutex));
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
pPeer = (SSyncPeer *)pNode->peerInfo[i];
if (pPeer->peerFd >= 0) {
syncRestartConnection(pPeer);
@@ -414,17 +385,22 @@ void syncRecover(void *param) {
}
pthread_mutex_unlock(&(pNode->mutex));
+
+ taosReleaseRef(tsSyncRefId, rid);
}
-int syncGetNodesRole(void *param, SNodesRole *pNodesRole) {
- SSyncNode *pNode = param;
+int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) {
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+ if (pNode == NULL) return -1;
pNodesRole->selfIndex = pNode->selfIndex;
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId;
pNodesRole->role[i] = pNode->peerInfo[i]->role;
}
+ taosReleaseRef(tsSyncRefId, rid);
+
return 0;
}
@@ -440,7 +416,7 @@ static void syncAddArbitrator(SSyncNode *pNode) {
SNodeInfo nodeInfo;
nodeInfo.nodeId = 0;
- int ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort);
+ int32_t ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort);
if (-1 == ret) {
nodeInfo.nodePort = tsArbitratorPort;
}
@@ -457,26 +433,24 @@ static void syncAddArbitrator(SSyncNode *pNode) {
pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo);
}
-static void syncAddNodeRef(SSyncNode *pNode) { atomic_add_fetch_8(&pNode->refCount, 1); }
+static void syncFreeNode(void *param) {
+ SSyncNode *pNode = param;
-static void syncDecNodeRef(SSyncNode *pNode) {
- if (atomic_sub_fetch_8(&pNode->refCount, 1) == 0) {
- pthread_mutex_destroy(&pNode->mutex);
- taosTFree(pNode->pRecv);
- taosTFree(pNode->pSyncFwds);
- taosTFree(pNode);
- }
+ pthread_mutex_destroy(&pNode->mutex);
+ tfree(pNode->pRecv);
+ tfree(pNode->pSyncFwds);
+ tfree(pNode);
}
void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); }
-int syncDecPeerRef(SSyncPeer *pPeer) {
+int32_t syncDecPeerRef(SSyncPeer *pPeer) {
if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) {
- syncDecNodeRef(pPeer->pSyncNode);
+ taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid);
sDebug("%s, resource is freed", pPeer->id);
- taosTFree(pPeer->watchFd);
- taosTFree(pPeer);
+ tfree(pPeer->watchFd);
+ tfree(pPeer);
return 0;
}
@@ -512,7 +486,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
pPeer->ip = ip;
pPeer->port = pInfo->nodePort;
pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0;
- snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d peer:%s:%u", pNode->vgId, pPeer->fqdn, pPeer->port);
+ snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, peer:%s:%u", pNode->vgId, pPeer->fqdn, pPeer->port);
pPeer->peerFd = -1;
pPeer->syncFd = -1;
@@ -521,22 +495,22 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
pPeer->refCount = 1;
sInfo("%s, it is configured", pPeer->id);
- int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
+ int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
int32_t checkMs = 100 + (pNode->vgId * 10) % 100;
if (pNode->vgId > 1) checkMs = tsStatusInterval * 2000 + checkMs;
sDebug("%s, start to check peer connection after %d ms", pPeer->id, checkMs);
- taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, syncTmrCtrl, &pPeer->timer);
+ taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer);
}
- syncAddNodeRef(pNode);
+ taosAcquireRef(tsSyncRefId, pNode->rid);
return pPeer;
}
void syncBroadcastStatus(SSyncNode *pNode) {
SSyncPeer *pPeer;
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
if (i == pNode->selfIndex) continue;
pPeer = pNode->peerInfo[i];
syncSendPeersStatusMsgToPeer(pPeer, 1);
@@ -544,7 +518,7 @@ void syncBroadcastStatus(SSyncNode *pNode) {
}
static void syncResetFlowCtrl(SSyncNode *pNode) {
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
pNode->peerInfo[i]->numOfRetrieves = 0;
}
@@ -555,13 +529,13 @@ static void syncResetFlowCtrl(SSyncNode *pNode) {
static void syncChooseMaster(SSyncNode *pNode) {
SSyncPeer *pPeer;
- int onlineNum = 0;
- int index = -1;
- int replica = pNode->replica;
+ int32_t onlineNum = 0;
+ int32_t index = -1;
+ int32_t replica = pNode->replica;
sDebug("vgId:%d, choose master", pNode->vgId);
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) {
onlineNum++;
}
@@ -570,7 +544,7 @@ static void syncChooseMaster(SSyncNode *pNode) {
if (onlineNum == pNode->replica) {
// if all peers are online, peer with highest version shall be master
index = 0;
- for (int i = 1; i < pNode->replica; ++i) {
+ for (int32_t i = 1; i < pNode->replica; ++i) {
if (pNode->peerInfo[i]->version > pNode->peerInfo[index]->version) {
index = i;
}
@@ -586,7 +560,7 @@ static void syncChooseMaster(SSyncNode *pNode) {
if (index < 0 && onlineNum > replica / 2.0) {
// over half of nodes are online
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
// slave with highest version shall be master
pPeer = pNode->peerInfo[i];
if (pPeer->role == TAOS_SYNC_ROLE_SLAVE || pPeer->role == TAOS_SYNC_ROLE_MASTER) {
@@ -613,11 +587,11 @@ static void syncChooseMaster(SSyncNode *pNode) {
}
static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
- int onlineNum = 0;
- int index = -1;
- int replica = pNode->replica;
+ int32_t onlineNum = 0;
+ int32_t index = -1;
+ int32_t replica = pNode->replica;
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) {
onlineNum++;
}
@@ -638,7 +612,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
}
} else {
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
SSyncPeer *pTemp = pNode->peerInfo[i];
if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue;
if (index < 0) {
@@ -657,9 +631,9 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
return pMaster;
}
-static int syncValidateMaster(SSyncPeer *pPeer) {
+static int32_t syncValidateMaster(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
- int code = 0;
+ int32_t code = 0;
if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) {
sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id);
@@ -667,7 +641,7 @@ static int syncValidateMaster(SSyncPeer *pPeer) {
(*pNode->notifyRole)(pNode->ahandle, nodeRole);
code = -1;
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
if (i == pNode->selfIndex) continue;
syncRestartPeer(pNode->peerInfo[i]);
}
@@ -709,7 +683,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne
}
} else {
// master not there, if all peer's state and version are consistent, choose the master
- int consistent = 0;
+ int32_t consistent = 0;
if (peersStatus) {
for (i = 0; i < pNode->replica; ++i) {
SSyncPeer *pTemp = pNode->peerInfo[i];
@@ -747,9 +721,9 @@ static void syncRestartPeer(SSyncPeer *pPeer) {
pPeer->sstatus = TAOS_SYNC_STATUS_INIT;
- int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
+ int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) {
- taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer);
+ taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
}
}
@@ -783,7 +757,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) {
pthread_t thread;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
- int ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer);
+ int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer);
pthread_attr_destroy(&thattr);
if (ret != 0) {
@@ -828,7 +802,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
// Ensure the sync of mnode not interrupted
if (pNode->vgId != 1 && tsSyncNum >= tsMaxSyncNum) {
sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum);
- taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, syncTmrCtrl, &pPeer->timer);
+ taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, tsSyncTmrCtrl, &pPeer->timer);
return;
}
@@ -841,7 +815,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
firstPkt.syncHead.len = sizeof(firstPkt) - sizeof(SSyncHead);
tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
firstPkt.port = tsSyncPort;
- taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer);
+ taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
if (write(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
sError("%s, failed to send sync-req to peer", pPeer->id);
@@ -862,7 +836,7 @@ static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) {
if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) {
// find the forwardInfo from first
- for (int i = 0; i < pSyncFwds->fwds; ++i) {
+ for (int32_t i = 0; i < pSyncFwds->fwds; ++i) {
pFwdInfo = pSyncFwds->fwdInfo + (i + pSyncFwds->first) % tsMaxFwdInfo;
if (pFwdRsp->version == pFwdInfo->version) break;
}
@@ -876,16 +850,16 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
SWalHead * pHead = (SWalHead *)cont;
- sDebug("%s, forward is received, ver:%" PRIu64, pPeer->id, pHead->version);
+ sDebug("%s, forward is received, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len);
if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
// nodeVersion = pHead->version;
- (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD);
+ (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL);
} else {
if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
syncSaveIntoBuffer(pPeer, pHead);
} else {
- sError("%s, forward discarded, ver:%" PRIu64, pPeer->id, pHead->version);
+ sError("%s, forward discarded, hver:%" PRIu64, pPeer->id, pHead->version);
}
}
}
@@ -905,10 +879,10 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) {
}
}
-static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
+static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
if (pPeer->peerFd < 0) return -1;
- int hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead));
+ int32_t hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead));
if (hlen != sizeof(SSyncHead)) {
sDebug("%s, failed to read msg, hlen:%d", pPeer->id, hlen);
return -1;
@@ -916,11 +890,12 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
// head.len = htonl(head.len);
if (pHead->len < 0) {
- sError("%s, invalid pkt length, len:%d", pPeer->id, pHead->len);
+ sError("%s, invalid pkt length, hlen:%d", pPeer->id, pHead->len);
return -1;
}
- int bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len);
+ assert(pHead->len <= TSDB_MAX_WAL_SIZE);
+ int32_t bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len);
if (bytes != pHead->len) {
sError("%s, failed to read, bytes:%d len:%d", pPeer->id, bytes, pHead->len);
return -1;
@@ -929,7 +904,7 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
return 0;
}
-static int syncProcessPeerMsg(void *param, void *buffer) {
+static int32_t syncProcessPeerMsg(void *param, void *buffer) {
SSyncPeer *pPeer = param;
SSyncHead head;
char * cont = buffer;
@@ -937,7 +912,7 @@ static int syncProcessPeerMsg(void *param, void *buffer) {
SSyncNode *pNode = pPeer->pSyncNode;
pthread_mutex_lock(&(pNode->mutex));
- int code = syncReadPeerMsg(pPeer, &head, cont);
+ int32_t code = syncReadPeerMsg(pPeer, &head, cont);
if (code == 0) {
if (head.type == TAOS_SMSG_FORWARD) {
@@ -974,12 +949,12 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) {
pPeersStatus->role = nodeRole;
pPeersStatus->ack = ack;
- for (int i = 0; i < pNode->replica; ++i) {
+ for (int32_t i = 0; i < pNode->replica; ++i) {
pPeersStatus->peersStatus[i].role = pNode->peerInfo[i]->role;
pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version;
}
- int retLen = write(pPeer->peerFd, msg, statusMsgLen);
+ int32_t retLen = write(pPeer->peerFd, msg, statusMsgLen);
if (retLen == statusMsgLen) {
sDebug("%s, status msg is sent, self:%s ver:%" PRIu64 ", ack:%d", pPeer->id, syncRole[pPeersStatus->role],
pPeersStatus->version, pPeersStatus->ack);
@@ -1001,10 +976,10 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
return;
}
- int connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
+ int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
if (connFd < 0) {
sDebug("%s, failed to open tcp socket(%s)", pPeer->id, strerror(errno));
- taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer);
+ taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
return;
}
@@ -1025,7 +1000,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
} else {
sDebug("try later");
close(connFd);
- taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer);
+ taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
}
}
@@ -1050,7 +1025,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
syncAddPeerRef(pPeer);
- int ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer);
+ int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer);
pthread_attr_destroy(&thattr);
if (ret < 0) {
@@ -1062,9 +1037,9 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
}
}
-static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) {
- char ipstr[24];
- int i;
+static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
+ char ipstr[24];
+ int32_t i;
tinet_ntoa(ipstr, sourceIp);
sDebug("peer TCP connection from ip:%s", ipstr);
@@ -1077,7 +1052,7 @@ static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) {
}
int32_t vgId = firstPkt.syncHead.vgId;
- SSyncNode **ppNode = (SSyncNode **)taosHashGet(vgIdHash, (const char *)&vgId, sizeof(int32_t));
+ SSyncNode **ppNode = (SSyncNode **)taosHashGet(tsVgIdHash, (const char *)&vgId, sizeof(int32_t));
if (ppNode == NULL || *ppNode == NULL) {
sError("vgId:%d, vgId could not be found", vgId);
taosCloseSocket(connFd);
@@ -1122,7 +1097,7 @@ static void syncProcessBrokenLink(void *param) {
SSyncPeer *pPeer = param;
SSyncNode *pNode = pPeer->pSyncNode;
- syncAddNodeRef(pNode);
+ if (taosAcquireRef(tsSyncRefId, pNode->rid) < 0) return;
pthread_mutex_lock(&(pNode->mutex));
sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno));
@@ -1133,7 +1108,7 @@ static void syncProcessBrokenLink(void *param) {
}
pthread_mutex_unlock(&(pNode->mutex));
- syncDecNodeRef(pNode);
+ taosReleaseRef(tsSyncRefId, pNode->rid);
}
static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
@@ -1163,8 +1138,8 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) {
SSyncFwds *pSyncFwds = pNode->pSyncFwds;
- int fwds = pSyncFwds->fwds;
- for (int i = 0; i < fwds; ++i) {
+ int32_t fwds = pSyncFwds->fwds;
+ for (int32_t i = 0; i < fwds; ++i) {
SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->first;
if (pFwdInfo->confirmed == 0) break;
@@ -1178,7 +1153,7 @@ static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) {
}
static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) {
- int confirm = 0;
+ int32_t confirm = 0;
if (pFwdInfo->code == 0) pFwdInfo->code = code;
if (code == 0) {
@@ -1201,23 +1176,90 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code
}
static void syncMonitorFwdInfos(void *param, void *tmrId) {
- SSyncNode *pNode = param;
+ int64_t rid = (int64_t) param;
+ SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+ if (pNode == NULL) return;
+
SSyncFwds *pSyncFwds = pNode->pSyncFwds;
- if (pSyncFwds == NULL) return;
- uint64_t time = taosGetTimestampMs();
+ if (pSyncFwds) {;
+ uint64_t time = taosGetTimestampMs();
- if (pSyncFwds->fwds > 0) {
- pthread_mutex_lock(&(pNode->mutex));
- for (int i = 0; i < pSyncFwds->fwds; ++i) {
- SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo;
- if (time - pFwdInfo->time < 2000) break;
- syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+ if (pSyncFwds->fwds > 0) {
+ pthread_mutex_lock(&(pNode->mutex));
+ for (int32_t i = 0; i < pSyncFwds->fwds; ++i) {
+ SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo;
+ if (time - pFwdInfo->time < 2000) break;
+ syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+ }
+
+ syncRemoveConfirmedFwdInfo(pNode);
+ pthread_mutex_unlock(&(pNode->mutex));
}
- syncRemoveConfirmedFwdInfo(pNode);
- pthread_mutex_unlock(&(pNode->mutex));
+ pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl);
}
- pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
+ taosReleaseRef(tsSyncRefId, rid);
}
+
+static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype) {
+ SSyncPeer *pPeer;
+ SSyncHead *pSyncHead;
+ SWalHead * pWalHead = data;
+ int32_t fwdLen;
+ int32_t code = 0;
+
+ if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) {
+ sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId,
+ pWalHead->version, nodeVersion);
+ for (int32_t i = 0; i < pNode->replica; ++i) {
+ pPeer = pNode->peerInfo[i];
+ syncRestartConnection(pPeer);
+ }
+ return TSDB_CODE_SYN_INVALID_VERSION;
+ }
+
+ // always update version
+ nodeVersion = pWalHead->version;
+ sDebug("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica,
+ syncRole[nodeRole], qtypeStr[qtype], pWalHead->version);
+
+ if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0;
+
+ // only pkt from RPC or CQ can be forwarded
+ if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0;
+
+ // a hacker way to improve the performance
+ pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead));
+ pSyncHead->type = TAOS_SMSG_FORWARD;
+ pSyncHead->pversion = 0;
+ pSyncHead->len = sizeof(SWalHead) + pWalHead->len;
+ fwdLen = pSyncHead->len + sizeof(SSyncHead); // include the WAL and SYNC head
+
+ pthread_mutex_lock(&(pNode->mutex));
+
+ for (int32_t i = 0; i < pNode->replica; ++i) {
+ pPeer = pNode->peerInfo[i];
+ if (pPeer == NULL || pPeer->peerFd < 0) continue;
+ if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue;
+
+ if (pNode->quorum > 1 && code == 0) {
+ syncSaveFwdInfo(pNode, pWalHead->version, mhandle);
+ code = 1;
+ }
+
+ int32_t retLen = write(pPeer->peerFd, pSyncHead, fwdLen);
+ if (retLen == fwdLen) {
+ sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len);
+ } else {
+ sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen);
+ syncRestartConnection(pPeer);
+ }
+ }
+
+ pthread_mutex_unlock(&(pNode->mutex));
+
+ return code;
+}
+
diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c
index ebb6c3a0a9edff5acfc5f2ce7da8b58f03d8ab4a..44aed220d7369f63c4e7cdb5b3ce86d75c91b73e 100644
--- a/src/sync/src/syncRestore.c
+++ b/src/sync/src/syncRestore.c
@@ -48,20 +48,21 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex
}
}
-static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
+static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
SSyncNode *pNode = pPeer->pSyncNode;
SFileInfo minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */ // master file info
SFileInfo sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */ // slave file info
SFileAck fileAck;
- int code = -1;
+ int32_t code = -1;
char name[TSDB_FILENAME_LEN * 2] = {0};
uint32_t pindex = 0; // index in last restore
+ bool fileChanged = false;
*fversion = 0;
sinfo.index = 0;
while (1) {
// read file info
- int ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo));
+ int32_t ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo));
if (ret < 0) break;
// if no more file from master, break;
@@ -103,7 +104,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
minfo.name[sizeof(minfo.name) - 1] = 0;
snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name);
- int dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
+ int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
if (dfd < 0) {
sError("%s, failed to open file:%s", pPeer->id, name);
break;
@@ -114,10 +115,11 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
close(dfd);
if (ret < 0) break;
+ fileChanged = true;
sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
}
- if (code == 0 && (minfo.fversion != sinfo.fversion)) {
+ if (code == 0 && fileChanged) {
// data file is changed, code shall be set to 1
*fversion = minfo.fversion;
code = 1;
@@ -130,9 +132,9 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
return code;
}
-static int syncRestoreWal(SSyncPeer *pPeer) {
+static int32_t syncRestoreWal(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
- int ret, code = -1;
+ int32_t ret, code = -1;
void *buffer = calloc(1024000, 1); // size for one record
if (buffer == NULL) return -1;
@@ -151,8 +153,8 @@ static int syncRestoreWal(SSyncPeer *pPeer) {
ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len);
if (ret < 0) break;
- sDebug("%s, restore a record, ver:%" PRIu64, pPeer->id, pHead->version);
- (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL);
+ sDebug("%s, restore a record, qtype:wal hver:%" PRIu64, pPeer->id, pHead->version);
+ (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL, NULL);
}
if (code < 0) {
@@ -167,16 +169,16 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) {
SSyncNode *pNode = pPeer->pSyncNode;
SWalHead * pHead = (SWalHead *)offset;
- (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD);
+ (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL);
offset += pHead->len + sizeof(SWalHead);
return offset;
}
-static int syncProcessBufferedFwd(SSyncPeer *pPeer) {
+static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) {
SSyncNode * pNode = pPeer->pSyncNode;
SRecvBuffer *pRecv = pNode->pRecv;
- int forwards = 0;
+ int32_t forwards = 0;
sDebug("%s, number of buffered forwards:%d", pPeer->id, pRecv->forwards);
@@ -201,12 +203,12 @@ static int syncProcessBufferedFwd(SSyncPeer *pPeer) {
return pRecv->code;
}
-int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
+int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
SSyncNode * pNode = pPeer->pSyncNode;
SRecvBuffer *pRecv = pNode->pRecv;
if (pRecv == NULL) return -1;
- int len = pHead->len + sizeof(SWalHead);
+ int32_t len = pHead->len + sizeof(SWalHead);
if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) {
memcpy(pRecv->offset, pHead, len);
@@ -223,13 +225,13 @@ int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
static void syncCloseRecvBuffer(SSyncNode *pNode) {
if (pNode->pRecv) {
- taosTFree(pNode->pRecv->buffer);
+ tfree(pNode->pRecv->buffer);
}
- taosTFree(pNode->pRecv);
+ tfree(pNode->pRecv);
}
-static int syncOpenRecvBuffer(SSyncNode *pNode) {
+static int32_t syncOpenRecvBuffer(SSyncNode *pNode) {
syncCloseRecvBuffer(pNode);
SRecvBuffer *pRecv = calloc(sizeof(SRecvBuffer), 1);
@@ -250,13 +252,13 @@ static int syncOpenRecvBuffer(SSyncNode *pNode) {
return 0;
}
-static int syncRestoreDataStepByStep(SSyncPeer *pPeer) {
+static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
nodeSStatus = TAOS_SYNC_STATUS_FILE;
uint64_t fversion = 0;
sDebug("%s, start to restore file", pPeer->id);
- int code = syncRestoreFile(pPeer, &fversion);
+ int32_t code = syncRestoreFile(pPeer, &fversion);
if (code < 0) {
sError("%s, failed to restore file", pPeer->id);
return -1;
diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c
index 60625d75eccdbe6bbb29f97b31ecd8e9855480a7..968f5becaddc7b06c06171a4d91cc0e0ffffc0da 100644
--- a/src/sync/src/syncRetrieve.c
+++ b/src/sync/src/syncRetrieve.c
@@ -27,7 +27,7 @@
#include "tsync.h"
#include "syncInt.h"
-static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
+static int32_t syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
sDebug("%s, start to monitor:%s", pPeer->id, name);
if (pPeer->notifyFd <= 0) {
@@ -38,16 +38,16 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
return -1;
}
- if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles);
+ if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles);
if (pPeer->watchFd == NULL) {
sError("%s, failed to allocate watchFd", pPeer->id);
return -1;
}
- memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles);
+ memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles);
}
- int *wd = pPeer->watchFd + pPeer->watchNum;
+ int32_t *wd = pPeer->watchFd + pPeer->watchNum;
if (*wd >= 0) {
if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) {
@@ -69,17 +69,17 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
return 0;
}
-static int syncAreFilesModified(SSyncPeer *pPeer) {
+static int32_t syncAreFilesModified(SSyncPeer *pPeer) {
if (pPeer->notifyFd <= 0) return 0;
- char buf[2048];
- int len = read(pPeer->notifyFd, buf, sizeof(buf));
+ char buf[2048];
+ int32_t len = read(pPeer->notifyFd, buf, sizeof(buf));
if (len < 0 && errno != EAGAIN) {
sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));
return -1;
}
- int code = 0;
+ int32_t code = 0;
if (len > 0) {
const struct inotify_event *event;
char *ptr;
@@ -97,11 +97,11 @@ static int syncAreFilesModified(SSyncPeer *pPeer) {
return code;
}
-static int syncRetrieveFile(SSyncPeer *pPeer) {
+static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
SFileInfo fileInfo;
SFileAck fileAck;
- int code = -1;
+ int32_t code = -1;
char name[TSDB_FILENAME_LEN * 2] = {0};
memset(&fileInfo, 0, sizeof(fileInfo));
@@ -146,10 +146,10 @@ static int syncRetrieveFile(SSyncPeer *pPeer) {
}
// send the file to peer
- int sfd = open(name, O_RDONLY);
+ int32_t sfd = open(name, O_RDONLY);
if (sfd < 0) break;
- ret = taosTSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size);
+ ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size);
close(sfd);
if (ret < 0) break;
@@ -169,8 +169,8 @@ static int syncRetrieveFile(SSyncPeer *pPeer) {
/* if only a partial record is read out, set the IN_MODIFY flag in event,
so upper layer will reload the file to get a complete record */
-static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) {
- int ret;
+static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead, uint32_t *pEvent) {
+ int32_t ret;
ret = read(sfd, pHead, sizeof(SWalHead));
if (ret < 0) return -1;
@@ -194,7 +194,7 @@ static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) {
return sizeof(SWalHead) + pHead->len;
}
-static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
+static int32_t syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
pPeer->watchNum = 0;
taosClose(pPeer->notifyFd);
pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
@@ -203,14 +203,14 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
return -1;
}
- if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles);
+ if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles);
if (pPeer->watchFd == NULL) {
sError("%s, failed to allocate watchFd", pPeer->id);
return -1;
}
- memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles);
- int *wd = pPeer->watchFd;
+ memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles);
+ int32_t *wd = pPeer->watchFd;
*wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE);
if (*wd == -1) {
@@ -222,8 +222,8 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
}
static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) {
- char buf[2048];
- int len = read(pPeer->notifyFd, buf, sizeof(buf));
+ char buf[2048];
+ int32_t len = read(pPeer->notifyFd, buf, sizeof(buf));
if (len < 0 && errno != EAGAIN) {
sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));
return -1;
@@ -243,11 +243,11 @@ static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) {
return 0;
}
-static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) {
- SWalHead *pHead = malloc(640000);
- int code = -1;
+static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) {
+ SWalHead *pHead = malloc(SYNC_MAX_SIZE);
+ int32_t code = -1;
int32_t bytes = 0;
- int sfd;
+ int32_t sfd;
sfd = open(name, O_RDONLY);
if (sfd < 0) {
@@ -256,10 +256,10 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion,
}
(void)lseek(sfd, offset, SEEK_SET);
- sDebug("%s, retrieve last wal, offset:%" PRId64 " fversion:%" PRIu64, pPeer->id, offset, fversion);
+ sDebug("%s, retrieve last wal, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, offset, fversion);
while (1) {
- int wsize = syncReadOneWalRecord(sfd, pHead, pEvent);
+ int32_t wsize = syncReadOneWalRecord(sfd, pHead, pEvent);
if (wsize < 0) break;
if (wsize == 0) {
code = 0;
@@ -267,7 +267,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion,
}
sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version);
- int ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
+ int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
if (ret != wsize) break;
pPeer->sversion = pHead->version;
@@ -287,9 +287,9 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion,
return -1;
}
-static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) {
+static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) {
SSyncNode *pNode = pPeer->pSyncNode;
- int code = -1;
+ int32_t code = -1;
char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file
if (syncAreFilesModified(pPeer) != 0) return -1;
@@ -325,7 +325,7 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) {
// if all data up to fversion is read out, it is over
if (pPeer->sversion >= fversion && fversion > 0) {
code = 0;
- sDebug("%s, data up to fversion:%" PRId64 " has been read out, bytes:%d", pPeer->id, fversion, bytes);
+ sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d", pPeer->id, fversion, bytes);
break;
}
@@ -370,14 +370,14 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) {
return code;
}
-static int syncRetrieveWal(SSyncPeer *pPeer) {
+static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
SSyncNode * pNode = pPeer->pSyncNode;
char fname[TSDB_FILENAME_LEN * 3];
char wname[TSDB_FILENAME_LEN * 2];
int32_t size;
struct stat fstat;
- int code = -1;
- uint32_t index = 0;
+ int32_t code = -1;
+ int64_t index = 0;
while (1) {
// retrieve wal info
@@ -403,10 +403,10 @@ static int syncRetrieveWal(SSyncPeer *pPeer) {
size = fstat.st_size;
sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size);
- int sfd = open(fname, O_RDONLY);
+ int32_t sfd = open(fname, O_RDONLY);
if (sfd < 0) break;
- code = taosTSendFile(pPeer->syncFd, sfd, NULL, size);
+ code = taosSendFile(pPeer->syncFd, sfd, NULL, size);
close(sfd);
if (code < 0) break;
@@ -428,7 +428,7 @@ static int syncRetrieveWal(SSyncPeer *pPeer) {
return code;
}
-static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
+static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
SFirstPkt firstPkt;
diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c
index 6a210a136ffe67b2e1394d26bac4cb5083452c8c..3024d7d4e31ba999972c533099710cad6650af8d 100644
--- a/src/sync/src/taosTcpPool.c
+++ b/src/sync/src/taosTcpPool.c
@@ -13,18 +13,22 @@
* along with this program. If not, see .
*/
+#define _DEFAULT_SOURCE
#include "os.h"
#include "tulog.h"
#include "tutil.h"
#include "tsocket.h"
#include "taoserror.h"
#include "taosTcpPool.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
typedef struct SThreadObj {
pthread_t thread;
bool stop;
- int pollFd;
- int numOfFds;
+ int32_t pollFd;
+ int32_t numOfFds;
struct SPoolObj *pPool;
} SThreadObj;
@@ -32,15 +36,15 @@ typedef struct SPoolObj {
SPoolInfo info;
SThreadObj **pThread;
pthread_t thread;
- int nextId;
- int acceptFd; // FD for accept new connection
+ int32_t nextId;
+ int32_t acceptFd; // FD for accept new connection
} SPoolObj;
typedef struct {
SThreadObj *pThread;
- void *ahandle;
- int fd;
- int closedByApp;
+ void * ahandle;
+ int32_t fd;
+ int32_t closedByApp;
} SConnObj;
static void *taosAcceptPeerTcpConnection(void *argv);
@@ -53,66 +57,66 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) {
SPoolObj *pPool = calloc(sizeof(SPoolObj), 1);
if (pPool == NULL) {
- uError("TCP server, no enough memory");
+ sError("failed to alloc pool for TCP server since no enough memory");
return NULL;
}
pPool->info = *pInfo;
- pPool->pThread = (SThreadObj **)calloc(sizeof(SThreadObj *), pInfo->numOfThreads);
+ pPool->pThread = calloc(sizeof(SThreadObj *), pInfo->numOfThreads);
if (pPool->pThread == NULL) {
- uError("TCP server, no enough memory");
- free(pPool);
+ sError("failed to alloc pool thread for TCP server since no enough memory");
+ tfree(pPool);
return NULL;
}
pPool->acceptFd = taosOpenTcpServerSocket(pInfo->serverIp, pInfo->port);
if (pPool->acceptFd < 0) {
- free(pPool->pThread);
- free(pPool);
- uError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno));
+ tfree(pPool->pThread);
+ tfree(pPool);
+ sError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno));
return NULL;
}
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pPool->thread), &thattr, (void *)taosAcceptPeerTcpConnection, pPool) != 0) {
- uError("TCP server, failed to create accept thread, reason:%s", strerror(errno));
+ sError("failed to create accept thread for TCP server since %s", strerror(errno));
close(pPool->acceptFd);
- free(pPool->pThread);
- free(pPool);
+ tfree(pPool->pThread);
+ tfree(pPool);
return NULL;
}
pthread_attr_destroy(&thattr);
- uDebug("%p TCP pool is created", pPool);
+ sDebug("%p TCP pool is created", pPool);
return pPool;
}
void taosCloseTcpThreadPool(void *param) {
- SPoolObj * pPool = (SPoolObj *)param;
+ SPoolObj * pPool = param;
SThreadObj *pThread;
shutdown(pPool->acceptFd, SHUT_RD);
pthread_join(pPool->thread, NULL);
- for (int i = 0; i < pPool->info.numOfThreads; ++i) {
+ for (int32_t i = 0; i < pPool->info.numOfThreads; ++i) {
pThread = pPool->pThread[i];
if (pThread) taosStopPoolThread(pThread);
}
- uDebug("%p TCP pool is closed", pPool);
+ sDebug("%p TCP pool is closed", pPool);
- taosTFree(pPool->pThread);
- free(pPool);
+ tfree(pPool->pThread);
+ tfree(pPool);
}
-void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) {
+void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) {
struct epoll_event event;
- SPoolObj *pPool = (SPoolObj *)param;
+ SPoolObj *pPool = param;
- SConnObj *pConn = (SConnObj *)calloc(sizeof(SConnObj), 1);
+ SConnObj *pConn = calloc(sizeof(SConnObj), 1);
if (pConn == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
@@ -120,7 +124,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) {
SThreadObj *pThread = taosGetTcpThread(pPool);
if (pThread == NULL) {
- free(pConn);
+ tfree(pConn);
return NULL;
}
@@ -133,13 +137,13 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) {
event.data.ptr = pConn;
if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
- uError("failed to add fd:%d(%s)", connFd, strerror(errno));
+ sError("failed to add fd:%d since %s", connFd, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
- free(pConn);
+ tfree(pConn);
pConn = NULL;
} else {
pThread->numOfFds++;
- uDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds);
+ sDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds);
}
return pConn;
@@ -149,7 +153,7 @@ void taosFreeTcpConn(void *param) {
SConnObj * pConn = (SConnObj *)param;
SThreadObj *pThread = pConn->pThread;
- uDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd);
+ sDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd);
pConn->closedByApp = 1;
shutdown(pConn->fd, SHUT_WR);
}
@@ -164,9 +168,9 @@ static void taosProcessBrokenLink(SConnObj *pConn) {
pThread->numOfFds--;
epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL);
- uDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds);
+ sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds);
taosClose(pConn->fd);
- free(pConn);
+ tfree(pConn);
}
#define maxEvents 10
@@ -183,18 +187,18 @@ static void *taosProcessTcpData(void *param) {
while (1) {
if (pThread->stop) break;
- int fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME);
+ int32_t fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME);
if (pThread->stop) {
- uDebug("%p TCP epoll thread is exiting...", pThread);
+ sDebug("%p TCP epoll thread is exiting...", pThread);
break;
}
if (fdNum < 0) {
- uError("epoll_wait failed (%s)", strerror(errno));
+ sError("epoll_wait failed since %s", strerror(errno));
continue;
}
- for (int i = 0; i < fdNum; ++i) {
+ for (int32_t i = 0; i < fdNum; ++i) {
pConn = events[i].data.ptr;
assert(pConn);
@@ -219,17 +223,16 @@ static void *taosProcessTcpData(void *param) {
continue;
}
}
-
}
if (pThread->stop) break;
}
- uDebug("%p TCP epoll thread exits", pThread);
+ sDebug("%p TCP epoll thread exits", pThread);
close(pThread->pollFd);
- free(pThread);
- free(buffer);
+ tfree(pThread);
+ tfree(buffer);
return NULL;
}
@@ -242,18 +245,18 @@ static void *taosAcceptPeerTcpConnection(void *argv) {
while (1) {
struct sockaddr_in clientAddr;
socklen_t addrlen = sizeof(clientAddr);
- int connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen);
+ int32_t connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen);
if (connFd < 0) {
if (errno == EINVAL) {
- uDebug("%p TCP server accept is exiting...", pPool);
+ sDebug("%p TCP server accept is exiting...", pPool);
break;
} else {
- uError("TCP accept failure, reason:%s", strerror(errno));
+ sError("TCP accept failure since %s", strerror(errno));
continue;
}
}
- // uDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port);
+ // sDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port);
taosKeepTcpAlive(connFd);
(*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr);
}
@@ -273,23 +276,23 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) {
pThread->pPool = pPool;
pThread->pollFd = epoll_create(10); // size does not matter
if (pThread->pollFd < 0) {
- free(pThread);
+ tfree(pThread);
return NULL;
}
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
- int ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread);
+ int32_t ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread);
pthread_attr_destroy(&thattr);
if (ret != 0) {
close(pThread->pollFd);
- free(pThread);
+ tfree(pThread);
return NULL;
}
- uDebug("%p TCP epoll thread is created", pThread);
+ sDebug("%p TCP epoll thread is created", pThread);
pPool->pThread[pPool->nextId] = pThread;
pPool->nextId++;
pPool->nextId = pPool->nextId % pPool->info.numOfThreads;
@@ -314,12 +317,12 @@ static void taosStopPoolThread(SThreadObj *pThread) {
eventfd_t fd = eventfd(1, 0);
if (fd == -1) {
// failed to create eventfd, call pthread_cancel instead, which may result in data corruption
- uError("failed to create eventfd(%s)", strerror(errno));
+ sError("failed to create eventfd since %s", strerror(errno));
pthread_cancel(pThread->thread);
pThread->stop = true;
} else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
// failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption
- uError("failed to call epoll_ctl(%s)", strerror(errno));
+ sError("failed to call epoll_ctl since %s", strerror(errno));
pthread_cancel(pThread->thread);
}
diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c
index 360ea93f6c3fcd41e1df03702a0c963029deaacc..b7f819a3cd8f890d3580352322d8313f7cb2828e 100644
--- a/src/sync/src/tarbitrator.c
+++ b/src/sync/src/tarbitrator.c
@@ -28,22 +28,22 @@
#include "syncInt.h"
static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context);
-static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp);
+static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp);
static void arbProcessBrokenLink(void *param);
-static int arbProcessPeerMsg(void *param, void *buffer);
+static int32_t arbProcessPeerMsg(void *param, void *buffer);
static tsem_t tsArbSem;
static ttpool_h tsArbTcpPool;
typedef struct {
- char id[TSDB_EP_LEN + 24];
- int nodeFd;
- void *pConn;
+ char id[TSDB_EP_LEN + 24];
+ int32_t nodeFd;
+ void * pConn;
} SNodeConn;
-int main(int argc, char *argv[]) {
+int32_t main(int32_t argc, char *argv[]) {
char arbLogPath[TSDB_FILENAME_LEN + 16] = {0};
- for (int i = 1; i < argc; ++i) {
+ for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
tsArbitratorPort = atoi(argv[++i]);
} else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) {
@@ -86,7 +86,7 @@ int main(int argc, char *argv[]) {
info.numOfThreads = 1;
info.serverIp = 0;
info.port = tsArbitratorPort;
- info.bufferSize = 640000;
+ info.bufferSize = SYNC_MAX_SIZE;
info.processBrokenLink = arbProcessBrokenLink;
info.processIncomingMsg = arbProcessPeerMsg;
info.processIncomingConn = arbProcessIncommingConnection;
@@ -108,7 +108,7 @@ int main(int argc, char *argv[]) {
return 0;
}
-static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) {
+static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
char ipstr[24];
tinet_ntoa(ipstr, sourceIp);
sDebug("peer TCP connection from ip:%s", ipstr);
@@ -128,10 +128,10 @@ static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) {
}
firstPkt.fqdn[sizeof(firstPkt.fqdn) - 1] = 0;
- snprintf(pNode->id, sizeof(pNode->id), "vgId:%d peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port);
+ snprintf(pNode->id, sizeof(pNode->id), "vgId:%d, peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port);
if (firstPkt.syncHead.vgId) {
sDebug("%s, vgId in head is not zero, close the connection", pNode->id);
- taosTFree(pNode);
+ tfree(pNode);
taosCloseSocket(connFd);
return;
}
@@ -147,16 +147,16 @@ static void arbProcessBrokenLink(void *param) {
SNodeConn *pNode = param;
sDebug("%s, TCP link is broken(%s), close connection", pNode->id, strerror(errno));
- taosTFree(pNode);
+ tfree(pNode);
}
-static int arbProcessPeerMsg(void *param, void *buffer) {
+static int32_t arbProcessPeerMsg(void *param, void *buffer) {
SNodeConn *pNode = param;
SSyncHead head;
- int bytes = 0;
+ int32_t bytes = 0;
char * cont = (char *)buffer;
- int hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head));
+ int32_t hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head));
if (hlen != sizeof(head)) {
sDebug("%s, failed to read msg, hlen:%d", pNode->id, hlen);
return -1;
diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c
index 380b971fa89bd1726e138c973a974bc995500693..9dd3feb4614105c13ba6a2d1069931345167b472 100644
--- a/src/sync/test/syncServer.c
+++ b/src/sync/test/syncServer.c
@@ -30,7 +30,7 @@ int dataFd = -1;
void * qhandle = NULL;
int walNum = 0;
uint64_t tversion = 0;
-void * syncHandle;
+int64_t syncHandle;
int role;
int nodeId;
char path[256];
@@ -254,7 +254,7 @@ uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex
return magic;
}
-int getWalInfo(void *ahandle, char *name, uint32_t *index) {
+int getWalInfo(void *ahandle, char *name, int64_t *index) {
struct stat fstat;
char aname[280];
diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h
index 256b8189f8fc69345b27fdf702fb705d22ac3c10..0962e7c2cbb7fa0576fe1f00495ebbcbd23959d3 100644
--- a/src/tsdb/inc/tsdbMain.h
+++ b/src/tsdb/inc/tsdbMain.h
@@ -220,8 +220,7 @@ typedef struct {
SMemTable* mem;
SMemTable* imem;
STsdbFileH* tsdbFileH;
- int commit;
- pthread_t commitThread;
+ sem_t readyToCommit;
pthread_mutex_t mutex;
bool repoLocked;
} STsdbRepo;
@@ -320,6 +319,15 @@ typedef struct {
void* compBuffer; // Buffer for temperary compress/decompress purpose
} SRWHelper;
+typedef struct {
+ int rowsInserted;
+ int rowsUpdated;
+ int rowsDeleteSucceed;
+ int rowsDeleteFailed;
+ int nOperations;
+ TSKEY keyFirst;
+ TSKEY keyLast;
+} SMergeInfo;
// ------------------ tsdbScan.c
typedef struct {
SFileGroup fGroup;
@@ -422,7 +430,7 @@ void tsdbCloseBufPool(STsdbRepo* pRepo);
SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
// ------------------ tsdbMemTable.c
-int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
+int tsdbUpdateRowInMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
@@ -430,7 +438,8 @@ void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem)
void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
int tsdbAsyncCommit(STsdbRepo* pRepo);
int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
- TSKEY* filterKeys, int nFilterKeys);
+ TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
+void* tsdbCommitData(STsdbRepo* pRepo);
static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
if (pIter == NULL) return NULL;
@@ -438,16 +447,23 @@ static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
SSkipListNode* node = tSkipListIterGet(pIter);
if (node == NULL) return NULL;
- return *(SDataRow *)SL_GET_NODE_DATA(node);
+ return (SDataRow)SL_GET_NODE_DATA(node);
}
static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
SDataRow row = tsdbNextIterRow(pIter);
- if (row == NULL) return -1;
+ if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
return dataRowKey(row);
}
+static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
+ SDataRow row = tsdbNextIterRow(pIter);
+ if (row == NULL) return TKEY_NULL;
+
+ return dataRowTKey(row);
+}
+
static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
ASSERT(pRepo != NULL);
if (pRepo->mem == NULL) return NULL;
@@ -572,6 +588,9 @@ int tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx);
int tsdbCloseScanFile(STsdbScanHandle* pScanHandle);
void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle);
+// ------------------ tsdbCommitQueue.c
+int tsdbScheduleCommit(STsdbRepo *pRepo);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c
index 2e097c6ff73543f2dd858f8424b1e942ebdd8c3e..7cea27658c80d689972e3cb0f5dda3269a34b720 100644
--- a/src/tsdb/src/tsdbBuffer.c
+++ b/src/tsdb/src/tsdbBuffer.c
@@ -110,7 +110,7 @@ void tsdbCloseBufPool(STsdbRepo *pRepo) {
}
}
- tsdbDebug("vgId:%d buffer pool is closed", REPO_ID(pRepo));
+ tsdbDebug("vgId:%d, buffer pool is closed", REPO_ID(pRepo));
}
SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) {
@@ -134,7 +134,7 @@ SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) {
pBufBlock->offset = 0;
pBufBlock->remain = pBufPool->bufBlockSize;
- tsdbDebug("vgId:%d buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId);
+ tsdbDebug("vgId:%d, buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId);
return pNode;
}
@@ -157,4 +157,4 @@ _err:
return NULL;
}
-static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { taosTFree(pBufBlock); }
\ No newline at end of file
+static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); }
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c
new file mode 100644
index 0000000000000000000000000000000000000000..3c158a2201c1641d57b8c0902b2b3a9c1c828c9e
--- /dev/null
+++ b/src/tsdb/src/tsdbCommitQueue.c
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "os.h"
+#include "tlist.h"
+#include "tsdbMain.h"
+
+typedef struct {
+ bool stop;
+ pthread_mutex_t lock;
+ pthread_cond_t queueNotEmpty;
+ int nthreads;
+ SList * queue;
+ pthread_t * threads;
+} SCommitQueue;
+
+typedef struct {
+ STsdbRepo *pRepo;
+} SCommitReq;
+
+static void *tsdbLoopCommit(void *arg);
+
+SCommitQueue tsCommitQueue = {0};
+
+int tsdbInitCommitQueue(int nthreads) {
+ SCommitQueue *pQueue = &tsCommitQueue;
+
+ if (nthreads < 1) nthreads = 1;
+
+ pQueue->stop = false;
+ pQueue->nthreads = nthreads;
+
+ pQueue->queue = tdListNew(0);
+ if (pQueue->queue == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ pQueue->threads = (pthread_t *)calloc(nthreads, sizeof(pthread_t));
+ if (pQueue->threads == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ tdListFree(pQueue->queue);
+ return -1;
+ }
+
+ pthread_mutex_init(&(pQueue->lock), NULL);
+ pthread_cond_init(&(pQueue->queueNotEmpty), NULL);
+
+ for (int i = 0; i < nthreads; i++) {
+ pthread_create(pQueue->threads + i, NULL, tsdbLoopCommit, NULL);
+ }
+
+ return 0;
+}
+
+void tsdbDestroyCommitQueue() {
+ SCommitQueue *pQueue = &tsCommitQueue;
+
+ pthread_mutex_lock(&(pQueue->lock));
+
+ if (pQueue->stop) {
+ pthread_mutex_unlock(&(pQueue->lock));
+ return;
+ }
+
+ pQueue->stop = true;
+ pthread_cond_broadcast(&(pQueue->queueNotEmpty));
+
+ pthread_mutex_unlock(&(pQueue->lock));
+
+ for (size_t i = 0; i < pQueue->nthreads; i++) {
+ pthread_join(pQueue->threads[i], NULL);
+ }
+
+ free(pQueue->threads);
+ tdListFree(pQueue->queue);
+ pthread_cond_destroy(&(pQueue->queueNotEmpty));
+ pthread_mutex_destroy(&(pQueue->lock));
+}
+
+int tsdbScheduleCommit(STsdbRepo *pRepo) {
+ SCommitQueue *pQueue = &tsCommitQueue;
+
+ SListNode *pNode = (SListNode *)calloc(1, sizeof(SListNode) + sizeof(SCommitReq));
+ if (pNode == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ ((SCommitReq *)pNode->data)->pRepo = pRepo;
+
+ pthread_mutex_lock(&(pQueue->lock));
+
+ ASSERT(!pQueue->stop);
+
+ tdListAppendNode(pQueue->queue, pNode);
+ pthread_cond_signal(&(pQueue->queueNotEmpty));
+
+ pthread_mutex_unlock(&(pQueue->lock));
+ return 0;
+}
+
+static void *tsdbLoopCommit(void *arg) {
+ SCommitQueue *pQueue = &tsCommitQueue;
+ SListNode * pNode = NULL;
+ STsdbRepo * pRepo = NULL;
+
+ while (true) {
+ pthread_mutex_lock(&(pQueue->lock));
+
+ while (true) {
+ pNode = tdListPopHead(pQueue->queue);
+ if (pNode == NULL) {
+ if (pQueue->stop) {
+ pthread_mutex_unlock(&(pQueue->lock));
+ goto _exit;
+ } else {
+ pthread_cond_wait(&(pQueue->queueNotEmpty), &(pQueue->lock));
+ }
+ } else {
+ break;
+ }
+ }
+
+ pthread_mutex_unlock(&(pQueue->lock));
+
+ pRepo = ((SCommitReq *)pNode->data)->pRepo;
+
+ tsdbCommitData(pRepo);
+ listNodeFree(pNode);
+ }
+
+_exit:
+ return NULL;
+}
diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c
index 626ad77da2eab4be9e94516c4e5c7c0e5a45837e..1efde49ed0dba50cf01967265bcef63ed36839b7 100644
--- a/src/tsdb/src/tsdbFile.c
+++ b/src/tsdb/src/tsdbFile.c
@@ -13,10 +13,8 @@
* along with this program. If not, see .
*/
#define _DEFAULT_SOURCE
-#include
-
#define TAOS_RANDOM_FILE_FAIL_TEST
-
+#include
#include "os.h"
#include "talgo.h"
#include "tchecksum.h"
@@ -67,7 +65,7 @@ _err:
void tsdbFreeFileH(STsdbFileH *pFileH) {
if (pFileH) {
pthread_rwlock_destroy(&pFileH->fhlock);
- taosTFree(pFileH->pFGroup);
+ tfree(pFileH->pFGroup);
free(pFileH);
}
}
@@ -79,7 +77,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
DIR * dir = NULL;
int fid = 0;
int vid = 0;
- regex_t regex1, regex2;
+ regex_t regex1 = {0}, regex2 = {0};
int code = 0;
char fname[TSDB_FILENAME_LEN] = "\0";
@@ -95,9 +93,27 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
dir = opendir(tDataDir);
if (dir == NULL) {
- tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- goto _err;
+ if (errno == ENOENT) {
+ tsdbError("vgId:%d directory %s not exist", REPO_ID(pRepo), tDataDir);
+ terrno = TAOS_SYSTEM_ERROR(errno);
+
+ if (taosMkDir(tDataDir, 0755) < 0) {
+ tsdbError("vgId:%d failed to create directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+
+ dir = opendir(tDataDir);
+ if (dir == NULL) {
+ tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
+ } else {
+ tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ goto _err;
+ }
}
code = regcomp(®ex1, "^v[0-9]+f[0-9]+\\.(head|data|last|stat)$", REG_EXTENDED);
@@ -183,7 +199,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
regfree(®ex1);
regfree(®ex2);
- taosTFree(tDataDir);
+ tfree(tDataDir);
closedir(dir);
return 0;
@@ -193,7 +209,7 @@ _err:
regfree(®ex1);
regfree(®ex2);
- taosTFree(tDataDir);
+ tfree(tDataDir);
if (dir != NULL) closedir(dir);
tsdbCloseFileH(pRepo);
return -1;
@@ -410,7 +426,7 @@ int tsdbUpdateFileHeader(SFile *pFile) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
- if (taosTWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+ if (taosWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@@ -475,7 +491,7 @@ int tsdbLoadFileHeader(SFile *pFile, uint32_t *version) {
return -1;
}
- if (taosTRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+ if (taosRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
tsdbError("failed to read file %s header part with %d bytes, reason:%s", pFile->fname, TSDB_FILE_HEAD_SIZE,
strerror(errno));
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c
index a1e6376304a54ea810f3662bd30e3f97fd53765c..2ded3b668b47ac3dcf35fef287bae8f056977cb2 100644
--- a/src/tsdb/src/tsdbMain.c
+++ b/src/tsdb/src/tsdbMain.c
@@ -163,7 +163,7 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) {
if (toCommit) {
tsdbAsyncCommit(pRepo);
- if (pRepo->commit) pthread_join(pRepo->commitThread, NULL);
+ sem_wait(&(pRepo->readyToCommit));
}
tsdbUnRefMemTable(pRepo, pRepo->mem);
tsdbUnRefMemTable(pRepo, pRepo->imem);
@@ -228,7 +228,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
int prefixLen = (int)strlen(prefix);
if (name[0] == 0) { // get the file from index or after, but not larger than eindex
- taosTFree(sdup);
+ tfree(sdup);
int fid = (*index) / TSDB_FILE_TYPE_MAX;
if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) {
@@ -260,8 +260,8 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
fname = malloc(prefixLen + strlen(name) + 2);
sprintf(fname, "%s/%s", prefix, name);
if (access(fname, F_OK) != 0) {
- taosFree(fname);
- taosFree(sdup);
+ tfree(fname);
+ tfree(sdup);
return 0;
}
if (*index == TSDB_META_FILE_INDEX) { // get meta file
@@ -269,20 +269,20 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
} else {
tsdbGetFileInfoImpl(fname, &magic, size);
}
- taosFree(fname);
- taosFree(sdup);
+ tfree(fname);
+ tfree(sdup);
return magic;
}
if (stat(fname, &fState) < 0) {
- taosTFree(fname);
+ tfree(fname);
return 0;
}
*size = fState.st_size;
// magic = *size;
- taosTFree(fname);
+ tfree(fname);
return magic;
}
@@ -512,6 +512,9 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
}
}
+ // update check
+ if (pCfg->update != 0) pCfg->update = 1;
+
return 0;
_err:
@@ -579,7 +582,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) {
taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
- if (taosTWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+ if (taosWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, TSDB_FILE_HEAD_SIZE, fname,
strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -597,7 +600,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) {
return 0;
_err:
- taosTFree(fname);
+ tfree(fname);
if (fd >= 0) close(fd);
return -1;
}
@@ -620,7 +623,7 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) {
goto _err;
}
- if (taosTRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+ if (taosRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
tsdbError("failed to read %d bytes from file %s since %s", TSDB_FILE_HEAD_SIZE, fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
@@ -634,13 +637,13 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) {
tsdbDecodeCfg(buf, pCfg);
- taosTFree(fname);
+ tfree(fname);
close(fd);
return 0;
_err:
- taosTFree(fname);
+ tfree(fname);
if (fd >= 0) close(fd);
return -1;
}
@@ -672,6 +675,12 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) {
goto _err;
}
+ code = sem_init(&(pRepo->readyToCommit), 0, 1);
+ if (code != 0) {
+ terrno = TAOS_SYSTEM_ERROR(code);
+ goto _err;
+ }
+
pRepo->repoLocked = false;
pRepo->rootDir = strdup(rootDir);
@@ -715,7 +724,8 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) {
tsdbFreeMeta(pRepo->tsdbMeta);
// tsdbFreeMemTable(pRepo->mem);
// tsdbFreeMemTable(pRepo->imem);
- taosTFree(pRepo->rootDir);
+ tfree(pRepo->rootDir);
+ sem_destroy(&(pRepo->readyToCommit));
pthread_mutex_destroy(&pRepo->mutex);
free(pRepo);
}
@@ -762,7 +772,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY
return -1;
}
- if (tsdbInsertRowToMem(pRepo, row, pTable) < 0) return -1;
+ if (tsdbUpdateRowInMem(pRepo, row, pTable) < 0) return -1;
(*affectedrows)++;
points++;
@@ -923,6 +933,7 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) {
tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock);
tlen += taosEncodeFixedI8(buf, pCfg->precision);
tlen += taosEncodeFixedI8(buf, pCfg->compression);
+ tlen += taosEncodeFixedI8(buf, pCfg->update);
return tlen;
}
@@ -939,6 +950,7 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) {
buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock));
buf = taosDecodeFixedI8(buf, &(pCfg->precision));
buf = taosDecodeFixedI8(buf, &(pCfg->compression));
+ buf = taosDecodeFixedI8(buf, &(pCfg->update));
return buf;
}
diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c
index 4cf8ddd4bd8df396352ad66b8499552018d5d322..5680abcc6f4bf7874b59583cbfe3815c8a72a0a2 100644
--- a/src/tsdb/src/tsdbMemTable.c
+++ b/src/tsdb/src/tsdbMemTable.c
@@ -24,7 +24,6 @@ static void tsdbFreeMemTable(SMemTable *pMemTable);
static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable);
static void tsdbFreeTableData(STableData *pTableData);
static char * tsdbGetTsTupleKey(const void *data);
-static void * tsdbCommitData(void *arg);
static int tsdbCommitMeta(STsdbRepo *pRepo);
static void tsdbEndCommit(STsdbRepo *pRepo);
static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey);
@@ -32,43 +31,40 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables);
+static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row);
// ---------------- INTERNAL FUNCTIONS ----------------
-int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
+int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
STsdbCfg * pCfg = &pRepo->config;
STsdbMeta * pMeta = pRepo->tsdbMeta;
- int32_t level = 0;
- int32_t headSize = 0;
+ TKEY tkey = dataRowTKey(row);
TSKEY key = dataRowKey(row);
SMemTable * pMemTable = pRepo->mem;
STableData *pTableData = NULL;
- SSkipList * pSList = NULL;
+ bool isRowDelete = TKEY_IS_DELETED(tkey);
- if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL &&
- pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) {
- pTableData = pMemTable->tData[TABLE_TID(pTable)];
- pSList = pTableData->pData;
- }
-
- tSkipListNewNodeInfo(pSList, &level, &headSize);
+ if (isRowDelete) {
+ if (!pCfg->update) {
+ tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo));
+ terrno = TSDB_CODE_TDB_INVALID_ACTION;
+ return -1;
+ }
- SSkipListNode *pNode = (SSkipListNode *)malloc(headSize + sizeof(SDataRow *));
- if (pNode == NULL) {
- terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
- return -1;
+ if (key > TABLE_LASTKEY(pTable)) {
+ tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64,
+ REPO_ID(pRepo), key, TABLE_LASTKEY(pTable));
+ return 0;
+ }
}
void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row));
if (pRow == NULL) {
tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s",
REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno));
- free(pNode);
return -1;
}
- pNode->level = level;
dataRowCpy(pRow, row);
- *(SDataRow *)SL_GET_NODE_DATA(pNode) = pRow;
// Operations above may change pRepo->mem, retake those values
ASSERT(pRepo->mem != NULL);
@@ -77,7 +73,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
if (TABLE_TID(pTable) >= pMemTable->maxTables) {
if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) {
tsdbFreeBytes(pRepo, pRow, dataRowLen(row));
- free(pNode);
return -1;
}
}
@@ -97,7 +92,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
" to table %s while create new table data object since %s",
REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno));
tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
- free(pNode);
return -1;
}
@@ -106,24 +100,31 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable));
- if (tSkipListPut(pTableData->pData, pNode) == NULL) {
+ int64_t oldSize = SL_SIZE(pTableData->pData);
+ if (tSkipListPut(pTableData->pData, pRow) == NULL) {
tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
- free(pNode);
} else {
- if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key;
+ int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize;
+ if (isRowDelete) {
+ if (TABLE_LASTKEY(pTable) == key) {
+ // TODO: need to update table last key here (may from file)
+ }
+ } else {
+ if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key;
+ }
+
if (pMemTable->keyFirst > key) pMemTable->keyFirst = key;
if (pMemTable->keyLast < key) pMemTable->keyLast = key;
- pMemTable->numOfRows++;
+ pMemTable->numOfRows += deltaSize;
if (pTableData->keyFirst > key) pTableData->keyFirst = key;
if (pTableData->keyLast < key) pTableData->keyLast = key;
- pTableData->numOfRows++;
-
- ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData));
+ pTableData->numOfRows += deltaSize;
}
- tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
- TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), key);
+ tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
+ isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable),
+ key);
return 0;
}
@@ -196,6 +197,8 @@ void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem)
if (pIMem != NULL) {
tsdbUnRefMemTable(pRepo, pIMem);
}
+
+ tsdbDebug("vgId:%d utake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem);
}
void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
@@ -258,100 +261,211 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
int tsdbAsyncCommit(STsdbRepo *pRepo) {
SMemTable *pIMem = pRepo->imem;
- int code = 0;
- if (pIMem != NULL) {
- ASSERT(pRepo->commit);
- tsdbDebug("vgId:%d waiting for the commit thread", REPO_ID(pRepo));
- code = pthread_join(pRepo->commitThread, NULL);
- tsdbDebug("vgId:%d commit thread is finished", REPO_ID(pRepo));
- if (code != 0) {
- tsdbError("vgId:%d failed to thread join since %s", REPO_ID(pRepo), strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- return -1;
- }
- pRepo->commit = 0;
- }
-
- ASSERT(pRepo->commit == 0);
if (pRepo->mem != NULL) {
+ sem_wait(&(pRepo->readyToCommit));
+
if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
if (tsdbLockRepo(pRepo) < 0) return -1;
pRepo->imem = pRepo->mem;
pRepo->mem = NULL;
- pRepo->commit = 1;
- code = pthread_create(&pRepo->commitThread, NULL, tsdbCommitData, (void *)pRepo);
- if (code != 0) {
- tsdbError("vgId:%d failed to create commit thread since %s", REPO_ID(pRepo), strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(code);
- tsdbUnlockRepo(pRepo);
- return -1;
- }
+ tsdbScheduleCommit(pRepo);
if (tsdbUnlockRepo(pRepo) < 0) return -1;
}
- if (pIMem && tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1;
+ if (tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1;
+
+ return 0;
+}
+int tsdbSyncCommit(TSDB_REPO_T *repo) {
+ STsdbRepo *pRepo = (STsdbRepo *)repo;
+ tsdbAsyncCommit(pRepo);
+ sem_wait(&(pRepo->readyToCommit));
+ sem_post(&(pRepo->readyToCommit));
return 0;
}
+/**
+ * This is an important function to load data or try to load data from memory skiplist iterator.
+ *
+ * This function load memory data until:
+ * 1. iterator ends
+ * 2. data key exceeds maxKey
+ * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead
+ * 4. operations in pCols not exceeds its max capacity if pCols is given
+ *
+ * The function tries to procceed AS MUSH AS POSSIBLE.
+ */
int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
- TSKEY *filterKeys, int nFilterKeys) {
- ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
+ TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
+ ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0 && pMergeInfo != NULL);
if (pIter == NULL) return 0;
STSchema *pSchema = NULL;
- int numOfRows = 0;
- TSKEY keyNext = 0;
+ TSKEY rowKey = 0;
+ TSKEY fKey = 0;
+ bool isRowDel = false;
int filterIter = 0;
+ SDataRow row = NULL;
+
+ memset(pMergeInfo, 0, sizeof(*pMergeInfo));
+ pMergeInfo->keyFirst = INT64_MAX;
+ pMergeInfo->keyLast = INT64_MIN;
+ if (pCols) tdResetDataCols(pCols);
+
+ row = tsdbNextIterRow(pIter);
+ if (row == NULL || dataRowKey(row) > maxKey) {
+ rowKey = INT64_MAX;
+ isRowDel = false;
+ } else {
+ rowKey = dataRowKey(row);
+ isRowDel = dataRowDeleted(row);
+ }
- if (nFilterKeys != 0) { // for filter purpose
- ASSERT(filterKeys != NULL);
- keyNext = tsdbNextIterKey(pIter);
- if (keyNext < 0 || keyNext > maxKey) return numOfRows;
- void *ptr = taosbsearch((void *)(&keyNext), (void *)filterKeys, nFilterKeys, sizeof(TSKEY), compTSKEY, TD_GE);
- filterIter = (ptr == NULL) ? nFilterKeys : (int)((POINTER_DISTANCE(ptr, filterKeys) / sizeof(TSKEY)));
- }
-
- do {
- SDataRow row = tsdbNextIterRow(pIter);
- if (row == NULL) break;
-
- keyNext = dataRowKey(row);
- if (keyNext > maxKey) break;
-
- bool keyFiltered = false;
- if (nFilterKeys != 0) {
- while (true) {
- if (filterIter >= nFilterKeys) break;
- if (keyNext == filterKeys[filterIter]) {
- keyFiltered = true;
- filterIter++;
- break;
- } else if (keyNext < filterKeys[filterIter]) {
- break;
+ if (filterIter >= nFilterKeys) {
+ fKey = INT64_MAX;
+ } else {
+ fKey = tdGetKey(filterKeys[filterIter]);
+ }
+
+ while (true) {
+ if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
+
+ if (fKey < rowKey) {
+ pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
+ pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
+
+ filterIter++;
+ if (filterIter >= nFilterKeys) {
+ fKey = INT64_MAX;
+ } else {
+ fKey = tdGetKey(filterKeys[filterIter]);
+ }
+ } else if (fKey > rowKey) {
+ if (isRowDel) {
+ pMergeInfo->rowsDeleteFailed++;
+ } else {
+ if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
+ if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+ pMergeInfo->rowsInserted++;
+ pMergeInfo->nOperations++;
+ pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
+ pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
+ tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
+ }
+
+ tSkipListIterNext(pIter);
+ row = tsdbNextIterRow(pIter);
+ if (row == NULL || dataRowKey(row) > maxKey) {
+ rowKey = INT64_MAX;
+ isRowDel = false;
+ } else {
+ rowKey = dataRowKey(row);
+ isRowDel = dataRowDeleted(row);
+ }
+ } else {
+ if (isRowDel) {
+ ASSERT(!keepDup);
+ if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+ pMergeInfo->rowsDeleteSucceed++;
+ pMergeInfo->nOperations++;
+ tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
+ } else {
+ if (keepDup) {
+ if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+ pMergeInfo->rowsUpdated++;
+ pMergeInfo->nOperations++;
+ pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
+ pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
+ tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
} else {
- filterIter++;
+ pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
+ pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
}
}
+
+ tSkipListIterNext(pIter);
+ row = tsdbNextIterRow(pIter);
+ if (row == NULL || dataRowKey(row) > maxKey) {
+ rowKey = INT64_MAX;
+ isRowDel = false;
+ } else {
+ rowKey = dataRowKey(row);
+ isRowDel = dataRowDeleted(row);
+ }
+
+ filterIter++;
+ if (filterIter >= nFilterKeys) {
+ fKey = INT64_MAX;
+ } else {
+ fKey = tdGetKey(filterKeys[filterIter]);
+ }
}
+ }
- if (!keyFiltered) {
- if (numOfRows >= maxRowsToRead) break;
- if (pCols) {
- if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
- pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
- if (pSchema == NULL) {
- ASSERT(0);
- }
- }
+ return 0;
+}
+
+void *tsdbCommitData(STsdbRepo *pRepo) {
+ SMemTable * pMem = pRepo->imem;
+ STsdbCfg * pCfg = &pRepo->config;
+ SDataCols * pDataCols = NULL;
+ STsdbMeta * pMeta = pRepo->tsdbMeta;
+ SCommitIter *iters = NULL;
+ SRWHelper whelper = {0};
+ ASSERT(pMem != NULL);
+
+ tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64, REPO_ID(pRepo),
+ pMem->keyFirst, pMem->keyLast, pMem->numOfRows);
+
+ // Create the iterator to read from cache
+ if (pMem->numOfRows > 0) {
+ iters = tsdbCreateCommitIters(pRepo);
+ if (iters == NULL) {
+ tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _exit;
+ }
+
+ if (tsdbInitWriteHelper(&whelper, pRepo) < 0) {
+ tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _exit;
+ }
+
+ if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) {
+ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+ tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s",
+ REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno));
+ goto _exit;
+ }
+
+ int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision));
+ int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision));
- tdAppendDataRowToDataCol(row, pSchema, pCols);
+ // Loop to commit to each file
+ for (int fid = sfid; fid <= efid; fid++) {
+ if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
+ tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
+ goto _exit;
}
- numOfRows++;
}
- } while (tSkipListIterNext(pIter));
+ }
+
+ // Commit to update meta file
+ if (tsdbCommitMeta(pRepo) < 0) {
+ tsdbError("vgId:%d failed to commit data while committing meta data since %s", REPO_ID(pRepo), tstrerror(terrno));
+ goto _exit;
+ }
- return numOfRows;
+ tsdbFitRetention(pRepo);
+
+_exit:
+ tdFreeDataCols(pDataCols);
+ tsdbDestroyCommitIters(iters, pMem->maxTables);
+ tsdbDestroyHelper(&whelper);
+ tsdbEndCommit(pRepo);
+ tsdbInfo("vgId:%d commit over", pRepo->config.tsdbId);
+
+ return NULL;
}
// ---------------- LOCAL FUNCTIONS ----------------
@@ -423,7 +537,7 @@ static void tsdbFreeMemTable(SMemTable* pMemTable) {
tdListFree(pMemTable->extraBuffList);
tdListFree(pMemTable->bufBlockList);
tdListFree(pMemTable->actList);
- taosTFree(pMemTable->tData);
+ tfree(pMemTable->tData);
free(pMemTable);
}
}
@@ -440,8 +554,9 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) {
pTableData->keyLast = 0;
pTableData->numOfRows = 0;
- pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP,
- TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 1, tsdbGetTsTupleKey);
+ pTableData->pData =
+ tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP],
+ tkeyComparFn, pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey);
if (pTableData->pData == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
@@ -461,71 +576,8 @@ static void tsdbFreeTableData(STableData *pTableData) {
}
}
-static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(*(SDataRow *)data); }
-
-static void *tsdbCommitData(void *arg) {
- STsdbRepo * pRepo = (STsdbRepo *)arg;
- SMemTable * pMem = pRepo->imem;
- STsdbCfg * pCfg = &pRepo->config;
- SDataCols * pDataCols = NULL;
- STsdbMeta * pMeta = pRepo->tsdbMeta;
- SCommitIter *iters = NULL;
- SRWHelper whelper = {0};
- ASSERT(pRepo->commit == 1);
- ASSERT(pMem != NULL);
+static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); }
- tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64, REPO_ID(pRepo),
- pMem->keyFirst, pMem->keyLast, pMem->numOfRows);
-
- // Create the iterator to read from cache
- if (pMem->numOfRows > 0) {
- iters = tsdbCreateCommitIters(pRepo);
- if (iters == NULL) {
- tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno));
- goto _exit;
- }
-
- if (tsdbInitWriteHelper(&whelper, pRepo) < 0) {
- tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno));
- goto _exit;
- }
-
- if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) {
- terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
- tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s",
- REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno));
- goto _exit;
- }
-
- int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision));
- int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision));
-
- // Loop to commit to each file
- for (int fid = sfid; fid <= efid; fid++) {
- if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
- tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
- goto _exit;
- }
- }
- }
-
- // Commit to update meta file
- if (tsdbCommitMeta(pRepo) < 0) {
- tsdbError("vgId:%d failed to commit data while committing meta data since %s", REPO_ID(pRepo), tstrerror(terrno));
- goto _exit;
- }
-
- tsdbFitRetention(pRepo);
-
-_exit:
- tdFreeDataCols(pDataCols);
- tsdbDestroyCommitIters(iters, pMem->maxTables);
- tsdbDestroyHelper(&whelper);
- tsdbEndCommit(pRepo);
- tsdbInfo("vgId:%d commit over", pRepo->config.tsdbId);
-
- return NULL;
-}
static int tsdbCommitMeta(STsdbRepo *pRepo) {
SMemTable *pMem = pRepo->imem;
@@ -576,14 +628,14 @@ _err:
}
static void tsdbEndCommit(STsdbRepo *pRepo) {
- ASSERT(pRepo->commit == 1);
if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER);
+ sem_post(&(pRepo->readyToCommit));
}
static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
for (int i = 0; i < nIters; i++) {
TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
- if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1;
+ if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1;
}
return 0;
}
@@ -680,7 +732,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
goto _err;
}
- taosTFree(dataDir);
+ tfree(dataDir);
tsdbCloseHelperFile(pHelper, 0, pGroup);
pthread_rwlock_wrlock(&(pFileH->fhlock));
@@ -702,7 +754,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
return 0;
_err:
- taosTFree(dataDir);
+ tfree(dataDir);
tsdbCloseHelperFile(pHelper, 1, NULL);
return -1;
}
@@ -777,7 +829,23 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
pMemTable->tData = pTableData;
taosWUnLockLatch(&(pMemTable->latch));
- taosTFree(tData);
+ tfree(tData);
+
+ return 0;
+}
+
+static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) {
+ if (pCols) {
+ if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) {
+ *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
+ if (*ppSchema == NULL) {
+ ASSERT(false);
+ return -1;
+ }
+ }
+
+ tdAppendDataRowToDataCol(row, *ppSchema, pCols);
+ }
return 0;
}
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c
index f3bd91f038cf209827b1c252160019a6b0aac27f..25c815b74e3bd5593bef9157cbd42ba869298cab 100644
--- a/src/tsdb/src/tsdbMeta.c
+++ b/src/tsdb/src/tsdbMeta.c
@@ -86,7 +86,8 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
if (pTable != NULL) {
tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
TABLE_TID(pTable), TABLE_UID(pTable));
- return TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
+ terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
+ goto _err;
}
if (pCfg->type == TSDB_CHILD_TABLE) {
@@ -191,7 +192,7 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) {
return 0;
_err:
- taosTFree(tbname);
+ tfree(tbname);
return -1;
}
@@ -461,7 +462,7 @@ void tsdbFreeMeta(STsdbMeta *pMeta) {
if (pMeta) {
taosHashCleanup(pMeta->uidMap);
tdListFree(pMeta->superList);
- taosTFree(pMeta->tables);
+ tfree(pMeta->tables);
pthread_rwlock_destroy(&pMeta->rwLock);
free(pMeta);
}
@@ -485,11 +486,11 @@ int tsdbOpenMeta(STsdbRepo *pRepo) {
}
tsdbDebug("vgId:%d open TSDB meta succeed", REPO_ID(pRepo));
- taosTFree(fname);
+ tfree(fname);
return 0;
_err:
- taosTFree(fname);
+ tfree(fname);
return -1;
}
@@ -562,12 +563,12 @@ int tsdbUnlockRepoMeta(STsdbRepo *pRepo) {
void tsdbRefTable(STable *pTable) {
int32_t ref = T_REF_INC(pTable);
UNUSED(ref);
- // tsdbDebug("ref table %"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref);
+ tsdbDebug("ref table %s uid %" PRIu64 " tid:%d, refCount:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), ref);
}
void tsdbUnRefTable(STable *pTable) {
int32_t ref = T_REF_DEC(pTable);
- tsdbDebug("unref table uid:%"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref);
+ tsdbDebug("unref table %s uid:%"PRIu64" tid:%d, refCount:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), ref);
if (ref == 0) {
// tsdbDebug("destory table name:%s uid:%"PRIu64", tid:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable));
@@ -643,7 +644,7 @@ static void tsdbOrgMeta(void *pHandle) {
}
static char *getTagIndexKey(const void *pData) {
- STable *pTable = *(STable **)pData;
+ STable *pTable = (STable *)pData;
STSchema *pSchema = tsdbGetTableTagSchema(pTable);
STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN);
@@ -700,7 +701,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) {
}
pTable->tagVal = NULL;
STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
- pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey);
+ pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, SL_ALLOW_DUP_KEY, getTagIndexKey);
if (pTable->pIndex == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
@@ -745,7 +746,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) {
T_REF_INC(pTable);
- tsdbTrace("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable),
+ tsdbDebug("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable),
TABLE_UID(pTable));
return pTable;
@@ -760,7 +761,7 @@ static void tsdbFreeTable(STable *pTable) {
if (pTable->name != NULL)
tsdbTrace("table %s tid %d uid %" PRIu64 " is freed", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable),
TABLE_UID(pTable));
- taosTFree(TABLE_NAME(pTable));
+ tfree(TABLE_NAME(pTable));
if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) {
for (int i = 0; i < TSDB_MAX_TABLE_SCHEMAS; i++) {
tdFreeSchema(pTable->schema[i]);
@@ -774,7 +775,7 @@ static void tsdbFreeTable(STable *pTable) {
kvRowFree(pTable->tagVal);
tSkipListDestroy(pTable->pIndex);
- taosTFree(pTable->sql);
+ tfree(pTable->sql);
free(pTable);
}
}
@@ -889,7 +890,7 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro
}
if (lock) tsdbUnlockRepoMeta(pRepo);
- tsdbDebug("vgId:%d table %s is removed from meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable));
+ tsdbDebug("vgId:%d table %s uid %" PRIu64 " is removed from meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_UID(pTable));
tsdbUnRefTable(pTable);
}
@@ -900,23 +901,8 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper
pTable->pSuper = pSTable;
- int32_t level = 0;
- int32_t headSize = 0;
-
- tSkipListNewNodeInfo(pSTable->pIndex, &level, &headSize);
-
- // NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the
- // actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function
- SSkipListNode *pNode = calloc(1, headSize + sizeof(STable *));
- if (pNode == NULL) {
- terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
- return -1;
- }
- pNode->level = level;
-
- memcpy(SL_GET_NODE_DATA(pNode), &pTable, sizeof(STable *));
+ tSkipListPut(pSTable->pIndex, (void *)pTable);
- tSkipListPut(pSTable->pIndex, pNode);
if (refSuper) T_REF_INC(pSTable);
return 0;
}
@@ -940,7 +926,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
SSkipListNode *pNode = taosArrayGetP(res, i);
// STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode);
- if (*(STable **)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need
+ if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need
tSkipListRemoveNode(pSTable->pIndex, pNode);
}
}
@@ -1080,9 +1066,9 @@ void tsdbClearTableCfg(STableCfg *config) {
if (config->schema) tdFreeSchema(config->schema);
if (config->tagSchema) tdFreeSchema(config->tagSchema);
if (config->tagValues) kvRowFree(config->tagValues);
- taosTFree(config->name);
- taosTFree(config->sname);
- taosTFree(config->sql);
+ tfree(config->name);
+ tfree(config->sname);
+ tfree(config->sql);
free(config);
}
}
@@ -1170,8 +1156,8 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) {
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
buf = tdDecodeSchema(buf, &(pTable->tagSchema));
STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
- pTable->pIndex =
- tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey);
+ pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL,
+ SL_ALLOW_DUP_KEY, getTagIndexKey);
if (pTable->pIndex == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbFreeTable(pTable);
@@ -1197,7 +1183,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) {
tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM);
} else {
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
- tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (tSkipListGetSize(pTable->pIndex) + 1));
+ tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_SIZE(pTable->pIndex) + 1));
} else {
tlen = sizeof(SListNode) + sizeof(SActObj);
}
@@ -1244,7 +1230,7 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) {
}
while (tSkipListIterNext(pIter)) {
- STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
+ STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE);
pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, tTable);
}
@@ -1269,7 +1255,7 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) {
tsdbWLockRepoMeta(pRepo);
while (tSkipListIterNext(pIter)) {
- STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
+ STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
tsdbRemoveTableFromMeta(pRepo, tTable, false, false);
}
@@ -1304,7 +1290,7 @@ static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) {
STable **tTables = pMeta->tables;
pMeta->tables = tables;
- taosTFree(tTables);
+ tfree(tTables);
tsdbDebug("vgId:%d tsdb meta maxTables is adjusted as %d", REPO_ID(pRepo), maxTables);
return 0;
diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c
index 357093bd9e125567ef3d3629c00877f54778b500..98f815f78acb5f8beb77b27bc43c88e93bf1b917 100644
--- a/src/tsdb/src/tsdbRWHelper.c
+++ b/src/tsdb/src/tsdbRWHelper.c
@@ -14,9 +14,7 @@
*/
#define _DEFAULT_SOURCE
-
#define TAOS_RANDOM_FILE_FAIL_TEST
-
#include "os.h"
#include "talgo.h"
#include "tchecksum.h"
@@ -27,6 +25,7 @@
#define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM))
#define TSDB_KEY_COL_OFFSET 0
#define TSDB_GET_COMPBLOCK_IDX(h, b) (POINTER_DISTANCE(b, (h)->pCompInfo->blocks)/sizeof(SCompBlock))
+#define TSDB_IS_LAST_BLOCK(pb) ((pb)->last)
static bool tsdbShouldCreateNewLast(SRWHelper *pHelper);
static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock,
@@ -34,7 +33,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pD
static int compareKeyBlock(const void *arg1, const void *arg2);
static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize);
static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
-static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded);
+static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo);
static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
static void tsdbResetHelperFileImpl(SRWHelper *pHelper);
static int tsdbInitHelperFile(SRWHelper *pHelper);
@@ -61,8 +60,10 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pComp
static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock);
static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey,
int *blkIdx);
-static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
- TSKEY maxKey, int maxRows);
+static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
+ TSKEY maxKey, int maxRows, int8_t update);
+static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps);
+static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx);
// ---------------------- INTERNAL FUNCTIONS ----------------------
int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
@@ -279,7 +280,7 @@ int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols
while (true) {
ASSERT(blkIdx <= (int)pIdx->numOfBlocks);
TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
- if (keyFirst < 0 || keyFirst > maxKey) break; // iter over
+ if (keyFirst == TSDB_DATA_TIMESTAMP_NULL || keyFirst > maxKey) break; // iter over
if (pIdx->len <= 0 || keyFirst > pIdx->maxKey) {
if (tsdbProcessAppendCommit(pHelper, pCommitIter, pDataCols, maxKey) < 0) return -1;
@@ -335,7 +336,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
return -1;
}
- if (taosTSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) {
+ if (taosSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) {
tsdbError("vgId:%d failed to sendfile from file %s to file %s since %s", REPO_ID(pHelper->pRepo),
helperLastF(pHelper)->fname, helperNewLastF(pHelper)->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -380,7 +381,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
pIdx->tid = pHelper->tableInfo.tid;
ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
- if (taosTWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) {
+ if (taosWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) {
tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
pFile->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -432,7 +433,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) {
ASSERT(offset == pFile->info.size);
- if (taosTWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < (int)pFile->info.len) {
+ if (taosWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < (int)pFile->info.len) {
tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len,
pFile->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -454,7 +455,7 @@ int tsdbLoadCompIdxImpl(SFile *pFile, uint32_t offset, uint32_t len, void *buffe
return -1;
}
- if (taosTRead(pFile->fd, buffer, len) < len) {
+ if (taosRead(pFile->fd, buffer, len) < len) {
tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, offset, len,
strerror(errno));
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
@@ -551,7 +552,7 @@ int tsdbLoadCompInfoImpl(SFile *pFile, SCompIdx *pIdx, SCompInfo **ppCompInfo) {
return -1;
}
- if (taosTRead(pFile->fd, (void *)(*ppCompInfo), pIdx->len) < (int)pIdx->len) {
+ if (taosRead(pFile->fd, (void *)(*ppCompInfo), pIdx->len) < (int)pIdx->len) {
tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, pIdx->offset, pIdx->len,
strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -608,7 +609,7 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) {
return -1;
}
- if (taosTRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) {
+ if (taosRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) {
tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname,
strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -823,7 +824,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
sizeof(TSCKSUM));
// Write the whole block to file
- if (taosTWrite(pFile->fd, (void *)pCompData, lsize) < lsize) {
+ if (taosWrite(pFile->fd, (void *)pCompData, lsize) < lsize) {
tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, pFile->fname,
strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -925,7 +926,7 @@ _err:
return -1;
}
-static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) {
+static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo) {
ASSERT(pCompBlock->numOfSubBlocks == 0);
SCompIdx *pIdx = &(pHelper->curCompIdx);
@@ -958,9 +959,9 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
pSCompBlock->numOfSubBlocks++;
ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
pSCompBlock->len += sizeof(SCompBlock);
- pSCompBlock->numOfRows += rowsAdded;
- pSCompBlock->keyFirst = MIN(pSCompBlock->keyFirst, pCompBlock->keyFirst);
- pSCompBlock->keyLast = MAX(pSCompBlock->keyLast, pCompBlock->keyLast);
+ pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
+ pSCompBlock->keyFirst = pMergeInfo->keyFirst;
+ pSCompBlock->keyLast = pMergeInfo->keyLast;
pIdx->len += sizeof(SCompBlock);
} else { // Need to create two sub-blocks
void *ptr = NULL;
@@ -989,11 +990,11 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
((SCompBlock *)ptr)[1] = *pCompBlock;
pSCompBlock->numOfSubBlocks = 2;
- pSCompBlock->numOfRows += rowsAdded;
+ pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo);
pSCompBlock->len = sizeof(SCompBlock) * 2;
- pSCompBlock->keyFirst = MIN(((SCompBlock *)ptr)[0].keyFirst, ((SCompBlock *)ptr)[1].keyFirst);
- pSCompBlock->keyLast = MAX(((SCompBlock *)ptr)[0].keyLast, ((SCompBlock *)ptr)[1].keyLast);
+ pSCompBlock->keyFirst = pMergeInfo->keyFirst;
+ pSCompBlock->keyLast = pMergeInfo->keyLast;
pIdx->len += (sizeof(SCompBlock) * 2);
}
@@ -1047,6 +1048,45 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
return 0;
}
+static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) {
+ SCompIdx *pCompIdx = &(pHelper->curCompIdx);
+
+ ASSERT(pCompIdx->numOfBlocks > 0 && blkIdx < pCompIdx->numOfBlocks);
+
+ SCompBlock *pCompBlock= blockAtIdx(pHelper, blkIdx);
+ SCompBlock compBlock = *pCompBlock;
+ ASSERT(pCompBlock->numOfSubBlocks > 0 && pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
+
+ if (pCompIdx->numOfBlocks == 1) {
+ memset(pCompIdx, 0, sizeof(*pCompIdx));
+ } else {
+ int tsize = 0;
+
+ if (compBlock.numOfSubBlocks > 1) {
+ tsize = (int)(pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks));
+
+ ASSERT(tsize > 0);
+ memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset),
+ POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks),
+ tsize);
+
+ pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks;
+ }
+
+ tsize = (int)(pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo));
+ ASSERT(tsize > 0);
+ memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize);
+
+ pCompIdx->len -= sizeof(SCompBlock);
+
+ pCompIdx->numOfBlocks--;
+ pCompIdx->hasLast = (uint32_t)(blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last);
+ pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast;
+ }
+
+ return 0;
+}
+
static void tsdbResetHelperFileImpl(SRWHelper *pHelper) {
pHelper->idxH.numOfIdx = 0;
pHelper->idxH.curIdx = 0;
@@ -1210,7 +1250,7 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl
return -1;
}
- if (taosTRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) {
+ if (taosRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) {
tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompCol->len, pFile->fname,
strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -1325,7 +1365,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
- if (taosTRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) {
+ if (taosRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) {
tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompBlock->len,
pFile->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -1439,51 +1479,62 @@ static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) {
}
static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) {
- STsdbCfg * pCfg = &(pHelper->pRepo->config);
- STable * pTable = pCommitIter->pTable;
- SCompIdx * pIdx = &(pHelper->curCompIdx);
- TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
- int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
- SCompBlock compBlock = {0};
+ STsdbCfg * pCfg = &(pHelper->pRepo->config);
+ STable * pTable = pCommitIter->pTable;
+ SCompIdx * pIdx = &(pHelper->curCompIdx);
+ TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
+ int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
+ SCompBlock compBlock = {0};
+ SMergeInfo mergeInfo = {0};
+ SMergeInfo *pMergeInfo = &mergeInfo;
ASSERT(pIdx->len <= 0 || keyFirst > pIdx->maxKey);
if (pIdx->hasLast) { // append to with last block
ASSERT(pIdx->len > 0);
SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1);
ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock);
- tdResetDataCols(pDataCols);
- int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows,
- pDataCols, NULL, 0);
- ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows);
- if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock &&
- pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
- if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
- if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, rowsRead) < 0) return -1;
- } else {
- if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
- ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows);
+ tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols,
+ NULL, 0, pCfg->update, pMergeInfo);
- if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1;
- ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows);
+ ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
- if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1;
- if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1;
- }
+ if (pDataCols->numOfRows > 0) {
+ ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
- if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
+ if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock &&
+ pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
+ if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
+ pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, pCompBlock->keyFirst);
+ pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, pCompBlock->keyLast);
+ if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1;
+ } else {
+ if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
+ ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows);
+
+ if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1;
+ ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows);
+
+ if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1;
+ if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1;
+ }
+
+ if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
+ }
} else {
ASSERT(!pHelper->hasOldLastBlock);
- tdResetDataCols(pDataCols);
- int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0);
- ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows);
+ tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo);
+ ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
- if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
- if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1;
+ if (pDataCols->numOfRows > 0) {
+ ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
+ if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
+ if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1;
+ }
}
#ifndef NDEBUG
TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter);
- ASSERT(keyNext < 0 || keyNext > pIdx->maxKey);
+ ASSERT(keyNext == TSDB_DATA_TIMESTAMP_NULL || keyNext > pIdx->maxKey);
#endif
return 0;
@@ -1491,13 +1542,16 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey,
int *blkIdx) {
- STsdbCfg * pCfg = &(pHelper->pRepo->config);
- STable * pTable = pCommitIter->pTable;
- SCompIdx * pIdx = &(pHelper->curCompIdx);
- SCompBlock compBlock = {0};
- TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
- int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
- SDataCols *pDataCols0 = pHelper->pDataCols[0];
+ STsdbCfg * pCfg = &(pHelper->pRepo->config);
+ STable * pTable = pCommitIter->pTable;
+ SCompIdx * pIdx = &(pHelper->curCompIdx);
+ SCompBlock compBlock = {0};
+ TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
+ int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
+ SDataCols * pDataCols0 = pHelper->pDataCols[0];
+ SMergeInfo mergeInfo = {0};
+ SMergeInfo *pMergeInfo = &mergeInfo;
+ SCompBlock oBlock = {0};
SSkipListIterator slIter = {0};
@@ -1507,123 +1561,82 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE);
ASSERT(pCompBlock != NULL);
int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock));
+ oBlock = *pCompBlock;
+
+ ASSERT((!TSDB_IS_LAST_BLOCK(&oBlock)) || (tblkIdx == pIdx->numOfBlocks - 1));
- if (pCompBlock->last) {
- ASSERT(pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && tblkIdx == pIdx->numOfBlocks - 1);
+ if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) {
+ while (true) {
+ tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0,
+ pCfg->update, pMergeInfo);
+ ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
+ if (pDataCols->numOfRows == 0) break;
+
+ if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
+ if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
+ tblkIdx++;
+ }
+ ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) == TSDB_DATA_TIMESTAMP_NULL ||
+ tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
+ } else {
int16_t colId = 0;
+ if (tsdbLoadBlockDataCols(pHelper, &oBlock, NULL, &colId, 1) < 0) return -1;
+
+ TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1);
+
slIter = *(pCommitIter->pIter);
- if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1;
- ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows);
+ tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows,
+ pCfg->update, pMergeInfo);
- int rows1 = defaultRowsInBlock - pCompBlock->numOfRows;
- int rows2 =
- tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows);
- if (rows2 == 0) { // all data filtered out
+ if (pMergeInfo->nOperations == 0) {
+ // Do nothing
+ ASSERT(pMergeInfo->rowsDeleteFailed >= 0);
*(pCommitIter->pIter) = slIter;
+ tblkIdx++;
+ } else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) {
+ // Delete the block and do some stuff
+ ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN);
+ if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1;
+ *pCommitIter->pIter = slIter;
+ if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
+ } else if (tsdbCheckAddSubBlockCond(pHelper, &oBlock, pMergeInfo, pDataCols->maxPoints)) {
+ // Append as a sub-block of the searched block
+ tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData,
+ pDataCols0->numOfRows, pCfg->update, pMergeInfo);
+ ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0);
+ if (tsdbWriteBlockToFile(pHelper, oBlock.last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols,
+ &compBlock, oBlock.last, false) < 0) {
+ return -1;
+ }
+ if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) {
+ return -1;
+ }
+ tblkIdx++;
} else {
- if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock &&
- pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
- tdResetDataCols(pDataCols);
- int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols,
- pDataCols0->cols[0].pData, pDataCols0->numOfRows);
- ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows);
- if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
- if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1;
- tblkIdx++;
- } else {
- if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
- int round = 0;
- int dIter = 0;
- while (true) {
- tdResetDataCols(pDataCols);
- int rowsRead =
- tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, defaultRowsInBlock);
- if (rowsRead == 0) break;
+ // load the block data, merge with the memory data
+ if (tsdbLoadBlockData(pHelper, &oBlock, NULL) < 0) return -1;
+ int round = 0;
+ int dIter = 0;
+ while (true) {
+ tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock,
+ pCfg->update);
+ if (pDataCols->numOfRows == 0) break;
+ if (tblkIdx == pIdx->numOfBlocks - 1) {
if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
- if (round == 0) {
- if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
- } else {
- if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
- }
-
- tblkIdx++;
- round++;
+ } else {
+ if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
}
- }
- if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
- }
- } else {
- TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (pCompBlock[1].keyFirst - 1);
- TSKEY blkKeyFirst = pCompBlock->keyFirst;
- TSKEY blkKeyLast = pCompBlock->keyLast;
- if (keyFirst < blkKeyFirst) {
- while (true) {
- tdResetDataCols(pDataCols);
- int rowsRead =
- tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, NULL, 0);
- if (rowsRead == 0) break;
-
- ASSERT(rowsRead == pDataCols->numOfRows);
- if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
- if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
- tblkIdx++;
- }
- ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
- tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
- } else {
- ASSERT(keyFirst <= blkKeyLast);
- int16_t colId = 0;
- if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1;
-
- slIter = *(pCommitIter->pIter);
- int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows);
- int rows2 = tsdbLoadDataFromCache(pTable, &slIter, blkKeyLast, INT_MAX, NULL, pDataCols0->cols[0].pData,
- pDataCols0->numOfRows);
-
- if (rows2 == 0) { // all filtered out
- *(pCommitIter->pIter) = slIter;
- ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
- tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
- } else {
- int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0) + rows2;
-
- if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) {
- int rows = (rows1 >= rows3) ? rows3 : rows2;
- tdResetDataCols(pDataCols);
- int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols,
- pDataCols0->cols[0].pData, pDataCols0->numOfRows);
- ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows);
- if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0)
- return -1;
- if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1;
- tblkIdx++;
- ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
- tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
+ if (round == 0) {
+ if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
+ if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
} else {
- if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
- int round = 0;
- int dIter = 0;
- while (true) {
- int rowsRead =
- tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock);
- if (rowsRead == 0) break;
-
- if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0)
- return -1;
- if (round == 0) {
- if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
- } else {
- if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
- }
-
- round++;
- tblkIdx++;
- }
- ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
- tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
+ if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
}
+
+ round++;
+ tblkIdx++;
}
}
}
@@ -1632,9 +1645,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
return 0;
}
-static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
- TSKEY maxKey, int maxRows) {
- int numOfRows = 0;
+static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
+ TSKEY maxKey, int maxRows, int8_t update) {
TSKEY key1 = INT64_MAX;
TSKEY key2 = INT64_MAX;
STSchema *pSchema = NULL;
@@ -1644,35 +1656,62 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte
while (true) {
key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
+ bool isRowDel = false;
SDataRow row = tsdbNextIterRow(pCommitIter->pIter);
- key2 = (row == NULL || dataRowKey(row) > maxKey) ? INT64_MAX : dataRowKey(row);
+ if (row == NULL || dataRowKey(row) > maxKey) {
+ key2 = INT64_MAX;
+ } else {
+ key2 = dataRowKey(row);
+ isRowDel = dataRowDeleted(row);
+ }
if (key1 == INT64_MAX && key2 == INT64_MAX) break;
- if (key1 <= key2) {
+ if (key1 < key2) {
for (int i = 0; i < pDataCols->numOfCols; i++) {
dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
pTarget->maxPoints);
}
+
pTarget->numOfRows++;
(*iter)++;
- if (key1 == key2) tSkipListIterNext(pCommitIter->pIter);
- } else {
- if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
- pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
- ASSERT(pSchema != NULL);
+ } else if (key1 > key2) {
+ if (!isRowDel) {
+ if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
+ pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
+ ASSERT(pSchema != NULL);
+ }
+
+ tdAppendDataRowToDataCol(row, pSchema, pTarget);
}
- tdAppendDataRowToDataCol(row, pSchema, pTarget);
+ tSkipListIterNext(pCommitIter->pIter);
+ } else {
+ if (update) {
+ if (!isRowDel) {
+ if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
+ pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
+ ASSERT(pSchema != NULL);
+ }
+
+ tdAppendDataRowToDataCol(row, pSchema, pTarget);
+ }
+ } else {
+ ASSERT(!isRowDel);
+
+ for (int i = 0; i < pDataCols->numOfCols; i++) {
+ dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
+ pTarget->maxPoints);
+ }
+
+ pTarget->numOfRows++;
+ }
+ (*iter)++;
tSkipListIterNext(pCommitIter->pIter);
}
- numOfRows++;
- if (numOfRows >= maxRows) break;
- ASSERT(numOfRows == pTarget->numOfRows && numOfRows <= pTarget->maxPoints);
+ if (pTarget->numOfRows >= maxRows) break;
}
-
- return numOfRows;
}
static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock) {
@@ -1695,3 +1734,20 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols,
return 0;
}
+
+static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps) {
+ STsdbCfg *pCfg = &(pHelper->pRepo->config);
+ int mergeRows = pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
+
+ ASSERT(mergeRows > 0);
+
+ if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pMergeInfo->nOperations <= maxOps) {
+ if (pCompBlock->last) {
+ if (!TSDB_NLAST_FILE_OPENED(pHelper) && mergeRows < pCfg->minRowsPerFileBlock) return true;
+ } else {
+ if (mergeRows < pCfg->maxRowsPerFileBlock) return true;
+ }
+ }
+
+ return false;
+}
\ No newline at end of file
diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c
index ac6c2e0c5a9d5590e7dd57863dd18d8726912de5..021c10ab6a53eac66d120d8b01574598ec6689ff 100644
--- a/src/tsdb/src/tsdbRead.c
+++ b/src/tsdb/src/tsdbRead.c
@@ -20,7 +20,6 @@
#include "exception.h"
#include "../../query/inc/qAst.h" // todo move to common module
-#include "../../query/inc/qExecutor.h" // todo move to common module
#include "tlosertree.h"
#include "tsdb.h"
#include "tsdbMain.h"
@@ -72,8 +71,8 @@ typedef struct STableCheckInfo {
STable* pTableObj;
SCompInfo* pCompInfo;
int32_t compSize;
- int32_t numOfBlocks; // number of qualified data blocks not the original blocks
- int32_t chosen; // indicate which iterator should move forward
+ int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks
+ int8_t chosen:2; // indicate which iterator should move forward
bool initBuf; // whether to initialize the in-memory skip list iterator or not
SSkipListIterator* iter; // mem buffer skip list iterator
SSkipListIterator* iiter; // imem buffer skip list iterator
@@ -120,8 +119,9 @@ typedef struct STsdbQueryHandle {
SDataCols *pDataCols; // in order to hold current file data block
int32_t allocSize; // allocated data block size
- SMemTable *mem; // mem-table
- SMemTable *imem; // imem-table, acquired from snapshot
+ SMemRef *pMemRef;
+// SMemTable *mem; // mem-table
+// SMemTable *imem; // imem-table, acquired from snapshot
SArray *defaultLoadColumn;// default load column
SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
@@ -184,26 +184,26 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS
return pLocalIdList;
}
-static void tsdbMayTakeMemSnapshot(TsdbQueryHandleT pHandle) {
- STsdbQueryHandle* pSecQueryHandle = (STsdbQueryHandle*) pHandle;
- SQInfo *pQInfo = (SQInfo *)(pSecQueryHandle->qinfo);
+static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
+ assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL);
- if (pQInfo->memRef.ref++ == 0) {
- tsdbTakeMemSnapshot(pSecQueryHandle->pTsdb, &pSecQueryHandle->mem, &pSecQueryHandle->imem);
- pQInfo->memRef.mem = pSecQueryHandle->mem;
- pQInfo->memRef.imem = pSecQueryHandle->imem;
- } else {
- pSecQueryHandle->mem = (SMemTable *)(pQInfo->memRef.mem);
- pSecQueryHandle->imem = (SMemTable *)(pQInfo->memRef.imem);
+ SMemRef* pMemRef = pQueryHandle->pMemRef;
+ if (pQueryHandle->pMemRef->ref++ == 0) {
+ tsdbTakeMemSnapshot(pQueryHandle->pTsdb, (SMemTable**)&(pMemRef->mem), (SMemTable**)&(pMemRef->imem));
}
}
-static void tsdbMayUnTakeMemSnapshot(TsdbQueryHandleT pHandle) {
- STsdbQueryHandle* pSecQueryHandle = (STsdbQueryHandle*) pHandle;
- SQInfo *pQInfo = (SQInfo *)(pSecQueryHandle->qinfo);
- if (--pQInfo->memRef.ref == 0) {
- tsdbUnTakeMemSnapShot(pSecQueryHandle->pTsdb, pSecQueryHandle->mem, pSecQueryHandle->imem);
+static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
+ assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL);
+
+ SMemRef* pMemRef = pQueryHandle->pMemRef;
+ if (--pMemRef->ref == 0) {
+ tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, pMemRef->mem, pMemRef->imem);
+ pMemRef->mem = NULL;
+ pMemRef->imem = NULL;
}
+
+ pQueryHandle->pMemRef = NULL;
}
static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta) {
size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList);
@@ -270,7 +270,7 @@ static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey)
return pNew;
}
-static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo) {
+static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo, SMemRef* pMemRef) {
STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle));
if (pQueryHandle == NULL) {
goto out_of_memory;
@@ -288,13 +288,14 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond*
pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock;
pQueryHandle->allocSize = 0;
pQueryHandle->locateStart = false;
+ pQueryHandle->pMemRef = pMemRef;
if (tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb) != 0) {
goto out_of_memory;
}
tsdbMayTakeMemSnapshot(pQueryHandle);
- assert(pCond != NULL && pCond->numOfCols > 0);
+ assert(pCond != NULL && pCond->numOfCols > 0 && pMemRef != NULL);
if (ASCENDING_TRAVERSE(pCond->order)) {
assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey);
@@ -348,8 +349,8 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond*
return NULL;
}
-TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) {
- STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo);
+TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo, SMemRef* pRef) {
+ STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo, pRef);
STsdbMeta* pMeta = tsdbGetMeta(tsdb);
assert(pMeta != NULL);
@@ -366,7 +367,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
return (TsdbQueryHandleT) pQueryHandle;
}
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) {
+TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) {
pCond->twindow = changeTableGroupByLastrow(groupList);
// no qualified table
@@ -374,7 +375,7 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
return NULL;
}
- STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
+ STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pMemRef);
assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey);
return pQueryHandle;
@@ -396,8 +397,8 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) {
return res;
}
-TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo) {
- STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
+TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) {
+ STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pRef);
if (pQueryHandle != NULL) {
pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
changeQueryHandleForInterpQuery(pQueryHandle);
@@ -417,7 +418,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
int32_t order = pHandle->order;
// no data in buffer, abort
- if (pHandle->mem == NULL && pHandle->imem == NULL) {
+ if (pHandle->pMemRef->mem == NULL && pHandle->pMemRef->imem == NULL) {
return false;
}
@@ -426,16 +427,19 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
STableData* pMem = NULL;
STableData* pIMem = NULL;
- if (pHandle->mem && pCheckInfo->tableId.tid < pHandle->mem->maxTables) {
- pMem = pHandle->mem->tData[pCheckInfo->tableId.tid];
+ SMemTable* pMemT = pHandle->pMemRef->mem;
+ SMemTable* pIMemT = pHandle->pMemRef->imem;
+
+ if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) {
+ pMem = pMemT->tData[pCheckInfo->tableId.tid];
if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid
pCheckInfo->iter =
tSkipListCreateIterFromVal(pMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
}
}
- if (pHandle->imem && pCheckInfo->tableId.tid < pHandle->imem->maxTables) {
- pIMem = pHandle->imem->tData[pCheckInfo->tableId.tid];
+ if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) {
+ pIMem = pIMemT->tData[pCheckInfo->tableId.tid];
if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid
pCheckInfo->iiter =
tSkipListCreateIterFromVal(pIMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
@@ -457,7 +461,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
assert(node != NULL);
- SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
+ SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
@@ -479,7 +483,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
assert(node != NULL);
- SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
+ SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
@@ -504,19 +508,19 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
tSkipListDestroyIter(pCheckInfo->iiter);
}
-static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) {
+static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) {
SDataRow rmem = NULL, rimem = NULL;
if (pCheckInfo->iter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
if (node != NULL) {
- rmem = *(SDataRow *)SL_GET_NODE_DATA(node);
+ rmem = (SDataRow)SL_GET_NODE_DATA(node);
}
}
if (pCheckInfo->iiter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
if (node != NULL) {
- rimem = *(SDataRow *)SL_GET_NODE_DATA(node);
+ rimem = (SDataRow)SL_GET_NODE_DATA(node);
}
}
@@ -538,9 +542,15 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order
TSKEY r2 = dataRowKey(rimem);
if (r1 == r2) { // data ts are duplicated, ignore the data in mem
- tSkipListIterNext(pCheckInfo->iter);
- pCheckInfo->chosen = 1;
- return rimem;
+ if (!update) {
+ tSkipListIterNext(pCheckInfo->iter);
+ pCheckInfo->chosen = 1;
+ return rimem;
+ } else {
+ tSkipListIterNext(pCheckInfo->iiter);
+ pCheckInfo->chosen = 0;
+ return rmem;
+ }
} else {
if (ASCENDING_TRAVERSE(order)) {
if (r1 < r2) {
@@ -594,6 +604,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
}
static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
+ STsdbCfg *pCfg = &pHandle->pTsdb->config;
size_t size = taosArrayGetSize(pHandle->pTableCheckInfo);
assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1);
pHandle->cur.fid = -1;
@@ -607,7 +618,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
initTableMemIterator(pHandle, pCheckInfo);
}
- SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order);
+ SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update);
if (row == NULL) {
return false;
}
@@ -827,11 +838,12 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl
static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){
SQueryFilePos* cur = &pQueryHandle->cur;
+ STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
int32_t code = TSDB_CODE_SUCCESS;
/*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
- SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
+ SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
assert(cur->pos >= 0 && cur->pos <= binfo.rows);
@@ -1263,7 +1275,6 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl
int32_t end = endPos;
if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
- assert(start >= end);
SWAP(start, end, int32_t);
}
@@ -1317,6 +1328,7 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl
static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
SQueryFilePos* cur = &pQueryHandle->cur;
SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
+ STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
initTableMemIterator(pQueryHandle, pCheckInfo);
@@ -1354,7 +1366,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
} else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
SSkipListNode* node = NULL;
do {
- SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
+ SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
if (row == NULL) {
break;
}
@@ -1384,7 +1396,22 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
moveToNextRowInMem(pCheckInfo);
} else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it
- moveToNextRowInMem(pCheckInfo);
+ if (pCfg->update) {
+ copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable);
+ numOfRows += 1;
+ if (cur->win.skey == TSKEY_INITIAL_VAL) {
+ cur->win.skey = key;
+ }
+
+ cur->win.ekey = key;
+ cur->lastKey = key + step;
+ cur->mixBlock = true;
+
+ moveToNextRowInMem(pCheckInfo);
+ pos += step;
+ } else {
+ moveToNextRowInMem(pCheckInfo);
+ }
} else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
if (cur->win.skey == TSKEY_INITIAL_VAL) {
@@ -1395,7 +1422,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
assert(end != -1);
if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
- moveToNextRowInMem(pCheckInfo);
+ if (!pCfg->update) {
+ moveToNextRowInMem(pCheckInfo);
+ } else {
+ end -= step;
+ }
}
int32_t qstart = 0, qend = 0;
@@ -1415,8 +1446,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
* copy them all to result buffer, since it may be overlapped with file data block.
*/
if (node == NULL ||
- ((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
- ((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
+ ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+ ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
// no data in cache or data in cache is greater than the ekey of time window, load data from file block
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = tsArray[pos];
@@ -1513,15 +1544,15 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
}
static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) {
- taosTFree(pSupporter->numOfBlocksPerTable);
- taosTFree(pSupporter->blockIndexArray);
+ tfree(pSupporter->numOfBlocksPerTable);
+ tfree(pSupporter->blockIndexArray);
for (int32_t i = 0; i < numOfTables; ++i) {
STableBlockInfo* pBlockInfo = pSupporter->pDataBlockInfo[i];
- taosTFree(pBlockInfo);
+ tfree(pBlockInfo);
}
- taosTFree(pSupporter->pDataBlockInfo);
+ tfree(pSupporter->pDataBlockInfo);
}
static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) {
@@ -1863,13 +1894,14 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
STsdbQueryHandle* pQueryHandle) {
int numOfRows = 0;
int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns);
+ STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
win->skey = TSKEY_INITIAL_VAL;
int64_t st = taosGetTimestampUs();
STable* pTable = pCheckInfo->pTableObj;
do {
- SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
+ SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
if (row == NULL) {
break;
}
@@ -1920,9 +1952,9 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
while (tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter);
- STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode);
+ STable* pTable = (STable*) SL_GET_NODE_DATA((SSkipListNode*) pNode);
- STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL};
+ STableKeyInfo info = {.pTable = pTable, .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(list, &info);
}
@@ -1938,7 +1970,7 @@ static void destroyHelper(void* param) {
tQueryInfo* pInfo = (tQueryInfo*)param;
if (pInfo->optr != TSDB_RELATION_IN) {
- taosTFree(pInfo->q);
+ tfree(pInfo->q);
}
free(param);
@@ -2001,9 +2033,9 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo));
}
- STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo);
+ STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pQueryHandle->pMemRef);
- taosTFree(cond.colList);
+ tfree(cond.colList);
pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey);
if (pSecQueryHandle->pTableCheckInfo == NULL) {
@@ -2122,7 +2154,16 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) {
}
}
- // clear current group
+ // clear current group, unref unused table
+ for (int32_t i = 0; i < numOfTables; ++i) {
+ STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pGroup, i);
+
+ // keyInfo.pTable may be NULL here.
+ if (pKeyInfo->pTable != keyInfo.pTable) {
+ tsdbUnRefTable(pKeyInfo->pTable);
+ }
+ }
+
taosArrayClear(pGroup);
// more than one table in each group, only one table left for each group
@@ -2303,7 +2344,11 @@ void filterPrepare(void* expr, void* param) {
if (pInfo->optr == TSDB_RELATION_IN) {
pInfo->q = (char*) pCond->arr;
} else {
- pInfo->q = calloc(1, pSchema->bytes + TSDB_NCHAR_SIZE); // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space.
+ uint32_t size = pCond->nLen * TSDB_NCHAR_SIZE;
+ if (size < (uint32_t)pSchema->bytes) {
+ size = pSchema->bytes;
+ }
+ pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE); // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space.
tVariantDump(pCond, pInfo->q, pSchema->type, true);
}
}
@@ -2452,7 +2497,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
static bool indexedNodeFilterFp(const void* pNode, void* param) {
tQueryInfo* pInfo = (tQueryInfo*) param;
- STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
+ STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
char* val = NULL;
@@ -2705,7 +2750,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
destroyTableMemIterator(pTableCheckInfo);
- taosTFree(pTableCheckInfo->pCompInfo);
+ tfree(pTableCheckInfo->pCompInfo);
}
taosArrayDestroy(pQueryHandle->pTableCheckInfo);
}
@@ -2714,14 +2759,14 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
size_t cols = taosArrayGetSize(pQueryHandle->pColumns);
for (int32_t i = 0; i < cols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
- taosTFree(pColInfo->pData);
+ tfree(pColInfo->pData);
}
taosArrayDestroy(pQueryHandle->pColumns);
}
taosArrayDestroy(pQueryHandle->defaultLoadColumn);
- taosTFree(pQueryHandle->pDataBlockInfo);
- taosTFree(pQueryHandle->statis);
+ tfree(pQueryHandle->pDataBlockInfo);
+ tfree(pQueryHandle->statis);
// todo check error
tsdbMayUnTakeMemSnapshot(pQueryHandle);
@@ -2735,7 +2780,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %p",
pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo);
- taosTFree(pQueryHandle);
+ tfree(pQueryHandle);
}
void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) {
diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp
index 605586515b7b1626751c01a550925c6e9ac4e183..ef5ed6f04459a4213e761f94ad00363ede9ecd26 100644
--- a/src/tsdb/tests/tsdbTests.cpp
+++ b/src/tsdb/tests/tsdbTests.cpp
@@ -80,7 +80,7 @@ static int insertData(SInsertInfo *pInfo) {
pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) {
- taosTFree(pMsg);
+ tfree(pMsg);
return -1;
}
}
@@ -88,7 +88,7 @@ static int insertData(SInsertInfo *pInfo) {
double etime = getCurTime();
printf("Spent %f seconds to write %d records\n", etime - stime, pInfo->totalRows);
- taosTFree(pMsg);
+ tfree(pMsg);
return 0;
}
diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt
index 89c8e3dc39211eca1b6c877f7789ad4313917ea2..78b9c90979fa74bdd0ce5ecfd53042db30e4e8fa 100644
--- a/src/util/CMakeLists.txt
+++ b/src/util/CMakeLists.txt
@@ -1,13 +1,14 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z)
IF (TD_LINUX)
TARGET_LINK_LIBRARIES(tutil m rt)
- ADD_SUBDIRECTORY(tests)
+ # ADD_SUBDIRECTORY(tests)
FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
IF (ICONV_INCLUDE_EXIST)
diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h
index 2a58bec8830faab4cf9909bd417a44e2e8881570..42bc136584d618e62826c307b698746ca387ff41 100644
--- a/src/util/inc/hash.h
+++ b/src/util/inc/hash.h
@@ -31,14 +31,16 @@ extern "C" {
typedef void (*_hash_free_fn_t)(void *param);
typedef struct SHashNode {
- char *key;
-// struct SHashNode *prev;
+// char *key;
struct SHashNode *next;
- uint32_t hashVal; // the hash value of key, if hashVal == HASH_VALUE_IN_TRASH, this node is moved to trash
+ uint32_t hashVal; // the hash value of key
uint32_t keyLen; // length of the key
- char *data;
+// char *data;
} SHashNode;
+#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode))
+#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->keyLen)
+
typedef enum SHashLockTypeE {
HASH_NO_LOCK = 0,
HASH_ENTRY_LOCK = 1,
@@ -175,6 +177,8 @@ void* taosHashDestroyIter(SHashMutableIterator* iter);
*/
int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj);
+size_t taosHashGetMemSize(const SHashObj *pHashObj);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h
index 0520cf29a87c9d4727ef6db48d8f5712ac845b89..33819f6a20ee64ada194d520ef09c6133d4dad96 100644
--- a/src/util/inc/tconfig.h
+++ b/src/util/inc/tconfig.h
@@ -53,7 +53,7 @@ enum {
TAOS_CFG_UTYPE_NONE,
TAOS_CFG_UTYPE_PERCENT,
TAOS_CFG_UTYPE_GB,
- TAOS_CFG_UTYPE_Mb,
+ TAOS_CFG_UTYPE_MB,
TAOS_CFG_UTYPE_BYTE,
TAOS_CFG_UTYPE_SECOND,
TAOS_CFG_UTYPE_MS
diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h
new file mode 100644
index 0000000000000000000000000000000000000000..10b7c1df35caaa675c9334106785b91af3fdeb05
--- /dev/null
+++ b/src/util/inc/tfile.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_TFILE_H
+#define TDENGINE_TFILE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include
+
+// init taos file module
+int32_t tfinit();
+
+// clean up taos file module
+void tfcleanup();
+
+// the same syntax as UNIX standard open/close/read/write
+// but FD is int64_t and will never be reused
+int64_t tfopen(const char *pathname, int32_t flags);
+int64_t tfclose(int64_t tfd);
+int64_t tfwrite(int64_t tfd, void *buf, int64_t count);
+int64_t tfread(int64_t tfd, void *buf, int64_t count);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TDENGINE_TREF_H
diff --git a/src/util/inc/tnettest.h b/src/util/inc/tnettest.h
index 3fe1dfa9204fbbf85f193078b17e0bb4f9643848..426df5cbb28b9c0fcada049c7242730359d2a3fc 100644
--- a/src/util/inc/tnettest.h
+++ b/src/util/inc/tnettest.h
@@ -20,7 +20,27 @@
extern "C" {
#endif
-void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole);
+typedef struct CmdArguments {
+ char* host;
+ char* password;
+ char* user;
+ char* auth;
+ char* database;
+ char* timezone;
+ bool is_raw_time;
+ bool is_use_passwd;
+ char file[TSDB_FILENAME_LEN];
+ char dir[TSDB_FILENAME_LEN];
+ int threadNum;
+ char* commands;
+ int abort;
+ int port;
+ int endPort;
+ int pktLen;
+ char* netTestRole;
+} CmdArguments;
+
+void taosNetTest(CmdArguments* args);
#ifdef __cplusplus
}
diff --git a/src/util/inc/tqueue.h b/src/util/inc/tqueue.h
index 8493a64315966aa32b58359652d0f429e8e0916a..c3051464e556860178be36f3473f5e4686f6082e 100644
--- a/src/util/inc/tqueue.h
+++ b/src/util/inc/tqueue.h
@@ -20,6 +20,23 @@
extern "C" {
#endif
+/*
+
+This set of API for queue is designed specially for vnode/mnode. The main purpose is to
+consume all the items instead of one item from a queue by one single read. Also, it can
+combine multiple queues into a queue set, a consumer thread can consume a queue set via
+a single API instead of looping every queue by itself.
+
+Notes:
+1: taosOpenQueue/taosCloseQueue, taosOpenQset/taosCloseQset is NOT multi-thread safe
+2: after taosCloseQueue/taosCloseQset is called, read/write operation APIs are not safe.
+3: read/write operation APIs are multi-thread safe
+
+To remove the limitation and make this set of queue APIs multi-thread safe, REF(tref.c)
+shall be used to set up the protection.
+
+*/
+
typedef void* taos_queue;
typedef void* taos_qset;
typedef void* taos_qall;
diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h
new file mode 100644
index 0000000000000000000000000000000000000000..cd5092f30a290c51de49e38b0226bbed637dc0e6
--- /dev/null
+++ b/src/util/inc/tref.h
@@ -0,0 +1,75 @@
+
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_TREF_H
+#define TDENGINE_TREF_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// open a reference set, max is the mod used by hash, fp is the pointer to free resource function
+// return rsetId which will be used by other APIs. On error, -1 is returned, and terrno is set appropriately
+int taosOpenRef(int max, void (*fp)(void *));
+
+// close the reference set, refId is the return value by taosOpenRef
+// return 0 if success. On error, -1 is returned, and terrno is set appropriately
+int taosCloseRef(int refId);
+
+// add ref, p is the pointer to resource or pointer ID
+// return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately
+int64_t taosAddRef(int refId, void *p);
+
+// remove ref, rid is the reference ID returned by taosAddRef
+// return 0 if success. On error, -1 is returned, and terrno is set appropriately
+int taosRemoveRef(int rsetId, int64_t rid);
+
+// acquire ref, rid is the reference ID returned by taosAddRef
+// return the resource p. On error, NULL is returned, and terrno is set appropriately
+void *taosAcquireRef(int rsetId, int64_t rid);
+
+// release ref, rid is the reference ID returned by taosAddRef
+// return 0 if success. On error, -1 is returned, and terrno is set appropriately
+int taosReleaseRef(int rsetId, int64_t rid);
+
+// return the first reference if rid is 0, otherwise return the next after current reference.
+// if return value is NULL, it means list is over(if terrno is set, it means error happens)
+void *taosIterateRef(int rsetId, int64_t rid);
+
+// return the number of references in system
+int taosListRef();
+
+/* sample code to iterate the refs
+
+void demoIterateRefs(int rsetId) {
+
+ void *p = taosIterateRef(refId, 0);
+ while (p) {
+ // process P
+
+ // get the rid from p
+
+ p = taosIterateRef(rsetId, rid);
+ }
+}
+
+*/
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TDENGINE_TREF_H
diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h
index a14a8565617bc08eefc989168a93f523f2e6caff..65b6482520c445fd65433300a5fcb569506c85e2 100644
--- a/src/util/inc/tskiplist.h
+++ b/src/util/inc/tskiplist.h
@@ -27,33 +27,25 @@ extern "C" {
#define MAX_SKIP_LIST_LEVEL 15
#define SKIP_LIST_RECORD_PERFORMANCE 0
+// For key property setting
+#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage)
+#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case)
+#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update=1 case)
+// For thread safety setting
+#define SL_THREAD_SAFE (uint8_t)0x4
+
typedef char *SSkipListKey;
typedef char *(*__sl_key_fn_t)(const void *);
-/**
- * the skip list node is located in a consecutive memory area,
- * the format of skip list node is as follows:
- * +------------+-----------------------+------------------------+-----+------+
- * | node level | forward pointer array | backward pointer array | key | data |
- * +------------+-----------------------+------------------------+-----+------+
- */
typedef struct SSkipListNode {
- uint8_t level;
+ uint8_t level;
+ void * pData;
+ struct SSkipListNode *forwards[];
} SSkipListNode;
-#define SL_NODE_HEADER_SIZE(_l) (sizeof(SSkipListNode) + ((_l) << 1u) * POINTER_BYTES)
-
-#define SL_GET_FORWARD_POINTER(n, _l) ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode)))[(_l)]
-#define SL_GET_BACKWARD_POINTER(n, _l) \
- ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)]
-
-#define SL_GET_NODE_DATA(n) ((char *)(n) + SL_NODE_HEADER_SIZE((n)->level))
-#define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n)))
-
-#define SL_GET_SL_MIN_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_FORWARD_POINTER((s)->pHead, 0)))
-#define SL_GET_SL_MAX_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_BACKWARD_POINTER((s)->pTail, 0)))
-
-#define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n))
+#define SL_GET_NODE_DATA(n) (n)->pData
+#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)]
+#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)]
/*
* @version 0.3
@@ -103,34 +95,23 @@ typedef struct tSkipListState {
uint64_t nTotalElapsedTimeForInsert;
} tSkipListState;
-typedef struct SSkipListKeyInfo {
- uint8_t dupKey : 2; // if allow duplicated key in the skip list
- uint8_t type : 4; // key type
- uint8_t freeNode:2; // free node when destroy the skiplist
- uint8_t len; // maximum key length, used in case of string key
-} SSkipListKeyInfo;
-
typedef struct SSkipList {
__compar_fn_t comparFn;
__sl_key_fn_t keyFn;
- uint32_t size;
+ pthread_rwlock_t *lock;
+ uint16_t len;
uint8_t maxLevel;
+ uint8_t flags;
+ uint8_t type; // static info above
uint8_t level;
- SSkipListKeyInfo keyInfo;
- pthread_rwlock_t *lock;
- SSkipListNode * pHead; // point to the first element
- SSkipListNode * pTail; // point to the last element
+ uint32_t size;
+ SSkipListNode * pHead; // point to the first element
+ SSkipListNode * pTail; // point to the last element
#if SKIP_LIST_RECORD_PERFORMANCE
tSkipListState state; // skiplist state
#endif
} SSkipList;
-/*
- * iterate the skiplist
- * this will cause the multi-thread problem, when the skiplist is destroyed, the iterate may
- * continue iterating the skiplist, so add the reference count for skiplist
- * TODO add the ref for skip list when one iterator is created
- */
typedef struct SSkipListIterator {
SSkipList * pSkipList;
SSkipListNode *cur;
@@ -139,114 +120,26 @@ typedef struct SSkipListIterator {
SSkipListNode *next; // next points to the true qualified node in skip list
} SSkipListIterator;
-/**
- *
- * @param nMaxLevel maximum skip list level
- * @param keyType type of key
- * @param dupKey allow the duplicated key in the skip list
- * @return
- */
-SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe,
- uint8_t freeNode, __sl_key_fn_t fn);
-
-/**
- *
- * @param pSkipList
- * @return NULL will always be returned
- */
-void *tSkipListDestroy(SSkipList *pSkipList);
-
-/**
- *
- * @param pSkipList
- * @param level
- * @param headSize
- */
-void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize);
-
-/**
- * put the skip list node into the skip list.
- * If failed, NULL will be returned, otherwise, the pNode will be returned.
- *
- * @param pSkipList
- * @param pNode
- * @return
- */
-SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode);
-
-/**
- * get *all* nodes which key are equivalent to pKey
- *
- * @param pSkipList
- * @param pKey
- * @return
- */
-SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
-
-/**
- * get the size of skip list
- * @param pSkipList
- * @return
- */
-size_t tSkipListGetSize(const SSkipList *pSkipList);
-
-/**
- * display skip list of the given level, for debug purpose only
- * @param pSkipList
- * @param nlevel
- */
-void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
-
-/**
- * create skiplist iterator
- * @param pSkipList
- * @return
- */
+#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE)
+#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1))
+#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData))
+#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0))
+#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0))
+#define SL_SIZE(s) (s)->size
+
+SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags,
+ __sl_key_fn_t fn);
+void tSkipListDestroy(SSkipList *pSkipList);
+SSkipListNode * tSkipListPut(SSkipList *pSkipList, void *pData);
+SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
+void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList);
-
-/**
- * create skip list iterator from the given node and specified the order
- * @param pSkipList
- * @param pNode start position, instead of the first node in skip list
- * @param order traverse order of the iterator
- * @return
- */
-SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order);
-
-/**
- * forward the skip list iterator
- * @param iter
- * @return
- */
-bool tSkipListIterNext(SSkipListIterator *iter);
-
-/**
- * get the element of skip list node
- * @param iter
- * @return
- */
-SSkipListNode *tSkipListIterGet(SSkipListIterator *iter);
-
-/**
- * destroy the skip list node
- * @param iter
- * @return
- */
-void *tSkipListDestroyIter(SSkipListIterator *iter);
-
-/*
- * remove nodes of the pKey value.
- * If more than one node has the same value, all will be removed
- *
- * @Return
- * the count of removed nodes
- */
-uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key);
-
-/*
- * remove the specified node in parameters
- */
-void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
+SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order);
+bool tSkipListIterNext(SSkipListIterator *iter);
+SSkipListNode * tSkipListIterGet(SSkipListIterator *iter);
+void * tSkipListDestroyIter(SSkipListIterator *iter);
+uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key);
+void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
#ifdef __cplusplus
}
diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h
index f14e8dbb356e131bbd580f8c5310bea944e71033..391cc44accadd785fb46e02aa79cde7617834c70 100644
--- a/src/util/inc/tsocket.h
+++ b/src/util/inc/tsocket.h
@@ -20,21 +20,21 @@
extern "C" {
#endif
-int taosReadn(SOCKET sock, char *buffer, int len);
-int taosWriteMsg(SOCKET fd, void *ptr, int nbytes);
-int taosReadMsg(SOCKET fd, void *ptr, int nbytes);
-int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes);
-int taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len);
-int taosSetNonblocking(SOCKET sock, int on);
+int32_t taosReadn(SOCKET sock, char *buffer, int32_t len);
+int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes);
+int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes);
+int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes);
+int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len);
+int32_t taosSetNonblocking(SOCKET sock, int32_t on);
-SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
-SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
-SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
-int taosKeepTcpAlive(SOCKET sockFd);
+SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
+SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
+SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
+int32_t taosKeepTcpAlive(SOCKET sockFd);
-int taosGetFqdn(char *);
+int32_t taosGetFqdn(char *);
uint32_t taosGetIpFromFqdn(const char *);
-void tinet_ntoa(char *ipstr, unsigned int ip);
+void tinet_ntoa(char *ipstr, uint32_t ip);
uint32_t ip2uint(const char *const ip_addr);
#ifdef __cplusplus
diff --git a/src/util/src/hash.c b/src/util/src/hash.c
index 625d4af1ac981c5d1f5f079f5b15533a4d63ef24..03a73424971ba7e4b77d508b12f71c729889a51a 100644
--- a/src/util/src/hash.c
+++ b/src/util/src/hash.c
@@ -22,14 +22,13 @@
#define DO_FREE_HASH_NODE(_n) \
do { \
- taosTFree((_n)->data); \
- taosTFree(_n); \
+ tfree(_n); \
} while (0)
#define FREE_HASH_NODE(_h, _n) \
do { \
if ((_h)->freeFp) { \
- (_h)->freeFp((_n)->data); \
+ (_h)->freeFp(GET_HASH_NODE_DATA(_n)); \
} \
\
DO_FREE_HASH_NODE(_n); \
@@ -77,7 +76,7 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) {
SHashNode *pNode = pe->next;
while (pNode) {
- if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) {
+ if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
assert(pNode->hashVal == hashVal);
break;
}
@@ -115,11 +114,15 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p
* @param dsize size of actual data
* @return hash node
*/
-static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode *pNode, SHashNode *pNewNode) {
+static FORCE_INLINE SHashNode *doUpdateHashNode(SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) {
assert(pNode->keyLen == pNewNode->keyLen);
- SWAP(pNode->key, pNewNode->key, void *);
- SWAP(pNode->data, pNewNode->data, void *);
+ if (prev != NULL) {
+ prev->next = pNewNode;
+ } else {
+ pe->next = pNewNode;
+ }
+ pNewNode->next = pNode->next;
return pNewNode;
}
@@ -208,12 +211,14 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
assert(pNode == NULL);
}
+ SHashNode* prev = NULL;
while (pNode) {
- if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) {
+ if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
assert(pNode->hashVal == hashVal);
break;
}
+ prev = pNode;
pNode = pNode->next;
}
@@ -239,7 +244,10 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
} else {
// not support the update operation, return error
if (pHashObj->enableUpdate) {
- doUpdateHashNode(pNode, pNewNode);
+ doUpdateHashNode(pe, prev, pNode, pNewNode);
+ DO_FREE_HASH_NODE(pNode);
+ } else {
+ DO_FREE_HASH_NODE(pNewNode);
}
if (pHashObj->type == HASH_ENTRY_LOCK) {
@@ -249,7 +257,6 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
// enable resize
__rd_unlock(&pHashObj->lock, pHashObj->type);
- DO_FREE_HASH_NODE(pNewNode);
return pHashObj->enableUpdate ? 0 : -1;
}
}
@@ -293,13 +300,13 @@ void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*f
SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal);
if (pNode != NULL) {
if (fp != NULL) {
- fp(pNode->data);
+ fp(GET_HASH_NODE_DATA(pNode));
}
if (d != NULL) {
- memcpy(d, pNode->data, dsize);
+ memcpy(d, GET_HASH_NODE_DATA(pNode), dsize);
} else {
- data = pNode->data;
+ data = GET_HASH_NODE_DATA(pNode);
}
}
@@ -357,13 +364,13 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
SHashNode *pRes = NULL;
// remove it
- if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) {
+ if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
pe->num -= 1;
pRes = pNode;
pe->next = pNode->next;
} else {
while (pNode->next != NULL) {
- if (((pNode->next)->keyLen == keyLen) && (memcmp((pNode->next)->key, key, keyLen) == 0)) {
+ if (((pNode->next)->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY((pNode->next)), key, keyLen) == 0)) {
assert((pNode->next)->hashVal == hashVal);
break;
}
@@ -392,7 +399,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
__rd_unlock(&pHashObj->lock, pHashObj->type);
if (data != NULL && pRes != NULL) {
- memcpy(data, pRes->data, dsize);
+ memcpy(data, GET_HASH_NODE_DATA(pRes), dsize);
}
if (pRes != NULL) {
@@ -426,7 +433,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi
// todo remove the first node
SHashNode *pNode = NULL;
while((pNode = pEntry->next) != NULL) {
- if (fp && (!fp(param, pNode->data))) {
+ if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) {
pEntry->num -= 1;
atomic_sub_fetch_64(&pHashObj->size, 1);
@@ -451,7 +458,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi
while ((pNext = pNode->next) != NULL) {
// not qualified, remove it
- if (fp && (!fp(param, pNext->data))) {
+ if (fp && (!fp(param, GET_HASH_NODE_DATA(pNext)))) {
pNode->next = pNext->next;
pEntry->num -= 1;
atomic_sub_fetch_64(&pHashObj->size, 1);
@@ -515,7 +522,7 @@ void taosHashCleanup(SHashObj *pHashObj) {
size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock);
for (int32_t i = 0; i < memBlock; ++i) {
void *p = taosArrayGetP(pHashObj->pMemBlock, i);
- taosTFree(p);
+ tfree(p);
}
taosArrayDestroy(pHashObj->pMemBlock);
@@ -605,7 +612,7 @@ bool taosHashIterNext(SHashMutableIterator *pIter) {
}
}
-void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : iter->pCur->data; }
+void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : GET_HASH_NODE_DATA(iter->pCur); }
void *taosHashDestroyIter(SHashMutableIterator *iter) {
if (iter == NULL) {
@@ -743,21 +750,19 @@ void taosHashTableResize(SHashObj *pHashObj) {
}
SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) {
- SHashNode *pNewNode = calloc(1, sizeof(SHashNode));
+ SHashNode *pNewNode = calloc(1, sizeof(SHashNode) + keyLen + dsize);
if (pNewNode == NULL) {
uError("failed to allocate memory, reason:%s", strerror(errno));
return NULL;
}
- pNewNode->data = malloc(dsize + keyLen);
- memcpy(pNewNode->data, pData, dsize);
-
- pNewNode->key = pNewNode->data + dsize;
- memcpy(pNewNode->key, key, keyLen);
-
pNewNode->keyLen = (uint32_t)keyLen;
pNewNode->hashVal = hashVal;
+
+ memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize);
+ memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen);
+
return pNewNode;
}
@@ -798,3 +803,11 @@ SHashNode *getNextHashNode(SHashMutableIterator *pIter) {
return NULL;
}
+
+size_t taosHashGetMemSize(const SHashObj *pHashObj) {
+ if (pHashObj == NULL) {
+ return 0;
+ }
+
+ return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj);
+}
diff --git a/src/util/src/talgo.c b/src/util/src/talgo.c
index 4b96e62e91ff440684328d7e7ea8c8c6cd783caa..278683539e3247b4b6dcd43687ac281368a7d31d 100644
--- a/src/util/src/talgo.c
+++ b/src/util/src/talgo.c
@@ -153,7 +153,7 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, size_t size, const
void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ext_compar_fn_t comparFn) {
char *buf = calloc(1, size); // prepare the swap buffer
tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf);
- taosTFree(buf);
+ tfree(buf);
}
void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t compar, int flags) {
diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c
index 6e20c1708dfc81728c6b961b9259d50e953b4b9d..9bdaf737009ea25f3d76353e394f43b5c070b027 100644
--- a/src/util/src/tcache.c
+++ b/src/util/src/tcache.c
@@ -228,7 +228,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v
pCacheObj->freeFp(p->data);
}
- taosTFree(p);
+ tfree(p);
} else {
taosAddToTrashcan(pCacheObj, p);
uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p->data);
@@ -335,7 +335,7 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) {
}
void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
- if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0) {
+ if (pCacheObj == NULL) {
return;
}
@@ -343,7 +343,12 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
uError("cache:%s, NULL data to release", pCacheObj->name);
return;
}
-
+
+
+ // The operation of removal from hash table and addition to trashcan is not an atomic operation,
+ // therefore the check for the empty of both the hash table and the trashcan has a race condition.
+ // It happens when there is only one object in the cache, and two threads which has referenced this object
+ // start to free the it simultaneously [TD-1569].
size_t offset = offsetof(SCacheDataNode, data);
SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset);
@@ -609,7 +614,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) {
__cache_lock_destroy(pCacheObj);
- taosTFree(pCacheObj->name);
+ tfree(pCacheObj->name);
memset(pCacheObj, 0, sizeof(SCacheObj));
free(pCacheObj);
}
diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c
index ba711ced8ff074e639a046c5dd9908865a3e9468..1090ab810121bfbd11ae3d5c26011924a2bdc77e 100644
--- a/src/util/src/tcompare.c
+++ b/src/util/src/tcompare.c
@@ -367,7 +367,14 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) {
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_BOOL: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2));
case TSDB_DATA_TYPE_NCHAR: {
- int32_t ret = wcsncmp((wchar_t*) f1, (wchar_t*) f2, size/TSDB_NCHAR_SIZE);
+ tstr* t1 = (tstr*) f1;
+ tstr* t2 = (tstr*) f2;
+
+ if (t1->len != t2->len) {
+ return t1->len > t2->len? 1:-1;
+ }
+
+ int32_t ret = wcsncmp((wchar_t*) t1->data, (wchar_t*) t2->data, t2->len/TSDB_NCHAR_SIZE);
if (ret == 0) {
return ret;
}
diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c
index 0ec55841a060a49f4aa9e29981fa426e42d29d5c..e89dea5a244acb5823c49d4b7c9aefb4c254db4c 100644
--- a/src/util/src/tconfig.c
+++ b/src/util/src/tconfig.c
@@ -288,7 +288,7 @@ void taosReadGlobalLogCfg() {
option = value = NULL;
olen = vlen = 0;
- taosGetline(&line, &len, fp);
+ tgetline(&line, &len, fp);
line[len - 1] = 0;
paGetToken(line, &option, &olen);
@@ -302,7 +302,7 @@ void taosReadGlobalLogCfg() {
taosReadLogOption(option, value);
}
- taosTFree(line);
+ tfree(line);
fclose(fp);
}
@@ -334,7 +334,7 @@ bool taosReadGlobalCfg() {
option = value = NULL;
olen = vlen = 0;
- taosGetline(&line, &len, fp);
+ tgetline(&line, &len, fp);
line[len - 1] = 0;
paGetToken(line, &option, &olen);
@@ -354,8 +354,12 @@ bool taosReadGlobalCfg() {
fclose(fp);
- taosTFree(line);
-
+ tfree(line);
+
+ if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) {
+ taosSetAllDebugFlag();
+ }
+
return true;
}
diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c
new file mode 100644
index 0000000000000000000000000000000000000000..27ba30fe8179ecd5e213d4cf78862fd4982b3ba4
--- /dev/null
+++ b/src/util/src/tfile.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "os.h"
+#include "taoserror.h"
+#include "tulog.h"
+#include "tutil.h"
+#include "tref.h"
+
+static int32_t tsFileRsetId = -1;
+
+static void taosCloseFile(void *p) {
+ close((int32_t)(uintptr_t)p);
+}
+
+int32_t tfinit() {
+ tsFileRsetId = taosOpenRef(2000, taosCloseFile);
+ return tsFileRsetId;
+}
+
+void tfcleanup() {
+ if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
+ tsFileRsetId = -1;
+}
+
+int64_t tfopen(const char *pathname, int32_t flags) {
+ int32_t fd = open(pathname, flags);
+
+ if (fd < 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return -1;
+ }
+
+ void *p = (void *)(int64_t)fd;
+ int64_t rid = taosAddRef(tsFileRsetId, p);
+ if (rid < 0) close(fd);
+
+ return rid;
+}
+
+int64_t tfclose(int64_t tfd) {
+ return taosRemoveRef(tsFileRsetId, tfd);
+}
+
+int64_t tfwrite(int64_t tfd, void *buf, int64_t count) {
+ void *p = taosAcquireRef(tsFileRsetId, tfd);
+ if (p == NULL) return -1;
+
+ int32_t fd = (int32_t)(uintptr_t)p;
+
+ int64_t ret = taosWrite(fd, buf, count);
+ if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
+
+ taosReleaseRef(tsFileRsetId, tfd);
+ return ret;
+}
+
+int64_t tfread(int64_t tfd, void *buf, int64_t count) {
+ void *p = taosAcquireRef(tsFileRsetId, tfd);
+ if (p == NULL) return -1;
+
+ int32_t fd = (int32_t)(uintptr_t)p;
+
+ int64_t ret = taosRead(fd, buf, count);
+ if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
+
+ taosReleaseRef(tsFileRsetId, tfd);
+ return ret;
+}
diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c
index 6ba1d87d92ecf216bfde346f9d3ff1563515d34d..101dd4a6f4e726dc2f74f96f4a56db5135af5c24 100644
--- a/src/util/src/tkvstore.c
+++ b/src/util/src/tkvstore.c
@@ -14,9 +14,7 @@
*/
#define _DEFAULT_SOURCE
-
#define TAOS_RANDOM_FILE_FAIL_TEST
-
#include "os.h"
#include "hash.h"
#include "taoserror.h"
@@ -188,7 +186,7 @@ int tdKVStoreStartCommit(SKVStore *pStore) {
goto _err;
}
- if (taosTSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+ if (taosSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
uError("failed to send file %d bytes since %s", TD_KVSTORE_HEADER_SIZE, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
@@ -248,13 +246,13 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe
ASSERT(tlen == POINTER_DISTANCE(pBuf, buf));
ASSERT(tlen == sizeof(SKVRecord));
- if (taosTWrite(pStore->fd, buf, tlen) < tlen) {
+ if (taosWrite(pStore->fd, buf, tlen) < tlen) {
uError("failed to write %d bytes to file %s since %s", tlen, pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
- if (taosTWrite(pStore->fd, cont, contLen) < contLen) {
+ if (taosWrite(pStore->fd, cont, contLen) < contLen) {
uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno));
return -1;
}
@@ -292,7 +290,7 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) {
void *pBuf = buf;
tdEncodeKVRecord(&pBuf, &rInfo);
- if (taosTWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
+ if (taosWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
uError("failed to write %" PRId64 " bytes to file %s since %s", (int64_t)(POINTER_DISTANCE(pBuf, buf)), pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@@ -339,7 +337,7 @@ void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size) {
int fd = open(fname, O_RDONLY);
if (fd < 0) goto _err;
- if (taosTRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err;
+ if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err;
if (!taosCheckChecksumWhole((uint8_t *)buf, TD_KVSTORE_HEADER_SIZE)) goto _err;
void *pBuf = (void *)buf;
@@ -368,7 +366,7 @@ static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo, uint32_t
return -1;
}
- if (taosTRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+ if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
uError("failed to read %d bytes from file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@@ -402,7 +400,7 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE);
taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE);
- if (taosTWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+ if (taosWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
@@ -477,9 +475,9 @@ _err:
static void tdFreeKVStore(SKVStore *pStore) {
if (pStore) {
- taosTFree(pStore->fname);
- taosTFree(pStore->fsnap);
- taosTFree(pStore->fnew);
+ tfree(pStore->fname);
+ tfree(pStore->fsnap);
+ tfree(pStore->fnew);
taosHashCleanup(pStore->map);
free(pStore);
}
@@ -535,7 +533,7 @@ static int tdRestoreKVStore(SKVStore *pStore) {
ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE);
while (true) {
- ssize_t tsize = taosTRead(pStore->fd, tbuf, sizeof(SKVRecord));
+ int64_t tsize = taosRead(pStore->fd, tbuf, sizeof(SKVRecord));
if (tsize == 0) break;
if (tsize < sizeof(SKVRecord)) {
uError("failed to read %" PRIzu " bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname,
@@ -598,7 +596,7 @@ static int tdRestoreKVStore(SKVStore *pStore) {
goto _err;
}
- if (taosTRead(pStore->fd, buf, (size_t)pRecord->size) < pRecord->size) {
+ if (taosRead(pStore->fd, buf, (size_t)pRecord->size) < pRecord->size) {
uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname,
strerror(errno), pRecord->offset);
terrno = TAOS_SYSTEM_ERROR(errno);
@@ -618,11 +616,11 @@ static int tdRestoreKVStore(SKVStore *pStore) {
if (pStore->aFunc) (*pStore->aFunc)(pStore->appH);
taosHashDestroyIter(pIter);
- taosTFree(buf);
+ tfree(buf);
return 0;
_err:
taosHashDestroyIter(pIter);
- taosTFree(buf);
+ tfree(buf);
return -1;
}
diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c
index 09b0933fd6e32e9b65c8c7acbb81fbfe7d5c005b..4157d88c45dda82d2ecef00a3aa52f1f492314a8 100644
--- a/src/util/src/tlog.c
+++ b/src/util/src/tlog.c
@@ -336,11 +336,11 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
lseek(tsLogObj.logHandle->fd, 0, SEEK_END);
sprintf(name, "==================================================\n");
- taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+ taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
sprintf(name, " new log file \n");
- taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+ taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
sprintf(name, "==================================================\n");
- taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+ taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
return 0;
}
@@ -390,7 +390,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
if (tsAsyncLog) {
taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
} else {
- taosTWrite(tsLogObj.logHandle->fd, buffer, len);
+ taosWrite(tsLogObj.logHandle->fd, buffer, len);
}
if (tsLogObj.maxLines > 0) {
@@ -400,7 +400,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
}
}
- if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len);
+ if (dflag & DEBUG_SCREEN) taosWrite(1, buffer, (uint32_t)len);
}
void taosDumpData(unsigned char *msg, int32_t len) {
@@ -419,7 +419,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
pos += 3;
if (c >= 16) {
temp[pos++] = '\n';
- taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
+ taosWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
c = 0;
pos = 0;
}
@@ -427,9 +427,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
temp[pos++] = '\n';
- taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
-
- return;
+ taosWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
}
void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) {
@@ -467,7 +465,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
if (tsAsyncLog) {
taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
} else {
- taosTWrite(tsLogObj.logHandle->fd, buffer, len);
+ taosWrite(tsLogObj.logHandle->fd, buffer, len);
}
if (tsLogObj.maxLines > 0) {
@@ -477,7 +475,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
}
}
- if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len);
+ if (dflag & DEBUG_SCREEN) taosWrite(1, buffer, (uint32_t)len);
}
#if 0
@@ -514,8 +512,8 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) {
return tLogBuff;
_err:
- taosTFree(LOG_BUF_BUFFER(tLogBuff));
- taosTFree(tLogBuff);
+ tfree(LOG_BUF_BUFFER(tLogBuff));
+ tfree(tLogBuff);
return NULL;
}
@@ -524,7 +522,7 @@ static void taosLogBuffDestroy(SLogBuff *tLogBuff) {
tsem_destroy(&(tLogBuff->buffNotEmpty));
pthread_mutex_destroy(&(tLogBuff->buffMutex));
free(tLogBuff->buffer);
- taosTFree(tLogBuff);
+ tfree(tLogBuff);
}
#endif
@@ -606,7 +604,7 @@ static void *taosAsyncOutputLog(void *param) {
while (1) {
log_size = taosPollLogBuffer(tLogBuff, tempBuffer, TSDB_DEFAULT_LOG_BUF_UNIT);
if (log_size) {
- taosTWrite(tLogBuff->fd, tempBuffer, log_size);
+ taosWrite(tLogBuff->fd, tempBuffer, log_size);
LOG_BUF_START(tLogBuff) = (LOG_BUF_START(tLogBuff) + log_size) % LOG_BUF_SIZE(tLogBuff);
} else {
break;
diff --git a/src/util/src/tmempool.c b/src/util/src/tmempool.c
index a3d10355003c85d993b3a37190081436c0fb19ab..678c965eb1a7315977616778c0e4b39ceb4c7525 100644
--- a/src/util/src/tmempool.c
+++ b/src/util/src/tmempool.c
@@ -52,9 +52,9 @@ mpool_h taosMemPoolInit(int numOfBlock, int blockSize) {
if (pool_p->pool == NULL || pool_p->freeList == NULL) {
uError("failed to allocate memory\n");
- taosTFree(pool_p->freeList);
- taosTFree(pool_p->pool);
- taosTFree(pool_p);
+ tfree(pool_p->freeList);
+ tfree(pool_p->pool);
+ tfree(pool_p);
return NULL;
}
diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c
index 3793f3d3a944cc5c8d86c0dc0c0fa5bfe3cee764..6fd526598365f831addd1bacb0b7f748d9552fdb 100644
--- a/src/util/src/tnettest.c
+++ b/src/util/src/tnettest.c
@@ -15,11 +15,16 @@
#include "os.h"
#include "taosdef.h"
+#include "taosmsg.h"
#include "taoserror.h"
#include "tulog.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tsocket.h"
+#include "trpc.h"
+#include "rpcHead.h"
+#include "tutil.h"
+#include "tnettest.h"
#define MAX_PKG_LEN (64*1000)
#define BUFFER_SIZE (MAX_PKG_LEN + 1024)
@@ -30,9 +35,15 @@ typedef struct {
uint16_t pktLen;
} info_s;
-static char serverFqdn[TSDB_FQDN_LEN];
+extern int tsRpcMaxUdpSize;
+
+static char g_user[TSDB_USER_LEN+1] = {0};
+static char g_pass[TSDB_PASSWORD_LEN+1] = {0};
+static char g_serverFqdn[TSDB_FQDN_LEN] = {0};
static uint16_t g_startPort = 0;
static uint16_t g_endPort = 6042;
+static uint32_t g_pktLen = 0;
+
static void *bindUdpPort(void *sarg) {
info_s *pinfo = (info_s *)sarg;
@@ -321,19 +332,145 @@ static void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uin
return ;
}
-static void taosNetTestClient(const char* serverFqdn, uint16_t startPort, uint16_t endPort, int pktLen) {
- uint32_t serverIp = taosGetIpFromFqdn(serverFqdn);
- if (serverIp == 0xFFFFFFFF) {
- printf("Failed to resolve FQDN:%s", serverFqdn);
- exit(-1);
+void* tnetInitRpc(char* secretEncrypt, char spi) {
+ SRpcInit rpcInit;
+ void* pRpcConn = NULL;
+
+ taosEncryptPass((uint8_t *)g_pass, strlen(g_pass), secretEncrypt);
+
+ memset(&rpcInit, 0, sizeof(rpcInit));
+ rpcInit.localPort = 0;
+ rpcInit.label = "NET-TEST";
+ rpcInit.numOfThreads = 1; // every DB connection has only one thread
+ rpcInit.cfp = NULL;
+ rpcInit.sessions = 16;
+ rpcInit.connType = TAOS_CONN_CLIENT;
+ rpcInit.user = g_user;
+ rpcInit.idleTime = 2000;
+ rpcInit.ckey = "key";
+ rpcInit.spi = spi;
+ rpcInit.secret = secretEncrypt;
+
+ pRpcConn = rpcOpen(&rpcInit);
+ return pRpcConn;
+}
+
+static int rpcCheckPortImpl(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi) {
+ SRpcEpSet epSet;
+ SRpcMsg reqMsg;
+ SRpcMsg rspMsg;
+ void* pRpcConn;
+
+ char secretEncrypt[32] = {0};
+
+ pRpcConn = tnetInitRpc(secretEncrypt, spi);
+ if (NULL == pRpcConn) {
+ return -1;
}
- checkPort(serverIp, startPort, endPort, pktLen);
+ memset(&epSet, 0, sizeof(SRpcEpSet));
+ epSet.inUse = 0;
+ epSet.numOfEps = 1;
+ epSet.port[0] = port;
+ strcpy(epSet.fqdn[0], serverFqdn);
+
+ reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST;
+ reqMsg.pCont = rpcMallocCont(pktLen);
+ reqMsg.contLen = pktLen;
+ reqMsg.code = 0;
+ reqMsg.handle = NULL; // rpc handle returned to app
+ reqMsg.ahandle = NULL; // app handle set by client
+
+ rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg);
+
+ // handle response
+ if ((rspMsg.code != 0) || (rspMsg.msgType != TSDB_MSG_TYPE_NETWORK_TEST + 1)) {
+ //printf("code:%d[%s]\n", rspMsg.code, tstrerror(rspMsg.code));
+ return -1;
+ }
+
+ rpcFreeCont(rspMsg.pCont);
- return;
+ rpcClose(pRpcConn);
+
+ return 0;
+}
+
+static void rpcCheckPort(uint32_t hostIp) {
+ int ret;
+ char spi;
+
+ for (uint16_t port = g_startPort; port <= g_endPort; port++) {
+ //printf("test: %s:%d\n", info.host, port);
+ printf("\n");
+
+ //================ check tcp port ================
+ int32_t pktLen;
+ if (g_pktLen <= tsRpcMaxUdpSize) {
+ pktLen = tsRpcMaxUdpSize + 1000;
+ } else {
+ pktLen = g_pktLen;
+ }
+
+ spi = 1;
+ ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
+ if (ret != 0) {
+ spi = 0;
+ ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
+ if (ret != 0) {
+ printf("TCP port:%d test fail.\t\t", port);
+ } else {
+ //printf("tcp port:%d test ok.\t\t", port);
+ printf("TCP port:\033[32m%d test OK\033[0m\t\t", port);
+ }
+ } else {
+ //printf("tcp port:%d test ok.\t\t", port);
+ printf("TCP port:\033[32m%d test OK\033[0m\t\t", port);
+ }
+
+ //================ check udp port ================
+ if (g_pktLen >= tsRpcMaxUdpSize) {
+ pktLen = tsRpcMaxUdpSize - 1000;
+ } else {
+ pktLen = g_pktLen;
+ }
+
+ spi = 0;
+ ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
+ if (ret != 0) {
+ spi = 1;
+ ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
+ if (ret != 0) {
+ printf("udp port:%d test fail.\t\n", port);
+ } else {
+ //printf("udp port:%d test ok.\t\n", port);
+ printf("UDP port:\033[32m%d test OK\033[0m\t\n", port);
+ }
+ } else {
+ //printf("udp port:%d test ok.\t\n", port);
+ printf("UDP port:\033[32m%d test OK\033[0m\t\n", port);
+ }
+ }
+
+ printf("\n");
+ return ;
}
+static void taosNetTestClient(int flag) {
+ uint32_t serverIp = taosGetIpFromFqdn(g_serverFqdn);
+ if (serverIp == 0xFFFFFFFF) {
+ printf("Failed to resolve FQDN:%s", g_serverFqdn);
+ exit(-1);
+ }
+ if (0 == flag) {
+ checkPort(serverIp, g_startPort, g_endPort, g_pktLen);
+ } else {
+ rpcCheckPort(serverIp);
+ }
+
+ return;
+}
static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen) {
@@ -375,49 +512,66 @@ static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen)
}
-void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole) {
- if (pktLen > MAX_PKG_LEN) {
- printf("test packet len overflow: %d, max len not greater than %d bytes\n", pktLen, MAX_PKG_LEN);
- exit(-1);
+void taosNetTest(CmdArguments *args) {
+ if (0 == args->pktLen) {
+ g_pktLen = 1000;
+ } else {
+ g_pktLen = args->pktLen;
}
- if (port && endPort) {
- if (port > endPort) {
- printf("endPort[%d] must not lesss port[%d]\n", endPort, port);
+ if (args->port && args->endPort) {
+ if (args->port > args->endPort) {
+ printf("endPort[%d] must not lesss port[%d]\n", args->endPort, args->port);
exit(-1);
}
}
- if (host && host[0] != 0) {
- if (strlen(host) >= TSDB_EP_LEN) {
- printf("host invalid: %s\n", host);
+ if (args->host && args->host[0] != 0) {
+ if (strlen(args->host) >= TSDB_EP_LEN) {
+ printf("host invalid: %s\n", args->host);
exit(-1);
}
- taosGetFqdnPortFromEp(host, serverFqdn, &g_startPort);
+ taosGetFqdnPortFromEp(args->host, g_serverFqdn, &g_startPort);
} else {
- tstrncpy(serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN);
+ tstrncpy(g_serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN);
g_startPort = tsServerPort;
}
- if (port) {
- g_startPort = port;
+ if (args->port) {
+ g_startPort = args->port;
}
- if (endPort) {
- g_endPort = endPort;
+ if (args->endPort) {
+ g_endPort = args->endPort;
}
- if (port > endPort) {
+ if (g_startPort > g_endPort) {
printf("endPort[%d] must not lesss port[%d]\n", g_endPort, g_startPort);
exit(-1);
}
+
+
+ if (args->is_use_passwd) {
+ if (args->password == NULL) args->password = getpass("Enter password: ");
+ } else {
+ args->password = TSDB_DEFAULT_PASS;
+ }
+ tstrncpy(g_pass, args->password, TSDB_PASSWORD_LEN);
+
+ if (args->user == NULL) {
+ args->user = TSDB_DEFAULT_USER;
+ }
+ tstrncpy(g_user, args->user, TSDB_USER_LEN);
- if (0 == strcmp("client", netTestRole)) {
- printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", serverFqdn, g_startPort, g_endPort, pktLen);
- taosNetTestClient(serverFqdn, g_startPort, g_endPort, pktLen);
- } else if (0 == strcmp("server", netTestRole)) {
- taosNetTestServer(g_startPort, g_endPort, pktLen);
+ if (0 == strcmp("client", args->netTestRole)) {
+ printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen);
+ taosNetTestClient(0);
+ } else if (0 == strcmp("clients", args->netTestRole)) {
+ printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen);
+ taosNetTestClient(1);
+ } else if (0 == strcmp("server", args->netTestRole)) {
+ taosNetTestServer(g_startPort, g_endPort, g_pktLen);
}
}
diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c
index 4f05277a847a07831371278915011ebc4da400c6..9536f6fb70f9fe6d74c981274c849cab0c00ef1f 100644
--- a/src/util/src/tnote.c
+++ b/src/util/src/tnote.c
@@ -265,7 +265,7 @@ void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...)
buffer[len] = 0;
if (pNote->taosNoteFd >= 0) {
- taosTWrite(pNote->taosNoteFd, buffer, (unsigned int)len);
+ taosWrite(pNote->taosNoteFd, buffer, (unsigned int)len);
if (pNote->taosNoteMaxLines > 0) {
pNote->taosNoteLines++;
diff --git a/src/util/src/tref.c b/src/util/src/tref.c
new file mode 100644
index 0000000000000000000000000000000000000000..760d1c0eb47c8af1991d6551cf063630a24ce2ab
--- /dev/null
+++ b/src/util/src/tref.c
@@ -0,0 +1,496 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "os.h"
+#include "taoserror.h"
+#include "tulog.h"
+#include "tutil.h"
+
+#define TSDB_REF_OBJECTS 50
+#define TSDB_REF_STATE_EMPTY 0
+#define TSDB_REF_STATE_ACTIVE 1
+#define TSDB_REF_STATE_DELETED 2
+
+typedef struct SRefNode {
+ struct SRefNode *prev; // previous node
+ struct SRefNode *next; // next node
+ void *p; // pointer to resource protected,
+ int64_t rid; // reference ID
+ int32_t count; // number of references
+ int removed; // 1: removed
+} SRefNode;
+
+typedef struct {
+ SRefNode **nodeList; // array of SRefNode linked list
+ int state; // 0: empty, 1: active; 2: deleted
+ int rsetId; // refSet ID, global unique
+ int64_t rid; // increase by one for each new reference
+ int max; // mod
+ int32_t count; // total number of SRefNodes in this set
+ int64_t *lockedBy;
+ void (*fp)(void *);
+} SRefSet;
+
+static SRefSet tsRefSetList[TSDB_REF_OBJECTS];
+static pthread_once_t tsRefModuleInit = PTHREAD_ONCE_INIT;
+static pthread_mutex_t tsRefMutex;
+static int tsRefSetNum = 0;
+static int tsNextId = 0;
+
+static void taosInitRefModule(void);
+static void taosLockList(int64_t *lockedBy);
+static void taosUnlockList(int64_t *lockedBy);
+static void taosIncRsetCount(SRefSet *pSet);
+static void taosDecRsetCount(SRefSet *pSet);
+static int taosDecRefCount(int rsetId, int64_t rid, int remove);
+
+int taosOpenRef(int max, void (*fp)(void *))
+{
+ SRefNode **nodeList;
+ SRefSet *pSet;
+ int64_t *lockedBy;
+ int i, rsetId;
+
+ pthread_once(&tsRefModuleInit, taosInitRefModule);
+
+ nodeList = calloc(sizeof(SRefNode *), (size_t)max);
+ if (nodeList == NULL) {
+ terrno = TSDB_CODE_REF_NO_MEMORY;
+ return -1;
+ }
+
+ lockedBy = calloc(sizeof(int64_t), (size_t)max);
+ if (lockedBy == NULL) {
+ free(nodeList);
+ terrno = TSDB_CODE_REF_NO_MEMORY;
+ return -1;
+ }
+
+ pthread_mutex_lock(&tsRefMutex);
+
+ for (i = 0; i < TSDB_REF_OBJECTS; ++i) {
+ tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS;
+ if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId
+ if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break;
+ }
+
+ if (i < TSDB_REF_OBJECTS) {
+ rsetId = tsNextId;
+ pSet = tsRefSetList + rsetId;
+ pSet->max = max;
+ pSet->nodeList = nodeList;
+ pSet->lockedBy = lockedBy;
+ pSet->fp = fp;
+ pSet->rid = 1;
+ pSet->rsetId = rsetId;
+ pSet->state = TSDB_REF_STATE_ACTIVE;
+ taosIncRsetCount(pSet);
+
+ tsRefSetNum++;
+ uTrace("rsetId:%d is opened, max:%d, fp:%p refSetNum:%d", rsetId, max, fp, tsRefSetNum);
+ } else {
+ rsetId = TSDB_CODE_REF_FULL;
+ free (nodeList);
+ free (lockedBy);
+ uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum);
+ }
+
+ pthread_mutex_unlock(&tsRefMutex);
+
+ return rsetId;
+}
+
+int taosCloseRef(int rsetId)
+{
+ SRefSet *pSet;
+ int deleted = 0;
+
+ if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+ uTrace("rsetId:%d is invalid, out of range", rsetId);
+ terrno = TSDB_CODE_REF_INVALID_ID;
+ return -1;
+ }
+
+ pSet = tsRefSetList + rsetId;
+
+ pthread_mutex_lock(&tsRefMutex);
+
+ if (pSet->state == TSDB_REF_STATE_ACTIVE) {
+ pSet->state = TSDB_REF_STATE_DELETED;
+ deleted = 1;
+ uTrace("rsetId:%d is closed, count:%d", rsetId, pSet->count);
+ } else {
+ uTrace("rsetId:%d is already closed, count:%d", rsetId, pSet->count);
+ }
+
+ pthread_mutex_unlock(&tsRefMutex);
+
+ if (deleted) taosDecRsetCount(pSet);
+
+ return 0;
+}
+
+int64_t taosAddRef(int rsetId, void *p)
+{
+ int hash;
+ SRefNode *pNode;
+ SRefSet *pSet;
+ int64_t rid = 0;
+
+ if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+ uTrace("rsetId:%d p:%p failed to add, rsetId not valid", rsetId, p);
+ terrno = TSDB_CODE_REF_INVALID_ID;
+ return -1;
+ }
+
+ pSet = tsRefSetList + rsetId;
+ taosIncRsetCount(pSet);
+ if (pSet->state != TSDB_REF_STATE_ACTIVE) {
+ taosDecRsetCount(pSet);
+ uTrace("rsetId:%d p:%p failed to add, not active", rsetId, p);
+ terrno = TSDB_CODE_REF_ID_REMOVED;
+ return -1;
+ }
+
+ pNode = calloc(sizeof(SRefNode), 1);
+ if (pNode == NULL) {
+ terrno = TSDB_CODE_REF_NO_MEMORY;
+ return -1;
+ }
+
+ rid = atomic_add_fetch_64(&pSet->rid, 1);
+ hash = rid % pSet->max;
+ taosLockList(pSet->lockedBy+hash);
+
+ pNode->p = p;
+ pNode->rid = rid;
+ pNode->count = 1;
+
+ pNode->prev = NULL;
+ pNode->next = pSet->nodeList[hash];
+ if (pSet->nodeList[hash]) pSet->nodeList[hash]->prev = pNode;
+ pSet->nodeList[hash] = pNode;
+
+ uTrace("rsetId:%d p:%p rid:%" PRId64 " is added, count:%d", rsetId, p, rid, pSet->count);
+
+ taosUnlockList(pSet->lockedBy+hash);
+
+ return rid;
+}
+
+int taosRemoveRef(int rsetId, int64_t rid)
+{
+ return taosDecRefCount(rsetId, rid, 1);
+}
+
+// if rid is 0, return the first p in hash list, otherwise, return the next after current rid
+void *taosAcquireRef(int rsetId, int64_t rid)
+{
+ int hash;
+ SRefNode *pNode;
+ SRefSet *pSet;
+ void *p = NULL;
+
+ if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid);
+ terrno = TSDB_CODE_REF_INVALID_ID;
+ return NULL;
+ }
+
+ if (rid <= 0) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rid not valid", rsetId, rid);
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ return NULL;
+ }
+
+ pSet = tsRefSetList + rsetId;
+ taosIncRsetCount(pSet);
+ if (pSet->state != TSDB_REF_STATE_ACTIVE) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, not active", rsetId, rid);
+ taosDecRsetCount(pSet);
+ terrno = TSDB_CODE_REF_ID_REMOVED;
+ return NULL;
+ }
+
+ hash = rid % pSet->max;
+ taosLockList(pSet->lockedBy+hash);
+
+ pNode = pSet->nodeList[hash];
+
+ while (pNode) {
+ if (pNode->rid == rid) {
+ break;
+ }
+
+ pNode = pNode->next;
+ }
+
+ if (pNode) {
+ if (pNode->removed == 0) {
+ pNode->count++;
+ p = pNode->p;
+ uTrace("rsetId:%d p:%p rid:%" PRId64 " is acquired", rsetId, pNode->p, rid);
+ } else {
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ uTrace("rsetId:%d p:%p rid:%" PRId64 " is already removed, failed to acquire", rsetId, pNode->p, rid);
+ }
+ } else {
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to acquire", rsetId, rid);
+ }
+
+ taosUnlockList(pSet->lockedBy+hash);
+
+ taosDecRsetCount(pSet);
+
+ return p;
+}
+
+int taosReleaseRef(int rsetId, int64_t rid)
+{
+ return taosDecRefCount(rsetId, rid, 0);
+}
+
+// if rid is 0, return the first p in hash list, otherwise, return the next after current rid
+void *taosIterateRef(int rsetId, int64_t rid) {
+ SRefNode *pNode = NULL;
+ SRefSet *pSet;
+
+ if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rsetId not valid", rsetId, rid);
+ terrno = TSDB_CODE_REF_INVALID_ID;
+ return NULL;
+ }
+
+ if (rid <= 0) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rid not valid", rsetId, rid);
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ return NULL;
+ }
+
+ pSet = tsRefSetList + rsetId;
+ taosIncRsetCount(pSet);
+ if (pSet->state != TSDB_REF_STATE_ACTIVE) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rset not active", rsetId, rid);
+ terrno = TSDB_CODE_REF_ID_REMOVED;
+ taosDecRsetCount(pSet);
+ return NULL;
+ }
+
+ int hash = 0;
+ if (rid > 0) {
+ hash = rid % pSet->max;
+ taosLockList(pSet->lockedBy+hash);
+
+ pNode = pSet->nodeList[hash];
+ while (pNode) {
+ if (pNode->rid == rid) break;
+ pNode = pNode->next;
+ }
+
+ if (pNode == NULL) {
+ uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid);
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ taosUnlockList(pSet->lockedBy+hash);
+ return NULL;
+ }
+
+ // rid is there
+ pNode = pNode->next;
+ if (pNode == NULL) {
+ taosUnlockList(pSet->lockedBy+hash);
+ hash++;
+ }
+ }
+
+ if (pNode == NULL) {
+ for (; hash < pSet->max; ++hash) {
+ taosLockList(pSet->lockedBy+hash);
+ pNode = pSet->nodeList[hash];
+ if (pNode) break;
+ taosUnlockList(pSet->lockedBy+hash);
+ }
+ }
+
+ void *newP = NULL;
+ if (pNode) {
+ pNode->count++; // acquire it
+ newP = pNode->p;
+ taosUnlockList(pSet->lockedBy+hash);
+ uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid);
+ } else {
+ uTrace("rsetId:%d the list is over", rsetId);
+ }
+
+ if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one
+
+ taosDecRsetCount(pSet);
+
+ return newP;
+}
+
+int taosListRef() {
+ SRefSet *pSet;
+ SRefNode *pNode;
+ int num = 0;
+
+ pthread_mutex_lock(&tsRefMutex);
+
+ for (int i = 0; i < TSDB_REF_OBJECTS; ++i) {
+ pSet = tsRefSetList + i;
+
+ if (pSet->state == TSDB_REF_STATE_EMPTY)
+ continue;
+
+ uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count);
+
+ for (int j=0; j < pSet->max; ++j) {
+ pNode = pSet->nodeList[j];
+
+ while (pNode) {
+ uInfo("rsetId:%d p:%p rid:%" PRId64 "count:%d", i, pNode->p, pNode->rid, pNode->count);
+ pNode = pNode->next;
+ num++;
+ }
+ }
+ }
+
+ pthread_mutex_unlock(&tsRefMutex);
+
+ return num;
+}
+
+static int taosDecRefCount(int rsetId, int64_t rid, int remove) {
+ int hash;
+ SRefSet *pSet;
+ SRefNode *pNode;
+ int released = 0;
+ int code = 0;
+
+ if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rsetId not valid", rsetId, rid);
+ terrno = TSDB_CODE_REF_INVALID_ID;
+ return -1;
+ }
+
+ if (rid <= 0) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rid not valid", rsetId, rid);
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ return -1;
+ }
+
+ pSet = tsRefSetList + rsetId;
+ if (pSet->state == TSDB_REF_STATE_EMPTY) {
+ uTrace("rsetId:%d rid:%" PRId64 " failed to remove, cleaned", rsetId, rid);
+ terrno = TSDB_CODE_REF_ID_REMOVED;
+ return -1;
+ }
+
+ hash = rid % pSet->max;
+ taosLockList(pSet->lockedBy+hash);
+
+ pNode = pSet->nodeList[hash];
+ while (pNode) {
+ if (pNode->rid == rid)
+ break;
+
+ pNode = pNode->next;
+ }
+
+ if (pNode) {
+ pNode->count--;
+ if (remove) pNode->removed = 1;
+
+ if (pNode->count <= 0) {
+ if (pNode->prev) {
+ pNode->prev->next = pNode->next;
+ } else {
+ pSet->nodeList[hash] = pNode->next;
+ }
+
+ if (pNode->next) {
+ pNode->next->prev = pNode->prev;
+ }
+
+ (*pSet->fp)(pNode->p);
+
+ uTrace("rsetId:%d p:%p rid:%" PRId64 "is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode);
+ free(pNode);
+ released = 1;
+ } else {
+ uTrace("rsetId:%d p:%p rid:%" PRId64 "is released, count:%d", rsetId, pNode->p, rid, pNode->count);
+ }
+ } else {
+ uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid);
+ terrno = TSDB_CODE_REF_NOT_EXIST;
+ code = -1;
+ }
+
+ taosUnlockList(pSet->lockedBy+hash);
+
+ if (released) taosDecRsetCount(pSet);
+
+ return code;
+}
+
+static void taosLockList(int64_t *lockedBy) {
+ int64_t tid = taosGetPthreadId();
+ int i = 0;
+ while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) {
+ if (++i % 100 == 0) {
+ sched_yield();
+ }
+ }
+}
+
+static void taosUnlockList(int64_t *lockedBy) {
+ int64_t tid = taosGetPthreadId();
+ if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) {
+ assert(false);
+ }
+}
+
+static void taosInitRefModule(void) {
+ pthread_mutex_init(&tsRefMutex, NULL);
+}
+
+static void taosIncRsetCount(SRefSet *pSet) {
+ atomic_add_fetch_32(&pSet->count, 1);
+ // uTrace("rsetId:%d inc count:%d", pSet->rsetId, count);
+}
+
+static void taosDecRsetCount(SRefSet *pSet) {
+ int32_t count = atomic_sub_fetch_32(&pSet->count, 1);
+ // uTrace("rsetId:%d dec count:%d", pSet->rsetId, count);
+
+ if (count > 0) return;
+
+ pthread_mutex_lock(&tsRefMutex);
+
+ if (pSet->state != TSDB_REF_STATE_EMPTY) {
+ pSet->state = TSDB_REF_STATE_EMPTY;
+ pSet->max = 0;
+ pSet->fp = NULL;
+
+ tfree(pSet->nodeList);
+ tfree(pSet->lockedBy);
+
+ tsRefSetNum--;
+ uTrace("rsetId:%d is cleaned, refSetNum:%d count:%d", pSet->rsetId, tsRefSetNum, pSet->count);
+ }
+
+ pthread_mutex_unlock(&tsRefMutex);
+}
+
diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c
index bacdaef6c8da1e42218331aaa34d25aa40ab5dc4..5ecc8ce119a925df47832684f58af34b79e8d773 100644
--- a/src/util/src/tskiplist.c
+++ b/src/util/src/tskiplist.c
@@ -13,13 +13,447 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*/
+#include "tskiplist.h"
#include "os.h"
+#include "tcompare.h"
#include "tulog.h"
-#include "tskiplist.h"
#include "tutil.h"
-#include "tcompare.h"
-UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level) { // record link count in each level
+static int initForwardBackwardPtr(SSkipList *pSkipList);
+static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur);
+static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode);
+static void tSkipListCorrectLevel(SSkipList *pSkipList);
+static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order);
+static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **backward, SSkipListNode *pNode);
+static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData);
+static SSkipListNode * tSkipListNewNode(uint8_t level);
+#define tSkipListFreeNode(n) tfree((n))
+
+static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList);
+static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList);
+static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList);
+static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList);
+
+SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags,
+ __sl_key_fn_t fn) {
+ SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList));
+ if (pSkipList == NULL) return NULL;
+
+ if (maxLevel > MAX_SKIP_LIST_LEVEL) {
+ maxLevel = MAX_SKIP_LIST_LEVEL;
+ }
+
+ pSkipList->maxLevel = maxLevel;
+ pSkipList->type = keyType;
+ pSkipList->len = keyLen;
+ pSkipList->flags = flags;
+ pSkipList->keyFn = fn;
+ if (comparFn == NULL) {
+ pSkipList->comparFn = getKeyComparFunc(keyType);
+ } else {
+ pSkipList->comparFn = comparFn;
+ }
+
+ if (initForwardBackwardPtr(pSkipList) < 0) {
+ tSkipListDestroy(pSkipList);
+ return NULL;
+ }
+
+ if (SL_IS_THREAD_SAFE(pSkipList)) {
+ pSkipList->lock = (pthread_rwlock_t *)calloc(1, sizeof(pthread_rwlock_t));
+ if (pSkipList->lock == NULL) {
+ tSkipListDestroy(pSkipList);
+ return NULL;
+ }
+
+ if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) {
+ tSkipListDestroy(pSkipList);
+ return NULL;
+ }
+ }
+
+ srand((uint32_t)time(NULL));
+
+#if SKIP_LIST_RECORD_PERFORMANCE
+ pSkipList->state.nTotalMemSize += sizeof(SSkipList);
+#endif
+
+ return pSkipList;
+}
+
+void tSkipListDestroy(SSkipList *pSkipList) {
+ if (pSkipList == NULL) return;
+
+ tSkipListWLock(pSkipList);
+
+ SSkipListNode *pNode = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, 0);
+
+ while (pNode != pSkipList->pTail) {
+ SSkipListNode *pTemp = pNode;
+ pNode = SL_NODE_GET_FORWARD_POINTER(pNode, 0);
+ tSkipListFreeNode(pTemp);
+ }
+
+ tSkipListUnlock(pSkipList);
+ if (pSkipList->lock != NULL) {
+ pthread_rwlock_destroy(pSkipList->lock);
+ tfree(pSkipList->lock);
+ }
+
+ tSkipListFreeNode(pSkipList->pHead);
+ tSkipListFreeNode(pSkipList->pTail);
+ tfree(pSkipList);
+}
+
+SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) {
+ if (pSkipList == NULL || pData == NULL) return NULL;
+
+ SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0};
+ uint8_t dupMode = SL_DUP_MODE(pSkipList);
+ SSkipListNode *pNode = NULL;
+
+ tSkipListWLock(pSkipList);
+
+ bool hasDup = tSkipListGetPosToPut(pSkipList, backward, pData);
+
+ if (hasDup && (dupMode == SL_DISCARD_DUP_KEY || dupMode == SL_UPDATE_DUP_KEY)) {
+ if (dupMode == SL_UPDATE_DUP_KEY) {
+ pNode = SL_NODE_GET_BACKWARD_POINTER(backward[0], 0);
+ atomic_store_ptr(&(pNode->pData), pData);
+ }
+ } else {
+ pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList));
+ if (pNode != NULL) {
+ pNode->pData = pData;
+
+ tSkipListDoInsert(pSkipList, backward, pNode);
+ }
+ }
+
+ tSkipListUnlock(pSkipList);
+
+ return pNode;
+}
+
+uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) {
+ uint32_t count = 0;
+
+ tSkipListWLock(pSkipList);
+
+ SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL);
+ while (1) {
+ SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0);
+ if (p == pSkipList->pTail) {
+ break;
+ }
+ if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) {
+ break;
+ }
+
+ tSkipListRemoveNodeImpl(pSkipList, p);
+
+ ++count;
+ }
+
+ tSkipListCorrectLevel(pSkipList);
+
+ tSkipListUnlock(pSkipList);
+
+ return count;
+}
+
+SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey key) {
+ SArray *sa = taosArrayInit(1, POINTER_BYTES);
+
+ tSkipListRLock(pSkipList);
+
+ SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL);
+ while (1) {
+ SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0);
+ if (p == pSkipList->pTail) {
+ break;
+ }
+ if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) {
+ break;
+ }
+ taosArrayPush(sa, &p);
+ pNode = p;
+ }
+
+ tSkipListUnlock(pSkipList);
+
+ return sa;
+}
+
+void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) {
+ tSkipListWLock(pSkipList);
+ tSkipListRemoveNodeImpl(pSkipList, pNode);
+ tSkipListCorrectLevel(pSkipList);
+ tSkipListUnlock(pSkipList);
+}
+
+SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList) {
+ if (pSkipList == NULL) return NULL;
+
+ return doCreateSkipListIterator(pSkipList, TSDB_ORDER_ASC);
+}
+
+SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order) {
+ ASSERT(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);
+ ASSERT(pSkipList != NULL);
+
+ SSkipListIterator *iter = doCreateSkipListIterator(pSkipList, order);
+ if (val == NULL) {
+ return iter;
+ }
+
+ tSkipListRLock(pSkipList);
+
+ iter->cur = getPriorNode(pSkipList, val, order, &(iter->next));
+
+ tSkipListUnlock(pSkipList);
+
+ return iter;
+}
+
+bool tSkipListIterNext(SSkipListIterator *iter) {
+ if (iter->pSkipList == NULL) return false;
+
+ SSkipList *pSkipList = iter->pSkipList;
+
+ tSkipListRLock(pSkipList);
+
+ if (iter->order == TSDB_ORDER_ASC) {
+ if (iter->cur == pSkipList->pTail) return false;
+ iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0);
+
+ // a new node is inserted into between iter->cur and iter->next, ignore it
+ if (iter->cur != iter->next && (iter->next != NULL)) {
+ iter->cur = iter->next;
+ }
+
+ iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0);
+ iter->step++;
+ } else {
+ if (iter->cur == pSkipList->pHead) return false;
+ iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0);
+
+ // a new node is inserted into between iter->cur and iter->next, ignore it
+ if (iter->cur != iter->next && (iter->next != NULL)) {
+ iter->cur = iter->next;
+ }
+
+ iter->next = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0);
+ iter->step++;
+ }
+
+ tSkipListUnlock(pSkipList);
+
+ return (iter->order == TSDB_ORDER_ASC) ? (iter->cur != pSkipList->pTail) : (iter->cur != pSkipList->pHead);
+}
+
+SSkipListNode *tSkipListIterGet(SSkipListIterator *iter) {
+ if (iter == NULL || iter->cur == iter->pSkipList->pTail || iter->cur == iter->pSkipList->pHead) {
+ return NULL;
+ } else {
+ return iter->cur;
+ }
+}
+
+void *tSkipListDestroyIter(SSkipListIterator *iter) {
+ if (iter == NULL) {
+ return NULL;
+ }
+
+ tfree(iter);
+ return NULL;
+}
+
+void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) {
+ if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) {
+ return;
+ }
+
+ SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1);
+
+ int32_t id = 1;
+ char * prev = NULL;
+
+ while (p != pSkipList->pTail) {
+ char *key = SL_GET_NODE_KEY(pSkipList, p);
+ if (prev != NULL) {
+ ASSERT(pSkipList->comparFn(prev, key) < 0);
+ }
+
+ switch (pSkipList->type) {
+ case TSDB_DATA_TYPE_INT:
+ fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key);
+ break;
+ case TSDB_DATA_TYPE_SMALLINT:
+ case TSDB_DATA_TYPE_TINYINT:
+ case TSDB_DATA_TYPE_BIGINT:
+ fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key);
+ break;
+ case TSDB_DATA_TYPE_BINARY:
+ fprintf(stdout, "%d: %s \n", id++, key);
+ break;
+ case TSDB_DATA_TYPE_DOUBLE:
+ fprintf(stdout, "%d: %lf \n", id++, *(double *)key);
+ break;
+ default:
+ fprintf(stdout, "\n");
+ }
+
+ prev = SL_GET_NODE_KEY(pSkipList, p);
+
+ p = SL_NODE_GET_FORWARD_POINTER(p, nlevel - 1);
+ }
+}
+
+static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **backward, SSkipListNode *pNode) {
+ for (int32_t i = 0; i < pNode->level; ++i) {
+ if (i >= pSkipList->level) {
+ SL_NODE_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail;
+ SL_NODE_GET_BACKWARD_POINTER(pNode, i) = pSkipList->pHead;
+ SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode;
+ SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode;
+ } else {
+ SSkipListNode *x = backward[i];
+ SL_NODE_GET_FORWARD_POINTER(pNode, i) = x;
+
+ SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(x, i);
+ SL_NODE_GET_FORWARD_POINTER(prev, i) = pNode;
+
+ SL_NODE_GET_BACKWARD_POINTER(x, i) = pNode;
+ SL_NODE_GET_BACKWARD_POINTER(pNode, i) = prev;
+ }
+ }
+
+ if (pSkipList->level < pNode->level) pSkipList->level = pNode->level;
+
+ pSkipList->size += 1;
+}
+
+static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) {
+ SSkipListIterator *iter = calloc(1, sizeof(SSkipListIterator));
+
+ iter->pSkipList = pSkipList;
+ iter->order = order;
+ if (order == TSDB_ORDER_ASC) {
+ iter->cur = pSkipList->pHead;
+ iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0);
+ } else {
+ iter->cur = pSkipList->pTail;
+ iter->next = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0);
+ }
+
+ return iter;
+}
+
+static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList) {
+ if (pSkipList->lock) {
+ return pthread_rwlock_wrlock(pSkipList->lock);
+ }
+ return 0;
+}
+
+static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList) {
+ if (pSkipList->lock) {
+ return pthread_rwlock_rdlock(pSkipList->lock);
+ }
+ return 0;
+}
+
+static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) {
+ if (pSkipList->lock) {
+ return pthread_rwlock_unlock(pSkipList->lock);
+ }
+ return 0;
+}
+
+static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData) {
+ int compare = 0;
+ bool hasDupKey = false;
+ char * pDataKey = pSkipList->keyFn(pData);
+
+ if (pSkipList->size == 0) {
+ for (int i = 0; i < pSkipList->level; i++) {
+ backward[i] = pSkipList->pTail;
+ }
+ } else {
+ char *pKey = NULL;
+
+ // Compare max key
+ pKey = SL_GET_MAX_KEY(pSkipList);
+ compare = pSkipList->comparFn(pDataKey, pKey);
+ if (compare >= 0) {
+ for (int i = 0; i < pSkipList->level; i++) {
+ backward[i] = pSkipList->pTail;
+ }
+
+ return (compare == 0);
+ }
+
+ // Compare min key
+ pKey = SL_GET_MIN_KEY(pSkipList);
+ compare = pSkipList->comparFn(pDataKey, pKey);
+ if (compare < 0) {
+ for (int i = 0; i < pSkipList->level; i++) {
+ backward[i] = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i);
+ }
+
+ return (compare == 0);
+ }
+
+ SSkipListNode *px = pSkipList->pTail;
+ for (int i = pSkipList->level - 1; i >= 0; --i) {
+ SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(px, i);
+ while (p != pSkipList->pHead) {
+ pKey = SL_GET_NODE_KEY(pSkipList, p);
+
+ compare = pSkipList->comparFn(pKey, pDataKey);
+ if (compare <= 0) {
+ if (compare == 0 && !hasDupKey) hasDupKey = true;
+ break;
+ } else {
+ px = p;
+ p = SL_NODE_GET_BACKWARD_POINTER(px, i);
+ }
+ }
+
+ backward[i] = px;
+ }
+ }
+
+ return hasDupKey;
+}
+
+static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) {
+ int32_t level = pNode->level;
+ uint8_t dupMode = SL_DUP_MODE(pSkipList);
+ ASSERT(dupMode != SL_DISCARD_DUP_KEY && dupMode != SL_UPDATE_DUP_KEY);
+
+ for (int32_t j = level - 1; j >= 0; --j) {
+ SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(pNode, j);
+ SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(pNode, j);
+
+ SL_NODE_GET_FORWARD_POINTER(prev, j) = next;
+ SL_NODE_GET_BACKWARD_POINTER(next, j) = prev;
+ }
+
+ tSkipListFreeNode(pNode);
+ pSkipList->size--;
+}
+
+// Function must be called after calling tSkipListRemoveNodeImpl() function
+static void tSkipListCorrectLevel(SSkipList *pSkipList) {
+ while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) {
+ pSkipList->level -= 1;
+ }
+}
+
+UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList,
+ int32_t level) { // record link count in each level
#if SKIP_LIST_RECORD_PERFORMANCE
for (int32_t i = 0; i < level; ++i) {
pSkipList->state.nLevelNodeCnt[i]++;
@@ -47,40 +481,28 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) {
}
static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) {
- int32_t level = getSkipListNodeRandomHeight(pSkipList);
+ int32_t level = 0;
if (pSkipList->size == 0) {
level = 1;
- pSkipList->level = 1;
} else {
+ level = getSkipListNodeRandomHeight(pSkipList);
if (level > pSkipList->level) {
if (pSkipList->level < pSkipList->maxLevel) {
- level = (++pSkipList->level);
+ level = pSkipList->level + 1;
} else {
level = pSkipList->level;
}
}
}
-
- assert(level <= pSkipList->maxLevel);
+
+ ASSERT(level <= pSkipList->maxLevel);
return level;
}
-#define DO_MEMSET_PTR_AREA(n) do {\
-int32_t _l = (n)->level;\
-memset(pNode, 0, SL_NODE_HEADER_SIZE(_l));\
-(n)->level = _l;\
-} while(0)
-
-static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode);
-static SSkipListNode* tSkipListPushBack(SSkipList *pSkipList, SSkipListNode *pNode);
-static SSkipListNode* tSkipListPushFront(SSkipList* pSkipList, SSkipListNode *pNode);
-static SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order);
-
-
// when order is TSDB_ORDER_ASC, return the last node with key less than val
// when order is TSDB_ORDER_DESC, return the first node with key large than val
-static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_t order, SSkipListNode** pCur) {
- __compar_fn_t comparFn = pSkipList->comparFn;
+static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur) {
+ __compar_fn_t comparFn = pSkipList->comparFn;
SSkipListNode *pNode = NULL;
if (pCur != NULL) {
*pCur = NULL;
@@ -89,12 +511,12 @@ static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_
if (order == TSDB_ORDER_ASC) {
pNode = pSkipList->pHead;
for (int32_t i = pSkipList->level - 1; i >= 0; --i) {
- SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, i);
+ SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, i);
while (p != pSkipList->pTail) {
char *key = SL_GET_NODE_KEY(pSkipList, p);
if (comparFn(key, val) < 0) {
pNode = p;
- p = SL_GET_FORWARD_POINTER(p, i);
+ p = SL_NODE_GET_FORWARD_POINTER(p, i);
} else {
if (pCur != NULL) {
*pCur = p;
@@ -106,12 +528,12 @@ static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_
} else {
pNode = pSkipList->pTail;
for (int32_t i = pSkipList->level - 1; i >= 0; --i) {
- SSkipListNode *p = SL_GET_BACKWARD_POINTER(pNode, i);
+ SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(pNode, i);
while (p != pSkipList->pHead) {
char *key = SL_GET_NODE_KEY(pSkipList, p);
if (comparFn(key, val) > 0) {
pNode = p;
- p = SL_GET_BACKWARD_POINTER(p, i);
+ p = SL_NODE_GET_BACKWARD_POINTER(p, i);
} else {
if (pCur != NULL) {
*pCur = p;
@@ -125,215 +547,38 @@ static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_
return pNode;
}
-
-static bool initForwardBackwardPtr(SSkipList* pSkipList) {
+static int initForwardBackwardPtr(SSkipList *pSkipList) {
uint32_t maxLevel = pSkipList->maxLevel;
-
- // head info
- pSkipList->pHead = (SSkipListNode *)calloc(1, SL_NODE_HEADER_SIZE(maxLevel) * 2);
- if (pSkipList->pHead == NULL) {
- return false;
- }
-
- pSkipList->pHead->level = pSkipList->maxLevel;
-
- // tail info
- pSkipList->pTail = (SSkipListNode*) ((char*) pSkipList->pHead + SL_NODE_HEADER_SIZE(maxLevel));
- pSkipList->pTail->level = pSkipList->maxLevel;
-
- for (uint32_t i = 0; i < maxLevel; ++i) {
- SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail;
- SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead;
- }
-
- return true;
-}
-
-SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t lock,
- uint8_t freeNode, __sl_key_fn_t fn) {
- SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList));
- if (pSkipList == NULL) {
- return NULL;
- }
-
- if (maxLevel > MAX_SKIP_LIST_LEVEL) {
- maxLevel = MAX_SKIP_LIST_LEVEL;
- }
-
- pSkipList->keyInfo.type = keyType;
- pSkipList->keyInfo.len = keyLen;
- pSkipList->keyInfo.dupKey = dupKey;
- pSkipList->keyInfo.freeNode = freeNode;
-
- pSkipList->keyFn = fn;
- pSkipList->comparFn = getKeyComparFunc(keyType);
- pSkipList->maxLevel = maxLevel;
- pSkipList->level = 1;
-
- if (!initForwardBackwardPtr(pSkipList)) {
- taosTFree(pSkipList);
- return NULL;
- }
-
- if (lock) {
- pSkipList->lock = calloc(1, sizeof(pthread_rwlock_t));
-
- if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) {
- taosTFree(pSkipList->pHead);
- taosTFree(pSkipList);
-
- return NULL;
- }
- }
-
- srand((uint32_t)time(NULL));
-
-#if SKIP_LIST_RECORD_PERFORMANCE
- pSkipList->state.nTotalMemSize += sizeof(SSkipList);
-#endif
-
- return pSkipList;
-}
-
-void *tSkipListDestroy(SSkipList *pSkipList) {
- if (pSkipList == NULL) {
- return NULL;
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_wrlock(pSkipList->lock);
- }
-
- if (pSkipList->keyInfo.freeNode) {
- SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0);
- while (pNode != pSkipList->pTail) {
- SSkipListNode *pTemp = pNode;
- pNode = SL_GET_FORWARD_POINTER(pNode, 0);
- taosTFree(pTemp);
- }
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- pthread_rwlock_destroy(pSkipList->lock);
+ // head info
+ pSkipList->pHead = tSkipListNewNode(maxLevel);
+ if (pSkipList->pHead == NULL) return -1;
- taosTFree(pSkipList->lock);
+ // tail info
+ pSkipList->pTail = tSkipListNewNode(maxLevel);
+ if (pSkipList->pTail == NULL) {
+ tSkipListFreeNode(pSkipList->pHead);
+ return -1;
}
- taosTFree(pSkipList->pHead);
- taosTFree(pSkipList);
- return NULL;
-}
-
-void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize) {
- if (pSkipList == NULL) {
- *level = 1;
- *headSize = SL_NODE_HEADER_SIZE(*level);
- return;
+ for (uint32_t i = 0; i < maxLevel; ++i) {
+ SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail;
+ SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead;
}
- *level = getSkipListRandLevel(pSkipList);
- *headSize = SL_NODE_HEADER_SIZE(*level);
+ return 0;
}
-SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) {
- if (pSkipList == NULL || pNode == NULL) {
- return NULL;
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_wrlock(pSkipList->lock);
- }
-
- // if the new key is greater than the maximum key of skip list, push back this node at the end of skip list
- char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode);
- if (pSkipList->size == 0 || pSkipList->comparFn(SL_GET_SL_MAX_KEY(pSkipList), newDatakey) < 0) {
- return tSkipListPushBack(pSkipList, pNode);
- }
-
- // if the new key is less than the minimum key of skip list, push front this node at the front of skip list
- assert(pSkipList->size > 0);
- char* minKey = SL_GET_SL_MIN_KEY(pSkipList);
- if (pSkipList->comparFn(newDatakey, minKey) < 0) {
- return tSkipListPushFront(pSkipList, pNode);
- }
-
- // find the appropriated position to insert data
- SSkipListNode *px = pSkipList->pHead;
- SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0};
-
- int32_t ret = -1;
- for (int32_t i = pSkipList->level - 1; i >= 0; --i) {
- SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i);
- while (p != pSkipList->pTail) {
- char *key = SL_GET_NODE_KEY(pSkipList, p);
-
- // if the forward element is less than the specified key, forward one step
- ret = pSkipList->comparFn(key, newDatakey);
- if (ret < 0) {
- px = p;
- p = SL_GET_FORWARD_POINTER(px, i);
- } else {
- break;
- }
- }
-
- forward[i] = px;
- }
+static SSkipListNode *tSkipListNewNode(uint8_t level) {
+ int32_t tsize = sizeof(SSkipListNode) + sizeof(SSkipListNode *) * level * 2;
- // if the skip list does not allowed identical key inserted, the new data will be discarded.
- if (pSkipList->keyInfo.dupKey == 0 && ret == 0) {
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
+ SSkipListNode *pNode = (SSkipListNode *)calloc(1, tsize);
+ if (pNode == NULL) return NULL;
- return NULL;
- }
-
- tSkipListDoInsert(pSkipList, forward, pNode);
+ pNode->level = level;
return pNode;
}
-
-
-SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey key) {
- SArray* sa = taosArrayInit(1, POINTER_BYTES);
-
- if (pSkipList->lock) {
- pthread_rwlock_wrlock(pSkipList->lock);
- }
-
- SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL);
- while (1) {
- SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0);
- if (p == pSkipList->pTail) {
- break;
- }
- if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) {
- break;
- }
- taosArrayPush(sa, &p);
- pNode = p;
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-
- return sa;
-}
-
-
-
-size_t tSkipListGetSize(const SSkipList* pSkipList) {
- if (pSkipList == NULL) {
- return 0;
- }
-
- return pSkipList->size;
-}
-
// static int32_t tSkipListEndParQuery(SSkipList *pSkipList, SSkipListNode *pStartNode, SSkipListKey *pEndKey,
// int32_t cond, SSkipListNode ***pRes) {
// pthread_rwlock_rdlock(&pSkipList->lock);
@@ -447,178 +692,13 @@ size_t tSkipListGetSize(const SSkipList* pSkipList) {
// }
//
// // compress the minimum level of skip list
-// while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) {
+// while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) {
// pSkipList->level -= 1;
// }
//
// return true;
//}
-
-uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) {
- uint32_t count = 0;
-
- if (pSkipList->lock) {
- pthread_rwlock_wrlock(pSkipList->lock);
- }
-
- SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL);
- while (1) {
- SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0);
- if (p == pSkipList->pTail) {
- break;
- }
- if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) {
- break;
- }
-
- for (int32_t j = p->level - 1; j >= 0; --j) {
- SSkipListNode* prev = SL_GET_BACKWARD_POINTER(p, j);
- SSkipListNode* next = SL_GET_FORWARD_POINTER(p, j);
- SL_GET_FORWARD_POINTER(prev, j) = next;
- SL_GET_BACKWARD_POINTER(next, j) = prev;
- }
-
- if (pSkipList->keyInfo.freeNode) {
- taosTFree(p);
- }
-
- ++count;
- }
-
- // compress the minimum level of skip list
- while (pSkipList->level > 0) {
- if (SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) != NULL) {
- break;
- }
- pSkipList->level--;
- }
-
- pSkipList->size -= count;
-
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-
- return count;
-}
-
-void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) {
- int32_t level = pNode->level;
-
- if (pSkipList->lock) {
- pthread_rwlock_wrlock(pSkipList->lock);
- }
-
- for (int32_t j = level - 1; j >= 0; --j) {
- SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pNode, j);
- SSkipListNode* next = SL_GET_FORWARD_POINTER(pNode, j);
-
- SL_GET_FORWARD_POINTER(prev, j) = next;
- SL_GET_BACKWARD_POINTER(next, j) = prev;
- }
-
- if (pSkipList->keyInfo.freeNode) {
- taosTFree(pNode);
- }
-
- atomic_sub_fetch_32(&pSkipList->size, 1);
-
- // compress the minimum level of skip list
- while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) {
- pSkipList->level -= 1;
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-}
-
-SSkipListIterator* tSkipListCreateIter(SSkipList *pSkipList) {
- if (pSkipList == NULL) {
- return NULL;
- }
-
- return doCreateSkipListIterator(pSkipList, TSDB_ORDER_ASC);
-}
-
-SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order) {
- assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);
- assert(pSkipList != NULL);
-
- SSkipListIterator* iter = doCreateSkipListIterator(pSkipList, order);
- if (val == NULL) {
- return iter;
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_rdlock(pSkipList->lock);
- }
-
- iter->cur = getPriorNode(pSkipList, val, order, &iter->next);
-
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-
- return iter;
-}
-
-bool tSkipListIterNext(SSkipListIterator *iter) {
- if (iter->pSkipList == NULL) {
- return false;
- }
-
- SSkipList *pSkipList = iter->pSkipList;
-
- if (pSkipList->lock) {
- pthread_rwlock_rdlock(pSkipList->lock);
- }
-
- if (iter->order == TSDB_ORDER_ASC) { // ascending order iterate
- iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0);
-
- // a new node is inserted into between iter->cur and iter->next, ignore it
- if (iter->cur != iter->next && (iter->next != NULL)) {
- iter->cur = iter->next;
- }
-
- iter->next = SL_GET_FORWARD_POINTER(iter->cur, 0);
- } else { // descending order iterate
- iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0);
-
- // a new node is inserted into between iter->cur and iter->next, ignore it
- if (iter->cur != iter->next && (iter->next != NULL)) {
- iter->cur = iter->next;
- }
-
- iter->next = SL_GET_BACKWARD_POINTER(iter->cur, 0);
- }
-
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-
- iter->step += 1;
- return (iter->order == TSDB_ORDER_ASC)? (iter->cur != pSkipList->pTail) : (iter->cur != pSkipList->pHead);
-}
-
-SSkipListNode *tSkipListIterGet(SSkipListIterator *iter) {
- if (iter == NULL || iter->cur == iter->pSkipList->pTail || iter->cur == iter->pSkipList->pHead) {
- return NULL;
- } else {
- return iter->cur;
- }
-}
-
-void* tSkipListDestroyIter(SSkipListIterator* iter) {
- if (iter == NULL) {
- return NULL;
- }
-
- taosTFree(iter);
- return NULL;
-}
-
+//
// bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey) {
// SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0};
// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType);
@@ -638,111 +718,3 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) {
//
// return ret;
//}
-
-void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) {
- if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) {
- return;
- }
-
- SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1);
-
- int32_t id = 1;
- char* prev = NULL;
-
- while (p != pSkipList->pTail) {
- char *key = SL_GET_NODE_KEY(pSkipList, p);
- if (prev != NULL) {
- assert(pSkipList->comparFn(prev, key) < 0);
- }
-
- switch (pSkipList->keyInfo.type) {
- case TSDB_DATA_TYPE_INT:
- fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key);
- break;
- case TSDB_DATA_TYPE_SMALLINT:
- case TSDB_DATA_TYPE_TINYINT:
- case TSDB_DATA_TYPE_BIGINT:
- fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key);
- break;
- case TSDB_DATA_TYPE_BINARY:
- fprintf(stdout, "%d: %s \n", id++, key);
- break;
- case TSDB_DATA_TYPE_DOUBLE:
- fprintf(stdout, "%d: %lf \n", id++, *(double *)key);
- break;
- default:
- fprintf(stdout, "\n");
- }
-
- prev = SL_GET_NODE_KEY(pSkipList, p);
-
- p = SL_GET_FORWARD_POINTER(p, nlevel - 1);
- }
-}
-
-void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) {
- DO_MEMSET_PTR_AREA(pNode);
-
- for (int32_t i = 0; i < pNode->level; ++i) {
- SSkipListNode *x = forward[i];
- SL_GET_BACKWARD_POINTER(pNode, i) = x;
-
- SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i);
- SL_GET_BACKWARD_POINTER(next, i) = pNode;
-
- SL_GET_FORWARD_POINTER(pNode, i) = next;
- SL_GET_FORWARD_POINTER(x, i) = pNode;
- }
-
- atomic_add_fetch_32(&pSkipList->size, 1);
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-}
-
-SSkipListNode* tSkipListPushFront(SSkipList* pSkipList, SSkipListNode *pNode) {
- SSkipListNode* forward[MAX_SKIP_LIST_LEVEL] = {0};
- for(int32_t i = 0; i < pSkipList->level; ++i) {
- forward[i] = pSkipList->pHead;
- }
-
- tSkipListDoInsert(pSkipList, forward, pNode);
- return pNode;
-}
-
-SSkipListNode* tSkipListPushBack(SSkipList *pSkipList, SSkipListNode *pNode) {
- // do clear pointer area
- DO_MEMSET_PTR_AREA(pNode);
-
- for(int32_t i = 0; i < pNode->level; ++i) {
- SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pSkipList->pTail, i);
- SL_GET_FORWARD_POINTER(prev, i) = pNode;
- SL_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail;
-
- SL_GET_BACKWARD_POINTER(pNode, i) = prev;
- SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode;
- }
-
- atomic_add_fetch_32(&pSkipList->size, 1);
- if (pSkipList->lock) {
- pthread_rwlock_unlock(pSkipList->lock);
- }
-
- return pNode;
-}
-
-SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) {
- SSkipListIterator* iter = calloc(1, sizeof(SSkipListIterator));
-
- iter->pSkipList = pSkipList;
- iter->order = order;
- if(order == TSDB_ORDER_ASC) {
- iter->cur = pSkipList->pHead;
- iter->next = SL_GET_FORWARD_POINTER(iter->cur, 0);
- } else {
- iter->cur = pSkipList->pTail;
- iter->next = SL_GET_BACKWARD_POINTER(iter->cur, 0);
- }
-
- return iter;
-}
\ No newline at end of file
diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c
index 4cf73e6dff8aa534e87145a3e1cb04e48d759c65..1003bc6178754e2eb6a3f8dbc61d0a878022e8f5 100644
--- a/src/util/src/tsocket.c
+++ b/src/util/src/tsocket.c
@@ -18,7 +18,7 @@
#include "tsocket.h"
#include "taoserror.h"
-int taosGetFqdn(char *fqdn) {
+int32_t taosGetFqdn(char *fqdn) {
char hostname[1024];
hostname[1023] = '\0';
if (gethostname(hostname, 1023) == -1) {
@@ -26,10 +26,10 @@ int taosGetFqdn(char *fqdn) {
return -1;
}
- struct addrinfo hints = {0};
+ struct addrinfo hints = {0};
struct addrinfo *result = NULL;
hints.ai_flags = AI_CANONNAME;
- int ret = getaddrinfo(hostname, NULL, &hints, &result);
+ int32_t ret = getaddrinfo(hostname, NULL, &hints, &result);
if (!result) {
uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
return -1;
@@ -49,10 +49,10 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) {
int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result);
if (result) {
- struct sockaddr *sa = result->ai_addr;
- struct sockaddr_in *si = (struct sockaddr_in*)sa;
- struct in_addr ia = si->sin_addr;
- uint32_t ip = ia.s_addr;
+ struct sockaddr * sa = result->ai_addr;
+ struct sockaddr_in *si = (struct sockaddr_in *)sa;
+ struct in_addr ia = si->sin_addr;
+ uint32_t ip = ia.s_addr;
freeaddrinfo(result);
return ip;
} else {
@@ -70,7 +70,7 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) {
}
}
-// Function converting an IP address string to an unsigned int.
+// Function converting an IP address string to an uint32_t.
uint32_t ip2uint(const char *const ip_addr) {
char ip_addr_cpy[20];
char ip[5];
@@ -81,7 +81,7 @@ uint32_t ip2uint(const char *const ip_addr) {
s_start = ip_addr_cpy;
s_end = ip_addr_cpy;
- int k;
+ int32_t k;
for (k = 0; *s_start != '\0'; s_start = s_end) {
for (s_end = s_start; *s_end != '.' && *s_end != '\0'; s_end++) {
@@ -95,17 +95,17 @@ uint32_t ip2uint(const char *const ip_addr) {
ip[k] = '\0';
- return *((unsigned int *)ip);
+ return *((uint32_t *)ip);
}
-int taosWriteMsg(SOCKET fd, void *buf, int nbytes) {
- int nleft, nwritten;
- char *ptr = (char *)buf;
+int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) {
+ int32_t nleft, nwritten;
+ char * ptr = (char *)buf;
nleft = nbytes;
while (nleft > 0) {
- nwritten = (int)taosWriteSocket(fd, (char *)ptr, (size_t)nleft);
+ nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft);
if (nwritten <= 0) {
if (errno == EINTR)
continue;
@@ -120,16 +120,16 @@ int taosWriteMsg(SOCKET fd, void *buf, int nbytes) {
return (nbytes - nleft);
}
-int taosReadMsg(SOCKET fd, void *buf, int nbytes) {
- int nleft, nread;
- char *ptr = (char *)buf;
+int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) {
+ int32_t nleft, nread;
+ char * ptr = (char *)buf;
nleft = nbytes;
if (fd < 0) return -1;
while (nleft > 0) {
- nread = (int)taosReadSocket(fd, ptr, (size_t)nleft);
+ nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft);
if (nread == 0) {
break;
} else if (nread < 0) {
@@ -147,11 +147,11 @@ int taosReadMsg(SOCKET fd, void *buf, int nbytes) {
return (nbytes - nleft);
}
-int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) {
+int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) {
taosSetNonblocking(fd, 1);
- int nleft, nwritten, nready;
- fd_set fset;
+ int32_t nleft, nwritten, nready;
+ fd_set fset;
struct timeval tv;
nleft = nbytes;
@@ -160,7 +160,7 @@ int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) {
tv.tv_usec = 0;
FD_ZERO(&fset);
FD_SET(fd, &fset);
- if ((nready = select((int)(fd + 1), NULL, &fset, NULL, &tv)) == 0) {
+ if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) {
errno = ETIMEDOUT;
uError("fd %d timeout, no enough space to write", fd);
break;
@@ -172,7 +172,7 @@ int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) {
return -1;
}
- nwritten = (int)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL);
+ nwritten = (int32_t)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL);
if (nwritten <= 0) {
if (errno == EAGAIN || errno == EINTR) continue;
@@ -189,10 +189,10 @@ int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) {
return (nbytes - nleft);
}
-int taosReadn(SOCKET fd, char *ptr, int nbytes) {
- int nread, nready, nleft = nbytes;
+int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) {
+ int32_t nread, nready, nleft = nbytes;
- fd_set fset;
+ fd_set fset;
struct timeval tv;
while (nleft > 0) {
@@ -200,7 +200,7 @@ int taosReadn(SOCKET fd, char *ptr, int nbytes) {
tv.tv_usec = 0;
FD_ZERO(&fset);
FD_SET(fd, &fset);
- if ((nready = select((int)(fd + 1), NULL, &fset, NULL, &tv)) == 0) {
+ if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) {
errno = ETIMEDOUT;
uError("fd %d timeout\n", fd);
break;
@@ -210,7 +210,7 @@ int taosReadn(SOCKET fd, char *ptr, int nbytes) {
return -1;
}
- if ((nread = (int)taosReadSocket(fd, ptr, (size_t)nleft)) < 0) {
+ if ((nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft)) < 0) {
if (errno == EINTR) continue;
uError("read error, %d (%s)", errno, strerror(errno));
return -1;
@@ -229,8 +229,8 @@ int taosReadn(SOCKET fd, char *ptr, int nbytes) {
SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) {
struct sockaddr_in localAddr;
- SOCKET sockFd;
- int bufSize = 1024000;
+ SOCKET sockFd;
+ int32_t bufSize = 1024000;
uDebug("open udp socket:0x%x:%hu", ip, port);
@@ -239,7 +239,7 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) {
localAddr.sin_addr.s_addr = ip;
localAddr.sin_port = (uint16_t)htons(port);
- if ((sockFd = (int)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) {
+ if ((sockFd = (int32_t)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) {
uError("failed to open udp socket: %d (%s)", errno, strerror(errno));
taosCloseSocketNoCheck(sockFd);
return -1;
@@ -268,9 +268,9 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) {
}
SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) {
- SOCKET sockFd = 0;
+ SOCKET sockFd = 0;
+ int32_t ret;
struct sockaddr_in serverAddr, clientAddr;
- int ret;
sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
@@ -281,7 +281,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie
}
/* set REUSEADDR option, so the portnumber can be re-used */
- int reuse = 1;
+ int32_t reuse = 1;
if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) {
uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno));
taosCloseSocket(sockFd);
@@ -296,8 +296,8 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie
/* bind socket to client address */
if (bind(sockFd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) {
- uError("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)",
- clientIp, destIp, destPort, strerror(errno));
+ uError("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort,
+ strerror(errno));
taosCloseSocket(sockFd);
return -1;
}
@@ -311,7 +311,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie
ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr));
if (ret != 0) {
- //uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno));
+ // uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno));
taosCloseSocket(sockFd);
sockFd = -1;
} else {
@@ -321,36 +321,36 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie
return sockFd;
}
-int taosKeepTcpAlive(SOCKET sockFd) {
- int alive = 1;
+int32_t taosKeepTcpAlive(SOCKET sockFd) {
+ int32_t alive = 1;
if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) {
uError("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno));
taosCloseSocket(sockFd);
return -1;
}
- int probes = 3;
+ int32_t probes = 3;
if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) {
uError("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno));
taosCloseSocket(sockFd);
return -1;
}
- int alivetime = 10;
+ int32_t alivetime = 10;
if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) {
uError("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno));
taosCloseSocket(sockFd);
return -1;
}
- int interval = 3;
+ int32_t interval = 3;
if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) {
uError("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno));
taosCloseSocket(sockFd);
return -1;
}
- int nodelay = 1;
+ int32_t nodelay = 1;
if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) {
uError("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno));
taosCloseSocket(sockFd);
@@ -371,8 +371,8 @@ int taosKeepTcpAlive(SOCKET sockFd) {
SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
struct sockaddr_in serverAdd;
- SOCKET sockFd;
- int reuse;
+ SOCKET sockFd;
+ int32_t reuse;
uDebug("open tcp server socket:0x%x:%hu", ip, port);
@@ -381,7 +381,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
serverAdd.sin_addr.s_addr = ip;
serverAdd.sin_port = (uint16_t)htons(port);
- if ((sockFd = (int)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) {
+ if ((sockFd = (int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) {
uError("failed to open TCP socket: %d (%s)", errno, strerror(errno));
taosCloseSocketNoCheck(sockFd);
return -1;
@@ -417,38 +417,38 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
return sockFd;
}
-void tinet_ntoa(char *ipstr, unsigned int ip) {
+void tinet_ntoa(char *ipstr, uint32_t ip) {
sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24);
}
#define COPY_SIZE 32768
// sendfile shall be used
-int taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len) {
+int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len) {
int64_t leftLen;
- int readLen, writeLen;
+ int32_t readLen, writeLen;
char temp[COPY_SIZE];
leftLen = len;
while (leftLen > 0) {
if (leftLen < COPY_SIZE)
- readLen = (int)leftLen;
+ readLen = (int32_t)leftLen;
else
readLen = COPY_SIZE; // 4K
- int retLen = taosReadMsg(sfd, temp, (int)readLen);
+ int32_t retLen = taosReadMsg(sfd, temp, (int32_t)readLen);
if (readLen != retLen) {
- uError("read error, readLen:%d retLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, retLen, len, leftLen,
- strerror(errno));
+ uError("read error, readLen:%d retLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, retLen, len,
+ leftLen, strerror(errno));
return -1;
}
writeLen = taosWriteMsg(dfd, temp, readLen);
if (readLen != writeLen) {
- uError("copy error, readLen:%d writeLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, writeLen, len, leftLen,
- strerror(errno));
+ uError("copy error, readLen:%d writeLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, writeLen,
+ len, leftLen, strerror(errno));
return -1;
}
diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c
index 6c4af437b27b6623aeebb404e0dad1c70241cf47..451976f563740b4ac933766d5c0c8f4075ad42d8 100644
--- a/src/util/src/tutil.c
+++ b/src/util/src/tutil.c
@@ -326,6 +326,7 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) {
return 0;
}
+// TODO move to comm module
bool taosGetVersionNumber(char *versionStr, int *versionNubmer) {
if (versionStr == NULL || versionNubmer == NULL) {
return false;
@@ -376,7 +377,8 @@ int taosCheckVersion(char *input_client_version, char *input_server_version, int
for(int32_t i = 0; i < comparedSegments; ++i) {
if (clientVersionNumber[i] != serverVersionNumber[i]) {
- uError("the %d-th number of server version:%s not matched with client version:%s", i, server_version, version);
+ uError("the %d-th number of server version:%s not matched with client version:%s", i, server_version,
+ client_version);
return TSDB_CODE_TSC_INVALID_VERSION;
}
}
diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt
index 8687a8005ddeda7320c60c9ef90dd221f56b971f..0c96ed2a2f3dfb7f03268c9f8fbb1b0afa2397b9 100644
--- a/src/util/tests/CMakeLists.txt
+++ b/src/util/tests/CMakeLists.txt
@@ -9,7 +9,22 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
-
+
+ LIST(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c)
ADD_EXECUTABLE(utilTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(utilTest tutil common osdetail gtest pthread gcov)
+
+ LIST(APPEND BIN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c)
+ ADD_EXECUTABLE(trefTest ${BIN_SRC})
+ TARGET_LINK_LIBRARIES(trefTest common tutil)
+
ENDIF()
+
+#IF (TD_LINUX)
+# ADD_EXECUTABLE(trefTest ./trefTest.c)
+# TARGET_LINK_LIBRARIES(trefTest tutil common)
+#ENDIF ()
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+
+
diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp
index 77174f69fda1a49975c44717d8993996274ca4c9..2203ae8e4fc75c2aa0267055f79884f05e3a65e9 100644
--- a/src/util/tests/skiplistTest.cpp
+++ b/src/util/tests/skiplistTest.cpp
@@ -59,7 +59,7 @@ void doubleSkipListTest() {
}
if (size > 0) {
- taosTFree(pNodes);
+ tfree(pNodes);
}
}
@@ -196,7 +196,7 @@ void stringKeySkiplistTest() {
tSkipListRemoveNode(pSkipList, pres[0]);
if (num > 0) {
- taosTFree(pres);
+ tfree(pres);
}
}
@@ -247,7 +247,7 @@ void skiplistPerformanceTest() {
printf("total:%" PRIu64 " ms, avg:%f\n", e - s, (e - s) / (double)size);
printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level);
- assert(tSkipListGetSize(pSkipList) == size);
+ assert(SL_GET_SIZE(pSkipList) == size);
// printf("the level of skiplist is:\n");
//
@@ -273,10 +273,10 @@ void skiplistPerformanceTest() {
int64_t et = taosGetTimestampMs();
printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st);
- assert(tSkipListGetSize(pSkipList) == size);
+ assert(SL_GET_SIZE(pSkipList) == size);
tSkipListDestroy(pSkipList);
- taosTFree(total);
+ tfree(total);
}
// todo not support duplicated key yet
@@ -357,7 +357,7 @@ TEST(testCase, skiplist_test) {
printf("-----%lf\n", pNodes[i]->key.dKey);
}
printf("the range query result size is: %d\n", size);
- taosTFree(pNodes);
+ tfree(pNodes);
SSkipListKey *pKeys = malloc(sizeof(SSkipListKey) * 20);
for (int32_t i = 0; i < 8; i += 2) {
@@ -371,7 +371,7 @@ TEST(testCase, skiplist_test) {
for (int32_t i = 0; i < r; ++i) {
// printf("%lf ", pNodes[i]->key.dKey);
}
- taosTFree(pNodes);
+ tfree(pNodes);
free(pKeys);*/
}
diff --git a/src/util/tests/trefTest.c b/src/util/tests/trefTest.c
new file mode 100644
index 0000000000000000000000000000000000000000..6887b24abdae9f5d9949fee0073a35eb717eb4f7
--- /dev/null
+++ b/src/util/tests/trefTest.c
@@ -0,0 +1,195 @@
+#include
+#include
+#include
+#include
+#include
+#include "os.h"
+#include "tref.h"
+#include "tlog.h"
+#include "tglobal.h"
+#include "taoserror.h"
+#include "tulog.h"
+
+typedef struct {
+ int refNum;
+ int steps;
+ int rsetId;
+ int64_t rid;
+ void **p;
+} SRefSpace;
+
+void iterateRefs(int rsetId) {
+ int count = 0;
+
+ void *p = taosIterateRef(rsetId, NULL);
+ while (p) {
+ // process P
+ count++;
+ p = taosIterateRef(rsetId, p);
+ }
+
+ printf(" %d ", count);
+}
+
+void *addRef(void *param) {
+ SRefSpace *pSpace = (SRefSpace *)param;
+ int id;
+ int64_t rid;
+
+ for (int i=0; i < pSpace->steps; ++i) {
+ printf("a");
+ id = random() % pSpace->refNum;
+ if (pSpace->rid[id] <= 0) {
+ pSpace->p[id] = malloc(128);
+ pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]);
+ }
+ usleep(100);
+ }
+
+ return NULL;
+}
+
+void *removeRef(void *param) {
+ SRefSpace *pSpace = (SRefSpace *)param;
+ int id;
+ int64_t rid;
+
+ for (int i=0; i < pSpace->steps; ++i) {
+ printf("d");
+ id = random() % pSpace->refNum;
+ if (pSpace->rid[id] > 0) {
+ code = taosRemoveRef(pSpace->rsetId, pSpace->rid[id]);
+ if (code == 0) pSpace->rid[id] = 0;
+ }
+
+ usleep(100);
+ }
+
+ return NULL;
+}
+
+void *acquireRelease(void *param) {
+ SRefSpace *pSpace = (SRefSpace *)param;
+ int id;
+ int64_t rid;
+
+ for (int i=0; i < pSpace->steps; ++i) {
+ printf("a");
+
+ id = random() % pSpace->refNum;
+ code = taosAcquireRef(pSpace->rsetId, pSpace->p[id]);
+ if (code >= 0) {
+ usleep(id % 5 + 1);
+ taosReleaseRef(pSpace->rsetId, pSpace->p[id]);
+ }
+ }
+
+ return NULL;
+}
+
+void myfree(void *p) {
+ free(p);
+}
+
+void *openRefSpace(void *param) {
+ SRefSpace *pSpace = (SRefSpace *)param;
+
+ printf("c");
+ pSpace->rsetId = taosOpenRef(50, myfree);
+
+ if (pSpace->rsetId < 0) {
+ printf("failed to open ref, reson:%s\n", tstrerror(pSpace->rsetId));
+ return NULL;
+ }
+
+ pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum);
+
+ pthread_attr_t thattr;
+ pthread_attr_init(&thattr);
+ pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+
+ pthread_t thread1, thread2, thread3;
+ pthread_create(&(thread1), &thattr, addRef, (void *)(pSpace));
+ pthread_create(&(thread2), &thattr, removeRef, (void *)(pSpace));
+ pthread_create(&(thread3), &thattr, acquireRelease, (void *)(pSpace));
+
+ pthread_join(thread1, NULL);
+ pthread_join(thread2, NULL);
+ pthread_join(thread3, NULL);
+
+ for (int i=0; irefNum; ++i) {
+ taosRemoveRef(pSpace->rsetId, pSpace->rid[i]);
+ }
+
+ taosCloseRef(pSpace->rsetId);
+
+ uInfo("rsetId:%d main thread exit", pSpace->rsetId);
+ free(pSpace->p);
+ pSpace->p = NULL;
+
+ return NULL;
+}
+
+int main(int argc, char *argv[]) {
+ int refNum = 100;
+ int threads = 10;
+ int steps = 10000;
+ int loops = 1;
+
+ uDebugFlag = 143;
+
+ for (int i=1; i
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_VNODE_CFG_H
+#define TDENGINE_VNODE_CFG_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t vnodeReadCfg(SVnodeObj *pVnode);
+int32_t vnodeWriteCfg(SCreateVnodeMsg *pVnodeCfg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h
index 169334c6119a35f53fd0a69dd1f7cb6952fbb727..317d4904cb6a632aad1ee83811194e7fd245ef21 100644
--- a/src/vnode/inc/vnodeInt.h
+++ b/src/vnode/inc/vnodeInt.h
@@ -41,13 +41,13 @@ typedef struct {
int8_t status;
int8_t role;
int8_t accessState;
- int64_t version; // current version
- int64_t fversion; // version on saved data file
+ uint64_t version; // current version
+ uint64_t fversion; // version on saved data file
void *wqueue;
void *rqueue;
void *wal;
void *tsdb;
- void *sync;
+ int64_t sync;
void *events;
void *cq; // continuous query
int32_t cfgVersion;
@@ -61,8 +61,6 @@ typedef struct {
char db[TSDB_DB_NAME_LEN];
} SVnodeObj;
-int vnodeWriteToQueue(void *param, void *pHead, int type);
-int vnodeWriteCqMsgToQueue(void *param, void *pHead, int type);
void vnodeInitWriteFp(void);
void vnodeInitReadFp(void);
diff --git a/src/vnode/inc/vnodeVersion.h b/src/vnode/inc/vnodeVersion.h
new file mode 100644
index 0000000000000000000000000000000000000000..1d086cb21fdab0247038c7f3d32d89b38c19d871
--- /dev/null
+++ b/src/vnode/inc/vnodeVersion.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_VNODE_VERSION_H
+#define TDENGINE_VNODE_VERSION_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t vnodeReadVersion(SVnodeObj *pVnode);
+int32_t vnodeSaveVersion(SVnodeObj *pVnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c
new file mode 100644
index 0000000000000000000000000000000000000000..e2e57a756640339febbbe11cdc231c5a216ff2b0
--- /dev/null
+++ b/src/vnode/src/vnodeCfg.c
@@ -0,0 +1,324 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taosmsg.h"
+#include "taoserror.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "tsdb.h"
+#include "dnode.h"
+#include "vnodeInt.h"
+#include "vnodeCfg.h"
+
+static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
+ strcpy(pVnode->db, vnodeMsg->db);
+ pVnode->cfgVersion = vnodeMsg->cfg.cfgVersion;
+ pVnode->tsdbCfg.cacheBlockSize = vnodeMsg->cfg.cacheBlockSize;
+ pVnode->tsdbCfg.totalBlocks = vnodeMsg->cfg.totalBlocks;
+ pVnode->tsdbCfg.daysPerFile = vnodeMsg->cfg.daysPerFile;
+ pVnode->tsdbCfg.keep = vnodeMsg->cfg.daysToKeep;
+ pVnode->tsdbCfg.keep1 = vnodeMsg->cfg.daysToKeep1;
+ pVnode->tsdbCfg.keep2 = vnodeMsg->cfg.daysToKeep2;
+ pVnode->tsdbCfg.minRowsPerFileBlock = vnodeMsg->cfg.minRowsPerFileBlock;
+ pVnode->tsdbCfg.maxRowsPerFileBlock = vnodeMsg->cfg.maxRowsPerFileBlock;
+ pVnode->tsdbCfg.precision = vnodeMsg->cfg.precision;
+ pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression;
+ pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel;
+ pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod;
+ pVnode->walCfg.keep = TAOS_WAL_NOT_KEEP;
+ pVnode->syncCfg.replica = vnodeMsg->cfg.replications;
+ pVnode->syncCfg.quorum = vnodeMsg->cfg.quorum;
+
+ for (int i = 0; i < pVnode->syncCfg.replica; ++i) {
+ SVnodeDesc *node = &vnodeMsg->nodes[i];
+ pVnode->syncCfg.nodeInfo[i].nodeId = node->nodeId;
+ taosGetFqdnPortFromEp(node->nodeEp, pVnode->syncCfg.nodeInfo[i].nodeFqdn, &pVnode->syncCfg.nodeInfo[i].nodePort);
+ pVnode->syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC;
+ }
+
+ vInfo("vgId:%d, load vnode cfg successfully, replcia:%d", pVnode->vgId, pVnode->syncCfg.replica);
+ for (int32_t i = 0; i < pVnode->syncCfg.replica; i++) {
+ SNodeInfo *node = &pVnode->syncCfg.nodeInfo[i];
+ vInfo("vgId:%d, dnode:%d, %s:%u", pVnode->vgId, node->nodeId, node->nodeFqdn, node->nodePort);
+ }
+}
+
+int32_t vnodeReadCfg(SVnodeObj *pVnode) {
+ int32_t ret = TSDB_CODE_VND_APP_ERROR;
+ int32_t len = 0;
+ int maxLen = 1000;
+ char * content = calloc(1, maxLen + 1);
+ cJSON * root = NULL;
+ FILE * fp = NULL;
+ bool nodeChanged = false;
+ SCreateVnodeMsg vnodeMsg;
+
+ char file[TSDB_FILENAME_LEN + 30] = {0};
+ sprintf(file, "%s/vnode%d/config.json", tsVnodeDir, pVnode->vgId);
+
+ vnodeMsg.cfg.vgId = pVnode->vgId;
+
+ fp = fopen(file, "r");
+ if (!fp) {
+ vError("vgId:%d, failed to open vnode cfg file:%s to read, error:%s", pVnode->vgId, file, strerror(errno));
+ ret = TAOS_SYSTEM_ERROR(errno);
+ goto PARSE_VCFG_ERROR;
+ }
+
+ len = fread(content, 1, maxLen, fp);
+ if (len <= 0) {
+ vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+
+ content[len] = 0;
+ root = cJSON_Parse(content);
+ if (root == NULL) {
+ vError("vgId:%d, failed to read %s, invalid json format", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+
+ cJSON *db = cJSON_GetObjectItem(root, "db");
+ if (!db || db->type != cJSON_String || db->valuestring == NULL) {
+ vError("vgId:%d, failed to read %s, db not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ strcpy(vnodeMsg.db, db->valuestring);
+
+ cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion");
+ if (!cfgVersion || cfgVersion->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, cfgVersion not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.cfgVersion = cfgVersion->valueint;
+
+ cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize");
+ if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, cacheBlockSize not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.cacheBlockSize = cacheBlockSize->valueint;
+
+ cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks");
+ if (!totalBlocks || totalBlocks->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, totalBlocks not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.totalBlocks = totalBlocks->valueint;
+
+ cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile");
+ if (!daysPerFile || daysPerFile->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, daysPerFile not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.daysPerFile = daysPerFile->valueint;
+
+ cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep");
+ if (!daysToKeep || daysToKeep->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, daysToKeep not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.daysToKeep = daysToKeep->valueint;
+
+ cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1");
+ if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, daysToKeep1 not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.daysToKeep1 = daysToKeep1->valueint;
+
+ cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2");
+ if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, daysToKeep2 not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.daysToKeep2 = daysToKeep2->valueint;
+
+ cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock");
+ if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, minRowsPerFileBlock not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.minRowsPerFileBlock = minRowsPerFileBlock->valueint;
+
+ cJSON *maxRowsPerFileBlock = cJSON_GetObjectItem(root, "maxRowsPerFileBlock");
+ if (!maxRowsPerFileBlock || maxRowsPerFileBlock->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, maxRowsPerFileBlock not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint;
+
+ cJSON *precision = cJSON_GetObjectItem(root, "precision");
+ if (!precision || precision->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, precision not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.precision = (int8_t)precision->valueint;
+
+ cJSON *compression = cJSON_GetObjectItem(root, "compression");
+ if (!compression || compression->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, compression not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.compression = (int8_t)compression->valueint;
+
+ cJSON *walLevel = cJSON_GetObjectItem(root, "walLevel");
+ if (!walLevel || walLevel->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, walLevel not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.walLevel = (int8_t)walLevel->valueint;
+
+ cJSON *fsyncPeriod = cJSON_GetObjectItem(root, "fsync");
+ if (!walLevel || walLevel->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, fsyncPeriod not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.fsyncPeriod = fsyncPeriod->valueint;
+
+ cJSON *wals = cJSON_GetObjectItem(root, "wals");
+ if (!wals || wals->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, wals not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.wals = (int8_t)wals->valueint;
+
+ cJSON *replica = cJSON_GetObjectItem(root, "replica");
+ if (!replica || replica->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, replica not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.replications = (int8_t)replica->valueint;
+
+ cJSON *quorum = cJSON_GetObjectItem(root, "quorum");
+ if (!quorum || quorum->type != cJSON_Number) {
+ vError("vgId: %d, failed to read %s, quorum not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ vnodeMsg.cfg.quorum = (int8_t)quorum->valueint;
+
+ cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
+ if (!nodeInfos || nodeInfos->type != cJSON_Array) {
+ vError("vgId:%d, failed to read %s, nodeInfos not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+
+ int size = cJSON_GetArraySize(nodeInfos);
+ if (size != vnodeMsg.cfg.replications) {
+ vError("vgId:%d, failed to read %s, nodeInfos size not matched", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+
+ for (int i = 0; i < size; ++i) {
+ cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
+ if (nodeInfo == NULL) continue;
+ SVnodeDesc *node = &vnodeMsg.nodes[i];
+
+ cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
+ if (!nodeId || nodeId->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, nodeId not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ node->nodeId = nodeId->valueint;
+
+ cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
+ if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
+ vError("vgId:%d, failed to read %s, nodeFqdn not found", pVnode->vgId, file);
+ goto PARSE_VCFG_ERROR;
+ }
+ tstrncpy(node->nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
+
+ if (!nodeChanged) {
+ nodeChanged = dnodeCheckEpChanged(node->nodeId, node->nodeEp);
+ }
+ }
+
+ ret = TSDB_CODE_SUCCESS;
+
+PARSE_VCFG_ERROR:
+ if (content != NULL) free(content);
+ if (root != NULL) cJSON_Delete(root);
+ if (fp != NULL) fclose(fp);
+
+ if (nodeChanged) {
+ vnodeWriteCfg(&vnodeMsg);
+ }
+
+ if (ret == TSDB_CODE_SUCCESS) {
+ vnodeLoadCfg(pVnode, &vnodeMsg);
+ }
+
+ terrno = 0;
+ return ret;
+}
+
+int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) {
+ char file[TSDB_FILENAME_LEN + 30] = {0};
+ sprintf(file, "%s/vnode%d/config.json", tsVnodeDir, pMsg->cfg.vgId);
+
+ FILE *fp = fopen(file, "w");
+ if (!fp) {
+ vError("vgId:%d, failed to write %s error:%s", pMsg->cfg.vgId, file, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return terrno;
+ }
+
+ int32_t len = 0;
+ int32_t maxLen = 1000;
+ char * content = calloc(1, maxLen + 1);
+
+ len += snprintf(content + len, maxLen - len, "{\n");
+ len += snprintf(content + len, maxLen - len, " \"db\": \"%s\",\n", pMsg->db);
+ len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pMsg->cfg.cfgVersion);
+ len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pMsg->cfg.cacheBlockSize);
+ len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pMsg->cfg.totalBlocks);
+ len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pMsg->cfg.daysPerFile);
+ len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pMsg->cfg.daysToKeep);
+ len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pMsg->cfg.daysToKeep1);
+ len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pMsg->cfg.daysToKeep2);
+ len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pMsg->cfg.minRowsPerFileBlock);
+ len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pMsg->cfg.maxRowsPerFileBlock);
+ len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pMsg->cfg.precision);
+ len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pMsg->cfg.compression);
+ len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pMsg->cfg.walLevel);
+ len += snprintf(content + len, maxLen - len, " \"fsync\": %d,\n", pMsg->cfg.fsyncPeriod);
+ len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pMsg->cfg.replications);
+ len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pMsg->cfg.wals);
+ len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pMsg->cfg.quorum);
+ len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
+ for (int32_t i = 0; i < pMsg->cfg.replications; i++) {
+ SVnodeDesc *node = &pMsg->nodes[i];
+ dnodeUpdateEp(node->nodeId, node->nodeEp, NULL, NULL);
+ len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", node->nodeId);
+ len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", node->nodeEp);
+ if (i < pMsg->cfg.replications - 1) {
+ len += snprintf(content + len, maxLen - len, " },{\n");
+ } else {
+ len += snprintf(content + len, maxLen - len, " }]\n");
+ }
+ }
+ len += snprintf(content + len, maxLen - len, "}\n");
+
+ fwrite(content, 1, len, fp);
+ fflush(fp);
+ fclose(fp);
+ free(content);
+ terrno = 0;
+
+ vInfo("vgId:%d, successed to write %s", pMsg->cfg.vgId, file);
+ return TSDB_CODE_SUCCESS;
+}
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index e529f27f55abe99e0a842fb839f2916a0f303bfc..199619e8514faac9e663914ddf59a3ea0462afde 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -15,45 +15,35 @@
#define _DEFAULT_SOURCE
#include "os.h"
-
-#include "tcache.h"
-#include "cJSON.h"
-#include "dnode.h"
-#include "hash.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tglobal.h"
#include "trpc.h"
#include "tsdb.h"
-#include "ttimer.h"
#include "tutil.h"
#include "vnode.h"
#include "vnodeInt.h"
#include "query.h"
#include "dnode.h"
+#include "vnodeCfg.h"
+#include "vnodeVersion.h"
-#define TSDB_VNODE_VERSION_CONTENT_LEN 31
-
-static SHashObj*tsDnodeVnodesHash;
+static SHashObj*tsVnodesHash;
static void vnodeCleanUp(SVnodeObj *pVnode);
-static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg);
-static int32_t vnodeReadCfg(SVnodeObj *pVnode);
-static int32_t vnodeSaveVersion(SVnodeObj *pVnode);
-static int32_t vnodeReadVersion(SVnodeObj *pVnode);
static int vnodeProcessTsdbStatus(void *arg, int status);
static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion);
-static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index);
+static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId);
static void vnodeNotifyRole(void *ahandle, int8_t role);
static void vnodeCtrlFlow(void *handle, int32_t mseconds);
static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion);
#ifndef _SYNC
-tsync_h syncStart(const SSyncInfo *info) { return NULL; }
-int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype) { return 0; }
-void syncStop(tsync_h shandle) {}
-int32_t syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; }
-int syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; }
-void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {}
+int64_t syncStart(const SSyncInfo *info) { return NULL; }
+int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype) { return 0; }
+void syncStop(int64_t rid) {}
+int32_t syncReconfig(int64_t rid, const SSyncCfg * cfg) { return 0; }
+int syncGetNodesRole(int64_t rid, SNodesRole * cfg) { return 0; }
+void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {}
#endif
char* vnodeStatus[] = {
@@ -71,26 +61,33 @@ int32_t vnodeInitResources() {
vnodeInitWriteFp();
vnodeInitReadFp();
- tsDnodeVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
- if (tsDnodeVnodesHash == NULL) {
+ tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
+ if (tsVnodesHash == NULL) {
vError("failed to init vnode list");
return TSDB_CODE_VND_OUT_OF_MEMORY;
}
+ if (tsdbInitCommitQueue(tsNumOfCommitThreads) < 0) {
+ vError("failed to init vnode commit queue");
+ return terrno;
+ }
+
return TSDB_CODE_SUCCESS;
}
void vnodeCleanupResources() {
- if (tsDnodeVnodesHash != NULL) {
+ tsdbDestroyCommitQueue();
+
+ if (tsVnodesHash != NULL) {
vDebug("vnode list is cleanup");
- taosHashCleanup(tsDnodeVnodesHash);
- tsDnodeVnodesHash = NULL;
+ taosHashCleanup(tsVnodesHash);
+ tsVnodesHash = NULL;
}
syncCleanUp();
}
-int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
+int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
int32_t code;
SVnodeObj *pVnode = vnodeAcquire(pVnodeCfg->cfg.vgId);
@@ -128,7 +125,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
}
}
- code = vnodeSaveCfg(pVnodeCfg);
+ code = vnodeWriteCfg(pVnodeCfg);
if (code != TSDB_CODE_SUCCESS) {
vError("vgId:%d, failed to save vnode cfg, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(code));
return code;
@@ -138,13 +135,13 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId;
tsdbCfg.cacheBlockSize = pVnodeCfg->cfg.cacheBlockSize;
tsdbCfg.totalBlocks = pVnodeCfg->cfg.totalBlocks;
- // tsdbCfg.maxTables = pVnodeCfg->cfg.maxTables;
tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile;
tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep;
tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock;
tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
tsdbCfg.precision = pVnodeCfg->cfg.precision;
tsdbCfg.compression = pVnodeCfg->cfg.compression;
+ tsdbCfg.update = pVnodeCfg->cfg.update;
char tsdbDir[TSDB_FILENAME_LEN] = {0};
sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId);
@@ -176,7 +173,7 @@ int32_t vnodeDrop(int32_t vgId) {
return TSDB_CODE_SUCCESS;
}
-int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) {
+int32_t vnodeAlter(void *param, SCreateVnodeMsg *pVnodeCfg) {
SVnodeObj *pVnode = param;
// vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS
@@ -186,7 +183,7 @@ int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) {
return TSDB_CODE_SUCCESS;
}
- int32_t code = vnodeSaveCfg(pVnodeCfg);
+ int32_t code = vnodeWriteCfg(pVnodeCfg);
if (code != TSDB_CODE_SUCCESS) {
pVnode->status = TAOS_VN_STATUS_READY;
return code;
@@ -237,6 +234,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
pVnode->vgId = vnode;
pVnode->status = TAOS_VN_STATUS_INIT;
+ pVnode->fversion = 0;
pVnode->version = 0;
pVnode->tsdbCfg.tsdbId = pVnode->vgId;
pVnode->rootDir = strdup(rootDir);
@@ -259,8 +257,8 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
pVnode->fversion = pVnode->version;
- pVnode->wqueue = dnodeAllocateVnodeWqueue(pVnode);
- pVnode->rqueue = dnodeAllocateVnodeRqueue(pVnode);
+ pVnode->wqueue = dnodeAllocVWriteQueue(pVnode);
+ pVnode->rqueue = dnodeAllocVReadQueue(pVnode);
if (pVnode->wqueue == NULL || pVnode->rqueue == NULL) {
vnodeCleanUp(pVnode);
return terrno;
@@ -271,7 +269,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
strcpy(cqCfg.pass, tsInternalPass);
strcpy(cqCfg.db, pVnode->db);
cqCfg.vgId = vnode;
- cqCfg.cqWrite = vnodeWriteCqMsgToQueue;
+ cqCfg.cqWrite = vnodeWriteToWQueue;
pVnode->cq = cqOpen(pVnode, &cqCfg);
if (pVnode->cq == NULL) {
vnodeCleanUp(pVnode);
@@ -291,25 +289,36 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
if (pVnode->tsdb == NULL) {
vnodeCleanUp(pVnode);
return terrno;
- } else if (terrno != TSDB_CODE_SUCCESS && pVnode->syncCfg.replica <= 1) {
+ } else if (terrno != TSDB_CODE_SUCCESS) {
vError("vgId:%d, failed to open tsdb, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
tstrerror(terrno));
- vnodeCleanUp(pVnode);
- return terrno;
+ if (pVnode->syncCfg.replica <= 1) {
+ vnodeCleanUp(pVnode);
+ return terrno;
+ } else {
+ pVnode->fversion = 0;
+ pVnode->version = 0;
+ }
}
sprintf(temp, "%s/wal", rootDir);
+ pVnode->walCfg.vgId = pVnode->vgId;
pVnode->wal = walOpen(temp, &pVnode->walCfg);
if (pVnode->wal == NULL) {
vnodeCleanUp(pVnode);
return terrno;
}
- walRestore(pVnode->wal, pVnode, vnodeWriteToQueue);
+ walRestore(pVnode->wal, pVnode, vnodeProcessWrite);
if (pVnode->version == 0) {
+ pVnode->fversion = 0;
pVnode->version = walGetVersion(pVnode->wal);
}
+ tsdbSyncCommit(pVnode->tsdb);
+ walRemoveAllOldFiles(pVnode->wal);
+ walRenew(pVnode->wal);
+
SSyncInfo syncInfo;
syncInfo.vgId = pVnode->vgId;
syncInfo.version = pVnode->version;
@@ -318,8 +327,8 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
syncInfo.ahandle = pVnode;
syncInfo.getWalInfo = vnodeGetWalInfo;
syncInfo.getFileInfo = vnodeGetFileInfo;
- syncInfo.writeToCache = vnodeWriteToQueue;
- syncInfo.confirmForward = dnodeSendRpcVnodeWriteRsp;
+ syncInfo.writeToCache = vnodeWriteToWQueue;
+ syncInfo.confirmForward = dnodeSendRpcVWriteRsp;
syncInfo.notifyRole = vnodeNotifyRole;
syncInfo.notifyFlowCtrl = vnodeCtrlFlow;
syncInfo.notifyFileSynced = vnodeNotifyFileSynced;
@@ -328,7 +337,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
#ifndef _SYNC
pVnode->role = TAOS_SYNC_ROLE_MASTER;
#else
- if (pVnode->sync == NULL) {
+ if (pVnode->sync <= 0) {
vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
tstrerror(terrno));
vnodeCleanUp(pVnode);
@@ -346,7 +355,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
pVnode->status = TAOS_VN_STATUS_READY;
vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode);
- taosHashPut(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *));
+ taosHashPut(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *));
return TSDB_CODE_SUCCESS;
}
@@ -363,6 +372,7 @@ int32_t vnodeClose(int32_t vgId) {
}
void vnodeRelease(void *pVnodeRaw) {
+ if (pVnodeRaw == NULL) return;
SVnodeObj *pVnode = pVnodeRaw;
int32_t vgId = pVnode->vgId;
@@ -384,6 +394,10 @@ void vnodeRelease(void *pVnodeRaw) {
pVnode->qMgmt = NULL;
}
+ if (pVnode->wal) {
+ walStop(pVnode->wal);
+ }
+
if (pVnode->tsdb) {
tsdbCloseRepo(pVnode->tsdb, 1);
pVnode->tsdb = NULL;
@@ -402,16 +416,16 @@ void vnodeRelease(void *pVnodeRaw) {
}
if (pVnode->wqueue) {
- dnodeFreeVnodeWqueue(pVnode->wqueue);
+ dnodeFreeVWriteQueue(pVnode->wqueue);
pVnode->wqueue = NULL;
}
if (pVnode->rqueue) {
- dnodeFreeVnodeRqueue(pVnode->rqueue);
+ dnodeFreeVReadQueue(pVnode->rqueue);
pVnode->rqueue = NULL;
}
- taosTFree(pVnode->rootDir);
+ tfree(pVnode->rootDir);
if (pVnode->dropped) {
char rootDir[TSDB_FILENAME_LEN] = {0};
@@ -433,7 +447,7 @@ void vnodeRelease(void *pVnodeRaw) {
tsem_destroy(&pVnode->sem);
free(pVnode);
- int32_t count = taosHashGetSize(tsDnodeVnodesHash);
+ int32_t count = taosHashGetSize(tsVnodesHash);
vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count);
}
@@ -450,7 +464,7 @@ static void vnodeIncRef(void *ptNode) {
}
void *vnodeAcquire(int32_t vgId) {
- SVnodeObj **ppVnode = taosHashGetCB(tsDnodeVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
+ SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
if (ppVnode == NULL || *ppVnode == NULL) {
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
@@ -461,41 +475,11 @@ void *vnodeAcquire(int32_t vgId) {
return *ppVnode;
}
-void *vnodeAcquireRqueue(int32_t vgId) {
- SVnodeObj *pVnode = vnodeAcquire(vgId);
- if (pVnode == NULL) return NULL;
-
- int32_t code = vnodeCheckRead(pVnode);
- if (code != TSDB_CODE_SUCCESS) {
- terrno = code;
- vInfo("vgId:%d, can not provide read service, status is %s", vgId, vnodeStatus[pVnode->status]);
- vnodeRelease(pVnode);
- return NULL;
- }
-
- return pVnode->rqueue;
-}
-
-void *vnodeAcquireWqueue(int32_t vgId) {
- SVnodeObj *pVnode = vnodeAcquire(vgId);
- if (pVnode == NULL) return NULL;
-
- int32_t code = vnodeCheckWrite(pVnode);
- if (code != TSDB_CODE_SUCCESS) {
- terrno = code;
- vInfo("vgId:%d, can not provide write service, status is %s", vgId, vnodeStatus[pVnode->status]);
- vnodeRelease(pVnode);
- return NULL;
- }
-
- return pVnode->wqueue;
-}
-
void *vnodeGetWal(void *pVnode) {
return ((SVnodeObj *)pVnode)->wal;
}
-static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) {
+static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) {
int64_t totalStorage = 0;
int64_t compStorage = 0;
int64_t pointsWritten = 0;
@@ -519,7 +503,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) {
}
int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
- SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash);
+ SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash);
while (taosHashIterNext(pIter)) {
SVnodeObj **pVnode = taosHashIterGet(pIter);
if (pVnode == NULL) continue;
@@ -539,8 +523,8 @@ int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
}
void vnodeBuildStatusMsg(void *param) {
- SDMStatusMsg *pStatus = param;
- SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash);
+ SStatusMsg *pStatus = param;
+ SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash);
while (taosHashIterNext(pIter)) {
SVnodeObj **pVnode = taosHashIterGet(pIter);
@@ -553,7 +537,7 @@ void vnodeBuildStatusMsg(void *param) {
taosHashDestroyIter(pIter);
}
-void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes) {
+void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) {
for (int32_t i = 0; i < numOfVnodes; ++i) {
pAccess[i].vgId = htonl(pAccess[i].vgId);
SVnodeObj *pVnode = vnodeAcquire(pAccess[i].vgId);
@@ -569,7 +553,7 @@ void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes) {
static void vnodeCleanUp(SVnodeObj *pVnode) {
// remove from hash, so new messages wont be consumed
- taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
+ taosHashRemove(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
if (pVnode->status != TAOS_VN_STATUS_INIT) {
// it may be in updateing or reset state, then it shall wait
@@ -583,9 +567,9 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
}
// stop replication module
- if (pVnode->sync) {
- void *sync = pVnode->sync;
- pVnode->sync = NULL;
+ if (pVnode->sync > 0) {
+ int64_t sync = pVnode->sync;
+ pVnode->sync = -1;
syncStop(sync);
}
@@ -601,25 +585,30 @@ static int vnodeProcessTsdbStatus(void *arg, int status) {
SVnodeObj *pVnode = arg;
if (status == TSDB_STATUS_COMMIT_START) {
- pVnode->fversion = pVnode->version;
+ pVnode->fversion = pVnode->version;
+ vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
return walRenew(pVnode->wal);
}
- if (status == TSDB_STATUS_COMMIT_OVER)
+ if (status == TSDB_STATUS_COMMIT_OVER) {
+ vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
+ walRemoveOneOldFile(pVnode->wal);
return vnodeSaveVersion(pVnode);
+ }
- return 0;
+ return 0;
}
-static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) {
+static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size,
+ uint64_t *fversion) {
SVnodeObj *pVnode = ahandle;
*fversion = pVnode->fversion;
return tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size);
}
-static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) {
+static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) {
SVnodeObj *pVnode = ahandle;
- return walGetWalFile(pVnode->wal, name, index);
+ return walGetWalFile(pVnode->wal, fileName, fileId);
}
static void vnodeNotifyRole(void *ahandle, int8_t role) {
@@ -636,17 +625,18 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) {
static void vnodeCtrlFlow(void *ahandle, int32_t mseconds) {
SVnodeObj *pVnode = ahandle;
- if (pVnode->delay != mseconds)
+ if (pVnode->delay != mseconds) {
vInfo("vgId:%d, sync flow control, mseconds:%d", pVnode->vgId, mseconds);
+ }
pVnode->delay = mseconds;
}
-static int vnodeResetTsdb(SVnodeObj *pVnode)
-{
+static int vnodeResetTsdb(SVnodeObj *pVnode) {
char rootDir[128] = "\0";
sprintf(rootDir, "%s/tsdb", pVnode->rootDir);
- if (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_RESET) != TAOS_VN_STATUS_READY) {
+ if (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_RESET) !=
+ TAOS_VN_STATUS_READY) {
return -1;
}
@@ -671,375 +661,19 @@ static int vnodeResetTsdb(SVnodeObj *pVnode)
pVnode->tsdb = tsdbOpenRepo(rootDir, &appH);
pVnode->status = TAOS_VN_STATUS_READY;
- vnodeRelease(pVnode);
+ vnodeRelease(pVnode);
return 0;
}
static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion) {
SVnodeObj *pVnode = ahandle;
- vDebug("vgId:%d, data file is synced, fversion:%" PRId64, pVnode->vgId, fversion);
pVnode->fversion = fversion;
pVnode->version = fversion;
vnodeSaveVersion(pVnode);
+ vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion,
+ pVnode->version);
return vnodeResetTsdb(pVnode);
}
-
-static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) {
- char cfgFile[TSDB_FILENAME_LEN + 30] = {0};
- sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnodeCfg->cfg.vgId);
- FILE *fp = fopen(cfgFile, "w");
- if (!fp) {
- vError("vgId:%d, failed to open vnode cfg file for write, file:%s error:%s", pVnodeCfg->cfg.vgId, cfgFile,
- strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- return terrno;
- }
-
- int32_t len = 0;
- int32_t maxLen = 1000;
- char * content = calloc(1, maxLen + 1);
- if (content == NULL) {
- fclose(fp);
- return TSDB_CODE_VND_OUT_OF_MEMORY;
- }
-
- len += snprintf(content + len, maxLen - len, "{\n");
- len += snprintf(content + len, maxLen - len, " \"db\": \"%s\",\n", pVnodeCfg->db);
- len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnodeCfg->cfg.cfgVersion);
- len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize);
- len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks);
- // len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables);
- len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pVnodeCfg->cfg.daysPerFile);
- len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep);
- len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pVnodeCfg->cfg.daysToKeep1);
- len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pVnodeCfg->cfg.daysToKeep2);
- len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.minRowsPerFileBlock);
- len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.maxRowsPerFileBlock);
- // len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnodeCfg->cfg.commitTime);
- len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision);
- len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression);
- len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pVnodeCfg->cfg.walLevel);
- len += snprintf(content + len, maxLen - len, " \"fsync\": %d,\n", pVnodeCfg->cfg.fsyncPeriod);
- len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnodeCfg->cfg.replications);
- len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnodeCfg->cfg.wals);
- len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum);
-
- len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
-
- vInfo("vgId:%d, save vnode cfg, replica:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.replications);
- for (int32_t i = 0; i < pVnodeCfg->cfg.replications; i++) {
- len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", pVnodeCfg->nodes[i].nodeId);
- len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", pVnodeCfg->nodes[i].nodeEp);
- vInfo("vgId:%d, save vnode cfg, nodeId:%d nodeEp:%s", pVnodeCfg->cfg.vgId, pVnodeCfg->nodes[i].nodeId,
- pVnodeCfg->nodes[i].nodeEp);
-
- if (i < pVnodeCfg->cfg.replications - 1) {
- len += snprintf(content + len, maxLen - len, " },{\n");
- } else {
- len += snprintf(content + len, maxLen - len, " }]\n");
- }
- }
- len += snprintf(content + len, maxLen - len, "}\n");
-
- fwrite(content, 1, len, fp);
- fflush(fp);
- fclose(fp);
- free(content);
-
- vInfo("vgId:%d, save vnode cfg successed", pVnodeCfg->cfg.vgId);
-
- return TSDB_CODE_SUCCESS;
-}
-
-static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
- cJSON *root = NULL;
- char *content = NULL;
- char cfgFile[TSDB_FILENAME_LEN + 30] = {0};
- int maxLen = 1000;
-
- terrno = TSDB_CODE_VND_APP_ERROR;
- sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnode->vgId);
- FILE *fp = fopen(cfgFile, "r");
- if (!fp) {
- vError("vgId:%d, failed to open vnode cfg file:%s to read, error:%s", pVnode->vgId,
- cfgFile, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- goto PARSE_OVER;
- }
-
- content = calloc(1, maxLen + 1);
- if (content == NULL) goto PARSE_OVER;
- int len = fread(content, 1, maxLen, fp);
- if (len <= 0) {
- vError("vgId:%d, failed to read vnode cfg, content is null", pVnode->vgId);
- free(content);
- fclose(fp);
- return errno;
- }
-
- root = cJSON_Parse(content);
- if (root == NULL) {
- vError("vgId:%d, failed to read vnode cfg, invalid json format", pVnode->vgId);
- goto PARSE_OVER;
- }
-
- cJSON *db = cJSON_GetObjectItem(root, "db");
- if (!db || db->type != cJSON_String || db->valuestring == NULL) {
- vError("vgId:%d, failed to read vnode cfg, db not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- strcpy(pVnode->db, db->valuestring);
-
- cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion");
- if (!cfgVersion || cfgVersion->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, cfgVersion not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->cfgVersion = cfgVersion->valueint;
-
- cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize");
- if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, cacheBlockSize not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.cacheBlockSize = cacheBlockSize->valueint;
-
- cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks");
- if (!totalBlocks || totalBlocks->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, totalBlocks not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.totalBlocks = totalBlocks->valueint;
-
- // cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables");
- // if (!maxTables || maxTables->type != cJSON_Number) {
- // vError("vgId:%d, failed to read vnode cfg, maxTables not found", pVnode->vgId);
- // goto PARSE_OVER;
- // }
- // pVnode->tsdbCfg.maxTables = maxTables->valueint;
-
- cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile");
- if (!daysPerFile || daysPerFile->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, daysPerFile not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.daysPerFile = daysPerFile->valueint;
-
- cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep");
- if (!daysToKeep || daysToKeep->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.keep = daysToKeep->valueint;
-
- cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1");
- if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, daysToKeep1 not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.keep1 = daysToKeep1->valueint;
-
- cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2");
- if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, daysToKeep2 not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.keep2 = daysToKeep2->valueint;
-
- cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock");
- if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, minRowsPerFileBlock not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.minRowsPerFileBlock = minRowsPerFileBlock->valueint;
-
- cJSON *maxRowsPerFileBlock = cJSON_GetObjectItem(root, "maxRowsPerFileBlock");
- if (!maxRowsPerFileBlock || maxRowsPerFileBlock->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, maxRowsPerFileBlock not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint;
-
- // cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime");
- // if (!commitTime || commitTime->type != cJSON_Number) {
- // vError("vgId:%d, failed to read vnode cfg, commitTime not found", pVnode->vgId);
- // goto PARSE_OVER;
- // }
- // pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint;
-
- cJSON *precision = cJSON_GetObjectItem(root, "precision");
- if (!precision || precision->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, precision not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.precision = (int8_t)precision->valueint;
-
- cJSON *compression = cJSON_GetObjectItem(root, "compression");
- if (!compression || compression->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, compression not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->tsdbCfg.compression = (int8_t)compression->valueint;
-
- cJSON *walLevel = cJSON_GetObjectItem(root, "walLevel");
- if (!walLevel || walLevel->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, walLevel not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->walCfg.walLevel = (int8_t) walLevel->valueint;
-
- cJSON *fsyncPeriod = cJSON_GetObjectItem(root, "fsync");
- if (!walLevel || walLevel->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, fsyncPeriod not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->walCfg.fsyncPeriod = fsyncPeriod->valueint;
-
- cJSON *wals = cJSON_GetObjectItem(root, "wals");
- if (!wals || wals->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, wals not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->walCfg.wals = (int8_t)wals->valueint;
- pVnode->walCfg.keep = 0;
-
- cJSON *replica = cJSON_GetObjectItem(root, "replica");
- if (!replica || replica->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, replica not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->syncCfg.replica = (int8_t)replica->valueint;
-
- cJSON *quorum = cJSON_GetObjectItem(root, "quorum");
- if (!quorum || quorum->type != cJSON_Number) {
- vError("vgId: %d, failed to read vnode cfg, quorum not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->syncCfg.quorum = (int8_t)quorum->valueint;
-
- cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
- if (!nodeInfos || nodeInfos->type != cJSON_Array) {
- vError("vgId:%d, failed to read vnode cfg, nodeInfos not found", pVnode->vgId);
- goto PARSE_OVER;
- }
-
- int size = cJSON_GetArraySize(nodeInfos);
- if (size != pVnode->syncCfg.replica) {
- vError("vgId:%d, failed to read vnode cfg, nodeInfos size not matched", pVnode->vgId);
- goto PARSE_OVER;
- }
-
- for (int i = 0; i < size; ++i) {
- cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
- if (nodeInfo == NULL) continue;
-
- cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
- if (!nodeId || nodeId->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode cfg, nodeId not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->syncCfg.nodeInfo[i].nodeId = nodeId->valueint;
-
- cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
- if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
- vError("vgId:%d, failed to read vnode cfg, nodeFqdn not found", pVnode->vgId);
- goto PARSE_OVER;
- }
-
- taosGetFqdnPortFromEp(nodeEp->valuestring, pVnode->syncCfg.nodeInfo[i].nodeFqdn, &pVnode->syncCfg.nodeInfo[i].nodePort);
- pVnode->syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC;
- }
-
- terrno = TSDB_CODE_SUCCESS;
-
- vInfo("vgId:%d, read vnode cfg successfully, replcia:%d", pVnode->vgId, pVnode->syncCfg.replica);
- for (int32_t i = 0; i < pVnode->syncCfg.replica; i++) {
- vInfo("vgId:%d, dnode:%d, %s:%d", pVnode->vgId, pVnode->syncCfg.nodeInfo[i].nodeId,
- pVnode->syncCfg.nodeInfo[i].nodeFqdn, pVnode->syncCfg.nodeInfo[i].nodePort);
- }
-
-PARSE_OVER:
- taosTFree(content);
- cJSON_Delete(root);
- if (fp) fclose(fp);
- return terrno;
-}
-
-static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
- char versionFile[TSDB_FILENAME_LEN + 30] = {0};
- sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
- FILE *fp = fopen(versionFile, "w");
- if (!fp) {
- vError("vgId:%d, failed to open vnode version file for write, file:%s error:%s", pVnode->vgId,
- versionFile, strerror(errno));
- return TAOS_SYSTEM_ERROR(errno);
- }
-
- int32_t len = 0;
- int32_t maxLen = 30;
- char content[TSDB_VNODE_VERSION_CONTENT_LEN] = {0};
-
- len += snprintf(content + len, maxLen - len, "{\n");
- len += snprintf(content + len, maxLen - len, " \"version\": %" PRId64 "\n", pVnode->fversion);
- len += snprintf(content + len, maxLen - len, "}\n");
-
- fwrite(content, 1, len, fp);
- fflush(fp);
- fclose(fp);
-
- vInfo("vgId:%d, save vnode version:%" PRId64 " succeed", pVnode->vgId, pVnode->fversion);
-
- return TSDB_CODE_SUCCESS;
-}
-
-static int32_t vnodeReadVersion(SVnodeObj *pVnode) {
- char versionFile[TSDB_FILENAME_LEN + 30] = {0};
- char *content = NULL;
- cJSON *root = NULL;
- int maxLen = 100;
-
- terrno = TSDB_CODE_VND_INVALID_VRESION_FILE;
- sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
- FILE *fp = fopen(versionFile, "r");
- if (!fp) {
- if (errno != ENOENT) {
- vError("vgId:%d, failed to open version file:%s error:%s", pVnode->vgId, versionFile, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- } else {
- terrno = TSDB_CODE_SUCCESS;
- }
- goto PARSE_OVER;
- }
-
- content = calloc(1, maxLen + 1);
- int len = fread(content, 1, maxLen, fp);
- if (len <= 0) {
- vError("vgId:%d, failed to read vnode version, content is null", pVnode->vgId);
- goto PARSE_OVER;
- }
-
- root = cJSON_Parse(content);
- if (root == NULL) {
- vError("vgId:%d, failed to read vnode version, invalid json format", pVnode->vgId);
- goto PARSE_OVER;
- }
-
- cJSON *ver = cJSON_GetObjectItem(root, "version");
- if (!ver || ver->type != cJSON_Number) {
- vError("vgId:%d, failed to read vnode version, version not found", pVnode->vgId);
- goto PARSE_OVER;
- }
- pVnode->version = ver->valueint;
-
- terrno = TSDB_CODE_SUCCESS;
- vInfo("vgId:%d, read vnode version successfully, version:%" PRId64, pVnode->vgId, pVnode->version);
-
-PARSE_OVER:
- taosTFree(content);
- cJSON_Delete(root);
- if (fp) fclose(fp);
- return terrno;
-}
diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c
index 99aed03e54ccd069e5879104f62eb01ff7bb3d05..9fa3a11c9c659b44a4edc12110969feb6779a8b7 100644
--- a/src/vnode/src/vnodeRead.c
+++ b/src/vnode/src/vnodeRead.c
@@ -15,13 +15,10 @@
#define _DEFAULT_SOURCE
#define _NON_BLOCKING_RETRIEVE 0
-
#include "os.h"
-
#include "tglobal.h"
#include "taoserror.h"
#include "taosmsg.h"
-#include "tcache.h"
#include "query.h"
#include "trpc.h"
#include "tsdb.h"
@@ -29,9 +26,9 @@
#include "vnodeInt.h"
#include "tqueue.h"
-static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SReadMsg *pReadMsg);
-static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg);
-static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg);
+static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pRead);
+static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead);
+static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead);
static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId);
void vnodeInitReadFp(void) {
@@ -44,19 +41,19 @@ void vnodeInitReadFp(void) {
// still required, or there will be a deadlock, so we don’t do any check here, but put the check codes before the
// request enters the queue
//
-int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) {
+int32_t vnodeProcessRead(void *param, SVReadMsg *pRead) {
SVnodeObj *pVnode = (SVnodeObj *)param;
- int msgType = pReadMsg->rpcMsg.msgType;
+ int32_t msgType = pRead->msgType;
if (vnodeProcessReadMsgFp[msgType] == NULL) {
vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]);
return TSDB_CODE_VND_MSG_NOT_PROCESSED;
}
- return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg);
+ return (*vnodeProcessReadMsgFp[msgType])(pVnode, pRead);
}
-int32_t vnodeCheckRead(void *param) {
+static int32_t vnodeCheckRead(void *param) {
SVnodeObj *pVnode = param;
if (pVnode->status != TAOS_VN_STATUS_READY) {
vDebug("vgId:%d, vnode status is %s, recCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status],
@@ -78,24 +75,58 @@ int32_t vnodeCheckRead(void *param) {
return TSDB_CODE_SUCCESS;
}
-static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) {
- int32_t code = vnodeCheckRead(pVnode);
- if (code != TSDB_CODE_SUCCESS) return code;
- SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg));
- pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
- pRead->pCont = qhandle;
- pRead->contLen = 0;
- pRead->rpcMsg.ahandle = ahandle;
+int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) {
+ SVnodeObj *pVnode = vparam;
- atomic_add_fetch_32(&pVnode->refCount, 1);
+ if (qtype == TAOS_QTYPE_RPC || qtype == TAOS_QTYPE_QUERY) {
+ int32_t code = vnodeCheckRead(pVnode);
+ if (code != TSDB_CODE_SUCCESS) return code;
+ }
+
+ int32_t size = sizeof(SVReadMsg) + contLen;
+ SVReadMsg *pRead = taosAllocateQitem(size);
+ if (pRead == NULL) {
+ return TSDB_CODE_VND_OUT_OF_MEMORY;
+ }
+
+ if (rparam != NULL) {
+ SRpcMsg *pRpcMsg = rparam;
+ pRead->rpcHandle = pRpcMsg->handle;
+ pRead->rpcAhandle = pRpcMsg->ahandle;
+ pRead->msgType = pRpcMsg->msgType;
+ pRead->code = pRpcMsg->code;
+ }
+
+ if (contLen != 0) {
+ pRead->contLen = contLen;
+ memcpy(pRead->pCont, pCont, contLen);
+ } else {
+ pRead->qhandle = pCont;
+ }
+
+ pRead->qtype = qtype;
- vDebug("QInfo:%p add to vread queue for exec query, msg:%p", *qhandle, pRead);
- taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead);
+ atomic_add_fetch_32(&pVnode->refCount, 1);
+ vTrace("vgId:%d, get vnode rqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
+ taosWriteQitem(pVnode->rqueue, qtype, pRead);
return TSDB_CODE_SUCCESS;
}
+static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) {
+ SRpcMsg rpcMsg = {0};
+ rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
+ rpcMsg.ahandle = ahandle;
+
+ int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg);
+ if (code == TSDB_CODE_SUCCESS) {
+ vDebug("QInfo:%p add to vread queue for exec query", *qhandle);
+ }
+
+ return code;
+}
+
/**
*
* @param pRet response message object
@@ -146,27 +177,27 @@ static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) {
pRsp->completed = true;
}
-static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
- void * pCont = pReadMsg->pCont;
- int32_t contLen = pReadMsg->contLen;
- SRspRet *pRet = &pReadMsg->rspRet;
+static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
+ void * pCont = pRead->pCont;
+ int32_t contLen = pRead->contLen;
+ SRspRet *pRet = &pRead->rspRet;
SQueryTableMsg *pQueryTableMsg = (SQueryTableMsg *)pCont;
memset(pRet, 0, sizeof(SRspRet));
// qHandle needs to be freed correctly
- if (pReadMsg->rpcMsg.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
- SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont;
+ if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
+ SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pRead->pCont;
killQueryMsg->free = htons(killQueryMsg->free);
killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle);
- vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
- assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1);
+ vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pRead->rpcHandle);
+ assert(pRead->contLen > 0 && killQueryMsg->free == 1);
void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle);
if (qhandle == NULL || *qhandle == NULL) {
vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle,
- pReadMsg->rpcMsg.handle);
+ pRead->rpcHandle);
} else {
assert(*qhandle == (void *)killQueryMsg->qhandle);
@@ -198,7 +229,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
if (handle == NULL) { // failed to register qhandle
pRsp->code = terrno;
terrno = 0;
- vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo,
+ vError("vgId:%d, QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo,
tstrerror(pRsp->code));
qDestroyQueryInfo(pQInfo); // destroy it directly
return pRsp->code;
@@ -208,9 +239,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
}
if (handle != NULL &&
- vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+ vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle,
- pReadMsg->rpcMsg.handle);
+ pRead->rpcHandle);
pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
return pRsp->code;
@@ -221,7 +252,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
if (handle != NULL) {
vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle);
- code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcMsg.ahandle);
+ code = vnodePutItemIntoReadQueue(pVnode, handle, pRead->rpcHandle);
if (code != TSDB_CODE_SUCCESS) {
pRsp->code = code;
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
@@ -230,7 +261,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
}
} else {
assert(pCont != NULL);
- void **qhandle = (void **)pCont;
+ void **qhandle = (void **)pRead->qhandle;
vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle);
@@ -242,14 +273,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
// build query rsp, the retrieve request has reached here already
if (buildRes) {
// update the connection info according to the retrieve connection
- pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*qhandle);
- assert(pReadMsg->rpcMsg.handle != NULL);
+ pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle);
+ assert(pRead->rpcHandle != NULL);
vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle,
- pReadMsg->rpcMsg.handle);
+ pRead->rpcHandle);
// set the real rsp error code
- pReadMsg->rpcMsg.code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcMsg.ahandle);
+ pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pRead->rpcHandle);
// NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client
code = TSDB_CODE_QRY_HAS_RSP;
@@ -274,16 +305,16 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
return code;
}
-static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
- void * pCont = pReadMsg->pCont;
- SRspRet *pRet = &pReadMsg->rspRet;
+static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
+ void * pCont = pRead->pCont;
+ SRspRet *pRet = &pRead->rspRet;
SRetrieveTableMsg *pRetrieve = pCont;
pRetrieve->free = htons(pRetrieve->free);
pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle,
- pRetrieve->free, pReadMsg->rpcMsg.handle);
+ pRetrieve->free, pRead->rpcHandle);
memset(pRet, 0, sizeof(SRspRet));
@@ -298,7 +329,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
}
if (code != TSDB_CODE_SUCCESS) {
- vDebug("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle);
+ vError("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle);
vnodeBuildNoResultQueryRsp(pRet);
return code;
}
@@ -314,9 +345,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
}
// register the qhandle to connect to quit query immediate if connection is broken
- if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
- vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle,
- pReadMsg->rpcMsg.handle);
+ if (vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+ vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pRead->rpcHandle);
code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
qKillQuery(*handle);
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
@@ -326,7 +356,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
bool freeHandle = true;
bool buildRes = false;
- code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle);
+ code = qRetrieveQueryResultInfo(*handle, &buildRes, pRead->rpcHandle);
if (code != TSDB_CODE_SUCCESS) {
// TODO handle malloc failure
pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
@@ -337,7 +367,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
assert(buildRes == true);
#if _NON_BLOCKING_RETRIEVE
if (!buildRes) {
- assert(pReadMsg->rpcMsg.handle != NULL);
+ assert(pRead->rpcHandle != NULL);
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false);
return TSDB_CODE_QRY_NOT_READY;
@@ -345,7 +375,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
#endif
// ahandle is the sqlObj pointer
- code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcMsg.ahandle);
+ code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pRead->rpcHandle);
}
// If qhandle is not added into vread queue, the query should be completed already or paused with error.
diff --git a/src/vnode/src/vnodeVersion.c b/src/vnode/src/vnodeVersion.c
new file mode 100644
index 0000000000000000000000000000000000000000..8f6360b4f98eac8f394f5078ed2b025cea4192b0
--- /dev/null
+++ b/src/vnode/src/vnodeVersion.c
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "cJSON.h"
+#include "tglobal.h"
+#include "tsdb.h"
+#include "vnodeInt.h"
+#include "vnodeVersion.h"
+
+int32_t vnodeReadVersion(SVnodeObj *pVnode) {
+ int32_t len = 0;
+ int32_t maxLen = 100;
+ char * content = calloc(1, maxLen + 1);
+ cJSON * root = NULL;
+ FILE * fp = NULL;
+
+ terrno = TSDB_CODE_VND_INVALID_VRESION_FILE;
+ char file[TSDB_FILENAME_LEN + 30] = {0};
+ sprintf(file, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
+
+ fp = fopen(file, "r");
+ if (!fp) {
+ if (errno != ENOENT) {
+ vError("vgId:%d, failed to read %s, error:%s", pVnode->vgId, file, strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ } else {
+ terrno = TSDB_CODE_SUCCESS;
+ }
+ goto PARSE_VER_ERROR;
+ }
+
+ len = fread(content, 1, maxLen, fp);
+ if (len <= 0) {
+ vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file);
+ goto PARSE_VER_ERROR;
+ }
+
+ root = cJSON_Parse(content);
+ if (root == NULL) {
+ vError("vgId:%d, failed to read %s, invalid json format", pVnode->vgId, file);
+ goto PARSE_VER_ERROR;
+ }
+
+ cJSON *ver = cJSON_GetObjectItem(root, "version");
+ if (!ver || ver->type != cJSON_Number) {
+ vError("vgId:%d, failed to read %s, version not found", pVnode->vgId, file);
+ goto PARSE_VER_ERROR;
+ }
+ pVnode->version = (uint64_t)ver->valueint;
+
+ terrno = TSDB_CODE_SUCCESS;
+ vInfo("vgId:%d, read %s successfully, fver:%" PRIu64, pVnode->vgId, file, pVnode->version);
+
+PARSE_VER_ERROR:
+ if (content != NULL) free(content);
+ if (root != NULL) cJSON_Delete(root);
+ if (fp != NULL) fclose(fp);
+
+ return terrno;
+}
+
+int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
+ char file[TSDB_FILENAME_LEN + 30] = {0};
+ sprintf(file, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
+
+ FILE *fp = fopen(file, "w");
+ if (!fp) {
+ vError("vgId:%d, failed to write %s, reason:%s", pVnode->vgId, file, strerror(errno));
+ return -1;
+ }
+
+ int32_t len = 0;
+ int32_t maxLen = 100;
+ char * content = calloc(1, maxLen + 1);
+
+ len += snprintf(content + len, maxLen - len, "{\n");
+ len += snprintf(content + len, maxLen - len, " \"version\": %" PRIu64 "\n", pVnode->fversion);
+ len += snprintf(content + len, maxLen - len, "}\n");
+
+ fwrite(content, 1, len, fp);
+ fflush(fp);
+ fclose(fp);
+ free(content);
+ terrno = 0;
+
+ vInfo("vgId:%d, successed to write %s, fver:%" PRIu64, pVnode->vgId, file, pVnode->fversion);
+ return TSDB_CODE_SUCCESS;
+}
\ No newline at end of file
diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c
index 855df81a1b9029d7ddd2f5e5ca6d42e8c680f7ac..e67c544fb23456d8775077e8a86f1745508bd0fc 100644
--- a/src/vnode/src/vnodeWrite.c
+++ b/src/vnode/src/vnodeWrite.c
@@ -19,7 +19,6 @@
#include "taoserror.h"
#include "tqueue.h"
#include "trpc.h"
-#include "tutil.h"
#include "tsdb.h"
#include "twal.h"
#include "tsync.h"
@@ -46,26 +45,31 @@ void vnodeInitWriteFp(void) {
vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessUpdateTagValMsg;
}
-int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
- int32_t code = 0;
- SVnodeObj *pVnode = (SVnodeObj *)param1;
- SWalHead * pHead = param2;
+int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) {
+ int32_t code = 0;
+ SVnodeObj * pVnode = vparam;
+ SWalHead * pHead = wparam;
+ SRspRet * pRspRet = rparam;
if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) {
- vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[pHead->msgType]);
+ vError("vgId:%d, msg:%s not processed since no handle, qtype:%s hver:%" PRIu64, pVnode->vgId,
+ taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version);
return TSDB_CODE_VND_MSG_NOT_PROCESSED;
}
+ vTrace("vgId:%d, msg:%s will be processed in vnode, qtype:%s hver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId,
+ taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version, pVnode->version);
+
if (pHead->version == 0) { // from client or CQ
if (pVnode->status != TAOS_VN_STATUS_READY) {
- vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[pHead->msgType],
- pVnode->status);
+ vDebug("vgId:%d, msg:%s not processed since vstatus:%d, qtype:%s hver:%" PRIu64, pVnode->vgId,
+ taosMsg[pHead->msgType], pVnode->status, qtypeStr[qtype], pHead->version);
return TSDB_CODE_APP_NOT_READY; // it may be in deleting or closing state
}
if (pVnode->role != TAOS_SYNC_ROLE_MASTER) {
- vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%s", pVnode->vgId, taosMsg[pHead->msgType],
- pVnode->syncCfg.replica, syncRole[pVnode->role]);
+ vDebug("vgId:%d, msg:%s not processed since replica:%d role:%s, qtype:%s hver:%" PRIu64, pVnode->vgId,
+ taosMsg[pHead->msgType], pVnode->syncCfg.replica, syncRole[pVnode->role], qtypeStr[qtype], pHead->version);
return TSDB_CODE_APP_NOT_READY;
}
@@ -80,7 +84,7 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
// forward to peers, even it is WAL/FWD, it shall be called to update version in sync
int32_t syncCode = 0;
- syncCode = syncForwardToPeer(pVnode->sync, pHead, item, qtype);
+ syncCode = syncForwardToPeer(pVnode->sync, pHead, pRspRet, qtype);
if (syncCode < 0) return syncCode;
// write into WAL
@@ -90,13 +94,13 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
pVnode->version = pHead->version;
// write data locally
- code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item);
+ code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, pRspRet);
if (code < 0) return code;
return syncCode;
}
-int32_t vnodeCheckWrite(void *param) {
+static int32_t vnodeCheckWrite(void *param) {
SVnodeObj *pVnode = param;
if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) {
vDebug("vgId:%d, no write auth, recCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
@@ -183,8 +187,8 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
}
static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
- SMDDropSTableMsg *pTable = pCont;
- int32_t code = TSDB_CODE_SUCCESS;
+ SDropSTableMsg *pTable = pCont;
+ int32_t code = TSDB_CODE_SUCCESS;
vDebug("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableId);
@@ -204,37 +208,37 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR
return TSDB_CODE_SUCCESS;
}
+int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) {
+ SVnodeObj *pVnode = vparam;
+ SWalHead * pHead = wparam;
-int vnodeWriteCqMsgToQueue(void *param, void *data, int type) {
- SVnodeObj *pVnode = param;
- SWalHead * pHead = data;
-
- int size = sizeof(SWalHead) + pHead->len;
- SSyncHead *pSync = (SSyncHead*) taosAllocateQitem(size + sizeof(SSyncHead));
- SWalHead *pWal = (SWalHead *)(pSync + 1);
- memcpy(pWal, pHead, size);
-
- atomic_add_fetch_32(&pVnode->refCount, 1);
- vTrace("CQ: vgId:%d, get vnode wqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
-
- taosWriteQitem(pVnode->wqueue, type, pSync);
+ if (qtype == TAOS_QTYPE_RPC) {
+ int32_t code = vnodeCheckWrite(pVnode);
+ if (code != TSDB_CODE_SUCCESS) return code;
+ }
- return 0;
-}
+ if (pHead->len > TSDB_MAX_WAL_SIZE) {
+ vError("vgId:%d, wal len:%d exceeds limit, hver:%" PRIu64, pVnode->vgId, pHead->len, pHead->version);
+ return TSDB_CODE_WAL_SIZE_LIMIT;
+ }
+ int32_t size = sizeof(SVWriteMsg) + sizeof(SWalHead) + pHead->len;
+ SVWriteMsg *pWrite = taosAllocateQitem(size);
+ if (pWrite == NULL) {
+ return TSDB_CODE_VND_OUT_OF_MEMORY;
+ }
-int vnodeWriteToQueue(void *param, void *data, int type) {
- SVnodeObj *pVnode = param;
- SWalHead * pHead = data;
+ if (rparam != NULL) {
+ SRpcMsg *pRpcMsg = rparam;
+ pWrite->rpcHandle = pRpcMsg->handle;
+ pWrite->rpcAhandle = pRpcMsg->ahandle;
+ }
- int size = sizeof(SWalHead) + pHead->len;
- SWalHead *pWal = (SWalHead *)taosAllocateQitem(size);
- memcpy(pWal, pHead, size);
+ memcpy(pWrite->pHead, pHead, sizeof(SWalHead) + pHead->len);
atomic_add_fetch_32(&pVnode->refCount, 1);
vTrace("vgId:%d, get vnode wqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
- taosWriteQitem(pVnode->wqueue, type, pWal);
-
- return 0;
+ taosWriteQitem(pVnode->wqueue, qtype, pWrite);
+ return TSDB_CODE_SUCCESS;
}
diff --git a/src/wal/inc/walInt.h b/src/wal/inc/walInt.h
new file mode 100644
index 0000000000000000000000000000000000000000..36311c8f5d92fe0ba97793224df59d4dd33b5da6
--- /dev/null
+++ b/src/wal/inc/walInt.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_WAL_INT_H
+#define TDENGINE_WAL_INT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tlog.h"
+
+extern int32_t wDebugFlag;
+
+#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }}
+#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }}
+#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }}
+#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }}
+#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
+#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
+
+#define WAL_PREFIX "wal"
+#define WAL_PREFIX_LEN 3
+#define WAL_REFRESH_MS 1000
+#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16)
+#define WAL_SIGNATURE ((uint32_t)(0xFAFBFDFE))
+#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12)
+#define WAL_FILE_LEN (TSDB_FILENAME_LEN + 32)
+#define WAL_FILE_NUM 3
+
+typedef struct {
+ uint64_t version;
+ int64_t fileId;
+ int64_t rid;
+ int32_t vgId;
+ int32_t fd;
+ int32_t keep;
+ int32_t level;
+ int32_t fsyncPeriod;
+ int32_t fsyncSeq;
+ int8_t stop;
+ int8_t reserved[3];
+ char path[WAL_PATH_LEN];
+ char name[WAL_FILE_LEN];
+ pthread_mutex_t mutex;
+} SWal;
+
+int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId);
+int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId);
+int32_t walGetNewFile(SWal *pWal, int64_t *newFileId);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c
deleted file mode 100644
index 182600204259e703d171d6598f46a2a16cdcb27b..0000000000000000000000000000000000000000
--- a/src/wal/src/walMain.c
+++ /dev/null
@@ -1,588 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc.
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-
-#define _DEFAULT_SOURCE
-
-#define TAOS_RANDOM_FILE_FAIL_TEST
-
-#include "os.h"
-#include "tlog.h"
-#include "tchecksum.h"
-#include "tutil.h"
-#include "ttimer.h"
-#include "taoserror.h"
-#include "twal.h"
-#include "tqueue.h"
-
-#define walPrefix "wal"
-
-#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }}
-#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }}
-#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }}
-#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }}
-#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
-#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
-
-typedef struct {
- uint64_t version;
- int fd;
- int keep;
- int level;
- int32_t fsyncPeriod;
- void *timer;
- void *signature;
- int max; // maximum number of wal files
- uint32_t id; // increase continuously
- int num; // number of wal files
- char path[TSDB_FILENAME_LEN];
- char name[TSDB_FILENAME_LEN+16];
- pthread_mutex_t mutex;
-} SWal;
-
-static void *walTmrCtrl = NULL;
-static int tsWalNum = 0;
-static pthread_once_t walModuleInit = PTHREAD_ONCE_INIT;
-static uint32_t walSignature = 0xFAFBFDFE;
-static int walHandleExistingFiles(const char *path);
-static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp);
-static int walRemoveWalFiles(const char *path);
-static void walProcessFsyncTimer(void *param, void *tmrId);
-static void walRelease(SWal *pWal);
-static int walGetMaxOldFileId(char *odir);
-
-static void walModuleInitFunc() {
- walTmrCtrl = taosTmrInit(1000, 100, 300000, "WAL");
- if (walTmrCtrl == NULL)
- walModuleInit = PTHREAD_ONCE_INIT;
- else
- wDebug("WAL module is initialized");
-}
-
-static inline bool walNeedFsyncTimer(SWal *pWal) {
- if (pWal->fsyncPeriod > 0 && pWal->level == TAOS_WAL_FSYNC) {
- return true;
- }
- return false;
-}
-
-void *walOpen(const char *path, const SWalCfg *pCfg) {
- SWal *pWal = calloc(sizeof(SWal), 1);
- if (pWal == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- return NULL;
- }
-
- pthread_once(&walModuleInit, walModuleInitFunc);
- if (walTmrCtrl == NULL) {
- free(pWal);
- terrno = TAOS_SYSTEM_ERROR(errno);
- return NULL;
- }
-
- atomic_add_fetch_32(&tsWalNum, 1);
- pWal->fd = -1;
- pWal->max = pCfg->wals;
- pWal->id = 0;
- pWal->num = 0;
- pWal->level = pCfg->walLevel;
- pWal->keep = pCfg->keep;
- pWal->fsyncPeriod = pCfg->fsyncPeriod;
- pWal->signature = pWal;
- tstrncpy(pWal->path, path, sizeof(pWal->path));
- pthread_mutex_init(&pWal->mutex, NULL);
-
- if (walNeedFsyncTimer(pWal)) {
- pWal->timer = taosTmrStart(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, walTmrCtrl);
- if (pWal->timer == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- walRelease(pWal);
- return NULL;
- }
- }
-
- if (taosMkDir(path, 0755) != 0) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- wError("wal:%s, failed to create directory(%s)", path, strerror(errno));
- walRelease(pWal);
- pWal = NULL;
- }
-
- if (pCfg->keep == 1) return pWal;
-
- if (walHandleExistingFiles(path) == 0) walRenew(pWal);
-
- if (pWal && pWal->fd < 0) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- wError("wal:%s, failed to open(%s)", path, strerror(errno));
- walRelease(pWal);
- pWal = NULL;
- }
-
- if (pWal) wDebug("wal:%s, it is open, level:%d fsyncPeriod:%d", path, pWal->level, pWal->fsyncPeriod);
- return pWal;
-}
-
-int walAlter(twalh wal, const SWalCfg *pCfg) {
- SWal *pWal = wal;
- if (pWal == NULL) {
- return TSDB_CODE_WAL_APP_ERROR;
- }
-
- if (pWal->level == pCfg->walLevel && pWal->fsyncPeriod == pCfg->fsyncPeriod) {
- wDebug("wal:%s, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->name, pWal->level,
- pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod);
- return TSDB_CODE_SUCCESS;
- }
-
- wInfo("wal:%s, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->name, pWal->level, pWal->fsyncPeriod,
- pCfg->walLevel, pCfg->fsyncPeriod);
-
- pthread_mutex_lock(&pWal->mutex);
- pWal->level = pCfg->walLevel;
- pWal->fsyncPeriod = pCfg->fsyncPeriod;
- if (walNeedFsyncTimer(pWal)) {
- wInfo("wal:%s, reset fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod);
- taosTmrReset(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, &pWal->timer, walTmrCtrl);
- } else {
- wInfo("wal:%s, stop fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod);
- taosTmrStop(pWal->timer);
- pWal->timer = NULL;
- }
- pthread_mutex_unlock(&pWal->mutex);
-
- return TSDB_CODE_SUCCESS;
-}
-
-void walClose(void *handle) {
- if (handle == NULL) return;
-
- SWal *pWal = handle;
- taosClose(pWal->fd);
- if (pWal->timer) taosTmrStopA(&pWal->timer);
-
- if (pWal->keep == 0) {
- // remove all files in the directory
- for (int i = 0; i < pWal->num; ++i) {
- snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id - i);
- if (remove(pWal->name) < 0) {
- wError("wal:%s, failed to remove", pWal->name);
- } else {
- wDebug("wal:%s, it is removed", pWal->name);
- }
- }
- } else {
- wDebug("wal:%s, it is closed and kept", pWal->name);
- }
-
- walRelease(pWal);
-}
-
-int walRenew(void *handle) {
- if (handle == NULL) return 0;
- SWal *pWal = handle;
-
- terrno = 0;
-
- pthread_mutex_lock(&pWal->mutex);
-
- if (pWal->fd >= 0) {
- close(pWal->fd);
- pWal->id++;
- wDebug("wal:%s, it is closed", pWal->name);
- }
-
- pWal->num++;
-
- snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id);
- pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
-
- if (pWal->fd < 0) {
- wError("wal:%s, failed to open(%s)", pWal->name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- } else {
- wDebug("wal:%s, it is created", pWal->name);
-
- if (pWal->num > pWal->max) {
- // remove the oldest wal file
- char name[TSDB_FILENAME_LEN * 3];
- snprintf(name, sizeof(name), "%s/%s%d", pWal->path, walPrefix, pWal->id - pWal->max);
- if (remove(name) < 0) {
- wError("wal:%s, failed to remove(%s)", name, strerror(errno));
- } else {
- wDebug("wal:%s, it is removed", name);
- }
-
- pWal->num--;
- }
- }
-
- pthread_mutex_unlock(&pWal->mutex);
-
- return terrno;
-}
-
-int walWrite(void *handle, SWalHead *pHead) {
- SWal *pWal = handle;
- if (pWal == NULL) return -1;
-
- terrno = 0;
-
- // no wal
- if (pWal->level == TAOS_WAL_NOLOG) return 0;
- if (pHead->version <= pWal->version) return 0;
-
- pHead->signature = walSignature;
- taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead));
- int contLen = pHead->len + sizeof(SWalHead);
-
- if (taosTWrite(pWal->fd, pHead, contLen) != contLen) {
- wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- } else {
- pWal->version = pHead->version;
- }
-
- return terrno;
-}
-
-void walFsync(void *handle) {
- SWal *pWal = handle;
- if (pWal == NULL || pWal->level != TAOS_WAL_FSYNC || pWal->fd < 0) return;
-
- if (pWal->fsyncPeriod == 0) {
- if (fsync(pWal->fd) < 0) {
- wError("wal:%s, fsync failed(%s)", pWal->name, strerror(errno));
- }
- }
-}
-
-int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) {
- SWal *pWal = handle;
- struct dirent *ent;
- int count = 0;
- uint32_t maxId = 0, minId = -1, index =0;
-
- terrno = 0;
- int plen = strlen(walPrefix);
- char opath[TSDB_FILENAME_LEN + 5];
-
- int slen = snprintf(opath, sizeof(opath), "%s", pWal->path);
- if (pWal->keep == 0) strcpy(opath + slen, "/old");
-
- DIR *dir = opendir(opath);
- if (dir == NULL && errno == ENOENT) return 0;
- if (dir == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- return terrno;
- }
-
- while ((ent = readdir(dir)) != NULL) {
- if (strncmp(ent->d_name, walPrefix, plen) == 0) {
- index = atol(ent->d_name + plen);
- if (index > maxId) maxId = index;
- if (index < minId) minId = index;
- count++;
- }
- }
-
- closedir(dir);
-
- if (count == 0) {
- if (pWal->keep) terrno = walRenew(pWal);
- return terrno;
- }
-
- if (count != (maxId - minId + 1)) {
- wError("wal:%s, messed up, count:%d max:%d min:%d", opath, count, maxId, minId);
- terrno = TSDB_CODE_WAL_APP_ERROR;
- } else {
- wDebug("wal:%s, %d files will be restored", opath, count);
-
- for (index = minId; index <= maxId; ++index) {
- snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, index);
- terrno = walRestoreWalFile(pWal, pVnode, writeFp);
- if (terrno < 0) continue;
- }
- }
-
- if (terrno == 0) {
- if (pWal->keep == 0) {
- terrno = walRemoveWalFiles(opath);
- if (terrno == 0) {
- if (remove(opath) < 0) {
- wError("wal:%s, failed to remove directory(%s)", opath, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- }
- }
- } else {
- // open the existing WAL file in append mode
- pWal->num = count;
- pWal->id = maxId;
- snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, maxId);
- pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
- if (pWal->fd < 0) {
- wError("wal:%s, failed to open file(%s)", pWal->name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- }
- }
- }
-
- return terrno;
-}
-
-int walGetWalFile(void *handle, char *name, uint32_t *index) {
- SWal * pWal = handle;
- int code = 1;
- int32_t first = 0;
-
- name[0] = 0;
- if (pWal == NULL || pWal->num == 0) return 0;
-
- pthread_mutex_lock(&(pWal->mutex));
-
- first = pWal->id + 1 - pWal->num;
- if (*index == 0) *index = first; // set to first one
-
- if (*index < first && *index > pWal->id) {
- code = -1; // index out of range
- } else {
- sprintf(name, "wal/%s%d", walPrefix, *index);
- code = (*index == pWal->id) ? 0 : 1;
- }
-
- pthread_mutex_unlock(&(pWal->mutex));
-
- return code;
-}
-
-static void walRelease(SWal *pWal) {
- pthread_mutex_destroy(&pWal->mutex);
- pWal->signature = NULL;
- free(pWal);
-
- if (atomic_sub_fetch_32(&tsWalNum, 1) == 0) {
- if (walTmrCtrl) taosTmrCleanUp(walTmrCtrl);
- walTmrCtrl = NULL;
- walModuleInit = PTHREAD_ONCE_INIT;
- wDebug("WAL module is cleaned up");
- }
-}
-
-static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
- char *name = pWal->name;
- int size = 1024 * 1024; // default 1M buffer size
-
- terrno = 0;
- char *buffer = malloc(size);
- if (buffer == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- return terrno;
- }
-
- SWalHead *pHead = (SWalHead *)buffer;
-
- int fd = open(name, O_RDWR);
- if (fd < 0) {
- wError("wal:%s, failed to open for restore(%s)", name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- free(buffer);
- return terrno;
- }
-
- wDebug("wal:%s, start to restore", name);
-
- size_t offset = 0;
- while (1) {
- int ret = taosTRead(fd, pHead, sizeof(SWalHead));
- if (ret == 0) break;
-
- if (ret < 0) {
- wError("wal:%s, failed to read wal head part since %s", name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- break;
- }
-
- if (ret < sizeof(SWalHead)) {
- wError("wal:%s, failed to read head, ret:%d, skip the rest of file", name, ret);
- taosFtruncate(fd, offset);
- fsync(fd);
- break;
- }
-
- if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) {
- wWarn("wal:%s, cksum is messed up, skip the rest of file", name);
- terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
- // ASSERT(false);
- break;
- }
-
- if (pHead->len > size - sizeof(SWalHead)) {
- size = sizeof(SWalHead) + pHead->len;
- buffer = realloc(buffer, size);
- if (buffer == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- break;
- }
-
- pHead = (SWalHead *)buffer;
- }
-
- ret = taosTRead(fd, pHead->cont, pHead->len);
- if (ret < 0) {
- wError("wal:%s failed to read wal body part since %s", name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- break;
- }
-
- if (ret < pHead->len) {
- wError("wal:%s, failed to read body, len:%d ret:%d, skip the rest of file", name, pHead->len, ret);
- taosFtruncate(fd, offset);
- fsync(fd);
- break;
- }
-
- offset = offset + sizeof(SWalHead) + pHead->len;
-
- if (pWal->keep) pWal->version = pHead->version;
- (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL);
- }
-
- close(fd);
- free(buffer);
-
- return terrno;
-}
-
-int walHandleExistingFiles(const char *path) {
- char oname[TSDB_FILENAME_LEN * 3];
- char nname[TSDB_FILENAME_LEN * 3];
- char opath[TSDB_FILENAME_LEN];
-
- snprintf(opath, sizeof(opath), "%s/old", path);
-
- struct dirent *ent;
- DIR *dir = opendir(path);
- int plen = strlen(walPrefix);
- terrno = 0;
-
- int midx = walGetMaxOldFileId(opath);
- int count = 0;
- while ((ent = readdir(dir)) != NULL) {
- if (strncmp(ent->d_name, walPrefix, plen) == 0) {
- midx++;
- snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name);
- snprintf(nname, sizeof(nname), "%s/old/wal%d", path, midx);
- if (taosMkDir(opath, 0755) != 0) {
- wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- break;
- }
-
- if (rename(oname, nname) < 0) {
- wError("wal:%s, failed to move to new:%s", oname, nname);
- terrno = TAOS_SYSTEM_ERROR(errno);
- break;
- }
-
- count++;
- }
-
- wDebug("wal:%s, %d files are moved for restoration", path, count);
- }
-
- closedir(dir);
- return terrno;
-}
-
-static int walRemoveWalFiles(const char *path) {
- int plen = strlen(walPrefix);
- char name[TSDB_FILENAME_LEN * 3];
-
- terrno = 0;
-
- struct dirent *ent;
- DIR *dir = opendir(path);
- if (dir == NULL && errno == ENOENT) return 0;
- if (dir == NULL) {
- terrno = TAOS_SYSTEM_ERROR(errno);
- return terrno;
- }
-
- while ((ent = readdir(dir)) != NULL) {
- if (strncmp(ent->d_name, walPrefix, plen) == 0) {
- snprintf(name, sizeof(name), "%s/%s", path, ent->d_name);
- if (remove(name) < 0) {
- wError("wal:%s, failed to remove(%s)", name, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- }
- }
- }
-
- closedir(dir);
-
- return terrno;
-}
-
-static void walProcessFsyncTimer(void *param, void *tmrId) {
- SWal *pWal = param;
-
- if (pWal->signature != pWal) return;
- if (pWal->fd < 0) return;
-
- if (fsync(pWal->fd) < 0) {
- wError("wal:%s, fsync failed(%s)", pWal->name, strerror(errno));
- }
-
- if (walNeedFsyncTimer(pWal)) {
- pWal->timer = taosTmrStart(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, walTmrCtrl);
- } else {
- wInfo("wal:%s, stop fsync timer for walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod);
- taosTmrStop(pWal->timer);
- pWal->timer = NULL;
- }
-}
-
-int64_t walGetVersion(twalh param) {
- SWal *pWal = param;
- if (pWal == 0) return 0;
-
- return pWal->version;
-}
-
-static int walGetMaxOldFileId(char *odir) {
- int midx = 0;
- DIR * dir = NULL;
- struct dirent *dp = NULL;
- int plen = strlen(walPrefix);
-
- if (access(odir, F_OK) != 0) return midx;
-
- dir = opendir(odir);
- if (dir == NULL) {
- wError("failed to open directory %s since %s", odir, strerror(errno));
- terrno = TAOS_SYSTEM_ERROR(errno);
- return -1;
- }
-
- while ((dp = readdir(dir)) != NULL) {
- if (strncmp(dp->d_name, walPrefix, plen) == 0) {
- int idx = atol(dp->d_name + plen);
- if (midx < idx) midx = idx;
- }
- }
-
- closedir(dir);
- return midx;
-}
\ No newline at end of file
diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c
new file mode 100644
index 0000000000000000000000000000000000000000..fb49f38217d62b9181f0eed92155434b516ab3e0
--- /dev/null
+++ b/src/wal/src/walMgmt.c
@@ -0,0 +1,225 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "tref.h"
+#include "twal.h"
+#include "walInt.h"
+
+typedef struct {
+ int32_t refId;
+ int32_t seq;
+ int8_t stop;
+ pthread_t thread;
+ pthread_mutex_t mutex;
+} SWalMgmt;
+
+static SWalMgmt tsWal = {0};
+static int32_t walCreateThread();
+static void walStopThread();
+static int32_t walInitObj(SWal *pWal);
+static void walFreeObj(void *pWal);
+
+int32_t walInit() {
+ tsWal.refId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj);
+
+ int32_t code = walCreateThread();
+ if (code != TSDB_CODE_SUCCESS) {
+ wError("failed to init wal module since %s", tstrerror(code));
+ return code;
+ }
+
+ wInfo("wal module is initialized, refId:%d", tsWal.refId);
+ return code;
+}
+
+void walCleanUp() {
+ walStopThread();
+ taosCloseRef(tsWal.refId);
+ wInfo("wal module is cleaned up");
+}
+
+void *walOpen(char *path, SWalCfg *pCfg) {
+ SWal *pWal = tcalloc(1, sizeof(SWal));
+ if (pWal == NULL) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return NULL;
+ }
+
+ pWal->vgId = pCfg->vgId;
+ pWal->fd = -1;
+ pWal->fileId = -1;
+ pWal->level = pCfg->walLevel;
+ pWal->keep = pCfg->keep;
+ pWal->fsyncPeriod = pCfg->fsyncPeriod;
+ tstrncpy(pWal->path, path, sizeof(pWal->path));
+ pthread_mutex_init(&pWal->mutex, NULL);
+
+ pWal->fsyncSeq = pCfg->fsyncPeriod / 1000;
+ if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1;
+
+ if (walInitObj(pWal) != TSDB_CODE_SUCCESS) {
+ walFreeObj(pWal);
+ return NULL;
+ }
+
+ pWal->rid = taosAddRef(tsWal.refId, pWal);
+ if (pWal->rid < 0) {
+ walFreeObj(pWal);
+ return NULL;
+ }
+
+ wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->vgId, pWal, pWal->level, pWal->fsyncPeriod);
+
+ return pWal;
+}
+
+int32_t walAlter(void *handle, SWalCfg *pCfg) {
+ if (handle == NULL) return TSDB_CODE_WAL_APP_ERROR;
+ SWal *pWal = handle;
+
+ if (pWal->level == pCfg->walLevel && pWal->fsyncPeriod == pCfg->fsyncPeriod) {
+ wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->vgId, pWal->level,
+ pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod);
+ return TSDB_CODE_SUCCESS;
+ }
+
+ wInfo("vgId:%d, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->vgId, pWal->level,
+ pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod);
+
+ pWal->level = pCfg->walLevel;
+ pWal->fsyncPeriod = pCfg->fsyncPeriod;
+ pWal->fsyncSeq = pCfg->fsyncPeriod % 1000;
+ if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1;
+
+ return TSDB_CODE_SUCCESS;
+}
+
+void walStop(void *handle) {
+ if (handle == NULL) return;
+ SWal *pWal = handle;
+
+ pthread_mutex_lock(&pWal->mutex);
+ pWal->stop = 1;
+ pthread_mutex_unlock(&pWal->mutex);
+ wDebug("vgId:%d, stop write wal", pWal->vgId);
+}
+
+void walClose(void *handle) {
+ if (handle == NULL) return;
+
+ SWal *pWal = handle;
+ pthread_mutex_lock(&pWal->mutex);
+
+ taosClose(pWal->fd);
+
+ if (pWal->keep != TAOS_WAL_KEEP) {
+ walRemoveAllOldFiles(pWal);
+ } else {
+ wDebug("vgId:%d, wal:%p file:%s, it is closed and kept", pWal->vgId, pWal, pWal->name);
+ }
+
+ pthread_mutex_unlock(&pWal->mutex);
+ taosRemoveRef(tsWal.refId, pWal->rid);
+}
+
+static int32_t walInitObj(SWal *pWal) {
+ if (taosMkDir(pWal->path, 0755) != 0) {
+ wError("vgId:%d, path:%s, failed to create directory since %s", pWal->vgId, pWal->path, strerror(errno));
+ return TAOS_SYSTEM_ERROR(errno);
+ }
+
+ wDebug("vgId:%d, object is initialized", pWal->vgId);
+ return TSDB_CODE_SUCCESS;
+}
+
+static void walFreeObj(void *wal) {
+ SWal *pWal = wal;
+ wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal);
+
+ taosClose(pWal->fd);
+ pthread_mutex_destroy(&pWal->mutex);
+ tfree(pWal);
+}
+
+static bool walNeedFsync(SWal *pWal) {
+ if (pWal->fsyncPeriod <= 0 || pWal->level != TAOS_WAL_FSYNC) {
+ return false;
+ }
+
+ if (tsWal.seq % pWal->fsyncSeq == 0) {
+ return true;
+ }
+
+ return false;
+}
+
+static void walUpdateSeq() {
+ taosMsleep(WAL_REFRESH_MS);
+ if (++tsWal.seq <= 0) {
+ tsWal.seq = 1;
+ }
+}
+
+static void walFsyncAll() {
+ SWal *pWal = taosIterateRef(tsWal.refId, 0);
+ while (pWal) {
+ if (walNeedFsync(pWal)) {
+ wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq);
+ int32_t code = fsync(pWal->fd);
+ if (code != 0) {
+ wError("vgId:%d, file:%s, failed to fsync since %s", pWal->vgId, pWal->name, strerror(code));
+ }
+ }
+ pWal = taosIterateRef(tsWal.refId, pWal->rid);
+ }
+}
+
+static void *walThreadFunc(void *param) {
+ while (1) {
+ walUpdateSeq();
+ walFsyncAll();
+ if (tsWal.stop) break;
+ }
+
+ return NULL;
+}
+
+static int32_t walCreateThread() {
+ pthread_attr_t thAttr;
+ pthread_attr_init(&thAttr);
+ pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+
+ if (pthread_create(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) {
+ wError("failed to create wal thread since %s", strerror(errno));
+ return TAOS_SYSTEM_ERROR(errno);
+ }
+
+ pthread_attr_destroy(&thAttr);
+ wDebug("wal thread is launched");
+
+ return TSDB_CODE_SUCCESS;
+}
+
+static void walStopThread() {
+ tsWal.stop = 1;
+ if (tsWal.thread) {
+ pthread_join(tsWal.thread, NULL);
+ }
+
+ wDebug("wal thread is stopped");
+}
diff --git a/src/wal/src/walUtil.c b/src/wal/src/walUtil.c
new file mode 100644
index 0000000000000000000000000000000000000000..e4d9a555b3a60cb6be1e6584652ec4a309b1c301
--- /dev/null
+++ b/src/wal/src/walUtil.c
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "walInt.h"
+
+int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId) {
+ int64_t curFileId = *nextFileId;
+ int64_t minFileId = INT64_MAX;
+
+ DIR *dir = opendir(pWal->path);
+ if (dir == NULL) {
+ wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno));
+ return -1;
+ }
+
+ struct dirent *ent;
+ while ((ent = readdir(dir)) != NULL) {
+ char *name = ent->d_name;
+
+ if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) {
+ int64_t id = atoll(name + WAL_PREFIX_LEN);
+ if (id <= curFileId) continue;
+
+ if (id < minFileId) {
+ minFileId = id;
+ }
+ }
+ }
+ closedir(dir);
+
+ if (minFileId == INT64_MAX) return -1;
+
+ *nextFileId = minFileId;
+ wTrace("vgId:%d, path:%s, curFileId:%" PRId64 " nextFileId:%" PRId64, pWal->vgId, pWal->path, curFileId, *nextFileId);
+
+ return 0;
+}
+
+int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId) {
+ int64_t minFileId = INT64_MAX;
+
+ DIR *dir = opendir(pWal->path);
+ if (dir == NULL) {
+ wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno));
+ return -1;
+ }
+
+ struct dirent *ent;
+ while ((ent = readdir(dir)) != NULL) {
+ char *name = ent->d_name;
+
+ if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) {
+ int64_t id = atoll(name + WAL_PREFIX_LEN);
+ if (id >= curFileId) continue;
+
+ minDiff--;
+ if (id < minFileId) {
+ minFileId = id;
+ }
+ }
+ }
+ closedir(dir);
+
+ if (minFileId == INT64_MAX) return -1;
+ if (minDiff > 0) return -1;
+
+ *oldFileId = minFileId;
+ wTrace("vgId:%d, path:%s, curFileId:%" PRId64 " oldFildId:%" PRId64, pWal->vgId, pWal->path, curFileId, *oldFileId);
+
+ return 0;
+}
+
+int32_t walGetNewFile(SWal *pWal, int64_t *newFileId) {
+ int64_t maxFileId = INT64_MIN;
+
+ DIR *dir = opendir(pWal->path);
+ if (dir == NULL) {
+ wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno));
+ return -1;
+ }
+
+ struct dirent *ent;
+ while ((ent = readdir(dir)) != NULL) {
+ char *name = ent->d_name;
+
+ if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) {
+ int64_t id = atoll(name + WAL_PREFIX_LEN);
+ if (id > maxFileId) {
+ maxFileId = id;
+ }
+ }
+ }
+ closedir(dir);
+
+ if (maxFileId == INT64_MIN) {
+ *newFileId = 0;
+ } else {
+ *newFileId = maxFileId;
+ }
+
+ wTrace("vgId:%d, path:%s, newFileId:%" PRId64, pWal->vgId, pWal->path, *newFileId);
+
+ return 0;
+}
\ No newline at end of file
diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c
new file mode 100644
index 0000000000000000000000000000000000000000..48021eecfc3523466f1e8e878cfb6b5344c8e9c4
--- /dev/null
+++ b/src/wal/src/walWrite.c
@@ -0,0 +1,342 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#define TAOS_RANDOM_FILE_FAIL_TEST
+#include "os.h"
+#include "taoserror.h"
+#include "tchecksum.h"
+#include "twal.h"
+#include "walInt.h"
+
+static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId);
+
+int32_t walRenew(void *handle) {
+ if (handle == NULL) return 0;
+
+ SWal * pWal = handle;
+ int32_t code = 0;
+
+ if (pWal->stop) {
+ wDebug("vgId:%d, do not create a new wal file", pWal->vgId);
+ return 0;
+ }
+
+ pthread_mutex_lock(&pWal->mutex);
+
+ if (pWal->fd >= 0) {
+ tclose(pWal->fd);
+ wDebug("vgId:%d, file:%s, it is closed", pWal->vgId, pWal->name);
+ }
+
+ if (pWal->keep == TAOS_WAL_KEEP) {
+ pWal->fileId = 0;
+ } else {
+ if (walGetNewFile(pWal, &pWal->fileId) != 0) pWal->fileId = 0;
+ pWal->fileId++;
+ }
+
+ snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId);
+ pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
+
+ if (pWal->fd < 0) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno));
+ } else {
+ wDebug("vgId:%d, file:%s, it is created", pWal->vgId, pWal->name);
+ }
+
+ pthread_mutex_unlock(&pWal->mutex);
+
+ return code;
+}
+
+void walRemoveOneOldFile(void *handle) {
+ SWal *pWal = handle;
+ if (pWal == NULL) return;
+ if (pWal->keep == TAOS_WAL_KEEP) return;
+
+ pthread_mutex_lock(&pWal->mutex);
+
+ // remove the oldest wal file
+ int64_t oldFileId = -1;
+ if (walGetOldFile(pWal, pWal->fileId, WAL_FILE_NUM, &oldFileId) == 0) {
+ char walName[WAL_FILE_LEN] = {0};
+ snprintf(walName, sizeof(walName), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, oldFileId);
+
+ if (remove(walName) < 0) {
+ wError("vgId:%d, file:%s, failed to remove since %s", pWal->vgId, walName, strerror(errno));
+ } else {
+ wInfo("vgId:%d, file:%s, it is removed", pWal->vgId, walName);
+ }
+ }
+
+ pthread_mutex_unlock(&pWal->mutex);
+}
+
+void walRemoveAllOldFiles(void *handle) {
+ if (handle == NULL) return;
+
+ SWal * pWal = handle;
+ int64_t fileId = -1;
+ while (walGetNextFile(pWal, &fileId) >= 0) {
+ snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId);
+
+ if (remove(pWal->name) < 0) {
+ wError("vgId:%d, wal:%p file:%s, failed to remove", pWal->vgId, pWal, pWal->name);
+ } else {
+ wInfo("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->name);
+ }
+ }
+}
+
+int32_t walWrite(void *handle, SWalHead *pHead) {
+ if (handle == NULL) return -1;
+
+ SWal * pWal = handle;
+ int32_t code = 0;
+
+ // no wal
+ if (pWal->fd <= 0) return 0;
+ if (pWal->level == TAOS_WAL_NOLOG) return 0;
+ if (pHead->version <= pWal->version) return 0;
+
+ pHead->signature = WAL_SIGNATURE;
+ taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead));
+ int32_t contLen = pHead->len + sizeof(SWalHead);
+
+ pthread_mutex_lock(&pWal->mutex);
+
+ if (taosWrite(pWal->fd, pHead, contLen) != contLen) {
+ code = TAOS_SYSTEM_ERROR(errno);
+ wError("vgId:%d, file:%s, failed to write since %s", pWal->vgId, pWal->name, strerror(errno));
+ } else {
+ wTrace("vgId:%d, write wal, fileId:%" PRId64 " fd:%d hver:%" PRId64 " wver:%" PRIu64 " len:%d", pWal->vgId,
+ pWal->fileId, pWal->fd, pHead->version, pWal->version, pHead->len);
+ pWal->version = pHead->version;
+ }
+
+ pthread_mutex_unlock(&pWal->mutex);
+
+ ASSERT(contLen == pHead->len + sizeof(SWalHead));
+
+ return code;
+}
+
+void walFsync(void *handle, bool forceFsync) {
+ SWal *pWal = handle;
+ if (pWal == NULL || pWal->fd < 0) return;
+
+ if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) {
+ wTrace("vgId:%d, file:%s, do fsync", pWal->vgId, pWal->name);
+ if (fsync(pWal->fd) < 0) {
+ wError("vgId:%d, file:%s, fsync failed since %s", pWal->vgId, pWal->name, strerror(errno));
+ }
+ }
+}
+
+int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) {
+ if (handle == NULL) return -1;
+
+ SWal * pWal = handle;
+ int32_t count = 0;
+ int32_t code = 0;
+ int64_t fileId = -1;
+
+ while ((code = walGetNextFile(pWal, &fileId)) >= 0) {
+ if (fileId == pWal->fileId) continue;
+
+ char walName[WAL_FILE_LEN];
+ snprintf(walName, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId);
+
+ wDebug("vgId:%d, file:%s, will be restored", pWal->vgId, walName);
+ int32_t code = walRestoreWalFile(pWal, pVnode, writeFp, walName, fileId);
+ if (code != TSDB_CODE_SUCCESS) {
+ wError("vgId:%d, file:%s, failed to restore since %s", pWal->vgId, walName, tstrerror(code));
+ continue;
+ }
+
+ wDebug("vgId:%d, file:%s, restore success", pWal->vgId, walName);
+
+ count++;
+ }
+
+ if (pWal->keep != TAOS_WAL_KEEP) return TSDB_CODE_SUCCESS;
+
+ if (count == 0) {
+ wDebug("vgId:%d, wal file not exist, renew it", pWal->vgId);
+ return walRenew(pWal);
+ } else {
+ // open the existing WAL file in append mode
+ pWal->fileId = 0;
+ snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId);
+ pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
+ if (pWal->fd < 0) {
+ wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno));
+ return TAOS_SYSTEM_ERROR(errno);
+ }
+ wDebug("vgId:%d, file:%s open success", pWal->vgId, pWal->name);
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) {
+ if (handle == NULL) return -1;
+ SWal *pWal = handle;
+
+ if (*fileId == 0) *fileId = -1;
+
+ pthread_mutex_lock(&(pWal->mutex));
+
+ int32_t code = walGetNextFile(pWal, fileId);
+ if (code >= 0) {
+ sprintf(fileName, "wal/%s%" PRId64, WAL_PREFIX, *fileId);
+ code = (*fileId == pWal->fileId) ? 0 : 1;
+ }
+
+ wTrace("vgId:%d, get wal file, code:%d curId:%" PRId64 " outId:%" PRId64, pWal->vgId, code, pWal->fileId, *fileId);
+ pthread_mutex_unlock(&(pWal->mutex));
+
+ return code;
+}
+
+static void walFtruncate(SWal *pWal, int32_t fd, int64_t offset) {
+ taosFtruncate(fd, offset);
+ fsync(fd);
+}
+
+static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int32_t fd, int64_t *offset) {
+ int64_t pos = *offset;
+ while (1) {
+ pos++;
+
+ if (lseek(fd, pos, SEEK_SET) < 0) {
+ wError("vgId:%d, failed to seek from corrupted wal file since %s", pWal->vgId, strerror(errno));
+ return TSDB_CODE_WAL_FILE_CORRUPTED;
+ }
+
+ if (taosRead(fd, pHead, sizeof(SWalHead)) <= 0) {
+ wError("vgId:%d, read to end of corrupted wal file, offset:%" PRId64, pWal->vgId, pos);
+ return TSDB_CODE_WAL_FILE_CORRUPTED;
+ }
+
+ if (pHead->signature != WAL_SIGNATURE) {
+ continue;
+ }
+
+ if (taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) {
+ wInfo("vgId:%d, wal head cksum check passed, offset:%" PRId64, pWal->vgId, pos);
+ *offset = pos;
+ return TSDB_CODE_SUCCESS;
+ }
+ }
+
+ return TSDB_CODE_WAL_FILE_CORRUPTED;
+}
+
+static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId) {
+ int32_t size = WAL_MAX_SIZE;
+ void * buffer = tmalloc(size);
+ if (buffer == NULL) {
+ wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno));
+ return TAOS_SYSTEM_ERROR(errno);
+ }
+
+ int32_t fd = open(name, O_RDWR);
+ if (fd < 0) {
+ wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno));
+ tfree(buffer);
+ return TAOS_SYSTEM_ERROR(errno);
+ }
+
+ wDebug("vgId:%d, file:%s, start to restore", pWal->vgId, name);
+
+ int32_t code = TSDB_CODE_SUCCESS;
+ int64_t offset = 0;
+ SWalHead *pHead = buffer;
+
+ while (1) {
+ int32_t ret = taosRead(fd, pHead, sizeof(SWalHead));
+ if (ret == 0) break;
+
+ if (ret < 0) {
+ wError("vgId:%d, file:%s, failed to read wal head since %s", pWal->vgId, name, strerror(errno));
+ code = TAOS_SYSTEM_ERROR(errno);
+ break;
+ }
+
+ if (ret < sizeof(SWalHead)) {
+ wError("vgId:%d, file:%s, failed to read wal head, ret is %d", pWal->vgId, name, ret);
+ walFtruncate(pWal, fd, offset);
+ break;
+ }
+
+ if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) {
+ wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name,
+ pHead->version, pHead->len, offset);
+ code = walSkipCorruptedRecord(pWal, pHead, fd, &offset);
+ if (code != TSDB_CODE_SUCCESS) {
+ walFtruncate(pWal, fd, offset);
+ break;
+ }
+ }
+
+ if (pHead->len > size - sizeof(SWalHead)) {
+ size = sizeof(SWalHead) + pHead->len;
+ buffer = realloc(buffer, size);
+ if (buffer == NULL) {
+ wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno));
+ code = TAOS_SYSTEM_ERROR(errno);
+ break;
+ }
+
+ pHead = buffer;
+ }
+
+ ret = taosRead(fd, pHead->cont, pHead->len);
+ if (ret < 0) {
+ wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno));
+ code = TAOS_SYSTEM_ERROR(errno);
+ break;
+ }
+
+ if (ret < pHead->len) {
+ wError("vgId:%d, file:%s, failed to read wal body, ret:%d len:%d", pWal->vgId, name, ret, pHead->len);
+ offset += sizeof(SWalHead);
+ continue;
+ }
+
+ offset = offset + sizeof(SWalHead) + pHead->len;
+
+ wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId,
+ fileId, pHead->version, pWal->version, pHead->len);
+
+ pWal->version = pHead->version;
+ (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL);
+ }
+
+ tclose(fd);
+ tfree(buffer);
+
+ return code;
+}
+
+uint64_t walGetVersion(twalh param) {
+ SWal *pWal = param;
+ if (pWal == 0) return 0;
+
+ return pWal->version;
+}
diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c
index bbee1347b8f92aa6cfad448fdfb369de8f5a6301..7a473ed18c958afa8be3c5b94b04d2fd548a56fd 100644
--- a/src/wal/test/waltest.c
+++ b/src/wal/test/waltest.c
@@ -23,7 +23,7 @@
int64_t ver = 0;
void *pWal = NULL;
-int writeToQueue(void *pVnode, void *data, int type) {
+int writeToQueue(void *pVnode, void *data, int type, void *pMsg) {
// do nothing
SWalHead *pHead = data;
@@ -37,7 +37,6 @@ int writeToQueue(void *pVnode, void *data, int type) {
int main(int argc, char *argv[]) {
char path[128] = "/home/jhtao/test/wal";
- int max = 3;
int level = 2;
int total = 5;
int rows = 10000;
@@ -47,8 +46,6 @@ int main(int argc, char *argv[]) {
for (int i=1; ijava -version
+java version "1.8.0_131"
+Java(TM) SE Runtime Environment (build 1.8.0_131-b11)
+Java HotSpot(TM) 64-Bit Server VM (build 25.131-b11, mixed mode)
+```
+
+
+(2)安装配置maven
+
+官网下载maven,下载地址:http://maven.apache.org/download.cgi
+
+配置环境变量MAVEN_HOME,将MAVEN_HOME/bin添加到PATH
+
+命令行里查看maven的版本
+
+```shell
+>mvn --version
+Apache Maven 3.5.0 (ff8f5e7444045639af65f6095c62210b5713f426; 2017-04-04T03:39:06+08:00)
+Maven home: D:\apache-maven-3.5.0\bin\..
+Java version: 1.8.0_131, vendor: Oracle Corporation
+Java home: C:\Program Files\Java\jdk1.8.0_131\jre
+Default locale: zh_CN, platform encoding: GBK
+OS name: "windows 10", version: "10.0", arch: "amd64", family: "windows"
+```
+
+为了加快maven下载依赖的速度,可以为maven配置mirror,修改MAVEN_HOME\config\settings.xml文件
+
+```xml
+
+
+ D:\apache-maven-localRepository
+
+
+
+
+ alimaven
+ aliyun maven
+ http://maven.aliyun.com/nexus/content/groups/public/
+ central
+
+
+
+
+
+
+ jdk-1.8
+
+ true
+ 1.8
+
+
+ 1.8
+ 1.8
+ 1.8
+
+
+
+
+```
+
+
+
+(3)在linux服务器上安装TDengine-server
+
+在taosdata官网下载TDengine-server,下载地址:https://www.taosdata.com/cn/all-downloads/
+
+在linux服务器上安装TDengine-server
+
+```shell
+# tar -zxvf package/TDengine-server-2.0.1.1-Linux-x64.tar.gz
+# cd TDengine-server/
+# ./install.sh
+```
+
+启动taosd
+
+```shell
+# systemctl start taosd
+```
+
+在server上用taos连接taosd
+
+```shell
+# taos
+taos> show dnodes;
+ id | end_point | vnodes | cores | status | role | create_time |
+==================================================================================================================
+ 1 | td01:6030 | 2 | 4 | ready | any | 2020-08-19 18:40:25.045 |
+Query OK, 1 row(s) in set (0.005765s)
+```
+
+如果可以正确连接到taosd实例,并打印出databases的信息,说明TDengine的server已经正确启动。这里查看server的hostname
+
+```shell
+# hostname -f
+td01
+```
+
+注意,如果安装TDengine后,使用默认的taos.cfg配置文件,taosd会使用当前server的hostname创建dnode实例。之后,在client也需要使用这个hostname来连接taosd。
+
+
+
+(4)在windows上安装TDengine-client
+
+在taosdata官网下载taos客户端,下载地址:
+https://www.taosdata.com/cn/all-downloads/
+下载后,双击exe安装。
+
+修改client的hosts文件(C:\Windows\System32\drivers\etc\hosts),将server的hostname和ip配置到client的hosts文件中
+
+```
+192.168.236.136 td01
+```
+
+配置完成后,在命令行内使用taos shell连接server端
+
+```shell
+C:\TDengine>taos
+Welcome to the TDengine shell from Linux, Client Version:2.0.1.1
+Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.
+
+taos> show databases;
+ name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | precision | status |
+===================================================================================================================================================================================================================================================================
+ test | 2020-08-19 18:43:50.731 | 1 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready |
+ log | 2020-08-19 18:40:28.064 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready |
+Query OK, 2 row(s) in set (0.068000s)
+```
+
+如果windows上的client能够正常连接,并打印database信息,说明client可以正常连接server了。
+
+
+
+## 应用开发
+
+(1)新建maven工程,在pom.xml中引入taos-jdbcdriver依赖。
+
+```xml
+
+
+ 4.0.0
+
+ com.taosdata.demo
+ JdbcDemo
+ 1.0-SNAPSHOT
+
+
+
+ com.taosdata.jdbc
+ taos-jdbcdriver
+ 2.0.8
+
+
+
+```
+
+(2)使用jdbc查询TDengine数据库
+
+下面是示例代码:
+
+```java
+public class JdbcDemo {
+
+ public static void main(String[] args) throws Exception {
+ Connection conn = getConn();
+ Statement stmt = conn.createStatement();
+ // create database
+ stmt.executeUpdate("create database if not exists db");
+ // use database
+ stmt.executeUpdate("use db");
+ // create table
+ stmt.executeUpdate("create table if not exists tb (ts timestamp, temperature int, humidity float)");
+ // insert data
+ int affectedRows = stmt.executeUpdate("insert into tb values(now, 23, 10.3) (now + 1s, 20, 9.3)");
+ System.out.println("insert " + affectedRows + " rows.");
+ // query data
+ ResultSet resultSet = stmt.executeQuery("select * from tb");
+ Timestamp ts = null;
+ int temperature = 0;
+ float humidity = 0;
+ while(resultSet.next()){
+ ts = resultSet.getTimestamp(1);
+ temperature = resultSet.getInt(2);
+ humidity = resultSet.getFloat("humidity");
+ System.out.printf("%s, %d, %s\n", ts, temperature, humidity);
+ }
+ }
+
+ public static Connection getConn() throws Exception{
+ Class.forName("com.taosdata.jdbc.TSDBDriver");
+ String jdbcUrl = "jdbc:TAOS://td01:0/log?user=root&password=taosdata";
+ Properties connProps = new Properties();
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
+ connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
+ Connection conn = DriverManager.getConnection(jdbcUrl, connProps);
+ return conn;
+ }
+
+}
+```
+
+(3)测试jdbc访问tdengine的sever实例
+
+console输出:
+
+```
+insert 2 rows.
+2020-08-26 00:06:34.575, 23, 10.3
+2020-08-26 00:06:35.575, 20, 9.3
+```
+
+
+
+## 指南
+
+(1)如何设置主机名和hosts
+
+在server上查看hostname和fqdn
+```shell
+查看hostname
+# hostname
+taos-server
+
+查看fqdn
+# hostname -f
+taos-server
+```
+
+windows下hosts文件位于:
+C:\\Windows\System32\drivers\etc\hosts
+修改hosts文件,添加server的ip和hostname
+
+```s
+192.168.56.101 node5
+```
+
+(2)什么是fqdn?
+
+
+> 什么是FQDN?
+>
+> FQDN(Full qualified domain name)全限定域名,fqdn由2部分组成:hostname+domainname。
+>
+> 例如,一个邮件服务器的fqdn可能是:mymail.somecollege.edu,其中mymail是hostname(主机名),somcollege.edu是domainname(域名)。本例中,.edu是顶级域名,.somecollege是二级域名。
+>
+> 当连接服务器时,必须指定fqdn,然后,dns服务器通过查看dns表,将hostname解析为相应的ip地址。如果只指定hostname(不指定domainname),应用程序可能服务解析主机名。因为如果你试图访问不在本地的远程服务器时,本地的dns服务器和可能没有远程服务器的hostname列表。
+>
+> 参考:https://kb.iu.edu/d/aiuv
diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c
index 759e16d1de285c9490d7f1af7a94de81dbadb6e4..be60a88ad70721bd6281d9ca8f1d73263a788532 100644
--- a/tests/examples/c/apitest.c
+++ b/tests/examples/c/apitest.c
@@ -9,26 +9,40 @@
static void prepare_data(TAOS* taos) {
- taos_query(taos, "drop database if exists test;");
+ TAOS_RES *result;
+ result = taos_query(taos, "drop database if exists test;");
+ taos_free_result(result);
usleep(100000);
- taos_query(taos, "create database test;");
+ result = taos_query(taos, "create database test;");
+ taos_free_result(result);
usleep(100000);
taos_select_db(taos, "test");
- taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);");
+ result = taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);");
+ taos_free_result(result);
- taos_query(taos, "create table t0 using meters tags(0);");
- taos_query(taos, "create table t1 using meters tags(1);");
- taos_query(taos, "create table t2 using meters tags(2);");
- taos_query(taos, "create table t3 using meters tags(3);");
- taos_query(taos, "create table t4 using meters tags(4);");
- taos_query(taos, "create table t5 using meters tags(5);");
- taos_query(taos, "create table t6 using meters tags(6);");
- taos_query(taos, "create table t7 using meters tags(7);");
- taos_query(taos, "create table t8 using meters tags(8);");
- taos_query(taos, "create table t9 using meters tags(9);");
+ result = taos_query(taos, "create table t0 using meters tags(0);");
+ taos_free_result(result);
+ result = taos_query(taos, "create table t1 using meters tags(1);");
+ taos_free_result(result);
+ result = taos_query(taos, "create table t2 using meters tags(2);");
+ taos_free_result(result);
+ result = taos_query(taos, "create table t3 using meters tags(3);");
+ taos_free_result(result);
+ result = taos_query(taos, "create table t4 using meters tags(4);");
+ taos_free_result(result);
+ result = taos_query(taos, "create table t5 using meters tags(5);");
+ taos_free_result(result);
+ result = taos_query(taos, "create table t6 using meters tags(6);");
+ taos_free_result(result);
+ result = taos_query(taos, "create table t7 using meters tags(7);");
+ taos_free_result(result);
+ result = taos_query(taos, "create table t8 using meters tags(8);");
+ taos_free_result(result);
+ result = taos_query(taos, "create table t9 using meters tags(9);");
+ taos_free_result(result);
- TAOS_RES* res = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)"
+ result = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)"
" ('2020-01-01 00:01:00.000', 0)"
" ('2020-01-01 00:02:00.000', 0)"
" t1 values('2020-01-01 00:00:00.000', 0)"
@@ -46,10 +60,11 @@ static void prepare_data(TAOS* taos) {
" t7 values('2020-01-01 00:01:02.000', 0)"
" t8 values('2020-01-01 00:01:02.000', 0)"
" t9 values('2020-01-01 00:01:02.000', 0)");
- int affected = taos_affected_rows(res);
+ int affected = taos_affected_rows(result);
if (affected != 18) {
printf("\033[31m%d rows affected by last insert statement, but it should be 18\033[0m\n", affected);
}
+ taos_free_result(result);
// super tables subscription
usleep(1000000);
}
@@ -135,6 +150,7 @@ static void verify_query(TAOS* taos) {
res = taos_query(taos, "select * from meters");
taos_stop_query(res);
+ taos_free_result(res);
}
@@ -153,23 +169,30 @@ static void verify_subscribe(TAOS* taos) {
res = taos_consume(tsub);
check_row_count(__LINE__, res, 0);
- taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);");
- taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);");
+ TAOS_RES *result;
+ result = taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);");
+ taos_free_result(result);
+ result = taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);");
+ taos_free_result(result);
res = taos_consume(tsub);
check_row_count(__LINE__, res, 2);
- taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);");
- taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);");
+ result = taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);");
+ taos_free_result(result);
+ result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);");
+ taos_free_result(result);
res = taos_consume(tsub);
check_row_count(__LINE__, res, 2);
- taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);");
+ result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);");
+ taos_free_result(result);
res = taos_consume(tsub);
check_row_count(__LINE__, res, 1);
// keep progress information and restart subscription
taos_unsubscribe(tsub, 1);
- taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);");
+ result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);");
+ taos_free_result(result);
tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0);
res = taos_consume(tsub);
check_row_count(__LINE__, res, 24);
@@ -196,7 +219,8 @@ static void verify_subscribe(TAOS* taos) {
res = taos_consume(tsub);
check_row_count(__LINE__, res, 0);
- taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);");
+ result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);");
+ taos_free_result(result);
res = taos_consume(tsub);
check_row_count(__LINE__, res, 1);
@@ -205,7 +229,8 @@ static void verify_subscribe(TAOS* taos) {
int blockFetch = 0;
tsub = taos_subscribe(taos, 1, "test", "select * from meters;", subscribe_callback, &blockFetch, 1000);
usleep(2000000);
- taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);");
+ result = taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);");
+ taos_free_result(result);
usleep(2000000);
taos_unsubscribe(tsub, 0);
}
@@ -213,8 +238,9 @@ static void verify_subscribe(TAOS* taos) {
void verify_prepare(TAOS* taos) {
TAOS_RES* result = taos_query(taos, "drop database if exists test;");
+ taos_free_result(result);
usleep(100000);
- taos_query(taos, "create database test;");
+ result = taos_query(taos, "create database test;");
int code = taos_errno(result);
if (code != 0) {
@@ -429,7 +455,8 @@ void verify_stream(TAOS* taos) {
NULL);
printf("waiting for stream data\n");
usleep(100000);
- taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);");
+ TAOS_RES* result = taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);");
+ taos_free_result(result);
usleep(200000000);
taos_close_stream(strm);
}
diff --git a/tests/examples/c/asyncdemo.c b/tests/examples/c/asyncdemo.c
index 1e523bd7fec9ba9fd90d1a71949f40c7be71384d..225c4f7541ec7cd7fb4784b8de94d8b47c4e3e36 100644
--- a/tests/examples/c/asyncdemo.c
+++ b/tests/examples/c/asyncdemo.c
@@ -46,6 +46,34 @@ void taos_insert_call_back(void *param, TAOS_RES *tres, int code);
void taos_select_call_back(void *param, TAOS_RES *tres, int code);
void taos_error(TAOS *taos);
+static void queryDB(TAOS *taos, char *command) {
+ int i;
+ TAOS_RES *pSql = NULL;
+ int32_t code = -1;
+
+ for (i = 0; i < 5; i++) {
+ if (NULL != pSql) {
+ taos_free_result(pSql);
+ pSql = NULL;
+ }
+
+ pSql = taos_query(taos, command);
+ code = taos_errno(pSql);
+ if (0 == code) {
+ break;
+ }
+ }
+
+ if (code != 0) {
+ fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
+ taos_free_result(pSql);
+ taos_close(taos);
+ exit(EXIT_FAILURE);
+ }
+
+ taos_free_result(pSql);
+}
+
int main(int argc, char *argv[])
{
TAOS *taos;
@@ -78,16 +106,14 @@ int main(int argc, char *argv[])
printf("success to connect to server\n");
- sprintf(sql, "drop database %s", db);
- taos_query(taos, sql);
+ sprintf(sql, "drop database if exists %s", db);
+ queryDB(taos, sql);
sprintf(sql, "create database %s", db);
- if (taos_query(taos, sql) != 0)
- taos_error(taos);
+ queryDB(taos, sql);
sprintf(sql, "use %s", db);
- if (taos_query(taos, sql) != 0)
- taos_error(taos);
+ queryDB(taos, sql);
strcpy(prefix, "asytbl_");
for (i = 0; i < numOfTables; ++i) {
@@ -95,8 +121,7 @@ int main(int argc, char *argv[])
tableList[i].taos = taos;
sprintf(tableList[i].name, "%s%d", prefix, i);
sprintf(sql, "create table %s%d (ts timestamp, volume bigint)", prefix, i);
- if (taos_query(taos, sql) != 0)
- taos_error(taos);
+ queryDB(taos, sql);
}
gettimeofday(&systemTime, NULL);
diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c
index 8f8a66a32593bc25d71b554808719ca42f5b32ac..74a49288e9a1aa7081db45925bf52d6516e4801a 100644
--- a/tests/examples/c/demo.c
+++ b/tests/examples/c/demo.c
@@ -22,10 +22,38 @@
#include
#include // TAOS header file
+static void queryDB(TAOS *taos, char *command) {
+ int i;
+ TAOS_RES *pSql = NULL;
+ int32_t code = -1;
+
+ for (i = 0; i < 5; i++) {
+ if (NULL != pSql) {
+ taos_free_result(pSql);
+ pSql = NULL;
+ }
+
+ pSql = taos_query(taos, command);
+ code = taos_errno(pSql);
+ if (0 == code) {
+ break;
+ }
+ }
+
+ if (code != 0) {
+ fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
+ taos_free_result(pSql);
+ taos_close(taos);
+ exit(EXIT_FAILURE);
+ }
+
+ taos_free_result(pSql);
+}
+
+void Test(char *qstr, const char *input, int i);
+
int main(int argc, char *argv[]) {
- TAOS * taos;
char qstr[1024];
- TAOS_RES *result;
// connect to server
if (argc < 2) {
@@ -35,37 +63,26 @@ int main(int argc, char *argv[]) {
// init TAOS
taos_init();
-
- taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
+ for (int i = 0; i < 4000000; i++) {
+ Test(qstr, argv[1], i);
+ }
+ taos_cleanup();
+}
+void Test(char *qstr, const char *input, int index) {
+ TAOS *taos = taos_connect(input, "root", "taosdata", NULL, 0);
+ printf("==================test at %d\n================================", index);
+ queryDB(taos, "drop database if exists demo");
+ queryDB(taos, "create database demo");
+ TAOS_RES *result;
if (taos == NULL) {
printf("failed to connect to server, reason:%s\n", "null taos"/*taos_errstr(taos)*/);
exit(1);
}
- printf("success to connect to server\n");
-
-
- taos_query(taos, "drop database demo");
-
- result = taos_query(taos, "create database demo");
- if (result == NULL) {
- printf("failed to create database, reason:%s\n", "null result"/*taos_errstr(taos)*/);
- exit(1);
- }
- printf("success to create database\n");
-
- taos_query(taos, "use demo");
+ queryDB(taos, "use demo");
- // create table
- if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) {
- printf("failed to create table, reason:%s\n", taos_errstr(result));
- exit(1);
- }
+ queryDB(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))");
printf("success to create table\n");
- // sleep for one second to make sure table is created on data node
- // taosMsleep(1000);
-
- // insert 10 records
int i = 0;
for (i = 0; i < 10; ++i) {
sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000 + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello");
@@ -80,10 +97,11 @@ int main(int argc, char *argv[]) {
printf("insert row: %i\n", i);
} else {
printf("failed to insert row: %i, reason:%s\n", i, "null result"/*taos_errstr(result)*/);
+ taos_free_result(result);
exit(1);
}
+ taos_free_result(result);
- //sleep(1);
}
printf("success to insert rows, total %d rows\n", i);
@@ -91,7 +109,8 @@ int main(int argc, char *argv[]) {
sprintf(qstr, "SELECT * FROM m1");
result = taos_query(taos, qstr);
if (result == NULL || taos_errno(result) != 0) {
- printf("failed to select, reason:%s\n", taos_errstr(result));
+ printf("failed to select, reason:%s\n", taos_errstr(result));
+ taos_free_result(result);
exit(1);
}
@@ -112,5 +131,6 @@ int main(int argc, char *argv[]) {
taos_free_result(result);
printf("====demo end====\n\n");
- return getchar();
+ taos_close(taos);
}
+
diff --git a/tests/examples/go/taosdemo.go b/tests/examples/go/taosdemo.go
index b42e1e6d703a96bb86454f177a7207577c6d4d4c..2c3a7d09b68d84feea1ae2771b90643dbbfbc063 100644
--- a/tests/examples/go/taosdemo.go
+++ b/tests/examples/go/taosdemo.go
@@ -87,7 +87,7 @@ func init() {
func printAllArgs() {
fmt.Printf("\n============= args parse result: =============\n")
- fmt.Printf("dbName: %v\n", configPara.hostName)
+ fmt.Printf("hostName: %v\n", configPara.hostName)
fmt.Printf("serverPort: %v\n", configPara.serverPort)
fmt.Printf("usr: %v\n", configPara.user)
fmt.Printf("password: %v\n", configPara.password)
@@ -107,7 +107,7 @@ func main() {
fmt.Scanln()
url = "root:taosdata@/tcp(" + configPara.hostName + ":" + strconv.Itoa(configPara.serverPort) + ")/"
- //url = fmt.Sprintf("%s:%s@/tcp(%s:%d)/%s?interpolateParams=true", configPara.user, configPara.password, configPara.hostName, configPara.serverPort, configPara.dbName)
+ //url = fmt.Sprintf("%s:%s@/tcp(%s:%d)/%s?interpolateParams=true", configPara.user, configPara.password, configPara.hostName, configPara.serverPort, configPara.dbName)
// open connect to taos server
//db, err := sql.Open(taosDriverName, url)
//if err != nil {
@@ -115,6 +115,7 @@ func main() {
// os.Exit(1)
//}
//defer db.Close()
+ rand.Seed(time.Now().Unix())
createDatabase(configPara.dbName, configPara.supTblName)
fmt.Printf("======== create database success! ========\n\n")
diff --git a/tests/examples/nodejs/README-win.md b/tests/examples/nodejs/README-win.md
new file mode 100644
index 0000000000000000000000000000000000000000..75fec69413af2bb49498118ec7235c9947e2f89e
--- /dev/null
+++ b/tests/examples/nodejs/README-win.md
@@ -0,0 +1,200 @@
+# 如何在windows上使用nodejs进行TDengine应用开发
+
+## 环境准备
+
+(1)安装nodejs-10.22.0
+
+下载链接:https://nodejs.org/dist/v10.22.0/node-v10.22.0-win-x64.zip
+解压安装,把node配置到环境变量里
+
+cmd启动命令行,查看node的版本
+
+```shell
+> node.exe --version
+v10.22.0
+
+> npm --version
+6.14.6
+```
+
+
+
+(2)安装python2.7
+
+下载链接:https://www.python.org/ftp/python/2.7.18/python-2.7.18.amd64.msi
+
+查看python版本
+
+```shell
+>python --version
+Python 2.7.18
+```
+
+
+(3)安装TDengine-client
+
+下载地址:https://www.taosdata.com/cn/all-downloads/,选择一个合适的windows-client下载(client应该尽量与server端的版本保持一致)
+
+使用client的taos shell连接server
+
+```shell
+>taos -h node5
+
+Welcome to the TDengine shell from Linux, Client Version:2.0.6.0
+Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.
+
+taos> show dnodes;
+ id | end_point | vnodes | cores | status | role | create_time | offline reason |
+============================================================================================================================================
+ 1 | node5:6030 | 7 | 1 | ready | any | 2020-10-26 09:45:26.308 | |
+Query OK, 1 row(s) in set (0.036000s)
+```
+
+注意:
+* 检查能否在client的机器上ping通server的fqdn
+* 如果你的dns server并没有提供到server的域名解析,可以将server的hostname配置到client的hosts文件中
+
+
+## 应用开发
+
+(1)建立nodejs项目
+
+```
+npm init
+```
+
+(2)安装windows-build-tools
+```
+npm install --global --production windows-build-tools
+```
+
+(3)安装td2.0-connector驱动
+
+``` tdshell
+npm install td2.0-connector
+```
+
+(4)nodejs访问tdengine的示例程序
+
+```javascript
+const taos = require('td2.0-connector');
+
+var host = null;
+var port = 6030;
+for (var i = 2; i < global.process.argv.length; i++) {
+ var key = global.process.argv[i].split("=")[0];
+ var value = global.process.argv[i].split("=")[1];
+
+ if ("host" == key) {
+ host = value;
+ }
+ if ("port" == key) {
+ port = value;
+ }
+}
+
+if (host == null) {
+ console.log("Usage: node nodejsChecker.js host= port=");
+ process.exit(0);
+}
+
+// establish connection
+var conn = taos.connect({host: host, user: "root", password: "taosdata", port: port});
+var cursor = conn.cursor();
+// create database
+executeSql("create database if not exists testnodejs", 0);
+// use db
+executeSql("use testnodejs", 0);
+// drop table
+executeSql("drop table if exists testnodejs.weather", 0);
+// create table
+executeSql("create table if not exists testnodejs.weather(ts timestamp, temperature float, humidity int)", 0);
+// insert
+executeSql("insert into testnodejs.weather (ts, temperature, humidity) values(now, 20.5, 34)", 1);
+// select
+executeQuery("select * from testnodejs.weather");
+// close connection
+conn.close();
+
+function executeQuery(sql) {
+ var start = new Date().getTime();
+ var promise = cursor.query(sql, true);
+ var end = new Date().getTime();
+ promise.then(function (result) {
+ printSql(sql, result != null, (end - start));
+ result.pretty();
+ });
+}
+
+function executeSql(sql, affectRows) {
+ var start = new Date().getTime();
+ var promise = cursor.execute(sql);
+ var end = new Date().getTime();
+ printSql(sql, promise == affectRows, (end - start));
+}
+
+function printSql(sql, succeed, cost) {
+ console.log("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql);
+}
+```
+
+(5)测试nodejs程序
+
+```shell
+>node nodejsChecker.js
+Usage: node nodejsChecker.js host= port=
+# 提示指定host
+
+>node nodejsChecker.js host=node5
+Successfully connected to TDengine
+Query OK, 0 row(s) affected (0.00997610s)
+[ OK ] time cost: 14 ms, execute statement ====> create database if not exists testnodejs
+Query OK, 0 row(s) affected (0.00235920s)
+[ OK ] time cost: 4 ms, execute statement ====> use testnodejs
+Query OK, 0 row(s) affected (0.06604280s)
+[ OK ] time cost: 67 ms, execute statement ====> drop table if exists testnodejs.weather
+Query OK, 0 row(s) affected (0.59403290s)
+[ OK ] time cost: 595 ms, execute statement ====> create table if not exists testnodejs.weather(ts timestamp, temperature float, humidity int)
+Query OK, 1 row(s) affected (0.01058950s)
+[ OK ] time cost: 12 ms, execute statement ====> insert into testnodejs.weather (ts, temperature, humidity) values(now, 20.5, 34)
+Query OK, 1 row(s) in set (0.00401490s)
+[ OK ] time cost: 10 ms, execute statement ====> select * from testnodejs.weather
+Connection is closed
+
+ ts | temperature | humidity |
+=====================================================================
+2020-10-27 18:49:15.547 | 20.5 | 34 |
+```
+
+## 指南
+
+### 如何设置主机名和hosts
+
+在server上查看hostname和fqdn
+```shell
+查看hostname
+# hostname
+taos-server
+
+查看fqdn
+# hostname -f
+taos-server
+```
+
+windows下hosts文件位于:
+C:\\Windows\System32\drivers\etc\hosts
+修改hosts文件,添加server的ip和hostname
+
+```
+192.168.56.101 node5
+```
+
+> 什么是FQDN?
+>
+> FQDN(Full qualified domain name)全限定域名,fqdn由2部分组成:hostname+domainname。
+>
+> 例如,一个邮件服务器的fqdn可能是:mymail.somecollege.edu,其中mymail是hostname(主机名),somcollege.edu是domainname(域名)。本例中,.edu是顶级域名,.somecollege是二级域名。
+>
+> 当连接服务器时,必须指定fqdn,然后,dns服务器通过查看dns表,将hostname解析为相应的ip地址。如果只指定hostname(不指定domainname),应用程序可能服务解析主机名。因为如果你试图访问不在本地的远程服务器时,本地的dns服务器和可能没有远程服务器的hostname列表。
+>
+> 参考:https://kb.iu.edu/d/aiuv
diff --git a/tests/examples/nodejs/nodejsChecker.js b/tests/examples/nodejs/nodejsChecker.js
index c77944f75243a50e6e2c738e659cb4e64f3e5574..f838d5cc8465dba70b5372a5d7720a8cff69544a 100644
--- a/tests/examples/nodejs/nodejsChecker.js
+++ b/tests/examples/nodejs/nodejsChecker.js
@@ -42,8 +42,8 @@ function executeQuery(sql){
var start = new Date().getTime();
var promise = cursor.query(sql, true);
var end = new Date().getTime();
- printSql(sql, promise != null,(end - start));
promise.then(function(result){
+ printSql(sql, result != null,(end - start));
result.pretty();
});
}
diff --git a/tests/gotest/batchtest.bat b/tests/gotest/batchtest.bat
old mode 100644
new mode 100755
index abe9a58f319068d5e11017abcd721a4c54d6aca9..efd8961bb0be2eb6f20e291114b92b00469b984f
--- a/tests/gotest/batchtest.bat
+++ b/tests/gotest/batchtest.bat
@@ -7,6 +7,9 @@ set serverPort=%2
if "%severIp%"=="" (set severIp=127.0.0.1)
if "%serverPort%"=="" (set serverPort=6030)
+go env -w GO111MODULE=on
+go env -w GOPROXY=https://goproxy.io,direct
+
cd case001
case001.bat %severIp% %serverPort%
diff --git a/tests/gotest/batchtest.sh b/tests/gotest/batchtest.sh
old mode 100644
new mode 100755
index e8ed9ecbed9f70c98e6b5db052c3e69082c1794d..0fbbf40714b3349651beea9302e66628b31a22ac
--- a/tests/gotest/batchtest.sh
+++ b/tests/gotest/batchtest.sh
@@ -13,6 +13,9 @@ if [ ! -n "$serverPort" ]; then
serverPort=6030
fi
+go env -w GO111MODULE=on
+go env -w GOPROXY=https://goproxy.io,direct
+
bash ./case001/case001.sh $severIp $serverPort
#bash ./case002/case002.sh $severIp $serverPort
#bash ./case003/case003.sh $severIp $serverPort
diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh
new file mode 100755
index 0000000000000000000000000000000000000000..bb5d9e0a9d16ccb232feeaea00a72a0972b31cc2
--- /dev/null
+++ b/tests/perftest-scripts/perftest-query.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+today=`date +"%Y%m%d"`
+WORK_DIR=/home/ubuntu/pxiao/
+PERFORMANCE_TEST_REPORT=$TDENGINE_DIR/tests/performance-test-report-$today.log
+
+# Coloured Echoes #
+function red_echo { echo -e "\033[31m$@\033[0m"; } #
+function green_echo { echo -e "\033[32m$@\033[0m"; } #
+function yellow_echo { echo -e "\033[33m$@\033[0m"; } #
+function white_echo { echo -e "\033[1;37m$@\033[0m"; } #
+# Coloured Printfs #
+function red_printf { printf "\033[31m$@\033[0m"; } #
+function green_printf { printf "\033[32m$@\033[0m"; } #
+function yellow_printf { printf "\033[33m$@\033[0m"; } #
+function white_printf { printf "\033[1;37m$@\033[0m"; } #
+# Debugging Outputs #
+function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } #
+function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } #
+function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } #
+function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } #
+
+
+function stopTaosd {
+ echo "Stop taosd"
+ systemctl stop taosd
+ snap stop tdengine
+ PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+ while [ -n "$PID" ]
+ do
+ pkill -TERM -x taosd
+ sleep 1
+ PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+ done
+}
+
+function buildTDengine {
+ echoInfo "Build TDengine"
+ cd $WORK_DIR/TDengine
+
+ git remote update > /dev/null
+ REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+ LOCAL_COMMIT=`git rev-parse --short @`
+
+ echo " LOCAL: $LOCAL_COMMIT"
+ echo "REMOTE: $REMOTE_COMMIT"
+ if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+ echo "repo up-to-date"
+ else
+ echo "repo need to pull"
+ git pull > /dev/null
+
+ LOCAL_COMMIT=`git rev-parse --short @`
+ cd debug
+ rm -rf *
+ cmake .. > /dev/null
+ make > /dev/null
+ make install
+ fi
+}
+
+function runQueryPerfTest {
+ nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taodperf/ > /dev/null 2>&1 &
+ echoInfo "Run Performance Test"
+ cd $WORK_DIR/TDengine/tests/pytest
+
+ python3 query/queryPerformance.py | tee -a $PERFORMANCE_TEST_REPORT
+}
+
+
+function sendReport {
+ echo "send report"
+ receiver="pxiao@taosdata.com"
+ mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
+
+ cd $TDENGINE_DIR
+
+ sed -i 's/\x1b\[[0-9;]*m//g' $PERFORMANCE_TEST_REPORT
+ BODY_CONTENT=`cat $PERFORMANCE_TEST_REPORT`
+ echo -e "to: ${receiver}\nsubject: Query Performace Report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+ (cat - && uuencode $PERFORMANCE_TEST_REPORT performance-test-report-$today.log) | \
+ ssmtp "${receiver}" && echo "Report Sent!"
+}
+
+
+stopTaosd
+buildTDengine
+runQueryPerfTest
+
+echoInfo "Send Report"
+sendReport
+echoInfo "End of Test"
diff --git a/tests/pytest/alter/db_update_options.py b/tests/pytest/alter/db_update_options.py
new file mode 100644
index 0000000000000000000000000000000000000000..224e0f25b074deed55802b9ba847f7f845716a23
--- /dev/null
+++ b/tests/pytest/alter/db_update_options.py
@@ -0,0 +1,71 @@
+
+# -*- coding: utf-8 -*-
+
+import random
+import string
+import subprocess
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+ def run(self):
+ tdLog.debug("check database")
+ tdSql.prepare()
+
+ # check default update value
+ sql = "create database if not exists db"
+ tdSql.execute(sql)
+ tdSql.query('show databases')
+ tdSql.checkRows(1)
+ tdSql.checkData(0,16,0)
+
+ sql = "alter database db update 1"
+
+ # check update value
+ tdSql.execute(sql)
+ tdSql.query('show databases')
+ tdSql.checkRows(1)
+ tdSql.checkData(0,16,1)
+
+
+ sql = "alter database db update 0"
+ tdSql.execute(sql)
+ tdSql.query('show databases')
+ tdSql.checkRows(1)
+ tdSql.checkData(0,16,0)
+
+ sql = "alter database db update -1"
+ tdSql.error(sql)
+
+ sql = "alter database db update 100"
+ tdSql.error(sql)
+
+ tdSql.query('show databases')
+ tdSql.checkRows(1)
+ tdSql.checkData(0,16,0)
+
+ tdSql.execute('drop database db')
+ tdSql.error('create database db update 100')
+ tdSql.error('create database db update -1')
+
+ tdSql.execute('create database db update 1')
+
+ tdSql.query('show databases')
+ tdSql.checkRows(1)
+ tdSql.checkData(0,16,1)
+
+ tdSql.execute('drop database db')
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/cluster/bananceTest.py b/tests/pytest/cluster/bananceTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..ef25afa7d2f7ea3b5358f8ba74d6702d28d54c85
--- /dev/null
+++ b/tests/pytest/cluster/bananceTest.py
@@ -0,0 +1,57 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from clusterSetup import *
+from util.sql import tdSql
+from util.log import tdLog
+import random
+import time
+
+class ClusterTestcase:
+
+ ## test case 32 ##
+ def run(self):
+
+ nodes = Nodes()
+ nodes.addConfigs("maxVgroupsPerDb", "10")
+ nodes.addConfigs("maxTablesPerVnode", "1000")
+ nodes.restartAllTaosd()
+
+ ctest = ClusterTest(nodes.node1.hostName)
+ ctest.connectDB()
+ ctest.createSTable(1)
+ ctest.run()
+ tdSql.init(ctest.conn.cursor(), False)
+
+ tdSql.execute("use %s" % ctest.dbName)
+ tdSql.query("show vgroups")
+ dnodes = []
+ for i in range(10):
+ dnodes.append(int(tdSql.getData(i, 4)))
+
+ s = set(dnodes)
+ if len(s) < 3:
+ tdLog.exit("cluster is not balanced")
+
+ tdLog.info("cluster is balanced")
+
+ nodes.removeConfigs("maxVgroupsPerDb", "10")
+ nodes.removeConfigs("maxTablesPerVnode", "1000")
+ nodes.restartAllTaosd()
+
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+ct = ClusterTestcase()
+ct.run()
diff --git a/tests/pytest/cluster/basicTest.py b/tests/pytest/cluster/basicTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..b990d7fd982a490383939707a32635d37e546b13
--- /dev/null
+++ b/tests/pytest/cluster/basicTest.py
@@ -0,0 +1,47 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from clusterSetup import *
+from util.sql import tdSql
+from util.log import tdLog
+import random
+
+class ClusterTestcase:
+
+ ## test case 1, 33 ##
+ def run(self):
+
+ nodes = Nodes()
+ ctest = ClusterTest(nodes.node1.hostName)
+
+ ctest.connectDB()
+ tdSql.init(ctest.conn.cursor(), False)
+
+ ## Test case 1 ##
+ tdLog.info("Test case 1 repeat %d times" % ctest.repeat)
+ for i in range(ctest.repeat):
+ tdLog.info("Start Round %d" % (i + 1))
+ replica = random.randint(1,3)
+ ctest.createSTable(replica)
+ ctest.run()
+ tdLog.sleep(10)
+ tdSql.query("select count(*) from %s.%s" %(ctest.dbName, ctest.stbName))
+ tdSql.checkData(0, 0, ctest.numberOfRecords * ctest.numberOfTables)
+ tdLog.info("Round %d completed" % (i + 1))
+
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+ct = ClusterTestcase()
+ct.run()
\ No newline at end of file
diff --git a/tests/pytest/cluster/changeReplicaTest.py b/tests/pytest/cluster/changeReplicaTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fa68edbfee2db599076befdf9bed5f4b4be3c83
--- /dev/null
+++ b/tests/pytest/cluster/changeReplicaTest.py
@@ -0,0 +1,51 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from clusterSetup import *
+from util.sql import tdSql
+from util.log import tdLog
+import random
+
+class ClusterTestcase:
+
+ ## test case 7, ##
+ def run(self):
+
+ nodes = Nodes()
+ ctest = ClusterTest(nodes.node1.hostName)
+ ctest.connectDB()
+ tdSql.init(ctest.conn.cursor(), False)
+
+ tdSql.execute("use %s" % ctest.dbName)
+ tdSql.query("show vgroups")
+ for i in range(10):
+ tdSql.checkData(i, 5, "master")
+
+ tdSql.execute("alter database %s replica 2" % ctest.dbName)
+ tdLog.sleep(30)
+ tdSql.query("show vgroups")
+ for i in range(10):
+ tdSql.checkData(i, 5, "master")
+ tdSql.checkData(i, 7, "slave")
+
+ tdSql.execute("alter database %s replica 3" % ctest.dbName)
+ tdLog.sleep(30)
+ tdSql.query("show vgroups")
+ for i in range(10):
+ tdSql.checkData(i, 5, "master")
+ tdSql.checkData(i, 7, "slave")
+ tdSql.checkData(i, 9, "slave")
+
+ct = ClusterTestcase()
+ct.run()
\ No newline at end of file
diff --git a/tests/pytest/cluster/clusterSetup.py b/tests/pytest/cluster/clusterSetup.py
new file mode 100644
index 0000000000000000000000000000000000000000..36af8ac42e56e1b8a7ab2237305a6bf286103552
--- /dev/null
+++ b/tests/pytest/cluster/clusterSetup.py
@@ -0,0 +1,202 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+sys.path.insert(0, os.getcwd())
+from fabric import Connection
+from util.sql import *
+from util.log import *
+import taos
+import random
+import threading
+import logging
+
+class Node:
+ def __init__(self, index, username, hostIP, hostName, password, homeDir):
+ self.index = index
+ self.username = username
+ self.hostIP = hostIP
+ self.hostName = hostName
+ self.homeDir = homeDir
+ self.conn = Connection("{}@{}".format(username, hostName), connect_kwargs={"password": "{}".format(password)})
+
+ def startTaosd(self):
+ try:
+ self.conn.run("sudo systemctl start taosd")
+ except Exception as e:
+ print("Start Taosd error for node %d " % self.index)
+ logging.exception(e)
+
+ def stopTaosd(self):
+ try:
+ self.conn.run("sudo systemctl stop taosd")
+ except Exception as e:
+ print("Stop Taosd error for node %d " % self.index)
+ logging.exception(e)
+
+ def restartTaosd(self):
+ try:
+ self.conn.run("sudo systemctl restart taosd")
+ except Exception as e:
+ print("Stop Taosd error for node %d " % self.index)
+ logging.exception(e)
+
+ def removeTaosd(self):
+ try:
+ self.conn.run("rmtaos")
+ except Exception as e:
+ print("remove taosd error for node %d " % self.index)
+ logging.exception(e)
+
+ def installTaosd(self, packagePath):
+ self.conn.put(packagePath, self.homeDir)
+ self.conn.cd(self.homeDir)
+ self.conn.run("tar -zxf $(basename '%s')" % packagePath)
+ with self.conn.cd("TDengine-enterprise-server"):
+ self.conn.run("yes|./install.sh")
+
+ def configTaosd(self, taosConfigKey, taosConfigValue):
+ self.conn.run("sudo echo '%s %s' >> %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg"))
+
+ def removeTaosConfig(self, taosConfigKey, taosConfigValue):
+ self.conn.run("sudo sed -in-place -e '/%s %s/d' %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg"))
+
+ def configHosts(self, ip, name):
+ self.conn.run("echo '%s %s' >> %s" % (ip, name, '/etc/hosts'))
+
+ def removeData(self):
+ try:
+ self.conn.run("sudo rm -rf /var/lib/taos/*")
+ except Exception as e:
+ print("remove taosd data error for node %d " % self.index)
+ logging.exception(e)
+
+ def removeLog(self):
+ try:
+ self.conn.run("sudo rm -rf /var/log/taos/*")
+ except Exception as e:
+ print("remove taosd error for node %d " % self.index)
+ logging.exception(e)
+
+ def removeDataForMnode(self):
+ try:
+ self.conn.run("sudo rm -rf /var/lib/taos/*")
+ except Exception as e:
+ print("remove taosd error for node %d " % self.index)
+ logging.exception(e)
+
+ def removeDataForVnode(self, id):
+ try:
+ self.conn.run("sudo rm -rf /var/lib/taos/vnode%d/*.data" % id)
+ except Exception as e:
+ print("remove taosd error for node %d " % self.index)
+ logging.exception(e)
+
+class Nodes:
+ def __init__(self):
+ self.node1 = Node(1, 'ubuntu', '192.168.1.52', 'node1', 'tbase125!', '/home/ubuntu')
+ self.node2 = Node(2, 'ubuntu', '192.168.1.53', 'node2', 'tbase125!', '/home/ubuntu')
+ self.node3 = Node(3, 'ubuntu', '192.168.1.54', 'node3', 'tbase125!', '/home/ubuntu')
+
+ def stopAllTaosd(self):
+ self.node1.stopTaosd()
+ self.node2.stopTaosd()
+ self.node3.stopTaosd()
+
+ def startAllTaosd(self):
+ self.node1.startTaosd()
+ self.node2.startTaosd()
+ self.node3.startTaosd()
+
+ def restartAllTaosd(self):
+ self.node1.restartTaosd()
+ self.node2.restartTaosd()
+ self.node3.restartTaosd()
+
+ def addConfigs(self, configKey, configValue):
+ self.node1.configTaosd(configKey, configValue)
+ self.node2.configTaosd(configKey, configValue)
+ self.node3.configTaosd(configKey, configValue)
+
+ def removeConfigs(self, configKey, configValue):
+ self.node1.removeTaosConfig(configKey, configValue)
+ self.node2.removeTaosConfig(configKey, configValue)
+ self.node3.removeTaosConfig(configKey, configValue)
+
+ def removeAllDataFiles(self):
+ self.node1.removeData()
+ self.node2.removeData()
+ self.node3.removeData()
+
+class ClusterTest:
+ def __init__(self, hostName):
+ self.host = hostName
+ self.user = "root"
+ self.password = "taosdata"
+ self.config = "/etc/taos"
+ self.dbName = "mytest"
+ self.stbName = "meters"
+ self.numberOfThreads = 20
+ self.numberOfTables = 10000
+ self.numberOfRecords = 1000
+ self.tbPrefix = "t"
+ self.ts = 1538548685000
+ self.repeat = 1
+
+ def connectDB(self):
+ self.conn = taos.connect(
+ host=self.host,
+ user=self.user,
+ password=self.password,
+ config=self.config)
+
+ def createSTable(self, replica):
+ cursor = self.conn.cursor()
+ tdLog.info("drop database if exists %s" % self.dbName)
+ cursor.execute("drop database if exists %s" % self.dbName)
+ tdLog.info("create database %s replica %d" % (self.dbName, replica))
+ cursor.execute("create database %s replica %d" % (self.dbName, replica))
+ tdLog.info("use %s" % self.dbName)
+ cursor.execute("use %s" % self.dbName)
+ tdLog.info("drop table if exists %s" % self.stbName)
+ cursor.execute("drop table if exists %s" % self.stbName)
+ tdLog.info("create table %s(ts timestamp, current float, voltage int, phase int) tags(id int)" % self.stbName)
+ cursor.execute("create table %s(ts timestamp, current float, voltage int, phase int) tags(id int)" % self.stbName)
+ cursor.close()
+
+ def insertData(self, threadID):
+ print("Thread %d: starting" % threadID)
+ cursor = self.conn.cursor()
+ tablesPerThread = int(self.numberOfTables / self.numberOfThreads)
+ baseTableID = tablesPerThread * threadID
+ for i in range (tablesPerThread):
+ cursor.execute("create table %s%d using %s tags(%d)" % (self.tbPrefix, baseTableID + i, self.stbName, baseTableID + i))
+ query = "insert into %s%d values" % (self.tbPrefix, baseTableID + i)
+ base = self.numberOfRecords * i
+ for j in range(self.numberOfRecords):
+ query += "(%d, %f, %d, %d)" % (self.ts + base + j, random.random(), random.randint(210, 230), random.randint(0, 10))
+ cursor.execute(query)
+ cursor.close()
+ print("Thread %d: finishing" % threadID)
+
+ def run(self):
+ threads = []
+ tdLog.info("Inserting data")
+ for i in range(self.numberOfThreads):
+ thread = threading.Thread(target=self.insertData, args=(i,))
+ threads.append(thread)
+ thread.start()
+
+ for i in range(self.numberOfThreads):
+ threads[i].join()
\ No newline at end of file
diff --git a/tests/pytest/cluster/dataFileRecoveryTest.py b/tests/pytest/cluster/dataFileRecoveryTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..089d3fffc1499a8d9cafc87a8d94252111fcd604
--- /dev/null
+++ b/tests/pytest/cluster/dataFileRecoveryTest.py
@@ -0,0 +1,53 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from clusterSetup import *
+from util.sql import tdSql
+from util.log import tdLog
+import random
+
+class ClusterTestcase:
+
+ ## test case 20, 21, 22 ##
+ def run(self):
+
+ nodes = Nodes()
+ ctest = ClusterTest(nodes.node1.hostName)
+ ctest.connectDB()
+ ctest.createSTable(3)
+ ctest.run()
+ tdSql.init(ctest.conn.cursor(), False)
+
+ nodes.node2.stopTaosd()
+ tdSql.execute("use %s" % ctest.dbName)
+ tdSql.query("show vgroups")
+ vnodeID = tdSql.getData(0, 0)
+ nodes.node2.removeDataForVnode(vnodeID)
+ nodes.node2.startTaosd()
+
+ # Wait for vnode file to recover
+ for i in range(10):
+ tdSql.query("select count(*) from t0")
+
+ tdLog.sleep(10)
+
+ for i in range(10):
+ tdSql.query("select count(*) from t0")
+ tdSql.checkData(0, 0, 1000)
+
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+ct = ClusterTestcase()
+ct.run()
diff --git a/tests/pytest/cluster/fullDnodesTest.py b/tests/pytest/cluster/fullDnodesTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c4b10d97a24dfbb156122aa0afdbb5d22ce3941
--- /dev/null
+++ b/tests/pytest/cluster/fullDnodesTest.py
@@ -0,0 +1,47 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from clusterSetup import *
+from util.sql import tdSql
+from util.log import tdLog
+import random
+
+class ClusterTestcase:
+
+ ##Cover test case 5 ##
+ def run(self):
+ # cluster environment set up
+ nodes = Nodes()
+ nodes.addConfigs("maxVgroupsPerDb", "10")
+ nodes.addConfigs("maxTablesPerVnode", "1000")
+ nodes.restartAllTaosd()
+
+ ctest = ClusterTest(nodes.node1.hostName)
+ ctest.connectDB()
+ ctest.createSTable(1)
+ ctest.run()
+
+ tdSql.init(ctest.conn.cursor(), False)
+ tdSql.execute("use %s" % ctest.dbName)
+ tdSql.error("create table tt1 using %s tags(1)" % ctest.stbName)
+
+ nodes.removeConfigs("maxVgroupsPerDb", "10")
+ nodes.removeConfigs("maxTablesPerVnode", "1000")
+ nodes.restartAllTaosd()
+
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+ct = ClusterTestcase()
+ct.run()
\ No newline at end of file
diff --git a/tests/pytest/cluster/killAndRestartDnodesTest.py b/tests/pytest/cluster/killAndRestartDnodesTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..be927e862f616c7fbe490e733a18984b6971ef1f
--- /dev/null
+++ b/tests/pytest/cluster/killAndRestartDnodesTest.py
@@ -0,0 +1,75 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from clusterSetup import *
+from util.sql import tdSql
+from util.log import tdLog
+import random
+
+class ClusterTestcase:
+
+ ## test case 7, 10 ##
+ def run(self):
+ # cluster environment set up
+ tdLog.info("Test case 7, 10")
+
+ nodes = Nodes()
+ ctest = ClusterTest(nodes.node1.hostName)
+ ctest.connectDB()
+ tdSql.init(ctest.conn.cursor(), False)
+
+ nodes.node1.stopTaosd()
+ tdSql.query("show dnodes")
+ tdSql.checkRows(3)
+ tdSql.checkData(0, 4, "offline")
+ tdSql.checkData(1, 4, "ready")
+ tdSql.checkData(2, 4, "ready")
+
+ nodes.node1.startTaosd()
+ tdSql.checkRows(3)
+ tdSql.checkData(0, 4, "ready")
+ tdSql.checkData(1, 4, "ready")
+ tdSql.checkData(2, 4, "ready")
+
+ nodes.node2.stopTaosd()
+ tdSql.query("show dnodes")
+ tdSql.checkRows(3)
+ tdSql.checkData(0, 4, "ready")
+ tdSql.checkData(1, 4, "offline")
+ tdSql.checkData(2, 4, "ready")
+
+ nodes.node2.startTaosd()
+ tdSql.checkRows(3)
+ tdSql.checkData(0, 4, "ready")
+ tdSql.checkData(1, 4, "ready")
+ tdSql.checkData(2, 4, "ready")
+
+ nodes.node3.stopTaosd()
+ tdSql.query("show dnodes")
+ tdSql.checkRows(3)
+ tdSql.checkData(0, 4, "ready")
+ tdSql.checkData(1, 4, "ready")
+ tdSql.checkData(2, 4, "offline")
+
+ nodes.node3.startTaosd()
+ tdSql.checkRows(3)
+ tdSql.checkData(0, 4, "ready")
+ tdSql.checkData(1, 4, "ready")
+ tdSql.checkData(2, 4, "ready")
+
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+ct = ClusterTestcase()
+ct.run()
\ No newline at end of file
diff --git a/tests/pytest/cluster/offlineThresholdTest.py b/tests/pytest/cluster/offlineThresholdTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..8373424f93c8217250907e09620c8523d63071ad
--- /dev/null
+++ b/tests/pytest/cluster/offlineThresholdTest.py
@@ -0,0 +1,54 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from clusterSetup import *
+from util.sql import tdSql
+from util.log import tdLog
+import random
+
+class ClusterTestcase:
+
+ ## cover test case 6, 8, 9, 11 ##
+ def run(self):
+ # cluster environment set up
+ nodes = Nodes()
+ ctest = ClusterTest(nodes.node1.hostName)
+ ctest.connectDB()
+ tdSql.init(ctest.conn.cursor(), False)
+
+ nodes.addConfigs("offlineThreshold", "10")
+ nodes.removeAllDataFiles()
+ nodes.restartAllTaosd()
+ nodes.node3.stopTaosd()
+
+ tdLog.sleep(10)
+ tdSql.query("show dnodes")
+ tdSql.checkRows(3)
+ tdSql.checkData(2, 4, "offline")
+
+ tdLog.sleep(60)
+ tdSql.checkRows(3)
+ tdSql.checkData(2, 4, "dropping")
+
+ tdLog.sleep(300)
+ tdSql.checkRows(2)
+
+ nodes.removeConfigs("offlineThreshold", "10")
+ nodes.restartAllTaosd()
+
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+ct = ClusterTestcase()
+ct.run()
\ No newline at end of file
diff --git a/tests/pytest/cluster/oneReplicaOfflineTest.py b/tests/pytest/cluster/oneReplicaOfflineTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..0223dfe01add9faca7987d7767f5c41a58b8edd2
--- /dev/null
+++ b/tests/pytest/cluster/oneReplicaOfflineTest.py
@@ -0,0 +1,65 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from clusterSetup import *
+from util.sql import tdSql
+from util.log import tdLog
+import random
+
+class ClusterTestcase:
+
+ ## test case 28, 29, 30, 31 ##
+ def run(self):
+
+ nodes = Nodes()
+ ctest = ClusterTest(nodes.node1.hostName)
+ ctest.connectDB()
+ ctest.createSTable(3)
+ ctest.run()
+ tdSql.init(ctest.conn.cursor(), False)
+
+ tdSql.execute("use %s" % ctest.dbName)
+
+ nodes.node2.stopTaosd()
+ for i in range(100):
+ tdSql.execute("drop table t%d" % i)
+
+ nodes.node2.startTaosd()
+ tdSql.query("show tables")
+ tdSql.checkRows(9900)
+
+ nodes.node2.stopTaosd()
+ for i in range(10):
+ tdSql.execute("create table a%d using meters tags(2)" % i)
+
+ nodes.node2.startTaosd()
+ tdSql.query("show tables")
+ tdSql.checkRows(9910)
+
+ nodes.node2.stopTaosd()
+ tdSql.execute("alter table meters add col col6 int")
+ nodes.node2.startTaosd()
+
+ nodes.node2.stopTaosd()
+ tdSql.execute("drop database %s" % ctest.dbName)
+
+ nodes.node2.startTaosd()
+ tdSql.query("show databases")
+ tdSql.checkRows(0)
+
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+ct = ClusterTestcase()
+ct.run()
diff --git a/tests/pytest/cluster/queryTimeTest.py b/tests/pytest/cluster/queryTimeTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..74a9081ccf4fd8abc175e2e0c82b0c6feedcbb26
--- /dev/null
+++ b/tests/pytest/cluster/queryTimeTest.py
@@ -0,0 +1,54 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from clusterSetup import *
+from util.sql import tdSql
+from util.log import tdLog
+import random
+import time
+
+class ClusterTestcase:
+
+ ## test case 32 ##
+ def run(self):
+
+ nodes = Nodes()
+ ctest = ClusterTest(nodes.node1.hostName)
+ ctest.connectDB()
+ ctest.createSTable(1)
+ ctest.run()
+ tdSql.init(ctest.conn.cursor(), False)
+
+ tdSql.execute("use %s" % ctest.dbName)
+ totalTime = 0
+ for i in range(10):
+ startTime = time.time()
+ tdSql.query("select * from %s" % ctest.stbName)
+ totalTime += time.time() - startTime
+ print("replica 1: avarage query time for %d records: %f seconds" % (ctest.numberOfTables * ctest.numberOfRecords,totalTime / 10))
+
+ tdSql.execute("alter database %s replica 3" % ctest.dbName)
+ tdLog.sleep(60)
+ totalTime = 0
+ for i in range(10):
+ startTime = time.time()
+ tdSql.query("select * from %s" % ctest.stbName)
+ totalTime += time.time() - startTime
+ print("replica 3: avarage query time for %d records: %f seconds" % (ctest.numberOfTables * ctest.numberOfRecords,totalTime / 10))
+
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+ct = ClusterTestcase()
+ct.run()
diff --git a/tests/pytest/cluster/stopAllDnodesTest.py b/tests/pytest/cluster/stopAllDnodesTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..a71ae52e3d7a640bb589f3bafe16b2e4d45c7b93
--- /dev/null
+++ b/tests/pytest/cluster/stopAllDnodesTest.py
@@ -0,0 +1,45 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from clusterSetup import *
+from util.sql import tdSql
+from util.log import tdLog
+import random
+
+class ClusterTestcase:
+
+ ## test case 19 ##
+ def run(self):
+
+ nodes = Nodes()
+ ctest = ClusterTest(nodes.node1.hostName)
+ tdSql.init(ctest.conn.cursor(), False)
+
+ tdSql.query("show databases")
+ count = tdSql.queryRows;
+
+ nodes.stopAllTaosd()
+ nodes.node1.startTaosd()
+ tdSql.error("show databases")
+
+ nodes.node2.startTaosd()
+ tdSql.error("show databases")
+
+ nodes.node3.startTaosd()
+ tdLog.sleep(10)
+ tdSql.query("show databases")
+ tdSql.checkRows(count)
+
+ct = ClusterTestcase()
+ct.run()
diff --git a/tests/pytest/cluster/stopTwoDnodesTest.py b/tests/pytest/cluster/stopTwoDnodesTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..9e9958e2d32018b6a89a3e0d08da2c1597151ff2
--- /dev/null
+++ b/tests/pytest/cluster/stopTwoDnodesTest.py
@@ -0,0 +1,48 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from clusterSetup import *
+from util.sql import tdSql
+from util.log import tdLog
+import random
+
+class ClusterTestcase:
+
+ ## test case 17, 18 ##
+ def run(self):
+
+ nodes = Nodes()
+ ctest = ClusterTest(nodes.node1.hostName)
+ ctest.connectDB()
+ ctest.createSTable(1)
+ ctest.run()
+ tdSql.init(ctest.conn.cursor(), False)
+
+ tdSql.query("show databases")
+ count = tdSql.queryRows;
+ tdSql.execute("use %s" % ctest.dbName)
+ tdSql.execute("alter database %s replica 3" % ctest.dbName)
+ nodes.node2.stopTaosd()
+ nodes.node3.stopTaosd()
+ tdSql.error("show databases")
+
+ nodes.node2.startTaosd()
+ tdSql.error("show databases")
+
+ nodes.node3.startTaosd()
+ tdSql.query("show databases")
+ tdSql.checkRows(count)
+
+ct = ClusterTestcase()
+ct.run()
diff --git a/tests/pytest/cluster/syncingTest.py b/tests/pytest/cluster/syncingTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..96be048d231e35f67e40fc4785d2e19337ed408b
--- /dev/null
+++ b/tests/pytest/cluster/syncingTest.py
@@ -0,0 +1,50 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from clusterSetup import *
+from util.sql import tdSql
+from util.log import tdLog
+import random
+
+class ClusterTestcase:
+
+ ## test case 24, 25, 26, 27 ##
+ def run(self):
+
+ nodes = Nodes()
+ ctest = ClusterTest(nodes.node1.hostName)
+ ctest.connectDB()
+ ctest.createSTable(1)
+ ctest.run()
+ tdSql.init(ctest.conn.cursor(), False)
+
+
+ tdSql.execute("use %s" % ctest.dbName)
+ tdSql.execute("alter database %s replica 3" % ctest.dbName)
+
+ for i in range(100):
+ tdSql.execute("drop table t%d" % i)
+
+ for i in range(100):
+ tdSql.execute("create table a%d using meters tags(1)" % i)
+
+ tdSql.execute("alter table meters add col col5 int")
+ tdSql.execute("alter table meters drop col col5 int")
+ tdSql.execute("drop database %s" % ctest.dbName)
+
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+ct = ClusterTestcase()
+ct.run()
diff --git a/tests/pytest/cluster/testcluster.sh b/tests/pytest/cluster/testcluster.sh
new file mode 100644
index 0000000000000000000000000000000000000000..6e15a498c0a73db450699fe66d63d07c3b18dbe5
--- /dev/null
+++ b/tests/pytest/cluster/testcluster.sh
@@ -0,0 +1,12 @@
+python3 basicTest.py
+python3 bananceTest.py
+python3 changeReplicaTest.py
+python3 dataFileRecoveryTest.py
+python3 fullDnodesTest.py
+python3 killAndRestartDnodesTest.py
+python3 offlineThresholdTest.py
+python3 oneReplicaOfflineTest.py
+python3 queryTimeTest.py
+python3 stopAllDnodesTest.py
+python3 stopTwoDnodesTest.py
+python3 syncingTest.py
\ No newline at end of file
diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh
index 4ffe35fc3c94edbdd194e03171696a1d681387c1..0af09634df5a5c418797ae4bd352c319fcbc74fa 100755
--- a/tests/pytest/crash_gen.sh
+++ b/tests/pytest/crash_gen.sh
@@ -54,6 +54,7 @@ export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3:$(pwd)
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR
# Now we are all let, and let's see if we can find a crash. Note we pass all params
+CRASH_GEN_EXEC=crash_gen_bootstrap.py
if [[ $1 == '--valgrind' ]]; then
shift
export PYTHONMALLOC=malloc
@@ -66,14 +67,16 @@ if [[ $1 == '--valgrind' ]]; then
--leak-check=yes \
--suppressions=crash_gen/valgrind_taos.supp \
$PYTHON_EXEC \
- ./crash_gen/crash_gen.py $@ > $VALGRIND_OUT 2> $VALGRIND_ERR
+ $CRASH_GEN_EXEC $@ > $VALGRIND_OUT 2> $VALGRIND_ERR
elif [[ $1 == '--helgrind' ]]; then
shift
+ HELGRIND_OUT=helgrind.out
+ HELGRIND_ERR=helgrind.err
valgrind \
--tool=helgrind \
$PYTHON_EXEC \
- ./crash_gen/crash_gen.py $@
+ $CRASH_GEN_EXEC $@ > $HELGRIND_OUT 2> $HELGRIND_ERR
else
- $PYTHON_EXEC ./crash_gen/crash_gen.py $@
+ $PYTHON_EXEC $CRASH_GEN_EXEC $@
fi
diff --git a/tests/pytest/crash_gen/README.md b/tests/pytest/crash_gen/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..6788ab1a63d0a7c515558695605d1ec8ac5fb7f9
--- /dev/null
+++ b/tests/pytest/crash_gen/README.md
@@ -0,0 +1,130 @@
+User's Guide to the Crash_Gen Tool
+
+# Introduction
+
+To effectively test and debug our TDengine product, we have developed a simple tool to
+exercise various functions of the system in a randomized fashion, hoping to expose
+maximum number of problems, hopefully without a pre-determined scenario.
+
+# Preparation
+
+To run this tool, please ensure the followed preparation work is done first.
+
+1. Fetch a copy of the TDengine source code, and build it successfully in the `build/`
+ directory
+1. Ensure that the system has Python3.8 or above properly installed. We use
+ Ubuntu 20.04LTS as our own development environment, and suggest you also use such
+ an environment if possible.
+
+# Simple Execution
+
+To run the tool with the simplest method, follow the steps below:
+
+1. Open a terminal window, start the `taosd` service in the `build/` directory
+ (or however you prefer to start the `taosd` service)
+1. Open another terminal window, go into the `tests/pytest/` directory, and
+ run `./crash_gen.sh -p -t 3 -s 10` (change the two parameters here as you wish)
+1. Watch the output to the end and see if you get a `SUCCESS` or `FAILURE`
+
+That's it!
+
+# Running Clusters
+
+This tool also makes it easy to test/verify the clustering capabilities of TDengine. You
+can start a cluster quite easily with the following command:
+
+```
+$ cd tests/pytest/
+$ ./crash_gen.sh -e -o 3
+```
+
+The `-e` option above tells the tool to start the service, and do not run any tests, while
+the `-o 3` option tells the tool to start 3 DNodes and join them together in a cluster.
+Obviously you can adjust the the number here.
+
+## Behind the Scenes
+
+When the tool runs a cluster, it users a number of directories, each holding the information
+for a single DNode, see:
+
+```
+$ ls build/cluster*
+build/cluster_dnode_0:
+cfg data log
+
+build/cluster_dnode_1:
+cfg data log
+
+build/cluster_dnode_2:
+cfg data log
+```
+
+Therefore, when something goes wrong and you want to reset everything with the cluster, simple
+erase all the files:
+
+```
+$ rm -rf build/cluster_dnode_*
+```
+
+## Addresses and Ports
+
+The DNodes in the cluster all binds the the `127.0.0.1` IP address (for now anyway), and
+uses port 6030 for the first DNode, and 6130 for the 2nd one, and so on.
+
+## Testing Against a Cluster
+
+In a separate terminal window, you can invoke the tool in client mode and test against
+a cluster, such as:
+
+```
+$ ./crash_gen.sh -p -t 10 -s 100 -i 3
+```
+
+Here the `-i` option tells the tool to always create tables with 3 replicas, and run
+all tests against such tables.
+
+# Additional Features
+
+The exhaustive features of the tool is available through the `-h` option:
+
+```
+$ ./crash_gen.sh -h
+usage: crash_gen_bootstrap.py [-h] [-a] [-b MAX_DBS] [-c CONNECTOR_TYPE] [-d] [-e] [-g IGNORE_ERRORS] [-i MAX_REPLICAS] [-l] [-n] [-o NUM_DNODES] [-p] [-r]
+ [-s MAX_STEPS] [-t NUM_THREADS] [-v] [-x]
+
+TDengine Auto Crash Generator (PLEASE NOTICE the Prerequisites Below)
+---------------------------------------------------------------------
+1. You build TDengine in the top level ./build directory, as described in offical docs
+2. You run the server there before this script: ./build/bin/taosd -c test/cfg
+
+optional arguments:
+ -h, --help show this help message and exit
+ -a, --auto-start-service
+ Automatically start/stop the TDengine service (default: false)
+ -b MAX_DBS, --max-dbs MAX_DBS
+ Maximum number of DBs to keep, set to disable dropping DB. (default: 0)
+ -c CONNECTOR_TYPE, --connector-type CONNECTOR_TYPE
+ Connector type to use: native, rest, or mixed (default: 10)
+ -d, --debug Turn on DEBUG mode for more logging (default: false)
+ -e, --run-tdengine Run TDengine service in foreground (default: false)
+ -g IGNORE_ERRORS, --ignore-errors IGNORE_ERRORS
+ Ignore error codes, comma separated, 0x supported (default: None)
+ -i MAX_REPLICAS, --max-replicas MAX_REPLICAS
+ Maximum number of replicas to use, when testing against clusters. (default: 1)
+ -l, --larger-data Write larger amount of data during write operations (default: false)
+ -n, --dynamic-db-table-names
+ Use non-fixed names for dbs/tables, useful for multi-instance executions (default: false)
+ -o NUM_DNODES, --num-dnodes NUM_DNODES
+ Number of Dnodes to initialize, used with -e option. (default: 1)
+ -p, --per-thread-db-connection
+ Use a single shared db connection (default: false)
+ -r, --record-ops Use a pair of always-fsynced fils to record operations performing + performed, for power-off tests (default: false)
+ -s MAX_STEPS, --max-steps MAX_STEPS
+ Maximum number of steps to run (default: 100)
+ -t NUM_THREADS, --num-threads NUM_THREADS
+ Number of threads to run (default: 10)
+ -v, --verify-data Verify data written in a number of places by reading back (default: false)
+ -x, --continue-on-exception
+ Continue execution after encountering unexpected/disallowed errors/exceptions (default: false)
+```
+
diff --git a/tests/pytest/crash_gen/crash_gen.py b/tests/pytest/crash_gen/crash_gen_main.py
similarity index 56%
rename from tests/pytest/crash_gen/crash_gen.py
rename to tests/pytest/crash_gen/crash_gen_main.py
index 48196ab383c974b5c5d3f5ebc54773cd846353e6..8d68457ec8d07c9c00f8b8fd0a11e2f25284ce4a 100755
--- a/tests/pytest/crash_gen/crash_gen.py
+++ b/tests/pytest/crash_gen/crash_gen_main.py
@@ -14,42 +14,36 @@
# For type hinting before definition, ref:
# https://stackoverflow.com/questions/33533148/how-do-i-specify-that-the-return-type-of-a-method-is-the-same-as-the-class-itsel
from __future__ import annotations
-import taos
-from util.sql import *
-from util.cases import *
-from util.dnodes import *
-from util.log import *
-from queue import Queue, Empty
-from typing import IO
+
from typing import Set
from typing import Dict
from typing import List
-from requests.auth import HTTPBasicAuth
+from typing import Optional # Type hinting, ref: https://stackoverflow.com/questions/19202633/python-3-type-hinting-for-none
+
import textwrap
-import datetime
-import logging
import time
+import datetime
import random
+import logging
import threading
-import requests
import copy
import argparse
import getopt
import sys
import os
-import io
import signal
import traceback
import resource
from guppy import hpy
import gc
-try:
- import psutil
-except:
- print("Psutil module needed, please install: sudo pip3 install psutil")
- sys.exit(-1)
+from crash_gen.service_manager import ServiceManager, TdeInstance
+from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress
+from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager
+
+import taos
+import requests
# Require Python 3
if sys.version_info[0] < 3:
@@ -59,41 +53,37 @@ if sys.version_info[0] < 3:
# Command-line/Environment Configurations, will set a bit later
# ConfigNameSpace = argparse.Namespace
-gConfig = argparse.Namespace() # Dummy value, will be replaced later
-gSvcMgr = None # TODO: refactor this hack, use dep injection
-logger = None # type: Logger
-
-def runThread(wt: WorkerThread):
- wt.run()
+gConfig: argparse.Namespace
+gSvcMgr: ServiceManager # TODO: refactor this hack, use dep injection
+# logger: logging.Logger
+gContainer: Container
-class CrashGenError(Exception):
- def __init__(self, msg=None, errno=None):
- self.msg = msg
- self.errno = errno
-
- def __str__(self):
- return self.msg
+# def runThread(wt: WorkerThread):
+# wt.run()
class WorkerThread:
- def __init__(self, pool: ThreadPool, tid, tc: ThreadCoordinator,
- # te: TaskExecutor,
- ): # note: main thread context!
+ def __init__(self, pool: ThreadPool, tid, tc: ThreadCoordinator):
+ """
+ Note: this runs in the main thread context
+ """
# self._curStep = -1
self._pool = pool
self._tid = tid
self._tc = tc # type: ThreadCoordinator
# self.threadIdent = threading.get_ident()
- self._thread = threading.Thread(target=runThread, args=(self,))
+ # self._thread = threading.Thread(target=runThread, args=(self,))
+ self._thread = threading.Thread(target=self.run)
self._stepGate = threading.Event()
# Let us have a DB connection of our own
if (gConfig.per_thread_db_connection): # type: ignore
# print("connector_type = {}".format(gConfig.connector_type))
- if gConfig.connector_type == 'native':
- self._dbConn = DbConn.createNative()
+ tInst = gContainer.defTdeInstance
+ if gConfig.connector_type == 'native':
+ self._dbConn = DbConn.createNative(tInst.getDbTarget())
elif gConfig.connector_type == 'rest':
- self._dbConn = DbConn.createRest()
+ self._dbConn = DbConn.createRest(tInst.getDbTarget())
elif gConfig.connector_type == 'mixed':
if Dice.throw(2) == 0: # 1/2 chance
self._dbConn = DbConn.createNative()
@@ -105,10 +95,10 @@ class WorkerThread:
# self._dbInUse = False # if "use db" was executed already
def logDebug(self, msg):
- logger.debug(" TRD[{}] {}".format(self._tid, msg))
+ Logging.debug(" TRD[{}] {}".format(self._tid, msg))
def logInfo(self, msg):
- logger.info(" TRD[{}] {}".format(self._tid, msg))
+ Logging.info(" TRD[{}] {}".format(self._tid, msg))
# def dbInUse(self):
# return self._dbInUse
@@ -127,10 +117,10 @@ class WorkerThread:
def run(self):
# initialization after thread starts, in the thread context
# self.isSleeping = False
- logger.info("Starting to run thread: {}".format(self._tid))
+ Logging.info("Starting to run thread: {}".format(self._tid))
if (gConfig.per_thread_db_connection): # type: ignore
- logger.debug("Worker thread openning database connection")
+ Logging.debug("Worker thread openning database connection")
self._dbConn.open()
self._doTaskLoop()
@@ -140,7 +130,7 @@ class WorkerThread:
if self._dbConn.isOpen: #sometimes it is not open
self._dbConn.close()
else:
- logger.warning("Cleaning up worker thread, dbConn already closed")
+ Logging.warning("Cleaning up worker thread, dbConn already closed")
def _doTaskLoop(self):
# while self._curStep < self._pool.maxSteps:
@@ -151,15 +141,15 @@ class WorkerThread:
tc.crossStepBarrier() # shared barrier first, INCLUDING the last one
except threading.BrokenBarrierError as err: # main thread timed out
print("_bto", end="")
- logger.debug("[TRD] Worker thread exiting due to main thread barrier time-out")
+ Logging.debug("[TRD] Worker thread exiting due to main thread barrier time-out")
break
- logger.debug("[TRD] Worker thread [{}] exited barrier...".format(self._tid))
+ Logging.debug("[TRD] Worker thread [{}] exited barrier...".format(self._tid))
self.crossStepGate() # then per-thread gate, after being tapped
- logger.debug("[TRD] Worker thread [{}] exited step gate...".format(self._tid))
+ Logging.debug("[TRD] Worker thread [{}] exited step gate...".format(self._tid))
if not self._tc.isRunning():
print("_wts", end="")
- logger.debug("[TRD] Thread Coordinator not running any more, worker thread now stopping...")
+ Logging.debug("[TRD] Thread Coordinator not running any more, worker thread now stopping...")
break
# Before we fetch the task and run it, let's ensure we properly "use" the database (not needed any more)
@@ -178,15 +168,15 @@ class WorkerThread:
raise
# Fetch a task from the Thread Coordinator
- logger.debug( "[TRD] Worker thread [{}] about to fetch task".format(self._tid))
+ Logging.debug( "[TRD] Worker thread [{}] about to fetch task".format(self._tid))
task = tc.fetchTask()
# Execute such a task
- logger.debug("[TRD] Worker thread [{}] about to execute task: {}".format(
+ Logging.debug("[TRD] Worker thread [{}] about to execute task: {}".format(
self._tid, task.__class__.__name__))
task.execute(self)
tc.saveExecutedTask(task)
- logger.debug("[TRD] Worker thread [{}] finished executing task".format(self._tid))
+ Logging.debug("[TRD] Worker thread [{}] finished executing task".format(self._tid))
# self._dbInUse = False # there may be changes between steps
# print("_wtd", end=None) # worker thread died
@@ -209,7 +199,7 @@ class WorkerThread:
self.verifyThreadSelf() # only allowed by ourselves
# Wait again at the "gate", waiting to be "tapped"
- logger.debug(
+ Logging.debug(
"[TRD] Worker thread {} about to cross the step gate".format(
self._tid))
self._stepGate.wait()
@@ -222,7 +212,7 @@ class WorkerThread:
self.verifyThreadMain() # only allowed for main thread
if self._thread.is_alive():
- logger.debug("[TRD] Tapping worker thread {}".format(self._tid))
+ Logging.debug("[TRD] Tapping worker thread {}".format(self._tid))
self._stepGate.set() # wake up!
time.sleep(0) # let the released thread run a bit
else:
@@ -253,7 +243,7 @@ class WorkerThread:
class ThreadCoordinator:
- WORKER_THREAD_TIMEOUT = 60 # one minute
+ WORKER_THREAD_TIMEOUT = 120 # Normal: 120
def __init__(self, pool: ThreadPool, dbManager: DbManager):
self._curStep = -1 # first step is 0
@@ -267,7 +257,7 @@ class ThreadCoordinator:
self._stepBarrier = threading.Barrier(
self._pool.numThreads + 1) # one barrier for all threads
self._execStats = ExecutionStats()
- self._runStatus = MainExec.STATUS_RUNNING
+ self._runStatus = Status.STATUS_RUNNING
self._initDbs()
def getTaskExecutor(self):
@@ -280,14 +270,14 @@ class ThreadCoordinator:
self._stepBarrier.wait(timeout)
def requestToStop(self):
- self._runStatus = MainExec.STATUS_STOPPING
+ self._runStatus = Status.STATUS_STOPPING
self._execStats.registerFailure("User Interruption")
def _runShouldEnd(self, transitionFailed, hasAbortedTask, workerTimeout):
maxSteps = gConfig.max_steps # type: ignore
if self._curStep >= (maxSteps - 1): # maxStep==10, last curStep should be 9
return True
- if self._runStatus != MainExec.STATUS_RUNNING:
+ if self._runStatus != Status.STATUS_RUNNING:
return True
if transitionFailed:
return True
@@ -308,7 +298,7 @@ class ThreadCoordinator:
def _releaseAllWorkerThreads(self, transitionFailed):
self._curStep += 1 # we are about to get into next step. TODO: race condition here!
# Now not all threads had time to go to sleep
- logger.debug(
+ Logging.debug(
"--\r\n\n--> Step {} starts with main thread waking up".format(self._curStep))
# A new TE for the new step
@@ -316,7 +306,7 @@ class ThreadCoordinator:
if not transitionFailed: # only if not failed
self._te = TaskExecutor(self._curStep)
- logger.debug("[TRD] Main thread waking up at step {}, tapping worker threads".format(
+ Logging.debug("[TRD] Main thread waking up at step {}, tapping worker threads".format(
self._curStep)) # Now not all threads had time to go to sleep
# Worker threads will wake up at this point, and each execute it's own task
self.tapAllThreads() # release all worker thread from their "gates"
@@ -325,10 +315,10 @@ class ThreadCoordinator:
# Now main thread (that's us) is ready to enter a step
# let other threads go past the pool barrier, but wait at the
# thread gate
- logger.debug("[TRD] Main thread about to cross the barrier")
+ Logging.debug("[TRD] Main thread about to cross the barrier")
self.crossStepBarrier(timeout=self.WORKER_THREAD_TIMEOUT)
self._stepBarrier.reset() # Other worker threads should now be at the "gate"
- logger.debug("[TRD] Main thread finished crossing the barrier")
+ Logging.debug("[TRD] Main thread finished crossing the barrier")
def _doTransition(self):
transitionFailed = False
@@ -336,11 +326,11 @@ class ThreadCoordinator:
for x in self._dbs:
db = x # type: Database
sm = db.getStateMachine()
- logger.debug("[STT] starting transitions for DB: {}".format(db.getName()))
+ Logging.debug("[STT] starting transitions for DB: {}".format(db.getName()))
# at end of step, transiton the DB state
tasksForDb = db.filterTasks(self._executedTasks)
sm.transition(tasksForDb, self.getDbManager().getDbConn())
- logger.debug("[STT] transition ended for DB: {}".format(db.getName()))
+ Logging.debug("[STT] transition ended for DB: {}".format(db.getName()))
# Due to limitation (or maybe not) of the TD Python library,
# we cannot share connections across threads
@@ -348,14 +338,14 @@ class ThreadCoordinator:
# Moving below to task loop
# if sm.hasDatabase():
# for t in self._pool.threadList:
- # logger.debug("[DB] use db for all worker threads")
+ # Logging.debug("[DB] use db for all worker threads")
# t.useDb()
# t.execSql("use db") # main thread executing "use
# db" on behalf of every worker thread
except taos.error.ProgrammingError as err:
if (err.msg == 'network unavailable'): # broken DB connection
- logger.info("DB connection broken, execution failed")
+ Logging.info("DB connection broken, execution failed")
traceback.print_stack()
transitionFailed = True
self._te = None # Not running any more
@@ -368,7 +358,7 @@ class ThreadCoordinator:
self.resetExecutedTasks() # clear the tasks after we are done
# Get ready for next step
- logger.debug("<-- Step {} finished, trasition failed = {}".format(self._curStep, transitionFailed))
+ Logging.debug("<-- Step {} finished, trasition failed = {}".format(self._curStep, transitionFailed))
return transitionFailed
def run(self):
@@ -382,8 +372,9 @@ class ThreadCoordinator:
hasAbortedTask = False
workerTimeout = False
while not self._runShouldEnd(transitionFailed, hasAbortedTask, workerTimeout):
- if not gConfig.debug: # print this only if we are not in debug mode
- print(".", end="", flush=True)
+ if not gConfig.debug: # print this only if we are not in debug mode
+ Progress.emit(Progress.STEP_BOUNDARY)
+ # print(".", end="", flush=True)
# if (self._curStep % 2) == 0: # print memory usage once every 10 steps
# memUsage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
# print("[m:{}]".format(memUsage), end="", flush=True) # print memory usage
@@ -395,10 +386,11 @@ class ThreadCoordinator:
try:
self._syncAtBarrier() # For now just cross the barrier
+ Progress.emit(Progress.END_THREAD_STEP)
except threading.BrokenBarrierError as err:
- logger.info("Main loop aborted, caused by worker thread time-out")
+ Logging.info("Main loop aborted, caused by worker thread(s) time-out")
self._execStats.registerFailure("Aborted due to worker thread timeout")
- print("\n\nWorker Thread time-out detected, important thread info:")
+ print("\n\nWorker Thread time-out detected, TAOS related threads are:")
ts = ThreadStacks()
ts.print(filterInternal=True)
workerTimeout = True
@@ -409,7 +401,7 @@ class ThreadCoordinator:
# threads are QUIET.
hasAbortedTask = self._hasAbortedTask() # from previous step
if hasAbortedTask:
- logger.info("Aborted task encountered, exiting test program")
+ Logging.info("Aborted task encountered, exiting test program")
self._execStats.registerFailure("Aborted Task Encountered")
break # do transition only if tasks are error free
@@ -420,29 +412,30 @@ class ThreadCoordinator:
transitionFailed = True
errno2 = Helper.convertErrno(err.errno) # correct error scheme
errMsg = "Transition failed: errno=0x{:X}, msg: {}".format(errno2, err)
- logger.info(errMsg)
+ Logging.info(errMsg)
traceback.print_exc()
self._execStats.registerFailure(errMsg)
# Then we move on to the next step
+ Progress.emit(Progress.BEGIN_THREAD_STEP)
self._releaseAllWorkerThreads(transitionFailed)
if hasAbortedTask or transitionFailed : # abnormal ending, workers waiting at "gate"
- logger.debug("Abnormal ending of main thraed")
+ Logging.debug("Abnormal ending of main thraed")
elif workerTimeout:
- logger.debug("Abnormal ending of main thread, due to worker timeout")
+ Logging.debug("Abnormal ending of main thread, due to worker timeout")
else: # regular ending, workers waiting at "barrier"
- logger.debug("Regular ending, main thread waiting for all worker threads to stop...")
+ Logging.debug("Regular ending, main thread waiting for all worker threads to stop...")
self._syncAtBarrier()
self._te = None # No more executor, time to end
- logger.debug("Main thread tapping all threads one last time...")
+ Logging.debug("Main thread tapping all threads one last time...")
self.tapAllThreads() # Let the threads run one last time
- logger.debug("\r\n\n--> Main thread ready to finish up...")
- logger.debug("Main thread joining all threads")
+ Logging.debug("\r\n\n--> Main thread ready to finish up...")
+ Logging.debug("Main thread joining all threads")
self._pool.joinAll() # Get all threads to finish
- logger.info("\nAll worker threads finished")
+ Logging.info(". . . All worker threads finished") # No CR/LF before
self._execStats.endExec()
def cleanup(self): # free resources
@@ -474,7 +467,7 @@ class ThreadCoordinator:
wakeSeq.append(i)
else:
wakeSeq.insert(0, i)
- logger.debug(
+ Logging.debug(
"[TRD] Main thread waking up worker threads: {}".format(
str(wakeSeq)))
# TODO: set dice seed to a deterministic value
@@ -492,9 +485,11 @@ class ThreadCoordinator:
dbc = self.getDbManager().getDbConn()
if gConfig.max_dbs == 0:
self._dbs.append(Database(0, dbc))
- else:
+ else:
+ baseDbNumber = int(datetime.datetime.now().timestamp( # Don't use Dice/random, as they are deterministic
+ )*333) % 888 if gConfig.dynamic_db_table_names else 0
for i in range(gConfig.max_dbs):
- self._dbs.append(Database(i, dbc))
+ self._dbs.append(Database(baseDbNumber + i, dbc))
def pickDatabase(self):
idxDb = 0
@@ -512,7 +507,7 @@ class ThreadCoordinator:
# pick a task type for current state
db = self.pickDatabase()
- taskType = db.getStateMachine().pickTaskType() # type: Task
+ taskType = db.getStateMachine().pickTaskType() # dynamic name of class
return taskType(self._execStats, db) # create a task from it
def resetExecutedTasks(self):
@@ -522,13 +517,6 @@ class ThreadCoordinator:
with self._lock:
self._executedTasks.append(task)
-# We define a class to run a number of threads in locking steps.
-
-class Helper:
- @classmethod
- def convertErrno(cls, errno):
- return errno if (errno > 0) else 0x80000000 + errno
-
class ThreadPool:
def __init__(self, numThreads, maxSteps):
self.numThreads = numThreads
@@ -546,7 +534,7 @@ class ThreadPool:
def joinAll(self):
for workerThread in self.threadList:
- logger.debug("Joining thread...")
+ Logging.debug("Joining thread...")
workerThread._thread.join()
def cleanup(self):
@@ -603,7 +591,7 @@ class LinearQueue():
def allocate(self, i):
with self._lock:
- # logger.debug("LQ allocating item {}".format(i))
+ # Logging.debug("LQ allocating item {}".format(i))
if (i in self.inUse):
raise RuntimeError(
"Cannot re-use same index in queue: {}".format(i))
@@ -611,7 +599,7 @@ class LinearQueue():
def release(self, i):
with self._lock:
- # logger.debug("LQ releasing item {}".format(i))
+ # Logging.debug("LQ releasing item {}".format(i))
self.inUse.remove(i) # KeyError possible, TODO: why?
def size(self):
@@ -633,357 +621,6 @@ class LinearQueue():
return ret
-class DbConn:
- TYPE_NATIVE = "native-c"
- TYPE_REST = "rest-api"
- TYPE_INVALID = "invalid"
-
- @classmethod
- def create(cls, connType):
- if connType == cls.TYPE_NATIVE:
- return DbConnNative()
- elif connType == cls.TYPE_REST:
- return DbConnRest()
- else:
- raise RuntimeError(
- "Unexpected connection type: {}".format(connType))
-
- @classmethod
- def createNative(cls):
- return cls.create(cls.TYPE_NATIVE)
-
- @classmethod
- def createRest(cls):
- return cls.create(cls.TYPE_REST)
-
- def __init__(self):
- self.isOpen = False
- self._type = self.TYPE_INVALID
- self._lastSql = None
-
- def getLastSql(self):
- return self._lastSql
-
- def open(self):
- if (self.isOpen):
- raise RuntimeError("Cannot re-open an existing DB connection")
-
- # below implemented by child classes
- self.openByType()
-
- logger.debug("[DB] data connection opened, type = {}".format(self._type))
- self.isOpen = True
-
- def queryScalar(self, sql) -> int:
- return self._queryAny(sql)
-
- def queryString(self, sql) -> str:
- return self._queryAny(sql)
-
- def _queryAny(self, sql): # actual query result as an int
- if (not self.isOpen):
- raise RuntimeError("Cannot query database until connection is open")
- nRows = self.query(sql)
- if nRows != 1:
- raise taos.error.ProgrammingError(
- "Unexpected result for query: {}, rows = {}".format(sql, nRows),
- (0x991 if nRows==0 else 0x992)
- )
- if self.getResultRows() != 1 or self.getResultCols() != 1:
- raise RuntimeError("Unexpected result set for query: {}".format(sql))
- return self.getQueryResult()[0][0]
-
- def use(self, dbName):
- self.execute("use {}".format(dbName))
-
- def existsDatabase(self, dbName: str):
- ''' Check if a certain database exists '''
- self.query("show databases")
- dbs = [v[0] for v in self.getQueryResult()] # ref: https://stackoverflow.com/questions/643823/python-list-transformation
- # ret2 = dbName in dbs
- # print("dbs = {}, str = {}, ret2={}, type2={}".format(dbs, dbName,ret2, type(dbName)))
- return dbName in dbs # TODO: super weird type mangling seen, once here
-
- def hasTables(self):
- return self.query("show tables") > 0
-
- def execute(self, sql):
- ''' Return the number of rows affected'''
- raise RuntimeError("Unexpected execution, should be overriden")
-
- def safeExecute(self, sql):
- '''Safely execute any SQL query, returning True/False upon success/failure'''
- try:
- self.execute(sql)
- return True # ignore num of results, return success
- except taos.error.ProgrammingError as err:
- return False # failed, for whatever TAOS reason
- # Not possile to reach here, non-TAOS exception would have been thrown
-
- def query(self, sql) -> int: # return num rows returned
- ''' Return the number of rows affected'''
- raise RuntimeError("Unexpected execution, should be overriden")
-
- def openByType(self):
- raise RuntimeError("Unexpected execution, should be overriden")
-
- def getQueryResult(self):
- raise RuntimeError("Unexpected execution, should be overriden")
-
- def getResultRows(self):
- raise RuntimeError("Unexpected execution, should be overriden")
-
- def getResultCols(self):
- raise RuntimeError("Unexpected execution, should be overriden")
-
-# Sample: curl -u root:taosdata -d "show databases" localhost:6020/rest/sql
-
-
-class DbConnRest(DbConn):
- def __init__(self):
- super().__init__()
- self._type = self.TYPE_REST
- self._url = "http://localhost:6041/rest/sql" # fixed for now
- self._result = None
-
- def openByType(self): # Open connection
- pass # do nothing, always open
-
- def close(self):
- if (not self.isOpen):
- raise RuntimeError("Cannot clean up database until connection is open")
- # Do nothing for REST
- logger.debug("[DB] REST Database connection closed")
- self.isOpen = False
-
- def _doSql(self, sql):
- self._lastSql = sql # remember this, last SQL attempted
- try:
- r = requests.post(self._url,
- data = sql,
- auth = HTTPBasicAuth('root', 'taosdata'))
- except:
- print("REST API Failure (TODO: more info here)")
- raise
- rj = r.json()
- # Sanity check for the "Json Result"
- if ('status' not in rj):
- raise RuntimeError("No status in REST response")
-
- if rj['status'] == 'error': # clearly reported error
- if ('code' not in rj): # error without code
- raise RuntimeError("REST error return without code")
- errno = rj['code'] # May need to massage this in the future
- # print("Raising programming error with REST return: {}".format(rj))
- raise taos.error.ProgrammingError(
- rj['desc'], errno) # todo: check existance of 'desc'
-
- if rj['status'] != 'succ': # better be this
- raise RuntimeError(
- "Unexpected REST return status: {}".format(
- rj['status']))
-
- nRows = rj['rows'] if ('rows' in rj) else 0
- self._result = rj
- return nRows
-
- def execute(self, sql):
- if (not self.isOpen):
- raise RuntimeError(
- "Cannot execute database commands until connection is open")
- logger.debug("[SQL-REST] Executing SQL: {}".format(sql))
- nRows = self._doSql(sql)
- logger.debug(
- "[SQL-REST] Execution Result, nRows = {}, SQL = {}".format(nRows, sql))
- return nRows
-
- def query(self, sql): # return rows affected
- return self.execute(sql)
-
- def getQueryResult(self):
- return self._result['data']
-
- def getResultRows(self):
- print(self._result)
- raise RuntimeError("TBD")
- # return self._tdSql.queryRows
-
- def getResultCols(self):
- print(self._result)
- raise RuntimeError("TBD")
-
- # Duplicate code from TDMySQL, TODO: merge all this into DbConnNative
-
-
-class MyTDSql:
- # Class variables
- _clsLock = threading.Lock() # class wide locking
- longestQuery = None # type: str
- longestQueryTime = 0.0 # seconds
- lqStartTime = 0.0
- # lqEndTime = 0.0 # Not needed, as we have the two above already
-
- def __init__(self, hostAddr, cfgPath):
- # Make the DB connection
- self._conn = taos.connect(host=hostAddr, config=cfgPath)
- self._cursor = self._conn.cursor()
-
- self.queryRows = 0
- self.queryCols = 0
- self.affectedRows = 0
-
- # def init(self, cursor, log=True):
- # self.cursor = cursor
- # if (log):
- # caller = inspect.getframeinfo(inspect.stack()[1][0])
- # self.cursor.log(caller.filename + ".sql")
-
- def close(self):
- self._cursor.close() # can we double close?
- self._conn.close() # TODO: very important, cursor close does NOT close DB connection!
- self._cursor.close()
-
- def _execInternal(self, sql):
- startTime = time.time()
- ret = self._cursor.execute(sql)
- # print("\nSQL success: {}".format(sql))
- queryTime = time.time() - startTime
- # Record the query time
- cls = self.__class__
- if queryTime > (cls.longestQueryTime + 0.01) :
- with cls._clsLock:
- cls.longestQuery = sql
- cls.longestQueryTime = queryTime
- cls.lqStartTime = startTime
- return ret
-
- def query(self, sql):
- self.sql = sql
- try:
- self._execInternal(sql)
- self.queryResult = self._cursor.fetchall()
- self.queryRows = len(self.queryResult)
- self.queryCols = len(self._cursor.description)
- except Exception as e:
- # caller = inspect.getframeinfo(inspect.stack()[1][0])
- # args = (caller.filename, caller.lineno, sql, repr(e))
- # tdLog.exit("%s(%d) failed: sql:%s, %s" % args)
- raise
- return self.queryRows
-
- def execute(self, sql):
- self.sql = sql
- try:
- self.affectedRows = self._execInternal(sql)
- except Exception as e:
- # caller = inspect.getframeinfo(inspect.stack()[1][0])
- # args = (caller.filename, caller.lineno, sql, repr(e))
- # tdLog.exit("%s(%d) failed: sql:%s, %s" % args)
- raise
- return self.affectedRows
-
-
-class DbConnNative(DbConn):
- # Class variables
- _lock = threading.Lock()
- _connInfoDisplayed = False
- totalConnections = 0 # Not private
-
- def __init__(self):
- super().__init__()
- self._type = self.TYPE_NATIVE
- self._conn = None
- # self._cursor = None
-
- def getBuildPath(self):
- selfPath = os.path.dirname(os.path.realpath(__file__))
- if ("community" in selfPath):
- projPath = selfPath[:selfPath.find("communit")]
- else:
- projPath = selfPath[:selfPath.find("tests")]
-
- buildPath = None
- for root, dirs, files in os.walk(projPath):
- if ("taosd" in files):
- rootRealPath = os.path.dirname(os.path.realpath(root))
- if ("packaging" not in rootRealPath):
- buildPath = root[:len(root) - len("/build/bin")]
- break
- if buildPath == None:
- raise RuntimeError("Failed to determine buildPath, selfPath={}, projPath={}"
- .format(selfPath, projPath))
- return buildPath
-
-
- def openByType(self): # Open connection
- cfgPath = self.getBuildPath() + "/test/cfg"
- hostAddr = "127.0.0.1"
-
- cls = self.__class__ # Get the class, to access class variables
- with cls._lock: # force single threading for opening DB connections. # TODO: whaaat??!!!
- if not cls._connInfoDisplayed:
- cls._connInfoDisplayed = True # updating CLASS variable
- logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath))
- # Make the connection
- # self._conn = taos.connect(host=hostAddr, config=cfgPath) # TODO: make configurable
- # self._cursor = self._conn.cursor()
- # Record the count in the class
- self._tdSql = MyTDSql(hostAddr, cfgPath) # making DB connection
- cls.totalConnections += 1
-
- self._tdSql.execute('reset query cache')
- # self._cursor.execute('use db') # do this at the beginning of every
-
- # Open connection
- # self._tdSql = MyTDSql()
- # self._tdSql.init(self._cursor)
-
- def close(self):
- if (not self.isOpen):
- raise RuntimeError("Cannot clean up database until connection is open")
- self._tdSql.close()
- # Decrement the class wide counter
- cls = self.__class__ # Get the class, to access class variables
- with cls._lock:
- cls.totalConnections -= 1
-
- logger.debug("[DB] Database connection closed")
- self.isOpen = False
-
- def execute(self, sql):
- if (not self.isOpen):
- raise RuntimeError("Cannot execute database commands until connection is open")
- logger.debug("[SQL] Executing SQL: {}".format(sql))
- self._lastSql = sql
- nRows = self._tdSql.execute(sql)
- logger.debug(
- "[SQL] Execution Result, nRows = {}, SQL = {}".format(
- nRows, sql))
- return nRows
-
- def query(self, sql): # return rows affected
- if (not self.isOpen):
- raise RuntimeError(
- "Cannot query database until connection is open")
- logger.debug("[SQL] Executing SQL: {}".format(sql))
- self._lastSql = sql
- nRows = self._tdSql.query(sql)
- logger.debug(
- "[SQL] Query Result, nRows = {}, SQL = {}".format(
- nRows, sql))
- return nRows
- # results are in: return self._tdSql.queryResult
-
- def getQueryResult(self):
- return self._tdSql.queryResult
-
- def getResultRows(self):
- return self._tdSql.queryRows
-
- def getResultCols(self):
- return self._tdSql.queryCols
-
-
class AnyState:
STATE_INVALID = -1
STATE_EMPTY = 0 # nothing there, no even a DB
@@ -1232,7 +869,7 @@ class StateMechine:
def init(self, dbc: DbConn): # late initailization, don't save the dbConn
self._curState = self._findCurrentState(dbc) # starting state
- logger.debug("Found Starting State: {}".format(self._curState))
+ Logging.debug("Found Starting State: {}".format(self._curState))
# TODO: seems no lnoger used, remove?
def getCurrentState(self):
@@ -1270,7 +907,7 @@ class StateMechine:
raise RuntimeError(
"No suitable task types found for state: {}".format(
self._curState))
- logger.debug(
+ Logging.debug(
"[OPS] Tasks found for state {}: {}".format(
self._curState,
typesToStrings(taskTypes)))
@@ -1280,27 +917,27 @@ class StateMechine:
ts = time.time() # we use this to debug how fast/slow it is to do the various queries to find the current DB state
dbName =self._db.getName()
if not dbc.existsDatabase(dbName): # dbc.hasDatabases(): # no database?!
- logger.debug( "[STT] empty database found, between {} and {}".format(ts, time.time()))
+ Logging.debug( "[STT] empty database found, between {} and {}".format(ts, time.time()))
return StateEmpty()
# did not do this when openning connection, and this is NOT the worker
# thread, which does this on their own
dbc.use(dbName)
if not dbc.hasTables(): # no tables
- logger.debug("[STT] DB_ONLY found, between {} and {}".format(ts, time.time()))
+ Logging.debug("[STT] DB_ONLY found, between {} and {}".format(ts, time.time()))
return StateDbOnly()
sTable = self._db.getFixedSuperTable()
if sTable.hasRegTables(dbc, dbName): # no regular tables
- logger.debug("[STT] SUPER_TABLE_ONLY found, between {} and {}".format(ts, time.time()))
+ Logging.debug("[STT] SUPER_TABLE_ONLY found, between {} and {}".format(ts, time.time()))
return StateSuperTableOnly()
else: # has actual tables
- logger.debug("[STT] HAS_DATA found, between {} and {}".format(ts, time.time()))
+ Logging.debug("[STT] HAS_DATA found, between {} and {}".format(ts, time.time()))
return StateHasData()
# We transition the system to a new state by examining the current state itself
def transition(self, tasks, dbc: DbConn):
if (len(tasks) == 0): # before 1st step, or otherwise empty
- logger.debug("[STT] Starting State: {}".format(self._curState))
+ Logging.debug("[STT] Starting State: {}".format(self._curState))
return # do nothing
# this should show up in the server log, separating steps
@@ -1336,7 +973,7 @@ class StateMechine:
# Nothing for sure
newState = self._findCurrentState(dbc)
- logger.debug("[STT] New DB state determined: {}".format(newState))
+ Logging.debug("[STT] New DB state determined: {}".format(newState))
# can old state move to new state through the tasks?
self._curState.verifyTasksToState(tasks, newState)
self._curState = newState
@@ -1354,7 +991,7 @@ class StateMechine:
# read data task, default to 10: TODO: change to a constant
weights.append(10)
i = self._weighted_choice_sub(weights)
- # logger.debug(" (weighted random:{}/{}) ".format(i, len(taskTypes)))
+ # Logging.debug(" (weighted random:{}/{}) ".format(i, len(taskTypes)))
return taskTypes[i]
# ref:
@@ -1372,6 +1009,8 @@ class Database:
possibly in a cluster environment.
For now we use it to manage state transitions in that database
+
+ TODO: consider moving, but keep in mind it contains "StateMachine"
'''
_clsLock = threading.Lock() # class wide lock
_lastInt = 101 # next one is initial integer
@@ -1433,17 +1072,18 @@ class Database:
t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years
t4 = datetime.datetime.fromtimestamp(
t3.timestamp() + elSec2) # see explanation above
- logger.info("Setting up TICKS to start from: {}".format(t4))
+ Logging.debug("Setting up TICKS to start from: {}".format(t4))
return t4
@classmethod
def getNextTick(cls):
with cls._clsLock: # prevent duplicate tick
- if cls._lastLaggingTick==0:
+ if cls._lastLaggingTick==0 or cls._lastTick==0 : # not initialized
# 10k at 1/20 chance, should be enough to avoid overlaps
- cls._lastLaggingTick = cls.setupLastTick() + datetime.timedelta(0, -10000)
- if cls._lastTick==0: # should be quite a bit into the future
- cls._lastTick = cls.setupLastTick()
+ tick = cls.setupLastTick()
+ cls._lastTick = tick
+ cls._lastLaggingTick = tick + datetime.timedelta(0, -10000)
+ # if : # should be quite a bit into the future
if Dice.throw(20) == 0: # 1 in 20 chance, return lagging tick
cls._lastLaggingTick += datetime.timedelta(0, 1) # Go back in time 100 seconds
@@ -1468,64 +1108,6 @@ class Database:
return ret
-class DbManager():
- ''' This is a wrapper around DbConn(), to make it easier to use.
-
- TODO: rename this to DbConnManager
- '''
- def __init__(self):
- self.tableNumQueue = LinearQueue() # TODO: delete?
- # self.openDbServerConnection()
- self._dbConn = DbConn.createNative() if (
- gConfig.connector_type == 'native') else DbConn.createRest()
- try:
- self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected
- except taos.error.ProgrammingError as err:
- # print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err))
- if (err.msg == 'client disconnected'): # cannot open DB connection
- print(
- "Cannot establish DB connection, please re-run script without parameter, and follow the instructions.")
- sys.exit(2)
- else:
- print("Failed to connect to DB, errno = {}, msg: {}"
- .format(Helper.convertErrno(err.errno), err.msg))
- raise
- except BaseException:
- print("[=] Unexpected exception")
- raise
-
- # Do this after dbConn is in proper shape
- # Moved to Database()
- # self._stateMachine = StateMechine(self._dbConn)
-
- def getDbConn(self):
- return self._dbConn
-
- # TODO: not used any more, to delete
- def pickAndAllocateTable(self): # pick any table, and "use" it
- return self.tableNumQueue.pickAndAllocate()
-
- # TODO: Not used any more, to delete
- def addTable(self):
- with self._lock:
- tIndex = self.tableNumQueue.push()
- return tIndex
-
- # Not used any more, to delete
- def releaseTable(self, i): # return the table back, so others can use it
- self.tableNumQueue.release(i)
-
- # TODO: not used any more, delete
- def getTableNameToDelete(self):
- tblNum = self.tableNumQueue.pop() # TODO: race condition!
- if (not tblNum): # maybe false
- return False
-
- return "table_{}".format(tblNum)
-
- def cleanUp(self):
- self._dbConn.close()
-
class TaskExecutor():
class BoundedList:
def __init__(self, size=10):
@@ -1584,10 +1166,10 @@ class TaskExecutor():
self._boundedList.add(n)
# def logInfo(self, msg):
- # logger.info(" T[{}.x]: ".format(self._curStep) + msg)
+ # Logging.info(" T[{}.x]: ".format(self._curStep) + msg)
# def logDebug(self, msg):
- # logger.debug(" T[{}.x]: ".format(self._curStep) + msg)
+ # Logging.debug(" T[{}.x]: ".format(self._curStep) + msg)
class Task():
@@ -1596,27 +1178,31 @@ class Task():
instead. But a task is always associated with a DB
'''
taskSn = 100
+ _lock = threading.Lock()
+ _tableLocks: Dict[str, threading.Lock] = {}
@classmethod
def allocTaskNum(cls):
Task.taskSn += 1 # IMPORTANT: cannot use cls.taskSn, since each sub class will have a copy
- # logger.debug("Allocating taskSN: {}".format(Task.taskSn))
+ # Logging.debug("Allocating taskSN: {}".format(Task.taskSn))
return Task.taskSn
def __init__(self, execStats: ExecutionStats, db: Database):
self._workerThread = None
- self._err = None # type: Exception
+ self._err: Optional[Exception] = None
self._aborted = False
self._curStep = None
self._numRows = None # Number of rows affected
# Assign an incremental task serial number
self._taskNum = self.allocTaskNum()
- # logger.debug("Creating new task {}...".format(self._taskNum))
+ # Logging.debug("Creating new task {}...".format(self._taskNum))
self._execStats = execStats
self._db = db # A task is always associated/for a specific DB
+
+
def isSuccess(self):
return self._err is None
@@ -1645,15 +1231,23 @@ class Task():
"To be implemeted by child classes, class name: {}".format(
self.__class__.__name__))
+ def _isServiceStable(self):
+ if not gSvcMgr:
+ return True # we don't run service, so let's assume it's stable
+ return gSvcMgr.isStable() # otherwise let's examine the service
+
def _isErrAcceptable(self, errno, msg):
if errno in [
0x05, # TSDB_CODE_RPC_NOT_READY
0x0B, # Unable to establish connection, more details in TD-1648
- # 0x200, # invalid SQL, TODO: re-examine with TD-934
+ 0x200, # invalid SQL, TODO: re-examine with TD-934
+ 0x20F, # query terminated, possibly due to vnoding being dropped, see TD-1776
+ 0x213, # "Disconnected from service", result of "kill connection ???"
0x217, # "db not selected", client side defined error code
- 0x218, # "Table does not exist" client side defined error code
- 0x360, 0x362,
- 0x369, # tag already exists
+ # 0x218, # "Table does not exist" client side defined error code
+ 0x360, # Table already exists
+ 0x362,
+ # 0x369, # tag already exists
0x36A, 0x36B, 0x36D,
0x381,
0x380, # "db not selected"
@@ -1662,12 +1256,17 @@ class Task():
0x503,
0x510, # vnode not in ready state
0x14, # db not ready, errno changed
- 0x600,
+ 0x600, # Invalid table ID, why?
1000 # REST catch-all error
]:
return True # These are the ALWAYS-ACCEPTABLE ones
- elif (errno in [ 0x0B ]) and gConfig.auto_start_service:
- return True # We may get "network unavilable" when restarting service
+ # This case handled below already.
+ # elif (errno in [ 0x0B ]) and gConfig.auto_start_service:
+ # return True # We may get "network unavilable" when restarting service
+ elif gConfig.ignore_errors: # something is specified on command line
+ moreErrnos = [int(v, 0) for v in gConfig.ignore_errors.split(',')]
+ if errno in moreErrnos:
+ return True
elif errno == 0x200 : # invalid SQL, we need to div in a bit more
if msg.find("invalid column name") != -1:
return True
@@ -1675,8 +1274,8 @@ class Task():
return True
elif msg.find("duplicated column names") != -1: # also alter table tag issues
return True
- elif (gSvcMgr!=None) and gSvcMgr.isRestarting():
- logger.info("Ignoring error when service is restarting: errno = {}, msg = {}".format(errno, msg))
+ elif not self._isServiceStable(): # We are managing service, and ...
+ Logging.info("Ignoring error when service starting/stopping: errno = {}, msg = {}".format(errno, msg))
return True
return False # Not an acceptable error
@@ -1725,7 +1324,7 @@ class Task():
self._err = err
self._aborted = True
except Exception as e:
- self.logInfo("Non-TAOS exception encountered")
+ Logging.info("Non-TAOS exception encountered with: {}".format(self.__class__.__name__))
self._err = e
self._aborted = True
traceback.print_exc()
@@ -1735,10 +1334,11 @@ class Task():
self._aborted = True
traceback.print_exc()
except BaseException: # TODO: what is this again??!!
- self.logDebug(
- "[=] Unexpected exception, SQL: {}".format(
- wt.getDbConn().getLastSql()))
- raise
+ raise RuntimeError("Punt")
+ # self.logDebug(
+ # "[=] Unexpected exception, SQL: {}".format(
+ # wt.getDbConn().getLastSql()))
+ # raise
self._execStats.endTaskType(self.__class__.__name__, self.isSuccess())
self.logDebug("[X] task execution completed, {}, status: {}".format(
@@ -1757,6 +1357,24 @@ class Task():
def getQueryResult(self, wt: WorkerThread): # execute an SQL on the worker thread
return wt.getQueryResult()
+ def lockTable(self, ftName): # full table name
+ # print(" <<" + ftName + '_', end="", flush=True)
+ with Task._lock:
+ if not ftName in Task._tableLocks:
+ Task._tableLocks[ftName] = threading.Lock()
+
+ Task._tableLocks[ftName].acquire()
+
+ def unlockTable(self, ftName):
+ # print('_' + ftName + ">> ", end="", flush=True)
+ with Task._lock:
+ if not ftName in self._tableLocks:
+ raise RuntimeError("Corrupt state, no such lock")
+ lock = Task._tableLocks[ftName]
+ if not lock.locked():
+ raise RuntimeError("Corrupte state, already unlocked")
+ lock.release()
+
class ExecutionStats:
def __init__(self):
@@ -1817,14 +1435,14 @@ class ExecutionStats:
self._failureReason = reason
def printStats(self):
- logger.info(
+ Logging.info(
"----------------------------------------------------------------------")
- logger.info(
+ Logging.info(
"| Crash_Gen test {}, with the following stats:". format(
"FAILED (reason: {})".format(
self._failureReason) if self._failed else "SUCCEEDED"))
- logger.info("| Task Execution Times (success/total):")
- execTimesAny = 0
+ Logging.info("| Task Execution Times (success/total):")
+ execTimesAny = 0.0
for k, n in self._execTimes.items():
execTimesAny += n[0]
errStr = None
@@ -1834,28 +1452,28 @@ class ExecutionStats:
errStrs = ["0x{:X}:{}".format(eno, n) for (eno, n) in errors.items()]
# print("error strings = {}".format(errStrs))
errStr = ", ".join(errStrs)
- logger.info("| {0:<24}: {1}/{2} (Errors: {3})".format(k, n[1], n[0], errStr))
+ Logging.info("| {0:<24}: {1}/{2} (Errors: {3})".format(k, n[1], n[0], errStr))
- logger.info(
+ Logging.info(
"| Total Tasks Executed (success or not): {} ".format(execTimesAny))
- logger.info(
+ Logging.info(
"| Total Tasks In Progress at End: {}".format(
self._tasksInProgress))
- logger.info(
+ Logging.info(
"| Total Task Busy Time (elapsed time when any task is in progress): {:.3f} seconds".format(
self._accRunTime))
- logger.info(
+ Logging.info(
"| Average Per-Task Execution Time: {:.3f} seconds".format(self._accRunTime / execTimesAny))
- logger.info(
+ Logging.info(
"| Total Elapsed Time (from wall clock): {:.3f} seconds".format(
self._elapsedTime))
- logger.info("| Top numbers written: {}".format(TaskExecutor.getBoundedList()))
- logger.info("| Active DB Native Connections (now): {}".format(DbConnNative.totalConnections))
- logger.info("| Longest native query time: {:.3f} seconds, started: {}".
+ Logging.info("| Top numbers written: {}".format(TaskExecutor.getBoundedList()))
+ Logging.info("| Active DB Native Connections (now): {}".format(DbConnNative.totalConnections))
+ Logging.info("| Longest native query time: {:.3f} seconds, started: {}".
format(MyTDSql.longestQueryTime,
time.strftime("%x %X", time.localtime(MyTDSql.lqStartTime))) )
- logger.info("| Longest native query: {}".format(MyTDSql.longestQuery))
- logger.info(
+ Logging.info("| Longest native query: {}".format(MyTDSql.longestQuery))
+ Logging.info(
"----------------------------------------------------------------------")
@@ -1865,11 +1483,14 @@ class StateTransitionTask(Task):
LARGE_NUMBER_OF_RECORDS = 50
SMALL_NUMBER_OF_RECORDS = 3
+ _baseTableNumber = None
+
+ _endState = None # TODO: no longter used?
+
@classmethod
def getInfo(cls): # each sub class should supply their own information
raise RuntimeError("Overriding method expected")
-
- _endState = None
+
@classmethod
def getEndState(cls): # TODO: optimize by calling it fewer times
raise RuntimeError("Overriding method expected")
@@ -1889,7 +1510,10 @@ class StateTransitionTask(Task):
@classmethod
def getRegTableName(cls, i):
- return "reg_table_{}".format(i)
+ if ( StateTransitionTask._baseTableNumber is None): # Set it one time
+ StateTransitionTask._baseTableNumber = Dice.throw(
+ 999) if gConfig.dynamic_db_table_names else 0
+ return "reg_table_{}".format(StateTransitionTask._baseTableNumber + i)
def execute(self, wt: WorkerThread):
super().execute(wt)
@@ -1909,7 +1533,8 @@ class TaskCreateDb(StateTransitionTask):
# was: self.execWtSql(wt, "create database db")
repStr = ""
if gConfig.max_replicas != 1:
- numReplica = Dice.throw(gConfig.max_replicas) + 1 # 1,2 ... N
+ # numReplica = Dice.throw(gConfig.max_replicas) + 1 # 1,2 ... N
+ numReplica = gConfig.max_replicas # fixed, always
repStr = "replica {}".format(numReplica)
self.execWtSql(wt, "create database {} {}"
.format(self._db.getName(), repStr) )
@@ -1925,7 +1550,7 @@ class TaskDropDb(StateTransitionTask):
def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
self.execWtSql(wt, "drop database {}".format(self._db.getName()))
- logger.debug("[OPS] database dropped at {}".format(time.time()))
+ Logging.debug("[OPS] database dropped at {}".format(time.time()))
class TaskCreateSuperTable(StateTransitionTask):
@classmethod
@@ -1938,13 +1563,16 @@ class TaskCreateSuperTable(StateTransitionTask):
def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
if not self._db.exists(wt.getDbConn()):
- logger.debug("Skipping task, no DB yet")
+ Logging.debug("Skipping task, no DB yet")
return
sTable = self._db.getFixedSuperTable() # type: TdSuperTable
# wt.execSql("use db") # should always be in place
+
sTable.create(wt.getDbConn(), self._db.getName(),
- {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'})
+ {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'},
+ dropIfExists = True
+ )
# self.execWtSql(wt,"create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName))
# No need to create the regular tables, INSERT will do that
# automatically
@@ -1957,14 +1585,41 @@ class TdSuperTable:
def getName(self):
return self._stName
+ def drop(self, dbc, dbName, skipCheck = False):
+ if self.exists(dbc, dbName) : # if myself exists
+ fullTableName = dbName + '.' + self._stName
+ dbc.execute("DROP TABLE {}".format(fullTableName))
+ else:
+ if not skipCheck:
+ raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName))
+
+ def exists(self, dbc, dbName):
+ dbc.execute("USE " + dbName)
+ return dbc.existsSuperTable(self._stName)
+
# TODO: odd semantic, create() method is usually static?
- def create(self, dbc, dbName, cols: dict, tags: dict):
+ def create(self, dbc, dbName, cols: dict, tags: dict,
+ dropIfExists = False
+ ):
+
'''Creating a super table'''
- sql = "CREATE TABLE {}.{} ({}) TAGS ({})".format(
- dbName,
- self._stName,
- ",".join(['%s %s'%(k,v) for (k,v) in cols.items()]),
- ",".join(['%s %s'%(k,v) for (k,v) in tags.items()])
+ dbc.execute("USE " + dbName)
+ fullTableName = dbName + '.' + self._stName
+ if dbc.existsSuperTable(self._stName):
+ if dropIfExists:
+ dbc.execute("DROP TABLE {}".format(fullTableName))
+ else: # error
+ raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName))
+
+ # Now let's create
+ sql = "CREATE TABLE {} ({})".format(
+ fullTableName,
+ ",".join(['%s %s'%(k,v) for (k,v) in cols.items()]))
+ if tags is None :
+ sql += " TAGS (dummy int) "
+ else:
+ sql += " TAGS ({})".format(
+ ",".join(['%s %s'%(k,v) for (k,v) in tags.items()])
)
dbc.execute(sql)
@@ -1973,7 +1628,7 @@ class TdSuperTable:
dbc.query("select TBNAME from {}.{}".format(dbName, self._stName)) # TODO: analyze result set later
except taos.error.ProgrammingError as err:
errno2 = Helper.convertErrno(err.errno)
- logger.debug("[=] Failed to get tables from super table: errno=0x{:X}, msg: {}".format(errno2, err))
+ Logging.debug("[=] Failed to get tables from super table: errno=0x{:X}, msg: {}".format(errno2, err))
raise
qr = dbc.getQueryResult()
@@ -1982,14 +1637,25 @@ class TdSuperTable:
def hasRegTables(self, dbc: DbConn, dbName: str):
return dbc.query("SELECT * FROM {}.{}".format(dbName, self._stName)) > 0
- def ensureTable(self, dbc: DbConn, dbName: str, regTableName: str):
+ def ensureTable(self, task: Task, dbc: DbConn, dbName: str, regTableName: str):
sql = "select tbname from {}.{} where tbname in ('{}')".format(dbName, self._stName, regTableName)
if dbc.query(sql) >= 1 : # reg table exists already
return
- sql = "CREATE TABLE {}.{} USING {}.{} tags ({})".format(
- dbName, regTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName)
- )
- dbc.execute(sql)
+
+ # acquire a lock first, so as to be able to *verify*. More details in TD-1471
+ fullTableName = dbName + '.' + regTableName
+ if task is not None: # optional lock
+ task.lockTable(fullTableName)
+ Progress.emit(Progress.CREATE_TABLE_ATTEMPT) # ATTEMPT to create a new table
+ # print("(" + fullTableName[-3:] + ")", end="", flush=True)
+ try:
+ sql = "CREATE TABLE {} USING {}.{} tags ({})".format(
+ fullTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName)
+ )
+ dbc.execute(sql)
+ finally:
+ if task is not None:
+ task.unlockTable(fullTableName) # no matter what
def _getTagStrForSql(self, dbc, dbName: str) :
tags = self._getTags(dbc, dbName)
@@ -2045,15 +1711,39 @@ class TaskReadData(StateTransitionTask):
def canBeginFrom(cls, state: AnyState):
return state.canReadData()
+ # def _canRestartService(self):
+ # if not gSvcMgr:
+ # return True # always
+ # return gSvcMgr.isActive() # only if it's running TODO: race condition here
+
def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
sTable = self._db.getFixedSuperTable()
- # 1 in 5 chance, simulate a broken connection.
- if random.randrange(5) == 0: # TODO: break connection in all situations
- wt.getDbConn().close()
- wt.getDbConn().open()
- print("_r", end="", flush=True)
-
+ # 1 in 5 chance, simulate a broken connection, only if service stable (not restarting)
+ if random.randrange(20)==0: # and self._canRestartService(): # TODO: break connection in all situations
+ # Logging.info("Attempting to reconnect to server") # TODO: change to DEBUG
+ Progress.emit(Progress.SERVICE_RECONNECT_START)
+ try:
+ wt.getDbConn().close()
+ wt.getDbConn().open()
+ except ConnectionError as err: # may fail
+ if not gSvcMgr:
+ Logging.error("Failed to reconnect in client-only mode")
+ raise # Not OK if we are running in client-only mode
+ if gSvcMgr.isRunning(): # may have race conditon, but low prob, due to
+ Logging.error("Failed to reconnect when managed server is running")
+ raise # Not OK if we are running normally
+
+ Progress.emit(Progress.SERVICE_RECONNECT_FAILURE)
+ # Logging.info("Ignoring DB reconnect error")
+
+ # print("_r", end="", flush=True)
+ Progress.emit(Progress.SERVICE_RECONNECT_SUCCESS)
+ # The above might have taken a lot of time, service might be running
+ # by now, causing error below to be incorrectly handled due to timing issue
+ return # TODO: fix server restart status race condtion
+
+
dbc = wt.getDbConn()
dbName = self._db.getName()
for rTbName in sTable.getRegTables(dbc, dbName): # regular tables
@@ -2088,7 +1778,7 @@ class TaskReadData(StateTransitionTask):
dbc.execute("select {} from {}.{}".format(aggExpr, dbName, sTable.getName()))
except taos.error.ProgrammingError as err:
errno2 = Helper.convertErrno(err.errno)
- logger.debug("[=] Read Failure: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, dbc.getLastSql()))
+ Logging.debug("[=] Read Failure: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, dbc.getLastSql()))
raise
class TaskDropSuperTable(StateTransitionTask):
@@ -2119,7 +1809,7 @@ class TaskDropSuperTable(StateTransitionTask):
errno2 = Helper.convertErrno(err.errno)
if (errno2 in [0x362]): # mnode invalid table name
isSuccess = False
- logger.debug("[DB] Acceptable error when dropping a table")
+ Logging.debug("[DB] Acceptable error when dropping a table")
continue # try to delete next regular table
if (not tickOutput):
@@ -2184,7 +1874,7 @@ class TaskRestartService(StateTransitionTask):
with self._classLock:
if self._isRunning:
- print("Skipping restart task, another running already")
+ Logging.info("Skipping restart task, another running already")
return
self._isRunning = True
@@ -2199,20 +1889,19 @@ class TaskAddData(StateTransitionTask):
# Track which table is being actively worked on
activeTable: Set[int] = set()
- # We use these two files to record operations to DB, useful for power-off
- # tests
- fAddLogReady = None
- fAddLogDone = None
+ # We use these two files to record operations to DB, useful for power-off tests
+ fAddLogReady = None # type: TextIOWrapper
+ fAddLogDone = None # type: TextIOWrapper
@classmethod
def prepToRecordOps(cls):
if gConfig.record_ops:
if (cls.fAddLogReady is None):
- logger.info(
+ Logging.info(
"Recording in a file operations to be performed...")
cls.fAddLogReady = open("add_log_ready.txt", "w")
if (cls.fAddLogDone is None):
- logger.info("Recording in a file operations completed...")
+ Logging.info("Recording in a file operations completed...")
cls.fAddLogDone = open("add_log_done.txt", "w")
@classmethod
@@ -2223,13 +1912,88 @@ class TaskAddData(StateTransitionTask):
def canBeginFrom(cls, state: AnyState):
return state.canAddData()
+ def _addDataInBatch(self, db, dbc, regTableName, te: TaskExecutor):
+ numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS
+ fullTableName = db.getName() + '.' + regTableName
+
+ sql = "insert into {} values ".format(fullTableName)
+ for j in range(numRecords): # number of records per table
+ nextInt = db.getNextInt()
+ nextTick = db.getNextTick()
+ sql += "('{}', {});".format(nextTick, nextInt)
+ dbc.execute(sql)
+
+ def _addData(self, db, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches
+ numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS
+
+ for j in range(numRecords): # number of records per table
+ nextInt = db.getNextInt()
+ nextTick = db.getNextTick()
+ if gConfig.record_ops:
+ self.prepToRecordOps()
+ self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName))
+ self.fAddLogReady.flush()
+ os.fsync(self.fAddLogReady)
+
+ # TODO: too ugly trying to lock the table reliably, refactor...
+ fullTableName = db.getName() + '.' + regTableName
+ if gConfig.verify_data:
+ self.lockTable(fullTableName)
+ # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written
+
+ try:
+ sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {})
+ fullTableName,
+ # ds.getFixedSuperTableName(),
+ # ds.getNextBinary(), ds.getNextFloat(),
+ nextTick, nextInt)
+ dbc.execute(sql)
+ except: # Any exception at all
+ if gConfig.verify_data:
+ self.unlockTable(fullTableName)
+ raise
+
+ # Now read it back and verify, we might encounter an error if table is dropped
+ if gConfig.verify_data: # only if command line asks for it
+ try:
+ readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts='{}'".
+ format(db.getName(), regTableName, nextTick))
+ if readBack != nextInt :
+ raise taos.error.ProgrammingError(
+ "Failed to read back same data, wrote: {}, read: {}"
+ .format(nextInt, readBack), 0x999)
+ except taos.error.ProgrammingError as err:
+ errno = Helper.convertErrno(err.errno)
+ if errno in [0x991, 0x992] : # not a single result
+ raise taos.error.ProgrammingError(
+ "Failed to read back same data for tick: {}, wrote: {}, read: {}"
+ .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"),
+ errno)
+ elif errno in [0x218, 0x362]: # table doesn't exist
+ # do nothing
+ dummy = 0
+ else:
+ # Re-throw otherwise
+ raise
+ finally:
+ self.unlockTable(fullTableName) # Unlock the table no matter what
+
+ # Successfully wrote the data into the DB, let's record it somehow
+ te.recordDataMark(nextInt)
+
+ if gConfig.record_ops:
+ self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName))
+ self.fAddLogDone.flush()
+ os.fsync(self.fAddLogDone)
+
def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
# ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access
db = self._db
dbc = wt.getDbConn()
- tblSeq = list(range(
- self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES))
- random.shuffle(tblSeq)
+ numTables = self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES
+ numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS
+ tblSeq = list(range(numTables ))
+ random.shuffle(tblSeq) # now we have random sequence
for i in tblSeq:
if (i in self.activeTable): # wow already active
print("x", end="", flush=True) # concurrent insertion
@@ -2237,541 +2001,19 @@ class TaskAddData(StateTransitionTask):
self.activeTable.add(i) # marking it active
sTable = db.getFixedSuperTable()
- regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i)
- sTable.ensureTable(wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists
+ regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i)
+ fullTableName = db.getName() + '.' + regTableName
+ # self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked"
+ sTable.ensureTable(self, wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists
+ # self._unlockTable(fullTableName)
- for j in range(self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS): # number of records per table
- nextInt = db.getNextInt()
- nextTick = db.getNextTick()
- if gConfig.record_ops:
- self.prepToRecordOps()
- self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName))
- self.fAddLogReady.flush()
- os.fsync(self.fAddLogReady)
- sql = "insert into {}.{} values ('{}', {});".format( # removed: tags ('{}', {})
- db.getName(),
- regTableName,
- # ds.getFixedSuperTableName(),
- # ds.getNextBinary(), ds.getNextFloat(),
- nextTick, nextInt)
- dbc.execute(sql)
- # Successfully wrote the data into the DB, let's record it
- # somehow
- te.recordDataMark(nextInt)
- if gConfig.record_ops:
- self.fAddLogDone.write(
- "Wrote {} to {}\n".format(
- nextInt, regTableName))
- self.fAddLogDone.flush()
- os.fsync(self.fAddLogDone)
-
- # Now read it back and verify, we might encounter an error if table is dropped
- if gConfig.verify_data: # only if command line asks for it
- try:
- readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts= '{}'".
- format(db.getName(), regTableName, nextTick))
- if readBack != nextInt :
- raise taos.error.ProgrammingError(
- "Failed to read back same data, wrote: {}, read: {}"
- .format(nextInt, readBack), 0x999)
- except taos.error.ProgrammingError as err:
- errno = Helper.convertErrno(err.errno)
- if errno in [0x991, 0x992] : # not a single result
- raise taos.error.ProgrammingError(
- "Failed to read back same data for tick: {}, wrote: {}, read: {}"
- .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"),
- errno)
- # Re-throw no matter what
- raise
-
-
- self.activeTable.discard(i) # not raising an error, unlike remove
-
-
-# Deterministic random number generator
-class Dice():
- seeded = False # static, uninitialized
-
- @classmethod
- def seed(cls, s): # static
- if (cls.seeded):
- raise RuntimeError(
- "Cannot seed the random generator more than once")
- cls.verifyRNG()
- random.seed(s)
- cls.seeded = True # TODO: protect against multi-threading
-
- @classmethod
- def verifyRNG(cls): # Verify that the RNG is determinstic
- random.seed(0)
- x1 = random.randrange(0, 1000)
- x2 = random.randrange(0, 1000)
- x3 = random.randrange(0, 1000)
- if (x1 != 864 or x2 != 394 or x3 != 776):
- raise RuntimeError("System RNG is not deterministic")
-
- @classmethod
- def throw(cls, stop): # get 0 to stop-1
- return cls.throwRange(0, stop)
-
- @classmethod
- def throwRange(cls, start, stop): # up to stop-1
- if (not cls.seeded):
- raise RuntimeError("Cannot throw dice before seeding it")
- return random.randrange(start, stop)
-
- @classmethod
- def choice(cls, cList):
- return random.choice(cList)
-
-
-class LoggingFilter(logging.Filter):
- def filter(self, record: logging.LogRecord):
- if (record.levelno >= logging.INFO):
- return True # info or above always log
-
- # Commenting out below to adjust...
-
- # if msg.startswith("[TRD]"):
- # return False
- return True
-
-
-class MyLoggingAdapter(logging.LoggerAdapter):
- def process(self, msg, kwargs):
- return "[{}]{}".format(threading.get_ident() % 10000, msg), kwargs
- # return '[%s] %s' % (self.extra['connid'], msg), kwargs
-
-
-class SvcManager:
- def __init__(self):
- print("Starting TDengine Service Manager")
- # signal.signal(signal.SIGTERM, self.sigIntHandler) # Moved to MainExec
- # signal.signal(signal.SIGINT, self.sigIntHandler)
- # signal.signal(signal.SIGUSR1, self.sigUsrHandler) # different handler!
-
- self.inSigHandler = False
- # self._status = MainExec.STATUS_RUNNING # set inside
- # _startTaosService()
- self.svcMgrThread = None # type: ServiceManagerThread
- self._lock = threading.Lock()
- self._isRestarting = False
-
- def _doMenu(self):
- choice = ""
- while True:
- print("\nInterrupting Service Program, Choose an Action: ")
- print("1: Resume")
- print("2: Terminate")
- print("3: Restart")
- # Remember to update the if range below
- # print("Enter Choice: ", end="", flush=True)
- while choice == "":
- choice = input("Enter Choice: ")
- if choice != "":
- break # done with reading repeated input
- if choice in ["1", "2", "3"]:
- break # we are done with whole method
- print("Invalid choice, please try again.")
- choice = "" # reset
- return choice
-
- def sigUsrHandler(self, signalNumber, frame):
- print("Interrupting main thread execution upon SIGUSR1")
- if self.inSigHandler: # already
- print("Ignoring repeated SIG...")
- return # do nothing if it's already not running
- self.inSigHandler = True
-
- choice = self._doMenu()
- if choice == "1":
- # TODO: can the sub-process be blocked due to us not reading from
- # queue?
- self.sigHandlerResume()
- elif choice == "2":
- self.stopTaosService()
- elif choice == "3": # Restart
- self.restart()
- else:
- raise RuntimeError("Invalid menu choice: {}".format(choice))
-
- self.inSigHandler = False
-
- def sigIntHandler(self, signalNumber, frame):
- print("SvcManager: INT Signal Handler starting...")
- if self.inSigHandler:
- print("Ignoring repeated SIG_INT...")
- return
- self.inSigHandler = True
-
- self.stopTaosService()
- print("SvcManager: INT Signal Handler returning...")
- self.inSigHandler = False
-
- def sigHandlerResume(self):
- print("Resuming TDengine service manager thread (main thread)...\n\n")
-
- def _checkServiceManagerThread(self):
- if self.svcMgrThread: # valid svc mgr thread
- if self.svcMgrThread.isStopped(): # done?
- self.svcMgrThread.procIpcBatch() # one last time. TODO: appropriate?
- self.svcMgrThread = None # no more
-
- def _procIpcAll(self):
- while self.isRunning() or self.isRestarting() : # for as long as the svc mgr thread is still here
- if self.isRunning():
- self.svcMgrThread.procIpcBatch() # regular processing,
- self._checkServiceManagerThread()
- elif self.isRetarting():
- print("Service restarting...")
- time.sleep(0.5) # pause, before next round
- print(
- "Service Manager Thread (with subprocess) has ended, main thread now exiting...")
-
- def startTaosService(self):
- with self._lock:
- if self.svcMgrThread:
- raise RuntimeError("Cannot start TAOS service when one may already be running")
-
- # Find if there's already a taosd service, and then kill it
- for proc in psutil.process_iter():
- if proc.name() == 'taosd':
- print("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupe")
- time.sleep(2.0)
- proc.kill()
- # print("Process: {}".format(proc.name()))
-
-
- self.svcMgrThread = ServiceManagerThread() # create the object
- print("Attempting to start TAOS service started, printing out output...")
- self.svcMgrThread.start()
- self.svcMgrThread.procIpcBatch(trimToTarget=10, forceOutput=True) # for printing 10 lines
- print("TAOS service started")
-
- def stopTaosService(self, outputLines=20):
- with self._lock:
- if not self.isRunning():
- logger.warning("Cannot stop TAOS service, not running")
- return
-
- print("Terminating Service Manager Thread (SMT) execution...")
- self.svcMgrThread.stop()
- if self.svcMgrThread.isStopped():
- self.svcMgrThread.procIpcBatch(outputLines) # one last time
- self.svcMgrThread = None
- print("End of TDengine Service Output")
- print("----- TDengine Service (managed by SMT) is now terminated -----\n")
+ if Dice.throw(1) == 0: # 1 in 2 chance
+ self._addData(db, dbc, regTableName, te)
else:
- print("WARNING: SMT did not terminate as expected")
-
- def run(self):
- self.startTaosService()
- self._procIpcAll() # pump/process all the messages, may encounter SIG + restart
- if self.isRunning(): # if sig handler hasn't destroyed it by now
- self.stopTaosService() # should have started already
-
- def restart(self):
- if self._isRestarting:
- logger.warning("Cannot restart service when it's already restarting")
- return
-
- self._isRestarting = True
- if self.isRunning():
- self.stopTaosService()
- else:
- logger.warning("Service not running when restart requested")
+ self._addDataInBatch(db, dbc, regTableName, te)
- self.startTaosService()
- self._isRestarting = False
-
- def isRunning(self):
- return self.svcMgrThread != None
-
- def isRestarting(self):
- return self._isRestarting
-
-class ServiceManagerThread:
- MAX_QUEUE_SIZE = 10000
-
- def __init__(self):
- self._tdeSubProcess = None # type: TdeSubProcess
- self._thread = None
- self._status = None
-
- def getStatus(self):
- return self._status
-
- def isRunning(self):
- # return self._thread and self._thread.is_alive()
- return self._status == MainExec.STATUS_RUNNING
-
- def isStopping(self):
- return self._status == MainExec.STATUS_STOPPING
-
- def isStopped(self):
- return self._status == MainExec.STATUS_STOPPED
-
- # Start the thread (with sub process), and wait for the sub service
- # to become fully operational
- def start(self):
- if self._thread:
- raise RuntimeError("Unexpected _thread")
- if self._tdeSubProcess:
- raise RuntimeError("TDengine sub process already created/running")
-
- self._status = MainExec.STATUS_STARTING
-
- self._tdeSubProcess = TdeSubProcess()
- self._tdeSubProcess.start()
-
- self._ipcQueue = Queue()
- self._thread = threading.Thread( # First thread captures server OUTPUT
- target=self.svcOutputReader,
- args=(self._tdeSubProcess.getStdOut(), self._ipcQueue))
- self._thread.daemon = True # thread dies with the program
- self._thread.start()
-
- self._thread2 = threading.Thread( # 2nd thread captures server ERRORs
- target=self.svcErrorReader,
- args=(self._tdeSubProcess.getStdErr(), self._ipcQueue))
- self._thread2.daemon = True # thread dies with the program
- self._thread2.start()
-
- # wait for service to start
- for i in range(0, 100):
- time.sleep(1.0)
- # self.procIpcBatch() # don't pump message during start up
- print("_zz_", end="", flush=True)
- if self._status == MainExec.STATUS_RUNNING:
- logger.info("[] TDengine service READY to process requests")
- return # now we've started
- # TODO: handle this better?
- self.procIpcBatch(100, True) # display output before cronking out, trim to last 20 msgs, force output
- raise RuntimeError("TDengine service did not start successfully")
-
- def stop(self):
- # can be called from both main thread or signal handler
- print("Terminating TDengine service running as the sub process...")
- if self.isStopped():
- print("Service already stopped")
- return
- if self.isStopping():
- print("Service is already being stopped")
- return
- # Linux will send Control-C generated SIGINT to the TDengine process
- # already, ref:
- # https://unix.stackexchange.com/questions/176235/fork-and-how-signals-are-delivered-to-processes
- if not self._tdeSubProcess:
- raise RuntimeError("sub process object missing")
-
- self._status = MainExec.STATUS_STOPPING
- retCode = self._tdeSubProcess.stop()
- print("Attempted to stop sub process, got return code: {}".format(retCode))
- if (retCode==-11): # SGV
- logger.error("[[--ERROR--]]: TDengine service SEGV fault (check core file!)")
-
- if self._tdeSubProcess.isRunning(): # still running
- print("FAILED to stop sub process, it is still running... pid = {}".format(
- self._tdeSubProcess.getPid()))
- else:
- self._tdeSubProcess = None # not running any more
- self.join() # stop the thread, change the status, etc.
-
- def join(self):
- # TODO: sanity check
- if not self.isStopping():
- raise RuntimeError(
- "Unexpected status when ending svc mgr thread: {}".format(
- self._status))
-
- if self._thread:
- self._thread.join()
- self._thread = None
- self._status = MainExec.STATUS_STOPPED
- # STD ERR thread
- self._thread2.join()
- self._thread2 = None
- else:
- print("Joining empty thread, doing nothing")
-
- def _trimQueue(self, targetSize):
- if targetSize <= 0:
- return # do nothing
- q = self._ipcQueue
- if (q.qsize() <= targetSize): # no need to trim
- return
-
- logger.debug("Triming IPC queue to target size: {}".format(targetSize))
- itemsToTrim = q.qsize() - targetSize
- for i in range(0, itemsToTrim):
- try:
- q.get_nowait()
- except Empty:
- break # break out of for loop, no more trimming
-
- TD_READY_MSG = "TDengine is initialized successfully"
-
- def procIpcBatch(self, trimToTarget=0, forceOutput=False):
- self._trimQueue(trimToTarget) # trim if necessary
- # Process all the output generated by the underlying sub process,
- # managed by IO thread
- print("<", end="", flush=True)
- while True:
- try:
- line = self._ipcQueue.get_nowait() # getting output at fast speed
- self._printProgress("_o")
- except Empty:
- # time.sleep(2.3) # wait only if there's no output
- # no more output
- print(".>", end="", flush=True)
- return # we are done with THIS BATCH
- else: # got line, printing out
- if forceOutput:
- logger.info(line)
- else:
- logger.debug(line)
- print(">", end="", flush=True)
-
- _ProgressBars = ["--", "//", "||", "\\\\"]
-
- def _printProgress(self, msg): # TODO: assuming 2 chars
- print(msg, end="", flush=True)
- pBar = self._ProgressBars[Dice.throw(4)]
- print(pBar, end="", flush=True)
- print('\b\b\b\b', end="", flush=True)
-
- def svcOutputReader(self, out: IO, queue):
- # Important Reference: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python
- # print("This is the svcOutput Reader...")
- # for line in out :
- for line in iter(out.readline, b''):
- # print("Finished reading a line: {}".format(line))
- # print("Adding item to queue...")
- try:
- line = line.decode("utf-8").rstrip()
- except UnicodeError:
- print("\nNon-UTF8 server output: {}\n".format(line))
-
- # This might block, and then causing "out" buffer to block
- queue.put(line)
- self._printProgress("_i")
-
- if self._status == MainExec.STATUS_STARTING: # we are starting, let's see if we have started
- if line.find(self.TD_READY_MSG) != -1: # found
- logger.info("Waiting for the service to become FULLY READY")
- time.sleep(1.0) # wait for the server to truly start. TODO: remove this
- logger.info("Service is now FULLY READY")
- self._status = MainExec.STATUS_RUNNING
-
- # Trim the queue if necessary: TODO: try this 1 out of 10 times
- self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10) # trim to 90% size
-
- if self.isStopping(): # TODO: use thread status instead
- # WAITING for stopping sub process to finish its outptu
- print("_w", end="", flush=True)
-
- # queue.put(line)
- # meaning sub process must have died
- print("\nNo more output from IO thread managing TDengine service")
- out.close()
-
- def svcErrorReader(self, err: IO, queue):
- for line in iter(err.readline, b''):
- print("\nTDengine Service (taosd) ERROR (from stderr): {}".format(line))
-
-
-class TdeSubProcess:
- def __init__(self):
- self.subProcess = None
-
- def getStdOut(self):
- return self.subProcess.stdout
-
- def getStdErr(self):
- return self.subProcess.stderr
-
- def isRunning(self):
- return self.subProcess is not None
-
- def getPid(self):
- return self.subProcess.pid
-
- def getBuildPath(self):
- selfPath = os.path.dirname(os.path.realpath(__file__))
- if ("community" in selfPath):
- projPath = selfPath[:selfPath.find("communit")]
- else:
- projPath = selfPath[:selfPath.find("tests")]
-
- for root, dirs, files in os.walk(projPath):
- if ("taosd" in files):
- rootRealPath = os.path.dirname(os.path.realpath(root))
- if ("packaging" not in rootRealPath):
- buildPath = root[:len(root) - len("/build/bin")]
- break
- return buildPath
+ self.activeTable.discard(i) # not raising an error, unlike remove
- def start(self):
- ON_POSIX = 'posix' in sys.builtin_module_names
-
- taosdPath = self.getBuildPath() + "/build/bin/taosd"
- cfgPath = self.getBuildPath() + "/test/cfg"
-
- # Delete the log files
- logPath = self.getBuildPath() + "/test/log"
- # ref: https://stackoverflow.com/questions/1995373/deleting-all-files-in-a-directory-with-python/1995397
- # filelist = [ f for f in os.listdir(logPath) ] # if f.endswith(".bak") ]
- # for f in filelist:
- # filePath = os.path.join(logPath, f)
- # print("Removing log file: {}".format(filePath))
- # os.remove(filePath)
- if os.path.exists(logPath):
- logPathSaved = logPath + "_" + time.strftime('%Y-%m-%d-%H-%M-%S')
- logger.info("Saving old log files to: {}".format(logPathSaved))
- os.rename(logPath, logPathSaved)
- # os.mkdir(logPath) # recreate, no need actually, TDengine will auto-create with proper perms
-
- svcCmd = [taosdPath, '-c', cfgPath]
- # svcCmdSingle = "{} -c {}".format(taosdPath, cfgPath)
- # svcCmd = ['vmstat', '1']
- if self.subProcess: # already there
- raise RuntimeError("Corrupt process state")
-
- # print("Starting service: {}".format(svcCmd))
- self.subProcess = subprocess.Popen(
- svcCmd, shell=False,
- # svcCmdSingle, shell=True, # capture core dump?
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- # bufsize=1, # not supported in binary mode
- close_fds=ON_POSIX
- ) # had text=True, which interferred with reading EOF
-
- def stop(self):
- if not self.subProcess:
- print("Sub process already stopped")
- return -1
-
- retCode = self.subProcess.poll() # contains real sub process return code
- if retCode: # valid return code, process ended
- self.subProcess = None
- else: # process still alive, let's interrupt it
- print(
- "Sub process is running, sending SIG_INT and waiting for it to terminate...")
- # sub process should end, then IPC queue should end, causing IO
- # thread to end
- self.subProcess.send_signal(signal.SIGINT)
- try:
- self.subProcess.wait(10)
- retCode = self.subProcess.returncode
- except subprocess.TimeoutExpired as err:
- print("Time out waiting for TDengine service process to exit")
- retCode = -3
- else:
- print("TDengine service process terminated successfully from SIG_INT")
- retCode = -4
- self.subProcess = None
- return retCode
class ThreadStacks: # stack info for all threads
def __init__(self):
@@ -2794,31 +2036,33 @@ class ThreadStacks: # stack info for all threads
'__init__']: # the thread that extracted the stack
continue # ignore
# Now print
- print("\n<----- Thread Info for ID: {}".format(thNid))
+ print("\n<----- Thread Info for LWP/ID: {} (Execution stopped at Bottom Frame) <-----".format(thNid))
+ stackFrame = 0
for frame in stack:
# print(frame)
- print("File {filename}, line {lineno}, in {name}".format(
- filename=frame.filename, lineno=frame.lineno, name=frame.name))
+ print("[{sf}] File {filename}, line {lineno}, in {name}".format(
+ sf=stackFrame, filename=frame.filename, lineno=frame.lineno, name=frame.name))
print(" {}".format(frame.line))
- print("-----> End of Thread Info\n")
+ stackFrame += 1
+ print("-----> End of Thread Info ----->\n")
class ClientManager:
def __init__(self):
- print("Starting service manager")
+ Logging.info("Starting service manager")
# signal.signal(signal.SIGTERM, self.sigIntHandler)
# signal.signal(signal.SIGINT, self.sigIntHandler)
- self._status = MainExec.STATUS_RUNNING
+ self._status = Status.STATUS_RUNNING
self.tc = None
self.inSigHandler = False
def sigIntHandler(self, signalNumber, frame):
- if self._status != MainExec.STATUS_RUNNING:
+ if self._status != Status.STATUS_RUNNING:
print("Repeated SIGINT received, forced exit...")
# return # do nothing if it's already not running
sys.exit(-1)
- self._status = MainExec.STATUS_STOPPING # immediately set our status
+ self._status = Status.STATUS_STOPPING # immediately set our status
print("ClientManager: Terminating program...")
self.tc.requestToStop()
@@ -2898,15 +2142,20 @@ class ClientManager:
# self._printLastNumbers()
global gConfig
- dbManager = DbManager() # Regular function
+ # Prepare Tde Instance
+ global gContainer
+ tInst = gContainer.defTdeInstance = TdeInstance() # "subdir to hold the instance"
+
+ dbManager = DbManager(gConfig.connector_type, tInst.getDbTarget()) # Regular function
thPool = ThreadPool(gConfig.num_threads, gConfig.max_steps)
self.tc = ThreadCoordinator(thPool, dbManager)
+ Logging.info("Starting client instance: {}".format(tInst))
self.tc.run()
# print("exec stats: {}".format(self.tc.getExecStats()))
# print("TC failed = {}".format(self.tc.isFailed()))
if svcMgr: # gConfig.auto_start_service:
- svcMgr.stopTaosService()
+ svcMgr.stopTaosServices()
svcMgr = None
# Print exec status, etc., AFTER showing messages from the server
self.conclude()
@@ -2936,18 +2185,10 @@ class ClientManager:
# self.tc.getDbManager().cleanUp() # clean up first, so we can show ZERO db connections
self.tc.printStats()
-
-
-
class MainExec:
- STATUS_STARTING = 1
- STATUS_RUNNING = 2
- STATUS_STOPPING = 3
- STATUS_STOPPED = 4
-
def __init__(self):
self._clientMgr = None
- self._svcMgr = None
+ self._svcMgr = None # type: ServiceManager
signal.signal(signal.SIGTERM, self.sigIntHandler)
signal.signal(signal.SIGINT, self.sigIntHandler)
@@ -2960,219 +2201,185 @@ class MainExec:
self._svcMgr.sigUsrHandler(signalNumber, frame)
def sigIntHandler(self, signalNumber, frame):
- if self._svcMgr:
+ if self._svcMgr:
self._svcMgr.sigIntHandler(signalNumber, frame)
- if self._clientMgr:
+ if self._clientMgr:
self._clientMgr.sigIntHandler(signalNumber, frame)
def runClient(self):
global gSvcMgr
if gConfig.auto_start_service:
- self._svcMgr = SvcManager()
- gSvcMgr = self._svcMgr # hack alert
- self._svcMgr.startTaosService() # we start, don't run
+ gSvcMgr = self._svcMgr = ServiceManager(1) # hack alert
+ gSvcMgr.startTaosServices() # we start, don't run
self._clientMgr = ClientManager()
ret = None
try:
ret = self._clientMgr.run(self._svcMgr) # stop TAOS service inside
except requests.exceptions.ConnectionError as err:
- logger.warning("Failed to open REST connection to DB: {}".format(err.getMessage()))
+ Logging.warning("Failed to open REST connection to DB: {}".format(err.getMessage()))
# don't raise
return ret
def runService(self):
global gSvcMgr
- self._svcMgr = SvcManager()
- gSvcMgr = self._svcMgr # save it in a global variable TODO: hack alert
-
- self._svcMgr.run() # run to some end state
- self._svcMgr = None
- gSvcMgr = None
-
- def runTemp(self): # for debugging purposes
- # # Hack to exercise reading from disk, imcreasing coverage. TODO: fix
- # dbc = dbState.getDbConn()
- # sTbName = dbState.getFixedSuperTableName()
- # dbc.execute("create database if not exists db")
- # if not dbState.getState().equals(StateEmpty()):
- # dbc.execute("use db")
-
- # rTables = None
- # try: # the super table may not exist
- # sql = "select TBNAME from db.{}".format(sTbName)
- # logger.info("Finding out tables in super table: {}".format(sql))
- # dbc.query(sql) # TODO: analyze result set later
- # logger.info("Fetching result")
- # rTables = dbc.getQueryResult()
- # logger.info("Result: {}".format(rTables))
- # except taos.error.ProgrammingError as err:
- # logger.info("Initial Super table OPS error: {}".format(err))
-
- # # sys.exit()
- # if ( not rTables == None):
- # # print("rTables[0] = {}, type = {}".format(rTables[0], type(rTables[0])))
- # try:
- # for rTbName in rTables : # regular tables
- # ds = dbState
- # logger.info("Inserting into table: {}".format(rTbName[0]))
- # sql = "insert into db.{} values ('{}', {});".format(
- # rTbName[0],
- # ds.getNextTick(), ds.getNextInt())
- # dbc.execute(sql)
- # for rTbName in rTables : # regular tables
- # dbc.query("select * from db.{}".format(rTbName[0])) # TODO: check success failure
- # logger.info("Initial READING operation is successful")
- # except taos.error.ProgrammingError as err:
- # logger.info("Initial WRITE/READ error: {}".format(err))
-
- # Sandbox testing code
- # dbc = dbState.getDbConn()
- # while True:
- # rows = dbc.query("show databases")
- # print("Rows: {}, time={}".format(rows, time.time()))
- return
-
-
-def main():
- # Super cool Python argument library:
- # https://docs.python.org/3/library/argparse.html
- parser = argparse.ArgumentParser(
- formatter_class=argparse.RawDescriptionHelpFormatter,
- description=textwrap.dedent('''\
- TDengine Auto Crash Generator (PLEASE NOTICE the Prerequisites Below)
- ---------------------------------------------------------------------
- 1. You build TDengine in the top level ./build directory, as described in offical docs
- 2. You run the server there before this script: ./build/bin/taosd -c test/cfg
-
- '''))
-
- # parser.add_argument('-a', '--auto-start-service', action='store_true',
- # help='Automatically start/stop the TDengine service (default: false)')
- # parser.add_argument('-c', '--connector-type', action='store', default='native', type=str,
- # help='Connector type to use: native, rest, or mixed (default: 10)')
- # parser.add_argument('-d', '--debug', action='store_true',
- # help='Turn on DEBUG mode for more logging (default: false)')
- # parser.add_argument('-e', '--run-tdengine', action='store_true',
- # help='Run TDengine service in foreground (default: false)')
- # parser.add_argument('-l', '--larger-data', action='store_true',
- # help='Write larger amount of data during write operations (default: false)')
- # parser.add_argument('-p', '--per-thread-db-connection', action='store_true',
- # help='Use a single shared db connection (default: false)')
- # parser.add_argument('-r', '--record-ops', action='store_true',
- # help='Use a pair of always-fsynced fils to record operations performing + performed, for power-off tests (default: false)')
- # parser.add_argument('-s', '--max-steps', action='store', default=1000, type=int,
- # help='Maximum number of steps to run (default: 100)')
- # parser.add_argument('-t', '--num-threads', action='store', default=5, type=int,
- # help='Number of threads to run (default: 10)')
- # parser.add_argument('-x', '--continue-on-exception', action='store_true',
- # help='Continue execution after encountering unexpected/disallowed errors/exceptions (default: false)')
-
- parser.add_argument(
- '-a',
- '--auto-start-service',
- action='store_true',
- help='Automatically start/stop the TDengine service (default: false)')
- parser.add_argument(
- '-b',
- '--max-dbs',
- action='store',
- default=0,
- type=int,
- help='Maximum number of DBs to keep, set to disable dropping DB. (default: 0)')
- parser.add_argument(
- '-c',
- '--connector-type',
- action='store',
- default='native',
- type=str,
- help='Connector type to use: native, rest, or mixed (default: 10)')
- parser.add_argument(
- '-d',
- '--debug',
- action='store_true',
- help='Turn on DEBUG mode for more logging (default: false)')
- parser.add_argument(
- '-e',
- '--run-tdengine',
- action='store_true',
- help='Run TDengine service in foreground (default: false)')
- parser.add_argument(
- '-i',
- '--max-replicas',
- action='store',
- default=1,
- type=int,
- help='Maximum number of replicas to use, when testing against clusters. (default: 1)')
- parser.add_argument(
- '-l',
- '--larger-data',
- action='store_true',
- help='Write larger amount of data during write operations (default: false)')
- parser.add_argument(
- '-p',
- '--per-thread-db-connection',
- action='store_true',
- help='Use a single shared db connection (default: false)')
- parser.add_argument(
- '-r',
- '--record-ops',
- action='store_true',
- help='Use a pair of always-fsynced fils to record operations performing + performed, for power-off tests (default: false)')
- parser.add_argument(
- '-s',
- '--max-steps',
- action='store',
- default=1000,
- type=int,
- help='Maximum number of steps to run (default: 100)')
- parser.add_argument(
- '-t',
- '--num-threads',
- action='store',
- default=5,
- type=int,
- help='Number of threads to run (default: 10)')
- parser.add_argument(
- '-v',
- '--verify-data',
- action='store_true',
- help='Verify data written in a number of places by reading back (default: false)')
- parser.add_argument(
- '-x',
- '--continue-on-exception',
- action='store_true',
- help='Continue execution after encountering unexpected/disallowed errors/exceptions (default: false)')
-
- global gConfig
- gConfig = parser.parse_args()
-
- # Logging Stuff
- global logger
- _logger = logging.getLogger('CrashGen') # real logger
- _logger.addFilter(LoggingFilter())
- ch = logging.StreamHandler()
- _logger.addHandler(ch)
-
- # Logging adapter, to be used as a logger
- logger = MyLoggingAdapter(_logger, [])
-
- if (gConfig.debug):
- logger.setLevel(logging.DEBUG) # default seems to be INFO
- else:
- logger.setLevel(logging.INFO)
-
- Dice.seed(0) # initial seeding of dice
-
- # Run server or client
- mExec = MainExec()
- if gConfig.run_tdengine: # run server
- mExec.runService()
- else:
- return mExec.runClient()
-
-
-if __name__ == "__main__":
- exitCode = main()
- # print("Exiting with code: {}".format(exitCode))
- sys.exit(exitCode)
+ gSvcMgr = self._svcMgr = ServiceManager(gConfig.num_dnodes) # save it in a global variable TODO: hack alert
+
+ gSvcMgr.run() # run to some end state
+ gSvcMgr = self._svcMgr = None
+
+ def init(self): # TODO: refactor
+ global gContainer
+ gContainer = Container() # micky-mouse DI
+
+ global gSvcMgr # TODO: refactor away
+ gSvcMgr = None
+
+ # Super cool Python argument library:
+ # https://docs.python.org/3/library/argparse.html
+ parser = argparse.ArgumentParser(
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description=textwrap.dedent('''\
+ TDengine Auto Crash Generator (PLEASE NOTICE the Prerequisites Below)
+ ---------------------------------------------------------------------
+ 1. You build TDengine in the top level ./build directory, as described in offical docs
+ 2. You run the server there before this script: ./build/bin/taosd -c test/cfg
+
+ '''))
+
+ parser.add_argument(
+ '-a',
+ '--auto-start-service',
+ action='store_true',
+ help='Automatically start/stop the TDengine service (default: false)')
+ parser.add_argument(
+ '-b',
+ '--max-dbs',
+ action='store',
+ default=0,
+ type=int,
+ help='Maximum number of DBs to keep, set to disable dropping DB. (default: 0)')
+ parser.add_argument(
+ '-c',
+ '--connector-type',
+ action='store',
+ default='native',
+ type=str,
+ help='Connector type to use: native, rest, or mixed (default: 10)')
+ parser.add_argument(
+ '-d',
+ '--debug',
+ action='store_true',
+ help='Turn on DEBUG mode for more logging (default: false)')
+ parser.add_argument(
+ '-e',
+ '--run-tdengine',
+ action='store_true',
+ help='Run TDengine service in foreground (default: false)')
+ parser.add_argument(
+ '-g',
+ '--ignore-errors',
+ action='store',
+ default=None,
+ type=str,
+ help='Ignore error codes, comma separated, 0x supported (default: None)')
+ parser.add_argument(
+ '-i',
+ '--max-replicas',
+ action='store',
+ default=1,
+ type=int,
+ help='Maximum number of replicas to use, when testing against clusters. (default: 1)')
+ parser.add_argument(
+ '-l',
+ '--larger-data',
+ action='store_true',
+ help='Write larger amount of data during write operations (default: false)')
+ parser.add_argument(
+ '-n',
+ '--dynamic-db-table-names',
+ action='store_true',
+ help='Use non-fixed names for dbs/tables, useful for multi-instance executions (default: false)')
+ parser.add_argument(
+ '-o',
+ '--num-dnodes',
+ action='store',
+ default=1,
+ type=int,
+ help='Number of Dnodes to initialize, used with -e option. (default: 1)')
+ parser.add_argument(
+ '-p',
+ '--per-thread-db-connection',
+ action='store_true',
+ help='Use a single shared db connection (default: false)')
+ parser.add_argument(
+ '-r',
+ '--record-ops',
+ action='store_true',
+ help='Use a pair of always-fsynced fils to record operations performing + performed, for power-off tests (default: false)')
+ parser.add_argument(
+ '-s',
+ '--max-steps',
+ action='store',
+ default=1000,
+ type=int,
+ help='Maximum number of steps to run (default: 100)')
+ parser.add_argument(
+ '-t',
+ '--num-threads',
+ action='store',
+ default=5,
+ type=int,
+ help='Number of threads to run (default: 10)')
+ parser.add_argument(
+ '-v',
+ '--verify-data',
+ action='store_true',
+ help='Verify data written in a number of places by reading back (default: false)')
+ parser.add_argument(
+ '-x',
+ '--continue-on-exception',
+ action='store_true',
+ help='Continue execution after encountering unexpected/disallowed errors/exceptions (default: false)')
+
+ global gConfig
+ gConfig = parser.parse_args()
+
+ Logging.clsInit(gConfig)
+
+ Dice.seed(0) # initial seeding of dice
+
+ def run(self):
+ if gConfig.run_tdengine: # run server
+ try:
+ self.runService()
+ return 0 # success
+ except ConnectionError as err:
+ Logging.error("Failed to make DB connection, please check DB instance manually")
+ return -1 # failure
+ else:
+ return self.runClient()
+
+
+class Container():
+ _propertyList = {'defTdeInstance'}
+
+ def __init__(self):
+ self._cargo = {} # No cargo at the beginning
+
+ def _verifyValidProperty(self, name):
+ if not name in self._propertyList:
+ raise CrashGenError("Invalid container property: {}".format(name))
+
+ # Called for an attribute, when other mechanisms fail (compare to __getattribute__)
+ def __getattr__(self, name):
+ self._verifyValidProperty(name)
+ return self._cargo[name] # just a simple lookup
+
+ def __setattr__(self, name, value):
+ if name == '_cargo' : # reserved vars
+ super().__setattr__(name, value)
+ return
+ self._verifyValidProperty(name)
+ self._cargo[name] = value
+
diff --git a/tests/pytest/crash_gen/db.py b/tests/pytest/crash_gen/db.py
new file mode 100644
index 0000000000000000000000000000000000000000..2a4b362f82c4516195becf78ef9771b7c62c3c41
--- /dev/null
+++ b/tests/pytest/crash_gen/db.py
@@ -0,0 +1,441 @@
+from __future__ import annotations
+
+import sys
+import time
+import threading
+import requests
+from requests.auth import HTTPBasicAuth
+
+import taos
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.log import *
+
+from .misc import Logging, CrashGenError, Helper, Dice
+import os
+import datetime
+# from .service_manager import TdeInstance
+
+class DbConn:
+ TYPE_NATIVE = "native-c"
+ TYPE_REST = "rest-api"
+ TYPE_INVALID = "invalid"
+
+ @classmethod
+ def create(cls, connType, dbTarget):
+ if connType == cls.TYPE_NATIVE:
+ return DbConnNative(dbTarget)
+ elif connType == cls.TYPE_REST:
+ return DbConnRest(dbTarget)
+ else:
+ raise RuntimeError(
+ "Unexpected connection type: {}".format(connType))
+
+ @classmethod
+ def createNative(cls, dbTarget) -> DbConn:
+ return cls.create(cls.TYPE_NATIVE, dbTarget)
+
+ @classmethod
+ def createRest(cls, dbTarget) -> DbConn:
+ return cls.create(cls.TYPE_REST, dbTarget)
+
+ def __init__(self, dbTarget):
+ self.isOpen = False
+ self._type = self.TYPE_INVALID
+ self._lastSql = None
+ self._dbTarget = dbTarget
+
+ def __repr__(self):
+ return "[DbConn: type={}, target={}]".format(self._type, self._dbTarget)
+
+ def getLastSql(self):
+ return self._lastSql
+
+ def open(self):
+ if (self.isOpen):
+ raise RuntimeError("Cannot re-open an existing DB connection")
+
+ # below implemented by child classes
+ self.openByType()
+
+ Logging.debug("[DB] data connection opened: {}".format(self))
+ self.isOpen = True
+
+ def close(self):
+ raise RuntimeError("Unexpected execution, should be overriden")
+
+ def queryScalar(self, sql) -> int:
+ return self._queryAny(sql)
+
+ def queryString(self, sql) -> str:
+ return self._queryAny(sql)
+
+ def _queryAny(self, sql): # actual query result as an int
+ if (not self.isOpen):
+ raise RuntimeError("Cannot query database until connection is open")
+ nRows = self.query(sql)
+ if nRows != 1:
+ raise taos.error.ProgrammingError(
+ "Unexpected result for query: {}, rows = {}".format(sql, nRows),
+ (0x991 if nRows==0 else 0x992)
+ )
+ if self.getResultRows() != 1 or self.getResultCols() != 1:
+ raise RuntimeError("Unexpected result set for query: {}".format(sql))
+ return self.getQueryResult()[0][0]
+
+ def use(self, dbName):
+ self.execute("use {}".format(dbName))
+
+ def existsDatabase(self, dbName: str):
+ ''' Check if a certain database exists '''
+ self.query("show databases")
+ dbs = [v[0] for v in self.getQueryResult()] # ref: https://stackoverflow.com/questions/643823/python-list-transformation
+ # ret2 = dbName in dbs
+ # print("dbs = {}, str = {}, ret2={}, type2={}".format(dbs, dbName,ret2, type(dbName)))
+ return dbName in dbs # TODO: super weird type mangling seen, once here
+
+ def existsSuperTable(self, stName):
+ self.query("show stables")
+ sts = [v[0] for v in self.getQueryResult()]
+ return stName in sts
+
+ def hasTables(self):
+ return self.query("show tables") > 0
+
+ def execute(self, sql):
+ ''' Return the number of rows affected'''
+ raise RuntimeError("Unexpected execution, should be overriden")
+
+ def safeExecute(self, sql):
+ '''Safely execute any SQL query, returning True/False upon success/failure'''
+ try:
+ self.execute(sql)
+ return True # ignore num of results, return success
+ except taos.error.ProgrammingError as err:
+ return False # failed, for whatever TAOS reason
+ # Not possile to reach here, non-TAOS exception would have been thrown
+
+ def query(self, sql) -> int: # return num rows returned
+ ''' Return the number of rows affected'''
+ raise RuntimeError("Unexpected execution, should be overriden")
+
+ def openByType(self):
+ raise RuntimeError("Unexpected execution, should be overriden")
+
+ def getQueryResult(self):
+ raise RuntimeError("Unexpected execution, should be overriden")
+
+ def getResultRows(self):
+ raise RuntimeError("Unexpected execution, should be overriden")
+
+ def getResultCols(self):
+ raise RuntimeError("Unexpected execution, should be overriden")
+
+# Sample: curl -u root:taosdata -d "show databases" localhost:6020/rest/sql
+
+
+class DbConnRest(DbConn):
+ REST_PORT_INCREMENT = 11
+
+ def __init__(self, dbTarget: DbTarget):
+ super().__init__(dbTarget)
+ self._type = self.TYPE_REST
+ restPort = dbTarget.port + 11
+ self._url = "http://{}:{}/rest/sql".format(
+ dbTarget.hostAddr, dbTarget.port + self.REST_PORT_INCREMENT)
+ self._result = None
+
+ def openByType(self): # Open connection
+ pass # do nothing, always open
+
+ def close(self):
+ if (not self.isOpen):
+ raise RuntimeError("Cannot clean up database until connection is open")
+ # Do nothing for REST
+ Logging.debug("[DB] REST Database connection closed")
+ self.isOpen = False
+
+ def _doSql(self, sql):
+ self._lastSql = sql # remember this, last SQL attempted
+ try:
+ r = requests.post(self._url,
+ data = sql,
+ auth = HTTPBasicAuth('root', 'taosdata'))
+ except:
+ print("REST API Failure (TODO: more info here)")
+ raise
+ rj = r.json()
+ # Sanity check for the "Json Result"
+ if ('status' not in rj):
+ raise RuntimeError("No status in REST response")
+
+ if rj['status'] == 'error': # clearly reported error
+ if ('code' not in rj): # error without code
+ raise RuntimeError("REST error return without code")
+ errno = rj['code'] # May need to massage this in the future
+ # print("Raising programming error with REST return: {}".format(rj))
+ raise taos.error.ProgrammingError(
+ rj['desc'], errno) # todo: check existance of 'desc'
+
+ if rj['status'] != 'succ': # better be this
+ raise RuntimeError(
+ "Unexpected REST return status: {}".format(
+ rj['status']))
+
+ nRows = rj['rows'] if ('rows' in rj) else 0
+ self._result = rj
+ return nRows
+
+ def execute(self, sql):
+ if (not self.isOpen):
+ raise RuntimeError(
+ "Cannot execute database commands until connection is open")
+ Logging.debug("[SQL-REST] Executing SQL: {}".format(sql))
+ nRows = self._doSql(sql)
+ Logging.debug(
+ "[SQL-REST] Execution Result, nRows = {}, SQL = {}".format(nRows, sql))
+ return nRows
+
+ def query(self, sql): # return rows affected
+ return self.execute(sql)
+
+ def getQueryResult(self):
+ return self._result['data']
+
+ def getResultRows(self):
+ print(self._result)
+ raise RuntimeError("TBD") # TODO: finish here to support -v under -c rest
+ # return self._tdSql.queryRows
+
+ def getResultCols(self):
+ print(self._result)
+ raise RuntimeError("TBD")
+
+ # Duplicate code from TDMySQL, TODO: merge all this into DbConnNative
+
+
+class MyTDSql:
+ # Class variables
+ _clsLock = threading.Lock() # class wide locking
+ longestQuery = None # type: str
+ longestQueryTime = 0.0 # seconds
+ lqStartTime = 0.0
+ # lqEndTime = 0.0 # Not needed, as we have the two above already
+
+ def __init__(self, hostAddr, cfgPath):
+ # Make the DB connection
+ self._conn = taos.connect(host=hostAddr, config=cfgPath)
+ self._cursor = self._conn.cursor()
+
+ self.queryRows = 0
+ self.queryCols = 0
+ self.affectedRows = 0
+
+ # def init(self, cursor, log=True):
+ # self.cursor = cursor
+ # if (log):
+ # caller = inspect.getframeinfo(inspect.stack()[1][0])
+ # self.cursor.log(caller.filename + ".sql")
+
+ def close(self):
+ self._cursor.close() # can we double close?
+ self._conn.close() # TODO: very important, cursor close does NOT close DB connection!
+ self._cursor.close()
+
+ def _execInternal(self, sql):
+ startTime = time.time()
+ # Logging.debug("Executing SQL: " + sql)
+ ret = self._cursor.execute(sql)
+ # print("\nSQL success: {}".format(sql))
+ queryTime = time.time() - startTime
+ # Record the query time
+ cls = self.__class__
+ if queryTime > (cls.longestQueryTime + 0.01) :
+ with cls._clsLock:
+ cls.longestQuery = sql
+ cls.longestQueryTime = queryTime
+ cls.lqStartTime = startTime
+ return ret
+
+ def query(self, sql):
+ self.sql = sql
+ try:
+ self._execInternal(sql)
+ self.queryResult = self._cursor.fetchall()
+ self.queryRows = len(self.queryResult)
+ self.queryCols = len(self._cursor.description)
+ except Exception as e:
+ # caller = inspect.getframeinfo(inspect.stack()[1][0])
+ # args = (caller.filename, caller.lineno, sql, repr(e))
+ # tdLog.exit("%s(%d) failed: sql:%s, %s" % args)
+ raise
+ return self.queryRows
+
+ def execute(self, sql):
+ self.sql = sql
+ try:
+ self.affectedRows = self._execInternal(sql)
+ except Exception as e:
+ # caller = inspect.getframeinfo(inspect.stack()[1][0])
+ # args = (caller.filename, caller.lineno, sql, repr(e))
+ # tdLog.exit("%s(%d) failed: sql:%s, %s" % args)
+ raise
+ return self.affectedRows
+
+class DbTarget:
+ def __init__(self, cfgPath, hostAddr, port):
+ self.cfgPath = cfgPath
+ self.hostAddr = hostAddr
+ self.port = port
+
+ def __repr__(self):
+ return "[DbTarget: cfgPath={}, host={}:{}]".format(
+ Helper.getFriendlyPath(self.cfgPath), self.hostAddr, self.port)
+
+ def getEp(self):
+ return "{}:{}".format(self.hostAddr, self.port)
+
+class DbConnNative(DbConn):
+ # Class variables
+ _lock = threading.Lock()
+ # _connInfoDisplayed = False # TODO: find another way to display this
+ totalConnections = 0 # Not private
+
+ def __init__(self, dbTarget):
+ super().__init__(dbTarget)
+ self._type = self.TYPE_NATIVE
+ self._conn = None
+ # self._cursor = None
+
+ def openByType(self): # Open connection
+ # global gContainer
+ # tInst = tInst or gContainer.defTdeInstance # set up in ClientManager, type: TdeInstance
+ # cfgPath = self.getBuildPath() + "/test/cfg"
+ # cfgPath = tInst.getCfgDir()
+ # hostAddr = tInst.getHostAddr()
+
+ cls = self.__class__ # Get the class, to access class variables
+ with cls._lock: # force single threading for opening DB connections. # TODO: whaaat??!!!
+ dbTarget = self._dbTarget
+ # if not cls._connInfoDisplayed:
+ # cls._connInfoDisplayed = True # updating CLASS variable
+ Logging.debug("Initiating TAOS native connection to {}".format(dbTarget))
+ # Make the connection
+ # self._conn = taos.connect(host=hostAddr, config=cfgPath) # TODO: make configurable
+ # self._cursor = self._conn.cursor()
+ # Record the count in the class
+ self._tdSql = MyTDSql(dbTarget.hostAddr, dbTarget.cfgPath) # making DB connection
+ cls.totalConnections += 1
+
+ self._tdSql.execute('reset query cache')
+ # self._cursor.execute('use db') # do this at the beginning of every
+
+ # Open connection
+ # self._tdSql = MyTDSql()
+ # self._tdSql.init(self._cursor)
+
+ def close(self):
+ if (not self.isOpen):
+ raise RuntimeError("Cannot clean up database until connection is open")
+ self._tdSql.close()
+ # Decrement the class wide counter
+ cls = self.__class__ # Get the class, to access class variables
+ with cls._lock:
+ cls.totalConnections -= 1
+
+ Logging.debug("[DB] Database connection closed")
+ self.isOpen = False
+
+ def execute(self, sql):
+ if (not self.isOpen):
+ raise RuntimeError("Cannot execute database commands until connection is open")
+ Logging.debug("[SQL] Executing SQL: {}".format(sql))
+ self._lastSql = sql
+ nRows = self._tdSql.execute(sql)
+ Logging.debug(
+ "[SQL] Execution Result, nRows = {}, SQL = {}".format(
+ nRows, sql))
+ return nRows
+
+ def query(self, sql): # return rows affected
+ if (not self.isOpen):
+ raise RuntimeError(
+ "Cannot query database until connection is open")
+ Logging.debug("[SQL] Executing SQL: {}".format(sql))
+ self._lastSql = sql
+ nRows = self._tdSql.query(sql)
+ Logging.debug(
+ "[SQL] Query Result, nRows = {}, SQL = {}".format(
+ nRows, sql))
+ return nRows
+ # results are in: return self._tdSql.queryResult
+
+ def getQueryResult(self):
+ return self._tdSql.queryResult
+
+ def getResultRows(self):
+ return self._tdSql.queryRows
+
+ def getResultCols(self):
+ return self._tdSql.queryCols
+
+
+class DbManager():
+ ''' This is a wrapper around DbConn(), to make it easier to use.
+
+ TODO: rename this to DbConnManager
+ '''
+ def __init__(self, cType, dbTarget):
+ # self.tableNumQueue = LinearQueue() # TODO: delete?
+ # self.openDbServerConnection()
+ self._dbConn = DbConn.createNative(dbTarget) if (
+ cType == 'native') else DbConn.createRest(dbTarget)
+ try:
+ self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected
+ except taos.error.ProgrammingError as err:
+ # print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err))
+ if (err.msg == 'client disconnected'): # cannot open DB connection
+ print(
+ "Cannot establish DB connection, please re-run script without parameter, and follow the instructions.")
+ sys.exit(2)
+ else:
+ print("Failed to connect to DB, errno = {}, msg: {}"
+ .format(Helper.convertErrno(err.errno), err.msg))
+ raise
+ except BaseException:
+ print("[=] Unexpected exception")
+ raise
+
+ # Do this after dbConn is in proper shape
+ # Moved to Database()
+ # self._stateMachine = StateMechine(self._dbConn)
+
+ def getDbConn(self):
+ return self._dbConn
+
+ # TODO: not used any more, to delete
+ def pickAndAllocateTable(self): # pick any table, and "use" it
+ return self.tableNumQueue.pickAndAllocate()
+
+ # TODO: Not used any more, to delete
+ def addTable(self):
+ with self._lock:
+ tIndex = self.tableNumQueue.push()
+ return tIndex
+
+ # Not used any more, to delete
+ def releaseTable(self, i): # return the table back, so others can use it
+ self.tableNumQueue.release(i)
+
+ # TODO: not used any more, delete
+ def getTableNameToDelete(self):
+ tblNum = self.tableNumQueue.pop() # TODO: race condition!
+ if (not tblNum): # maybe false
+ return False
+
+ return "table_{}".format(tblNum)
+
+ def cleanUp(self):
+ self._dbConn.close()
+
diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py
new file mode 100644
index 0000000000000000000000000000000000000000..2d2ce99d95582809a8940a8d777bc166b633747c
--- /dev/null
+++ b/tests/pytest/crash_gen/misc.py
@@ -0,0 +1,186 @@
+import threading
+import random
+import logging
+import os
+
+
+class CrashGenError(Exception):
+ def __init__(self, msg=None, errno=None):
+ self.msg = msg
+ self.errno = errno
+
+ def __str__(self):
+ return self.msg
+
+
+class LoggingFilter(logging.Filter):
+ def filter(self, record: logging.LogRecord):
+ if (record.levelno >= logging.INFO):
+ return True # info or above always log
+
+ # Commenting out below to adjust...
+
+ # if msg.startswith("[TRD]"):
+ # return False
+ return True
+
+
+class MyLoggingAdapter(logging.LoggerAdapter):
+ def process(self, msg, kwargs):
+ return "[{:04d}] {}".format(threading.get_ident() % 10000, msg), kwargs
+ # return '[%s] %s' % (self.extra['connid'], msg), kwargs
+
+
+class Logging:
+ logger = None
+
+ @classmethod
+ def getLogger(cls):
+ return logger
+
+ @classmethod
+ def clsInit(cls, gConfig): # TODO: refactor away gConfig
+ if cls.logger:
+ return
+
+ # Logging Stuff
+ # global misc.logger
+ _logger = logging.getLogger('CrashGen') # real logger
+ _logger.addFilter(LoggingFilter())
+ ch = logging.StreamHandler()
+ _logger.addHandler(ch)
+
+ # Logging adapter, to be used as a logger
+ # print("setting logger variable")
+ # global logger
+ cls.logger = MyLoggingAdapter(_logger, [])
+
+ if (gConfig.debug):
+ cls.logger.setLevel(logging.DEBUG) # default seems to be INFO
+ else:
+ cls.logger.setLevel(logging.INFO)
+
+ @classmethod
+ def info(cls, msg):
+ cls.logger.info(msg)
+
+ @classmethod
+ def debug(cls, msg):
+ cls.logger.debug(msg)
+
+ @classmethod
+ def warning(cls, msg):
+ cls.logger.warning(msg)
+
+ @classmethod
+ def error(cls, msg):
+ cls.logger.error(msg)
+
+class Status:
+ STATUS_STARTING = 1
+ STATUS_RUNNING = 2
+ STATUS_STOPPING = 3
+ STATUS_STOPPED = 4
+
+ def __init__(self, status):
+ self.set(status)
+
+ def __repr__(self):
+ return "[Status: v={}]".format(self._status)
+
+ def set(self, status):
+ self._status = status
+
+ def get(self):
+ return self._status
+
+ def isStarting(self):
+ return self._status == Status.STATUS_STARTING
+
+ def isRunning(self):
+ # return self._thread and self._thread.is_alive()
+ return self._status == Status.STATUS_RUNNING
+
+ def isStopping(self):
+ return self._status == Status.STATUS_STOPPING
+
+ def isStopped(self):
+ return self._status == Status.STATUS_STOPPED
+
+ def isStable(self):
+ return self.isRunning() or self.isStopped()
+
+# Deterministic random number generator
+class Dice():
+ seeded = False # static, uninitialized
+
+ @classmethod
+ def seed(cls, s): # static
+ if (cls.seeded):
+ raise RuntimeError(
+ "Cannot seed the random generator more than once")
+ cls.verifyRNG()
+ random.seed(s)
+ cls.seeded = True # TODO: protect against multi-threading
+
+ @classmethod
+ def verifyRNG(cls): # Verify that the RNG is determinstic
+ random.seed(0)
+ x1 = random.randrange(0, 1000)
+ x2 = random.randrange(0, 1000)
+ x3 = random.randrange(0, 1000)
+ if (x1 != 864 or x2 != 394 or x3 != 776):
+ raise RuntimeError("System RNG is not deterministic")
+
+ @classmethod
+ def throw(cls, stop): # get 0 to stop-1
+ return cls.throwRange(0, stop)
+
+ @classmethod
+ def throwRange(cls, start, stop): # up to stop-1
+ if (not cls.seeded):
+ raise RuntimeError("Cannot throw dice before seeding it")
+ return random.randrange(start, stop)
+
+ @classmethod
+ def choice(cls, cList):
+ return random.choice(cList)
+
+class Helper:
+ @classmethod
+ def convertErrno(cls, errno):
+ return errno if (errno > 0) else 0x80000000 + errno
+
+ @classmethod
+ def getFriendlyPath(cls, path): # returns .../xxx/yyy
+ ht1 = os.path.split(path)
+ ht2 = os.path.split(ht1[0])
+ return ".../" + ht2[1] + '/' + ht1[1]
+
+
+class Progress:
+ STEP_BOUNDARY = 0
+ BEGIN_THREAD_STEP = 1
+ END_THREAD_STEP = 2
+ SERVICE_HEART_BEAT= 3
+ SERVICE_RECONNECT_START = 4
+ SERVICE_RECONNECT_SUCCESS = 5
+ SERVICE_RECONNECT_FAILURE = 6
+ SERVICE_START_NAP = 7
+ CREATE_TABLE_ATTEMPT = 8
+
+ tokens = {
+ STEP_BOUNDARY: '.',
+ BEGIN_THREAD_STEP: '[',
+ END_THREAD_STEP: '] ',
+ SERVICE_HEART_BEAT: '.Y.',
+ SERVICE_RECONNECT_START: '',
+ SERVICE_RECONNECT_FAILURE: '.xr>',
+ SERVICE_START_NAP: '_zz',
+ CREATE_TABLE_ATTEMPT: '_c',
+ }
+
+ @classmethod
+ def emit(cls, token):
+ print(cls.tokens[token], end="", flush=True)
diff --git a/tests/pytest/crash_gen/service_manager.py b/tests/pytest/crash_gen/service_manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..d249abc4396462b6dbacbfcbb1f6619e48161c3b
--- /dev/null
+++ b/tests/pytest/crash_gen/service_manager.py
@@ -0,0 +1,773 @@
+import os
+import io
+import sys
+import threading
+import signal
+import logging
+import time
+import subprocess
+
+from typing import IO, List
+
+try:
+ import psutil
+except:
+ print("Psutil module needed, please install: sudo pip3 install psutil")
+ sys.exit(-1)
+
+from queue import Queue, Empty
+
+from .misc import Logging, Status, CrashGenError, Dice, Helper, Progress
+from .db import DbConn, DbTarget
+
+class TdeInstance():
+ """
+ A class to capture the *static* information of a TDengine instance,
+ including the location of the various files/directories, and basica
+ configuration.
+ """
+
+ @classmethod
+ def _getBuildPath(cls):
+ selfPath = os.path.dirname(os.path.realpath(__file__))
+ if ("community" in selfPath):
+ projPath = selfPath[:selfPath.find("communit")]
+ else:
+ projPath = selfPath[:selfPath.find("tests")]
+
+ buildPath = None
+ for root, dirs, files in os.walk(projPath):
+ if ("taosd" in files):
+ rootRealPath = os.path.dirname(os.path.realpath(root))
+ if ("packaging" not in rootRealPath):
+ buildPath = root[:len(root) - len("/build/bin")]
+ break
+ if buildPath == None:
+ raise RuntimeError("Failed to determine buildPath, selfPath={}, projPath={}"
+ .format(selfPath, projPath))
+ return buildPath
+
+ @classmethod
+ def prepareGcovEnv(cls, env):
+ # Ref: https://gcc.gnu.org/onlinedocs/gcc/Cross-profiling.html
+ bPath = cls._getBuildPath() # build PATH
+ numSegments = len(bPath.split('/')) - 1 # "/x/TDengine/build" should yield 3
+ numSegments = numSegments - 1 # DEBUG only
+ env['GCOV_PREFIX'] = bPath + '/svc_gcov'
+ env['GCOV_PREFIX_STRIP'] = str(numSegments) # Strip every element, plus, ENV needs strings
+ Logging.info("Preparing GCOV environement to strip {} elements and use path: {}".format(
+ numSegments, env['GCOV_PREFIX'] ))
+
+ def __init__(self, subdir='test', tInstNum=0, port=6030, fepPort=6030):
+ self._buildDir = self._getBuildPath()
+ self._subdir = '/' + subdir # TODO: tolerate "/"
+ self._port = port # TODO: support different IP address too
+ self._fepPort = fepPort
+
+ self._tInstNum = tInstNum
+ self._smThread = ServiceManagerThread()
+
+ def getDbTarget(self):
+ return DbTarget(self.getCfgDir(), self.getHostAddr(), self._port)
+
+ def getPort(self):
+ return self._port
+
+ def __repr__(self):
+ return "[TdeInstance: {}, subdir={}]".format(
+ self._buildDir, Helper.getFriendlyPath(self._subdir))
+
+ def generateCfgFile(self):
+ # print("Logger = {}".format(logger))
+ # buildPath = self.getBuildPath()
+ # taosdPath = self._buildPath + "/build/bin/taosd"
+
+ cfgDir = self.getCfgDir()
+ cfgFile = cfgDir + "/taos.cfg" # TODO: inquire if this is fixed
+ if os.path.exists(cfgFile):
+ if os.path.isfile(cfgFile):
+ Logging.warning("Config file exists already, skip creation: {}".format(cfgFile))
+ return # cfg file already exists, nothing to do
+ else:
+ raise CrashGenError("Invalid config file: {}".format(cfgFile))
+ # Now that the cfg file doesn't exist
+ if os.path.exists(cfgDir):
+ if not os.path.isdir(cfgDir):
+ raise CrashGenError("Invalid config dir: {}".format(cfgDir))
+ # else: good path
+ else:
+ os.makedirs(cfgDir, exist_ok=True) # like "mkdir -p"
+ # Now we have a good cfg dir
+ cfgValues = {
+ 'runDir': self.getRunDir(),
+ 'ip': '127.0.0.1', # TODO: change to a network addressable ip
+ 'port': self._port,
+ 'fepPort': self._fepPort,
+ }
+ cfgTemplate = """
+dataDir {runDir}/data
+logDir {runDir}/log
+
+charset UTF-8
+
+firstEp {ip}:{fepPort}
+fqdn {ip}
+serverPort {port}
+
+# was all 135 below
+dDebugFlag 135
+cDebugFlag 135
+rpcDebugFlag 135
+qDebugFlag 135
+# httpDebugFlag 143
+# asyncLog 0
+# tables 10
+maxtablesPerVnode 10
+rpcMaxTime 101
+# cache 2
+keep 36500
+# walLevel 2
+walLevel 1
+#
+# maxConnections 100
+"""
+ cfgContent = cfgTemplate.format_map(cfgValues)
+ f = open(cfgFile, "w")
+ f.write(cfgContent)
+ f.close()
+
+ def rotateLogs(self):
+ logPath = self.getLogDir()
+ # ref: https://stackoverflow.com/questions/1995373/deleting-all-files-in-a-directory-with-python/1995397
+ if os.path.exists(logPath):
+ logPathSaved = logPath + "_" + time.strftime('%Y-%m-%d-%H-%M-%S')
+ Logging.info("Saving old log files to: {}".format(logPathSaved))
+ os.rename(logPath, logPathSaved)
+ # os.mkdir(logPath) # recreate, no need actually, TDengine will auto-create with proper perms
+
+
+ def getExecFile(self): # .../taosd
+ return self._buildDir + "/build/bin/taosd"
+
+ def getRunDir(self): # TODO: rename to "root dir" ?!
+ return self._buildDir + self._subdir
+
+ def getCfgDir(self): # path, not file
+ return self.getRunDir() + "/cfg"
+
+ def getLogDir(self):
+ return self.getRunDir() + "/log"
+
+ def getHostAddr(self):
+ return "127.0.0.1"
+
+ def getServiceCmdLine(self): # to start the instance
+ return [self.getExecFile(), '-c', self.getCfgDir()] # used in subproce.Popen()
+
+ def _getDnodes(self, dbc):
+ dbc.query("show dnodes")
+ cols = dbc.getQueryResult() # id,end_point,vnodes,cores,status,role,create_time,offline reason
+ return {c[1]:c[4] for c in cols} # {'xxx:6030':'ready', 'xxx:6130':'ready'}
+
+ def createDnode(self, dbt: DbTarget):
+ """
+ With a connection to the "first" EP, let's create a dnode for someone else who
+ wants to join.
+ """
+ dbc = DbConn.createNative(self.getDbTarget())
+ dbc.open()
+
+ if dbt.getEp() in self._getDnodes(dbc):
+ Logging.info("Skipping DNode creation for: {}".format(dbt))
+ dbc.close()
+ return
+
+ sql = "CREATE DNODE \"{}\"".format(dbt.getEp())
+ dbc.execute(sql)
+ dbc.close()
+
+ def getStatus(self):
+ return self._smThread.getStatus()
+
+ def getSmThread(self):
+ return self._smThread
+
+ def start(self):
+ if not self.getStatus().isStopped():
+ raise CrashGenError("Cannot start instance from status: {}".format(self.getStatus()))
+
+ Logging.info("Starting TDengine instance: {}".format(self))
+ self.generateCfgFile() # service side generates config file, client does not
+ self.rotateLogs()
+
+ self._smThread.start(self.getServiceCmdLine())
+
+ def stop(self):
+ self._smThread.stop()
+
+ def isFirst(self):
+ return self._tInstNum == 0
+
+
+class TdeSubProcess:
+ """
+ A class to to represent the actual sub process that is the run-time
+ of a TDengine instance.
+
+ It takes a TdeInstance object as its parameter, with the rationale being
+ "a sub process runs an instance".
+ """
+
+ # RET_ALREADY_STOPPED = -1
+ # RET_TIME_OUT = -3
+ # RET_SUCCESS = -4
+
+ def __init__(self):
+ self.subProcess = None
+ # if tInst is None:
+ # raise CrashGenError("Empty instance not allowed in TdeSubProcess")
+ # self._tInst = tInst # Default create at ServiceManagerThread
+
+ def __repr__(self):
+ if self.subProcess is None:
+ return '[TdeSubProc: Empty]'
+ return '[TdeSubProc: pid = {}]'.format(self.getPid())
+
+ def getStdOut(self):
+ return self.subProcess.stdout
+
+ def getStdErr(self):
+ return self.subProcess.stderr
+
+ def isRunning(self):
+ return self.subProcess is not None
+
+ def getPid(self):
+ return self.subProcess.pid
+
+ def start(self, cmdLine):
+ ON_POSIX = 'posix' in sys.builtin_module_names
+
+ # Sanity check
+ if self.subProcess: # already there
+ raise RuntimeError("Corrupt process state")
+
+ # Prepare environment variables for coverage information
+ # Ref: https://stackoverflow.com/questions/2231227/python-subprocess-popen-with-a-modified-environment
+ myEnv = os.environ.copy()
+ TdeInstance.prepareGcovEnv(myEnv)
+
+ # print(myEnv)
+ # print(myEnv.items())
+ # print("Starting TDengine via Shell: {}".format(cmdLineStr))
+
+ useShell = True
+ self.subProcess = subprocess.Popen(
+ ' '.join(cmdLine) if useShell else cmdLine,
+ shell=useShell,
+ # svcCmdSingle, shell=True, # capture core dump?
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ # bufsize=1, # not supported in binary mode
+ close_fds=ON_POSIX,
+ env=myEnv
+ ) # had text=True, which interferred with reading EOF
+
+ STOP_SIGNAL = signal.SIGKILL # What signal to use (in kill) to stop a taosd process?
+
+ def stop(self):
+ """
+ Stop a sub process, and try to return a meaningful return code.
+
+ Common POSIX signal values (from man -7 signal):
+ SIGHUP 1
+ SIGINT 2
+ SIGQUIT 3
+ SIGILL 4
+ SIGTRAP 5
+ SIGABRT 6
+ SIGIOT 6
+ SIGBUS 7
+ SIGEMT -
+ SIGFPE 8
+ SIGKILL 9
+ SIGUSR1 10
+ SIGSEGV 11
+ SIGUSR2 12
+ """
+ if not self.subProcess:
+ Logging.error("Sub process already stopped")
+ return # -1
+
+ retCode = self.subProcess.poll() # ret -N means killed with signal N, otherwise it's from exit(N)
+ if retCode: # valid return code, process ended
+ retCode = -retCode # only if valid
+ Logging.warning("TSP.stop(): process ended itself")
+ self.subProcess = None
+ return retCode
+
+ # process still alive, let's interrupt it
+ Logging.info("Terminate running process, send SIG_{} and wait...".format(self.STOP_SIGNAL))
+ # sub process should end, then IPC queue should end, causing IO thread to end
+ topSubProc = psutil.Process(self.subProcess.pid)
+ for child in topSubProc.children(recursive=True): # or parent.children() for recursive=False
+ child.send_signal(self.STOP_SIGNAL)
+ time.sleep(0.2) # 200 ms
+ # topSubProc.send_signal(sig) # now kill the main sub process (likely the Shell)
+
+ self.subProcess.send_signal(self.STOP_SIGNAL) # main sub process (likely the Shell)
+ self.subProcess.wait(20)
+ retCode = self.subProcess.returncode # should always be there
+ # May throw subprocess.TimeoutExpired exception above, therefore
+ # The process is guranteed to have ended by now
+ self.subProcess = None
+ if retCode != 0: # != (- signal.SIGINT):
+ Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format(
+ self.STOP_SIGNAL, retCode))
+ else:
+ Logging.info("TSP.stop(): sub proc successfully terminated with SIG {}".format(self.STOP_SIGNAL))
+ return - retCode
+
+class ServiceManager:
+ PAUSE_BETWEEN_IPC_CHECK = 1.2 # seconds between checks on STDOUT of sub process
+
+ def __init__(self, numDnodes): # >1 when we run a cluster
+ Logging.info("TDengine Service Manager (TSM) created")
+ self._numDnodes = numDnodes # >1 means we have a cluster
+ self._lock = threading.Lock()
+ # signal.signal(signal.SIGTERM, self.sigIntHandler) # Moved to MainExec
+ # signal.signal(signal.SIGINT, self.sigIntHandler)
+ # signal.signal(signal.SIGUSR1, self.sigUsrHandler) # different handler!
+
+ self.inSigHandler = False
+ # self._status = MainExec.STATUS_RUNNING # set inside
+ # _startTaosService()
+ self._runCluster = (numDnodes > 1)
+ self._tInsts : List[TdeInstance] = []
+ for i in range(0, numDnodes):
+ ti = self._createTdeInstance(i) # construct tInst
+ self._tInsts.append(ti)
+
+ # self.svcMgrThreads : List[ServiceManagerThread] = []
+ # for i in range(0, numDnodes):
+ # thread = self._createThread(i) # construct tInst
+ # self.svcMgrThreads.append(thread)
+
+ def _createTdeInstance(self, dnIndex):
+ if not self._runCluster: # single instance
+ subdir = 'test'
+ else: # Create all threads in a cluster
+ subdir = 'cluster_dnode_{}'.format(dnIndex)
+ fepPort= 6030 # firstEP Port
+ port = fepPort + dnIndex * 100
+ return TdeInstance(subdir, dnIndex, port, fepPort)
+ # return ServiceManagerThread(dnIndex, ti)
+
+ def _doMenu(self):
+ choice = ""
+ while True:
+ print("\nInterrupting Service Program, Choose an Action: ")
+ print("1: Resume")
+ print("2: Terminate")
+ print("3: Restart")
+ # Remember to update the if range below
+ # print("Enter Choice: ", end="", flush=True)
+ while choice == "":
+ choice = input("Enter Choice: ")
+ if choice != "":
+ break # done with reading repeated input
+ if choice in ["1", "2", "3"]:
+ break # we are done with whole method
+ print("Invalid choice, please try again.")
+ choice = "" # reset
+ return choice
+
+ def sigUsrHandler(self, signalNumber, frame):
+ print("Interrupting main thread execution upon SIGUSR1")
+ if self.inSigHandler: # already
+ print("Ignoring repeated SIG...")
+ return # do nothing if it's already not running
+ self.inSigHandler = True
+
+ choice = self._doMenu()
+ if choice == "1":
+ self.sigHandlerResume() # TODO: can the sub-process be blocked due to us not reading from queue?
+ elif choice == "2":
+ self.stopTaosServices()
+ elif choice == "3": # Restart
+ self.restart()
+ else:
+ raise RuntimeError("Invalid menu choice: {}".format(choice))
+
+ self.inSigHandler = False
+
+ def sigIntHandler(self, signalNumber, frame):
+ print("ServiceManager: INT Signal Handler starting...")
+ if self.inSigHandler:
+ print("Ignoring repeated SIG_INT...")
+ return
+ self.inSigHandler = True
+
+ self.stopTaosServices()
+ print("ServiceManager: INT Signal Handler returning...")
+ self.inSigHandler = False
+
+ def sigHandlerResume(self):
+ print("Resuming TDengine service manager (main thread)...\n\n")
+
+ # def _updateThreadStatus(self):
+ # if self.svcMgrThread: # valid svc mgr thread
+ # if self.svcMgrThread.isStopped(): # done?
+ # self.svcMgrThread.procIpcBatch() # one last time. TODO: appropriate?
+ # self.svcMgrThread = None # no more
+
+ def isActive(self):
+ """
+ Determine if the service/cluster is active at all, i.e. at least
+ one thread is not "stopped".
+ """
+ for ti in self._tInsts:
+ if not ti.getStatus().isStopped():
+ return True
+ return False
+
+ def isRunning(self):
+ for ti in self._tInsts:
+ if not ti.getStatus().isRunning():
+ return False
+ return True
+
+
+ # def isRestarting(self):
+ # """
+ # Determine if the service/cluster is being "restarted", i.e., at least
+ # one thread is in "restarting" status
+ # """
+ # for thread in self.svcMgrThreads:
+ # if thread.isRestarting():
+ # return True
+ # return False
+
+ def isStable(self):
+ """
+ Determine if the service/cluster is "stable", i.e. all of the
+ threads are in "stable" status.
+ """
+ for ti in self._tInsts:
+ if not ti.getStatus().isStable():
+ return False
+ return True
+
+ def _procIpcAll(self):
+ while self.isActive():
+ Progress.emit(Progress.SERVICE_HEART_BEAT)
+ for ti in self._tInsts: # all thread objects should always be valid
+ # while self.isRunning() or self.isRestarting() : # for as long as the svc mgr thread is still here
+ status = ti.getStatus()
+ if status.isRunning():
+ th = ti.getSmThread()
+ th.procIpcBatch() # regular processing,
+ if status.isStopped():
+ th.procIpcBatch() # one last time?
+ # self._updateThreadStatus()
+
+ time.sleep(self.PAUSE_BETWEEN_IPC_CHECK) # pause, before next round
+ # raise CrashGenError("dummy")
+ Logging.info("Service Manager Thread (with subprocess) ended, main thread exiting...")
+
+ def _getFirstInstance(self):
+ return self._tInsts[0]
+
+ def startTaosServices(self):
+ with self._lock:
+ if self.isActive():
+ raise RuntimeError("Cannot start TAOS service(s) when one/some may already be running")
+
+ # Find if there's already a taosd service, and then kill it
+ for proc in psutil.process_iter():
+ if proc.name() == 'taosd':
+ Logging.info("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupt")
+ time.sleep(2.0)
+ proc.kill()
+ # print("Process: {}".format(proc.name()))
+
+ # self.svcMgrThread = ServiceManagerThread() # create the object
+
+ for ti in self._tInsts:
+ ti.start()
+ if not ti.isFirst():
+ tFirst = self._getFirstInstance()
+ tFirst.createDnode(ti.getDbTarget())
+ ti.getSmThread().procIpcBatch(trimToTarget=10, forceOutput=True) # for printing 10 lines
+
+ def stopTaosServices(self):
+ with self._lock:
+ if not self.isActive():
+ Logging.warning("Cannot stop TAOS service(s), already not active")
+ return
+
+ for ti in self._tInsts:
+ ti.stop()
+
+ def run(self):
+ self.startTaosServices()
+ self._procIpcAll() # pump/process all the messages, may encounter SIG + restart
+ if self.isActive(): # if sig handler hasn't destroyed it by now
+ self.stopTaosServices() # should have started already
+
+ def restart(self):
+ if not self.isStable():
+ Logging.warning("Cannot restart service/cluster, when not stable")
+ return
+
+ # self._isRestarting = True
+ if self.isActive():
+ self.stopTaosServices()
+ else:
+ Logging.warning("Service not active when restart requested")
+
+ self.startTaosServices()
+ # self._isRestarting = False
+
+ # def isRunning(self):
+ # return self.svcMgrThread != None
+
+ # def isRestarting(self):
+ # return self._isRestarting
+
+class ServiceManagerThread:
+ """
+ A class representing a dedicated thread which manages the "sub process"
+ of the TDengine service, interacting with its STDOUT/ERR.
+
+ It takes a TdeInstance parameter at creation time, or create a default
+ """
+ MAX_QUEUE_SIZE = 10000
+
+ def __init__(self):
+ # Set the sub process
+ self._tdeSubProcess = None # type: TdeSubProcess
+
+ # Arrange the TDengine instance
+ # self._tInstNum = tInstNum # instance serial number in cluster, ZERO based
+ # self._tInst = tInst or TdeInstance() # Need an instance
+
+ self._thread = None # The actual thread, # type: threading.Thread
+ self._status = Status(Status.STATUS_STOPPED) # The status of the underlying service, actually.
+
+ def __repr__(self):
+ return "[SvcMgrThread: status={}, subProc={}]".format(
+ self.getStatus(), self._tdeSubProcess)
+
+ def getStatus(self):
+ return self._status
+
+ # Start the thread (with sub process), and wait for the sub service
+ # to become fully operational
+ def start(self, cmdLine):
+ if self._thread:
+ raise RuntimeError("Unexpected _thread")
+ if self._tdeSubProcess:
+ raise RuntimeError("TDengine sub process already created/running")
+
+ Logging.info("Attempting to start TAOS service: {}".format(self))
+
+ self._status.set(Status.STATUS_STARTING)
+ self._tdeSubProcess = TdeSubProcess()
+ self._tdeSubProcess.start(cmdLine)
+
+ self._ipcQueue = Queue()
+ self._thread = threading.Thread( # First thread captures server OUTPUT
+ target=self.svcOutputReader,
+ args=(self._tdeSubProcess.getStdOut(), self._ipcQueue))
+ self._thread.daemon = True # thread dies with the program
+ self._thread.start()
+
+ self._thread2 = threading.Thread( # 2nd thread captures server ERRORs
+ target=self.svcErrorReader,
+ args=(self._tdeSubProcess.getStdErr(), self._ipcQueue))
+ self._thread2.daemon = True # thread dies with the program
+ self._thread2.start()
+
+ # wait for service to start
+ for i in range(0, 100):
+ time.sleep(1.0)
+ # self.procIpcBatch() # don't pump message during start up
+ Progress.emit(Progress.SERVICE_START_NAP)
+ # print("_zz_", end="", flush=True)
+ if self._status.isRunning():
+ Logging.info("[] TDengine service READY to process requests")
+ Logging.info("[] TAOS service started: {}".format(self))
+ # self._verifyDnode(self._tInst) # query and ensure dnode is ready
+ # Logging.debug("[] TAOS Dnode verified: {}".format(self))
+ return # now we've started
+ # TODO: handle failure-to-start better?
+ self.procIpcBatch(100, True) # display output before cronking out, trim to last 20 msgs, force output
+ raise RuntimeError("TDengine service did not start successfully: {}".format(self))
+
+ def _verifyDnode(self, tInst: TdeInstance):
+ dbc = DbConn.createNative(tInst.getDbTarget())
+ dbc.open()
+ dbc.query("show dnodes")
+ # dbc.query("DESCRIBE {}.{}".format(dbName, self._stName))
+ cols = dbc.getQueryResult() # id,end_point,vnodes,cores,status,role,create_time,offline reason
+ # ret = {row[0]:row[1] for row in stCols if row[3]=='TAG'} # name:type
+ isValid = False
+ for col in cols:
+ # print("col = {}".format(col))
+ ep = col[1].split(':') # 10.1.30.2:6030
+ print("Found ep={}".format(ep))
+ if tInst.getPort() == int(ep[1]): # That's us
+ # print("Valid Dnode matched!")
+ isValid = True # now we are valid
+ break
+ if not isValid:
+ print("Failed to start dnode, sleep for a while")
+ time.sleep(600)
+ raise RuntimeError("Failed to start Dnode, expected port not found: {}".
+ format(tInst.getPort()))
+ dbc.close()
+
+ def stop(self):
+ # can be called from both main thread or signal handler
+ Logging.info("Terminating TDengine service running as the sub process...")
+ if self.getStatus().isStopped():
+ Logging.info("Service already stopped")
+ return
+ if self.getStatus().isStopping():
+ Logging.info("Service is already being stopped")
+ return
+ # Linux will send Control-C generated SIGINT to the TDengine process
+ # already, ref:
+ # https://unix.stackexchange.com/questions/176235/fork-and-how-signals-are-delivered-to-processes
+ if not self._tdeSubProcess:
+ raise RuntimeError("sub process object missing")
+
+ self._status.set(Status.STATUS_STOPPING)
+ # retCode = self._tdeSubProcess.stop()
+ try:
+ retCode = self._tdeSubProcess.stop()
+ # print("Attempted to stop sub process, got return code: {}".format(retCode))
+ if retCode == signal.SIGSEGV : # SGV
+ Logging.error("[[--ERROR--]]: TDengine service SEGV fault (check core file!)")
+ except subprocess.TimeoutExpired as err:
+ Logging.info("Time out waiting for TDengine service process to exit")
+ else:
+ if self._tdeSubProcess.isRunning(): # still running, should now never happen
+ Logging.error("FAILED to stop sub process, it is still running... pid = {}".format(
+ self._tdeSubProcess.getPid()))
+ else:
+ self._tdeSubProcess = None # not running any more
+ self.join() # stop the thread, change the status, etc.
+
+ # Check if it's really stopped
+ outputLines = 10 # for last output
+ if self.getStatus().isStopped():
+ self.procIpcBatch(outputLines) # one last time
+ Logging.debug("End of TDengine Service Output: {}".format(self))
+ Logging.info("----- TDengine Service (managed by SMT) is now terminated -----\n")
+ else:
+ print("WARNING: SMT did not terminate as expected: {}".format(self))
+
+ def join(self):
+ # TODO: sanity check
+ if not self.getStatus().isStopping():
+ raise RuntimeError(
+ "SMT.Join(): Unexpected status: {}".format(self._status))
+
+ if self._thread:
+ self._thread.join()
+ self._thread = None
+ self._status.set(Status.STATUS_STOPPED)
+ # STD ERR thread
+ self._thread2.join()
+ self._thread2 = None
+ else:
+ print("Joining empty thread, doing nothing")
+
+ def _trimQueue(self, targetSize):
+ if targetSize <= 0:
+ return # do nothing
+ q = self._ipcQueue
+ if (q.qsize() <= targetSize): # no need to trim
+ return
+
+ Logging.debug("Triming IPC queue to target size: {}".format(targetSize))
+ itemsToTrim = q.qsize() - targetSize
+ for i in range(0, itemsToTrim):
+ try:
+ q.get_nowait()
+ except Empty:
+ break # break out of for loop, no more trimming
+
+ TD_READY_MSG = "TDengine is initialized successfully"
+
+ def procIpcBatch(self, trimToTarget=0, forceOutput=False):
+ self._trimQueue(trimToTarget) # trim if necessary
+ # Process all the output generated by the underlying sub process,
+ # managed by IO thread
+ print("<", end="", flush=True)
+ while True:
+ try:
+ line = self._ipcQueue.get_nowait() # getting output at fast speed
+ self._printProgress("_o")
+ except Empty:
+ # time.sleep(2.3) # wait only if there's no output
+ # no more output
+ print(".>", end="", flush=True)
+ return # we are done with THIS BATCH
+ else: # got line, printing out
+ if forceOutput:
+ Logging.info('[TAOSD] ' + line)
+ else:
+ Logging.debug('[TAOSD] ' + line)
+ print(">", end="", flush=True)
+
+ _ProgressBars = ["--", "//", "||", "\\\\"]
+
+ def _printProgress(self, msg): # TODO: assuming 2 chars
+ print(msg, end="", flush=True)
+ pBar = self._ProgressBars[Dice.throw(4)]
+ print(pBar, end="", flush=True)
+ print('\b\b\b\b', end="", flush=True)
+
+ def svcOutputReader(self, out: IO, queue):
+ # Important Reference: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python
+ # print("This is the svcOutput Reader...")
+ # for line in out :
+ for line in iter(out.readline, b''):
+ # print("Finished reading a line: {}".format(line))
+ # print("Adding item to queue...")
+ try:
+ line = line.decode("utf-8").rstrip()
+ except UnicodeError:
+ print("\nNon-UTF8 server output: {}\n".format(line))
+
+ # This might block, and then causing "out" buffer to block
+ queue.put(line)
+ self._printProgress("_i")
+
+ if self._status.isStarting(): # we are starting, let's see if we have started
+ if line.find(self.TD_READY_MSG) != -1: # found
+ Logging.info("Waiting for the service to become FULLY READY")
+ time.sleep(1.0) # wait for the server to truly start. TODO: remove this
+ Logging.info("Service is now FULLY READY") # TODO: more ID info here?
+ self._status.set(Status.STATUS_RUNNING)
+
+ # Trim the queue if necessary: TODO: try this 1 out of 10 times
+ self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10) # trim to 90% size
+
+ if self._status.isStopping(): # TODO: use thread status instead
+ # WAITING for stopping sub process to finish its outptu
+ print("_w", end="", flush=True)
+
+ # queue.put(line)
+ # meaning sub process must have died
+ Logging.info("EOF for TDengine STDOUT: {}".format(self))
+ out.close()
+
+ def svcErrorReader(self, err: IO, queue):
+ for line in iter(err.readline, b''):
+ Logging.info("TDengine STDERR: {}".format(line))
+ Logging.info("EOF for TDengine STDERR: {}".format(self))
+ err.close()
\ No newline at end of file
diff --git a/tests/pytest/crash_gen_bootstrap.py b/tests/pytest/crash_gen_bootstrap.py
new file mode 100644
index 0000000000000000000000000000000000000000..fd12284b9d7782ac7df89c37fcb653ca3bebe82b
--- /dev/null
+++ b/tests/pytest/crash_gen_bootstrap.py
@@ -0,0 +1,23 @@
+# -----!/usr/bin/python3.7
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+import sys
+from crash_gen.crash_gen_main import MainExec
+
+if __name__ == "__main__":
+
+ mExec = MainExec()
+ mExec.init()
+ exitCode = mExec.run()
+
+ print("Exiting with code: {}".format(exitCode))
+ sys.exit(exitCode)
diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh
index 39d0fa3d94d36aab88366cc9f428a08f6fd3d1dc..a48dbdc48024ea7348f11b70129ebbf1a486e53d 100755
--- a/tests/pytest/fulltest.sh
+++ b/tests/pytest/fulltest.sh
@@ -24,6 +24,7 @@ python3 ./test.py -f table/alter_wal0.py
python3 ./test.py -f table/column_name.py
python3 ./test.py -f table/column_num.py
python3 ./test.py -f table/db_table.py
+python3 ./test.py -f table/create_sensitive.py
#python3 ./test.py -f table/tablename-boundary.py
# tag
@@ -150,15 +151,17 @@ python3 ./test.py -f query/select_last_crash.py
python3 ./test.py -f query/queryNullValueTest.py
python3 ./test.py -f query/queryInsertValue.py
python3 ./test.py -f query/queryConnection.py
+python3 ./test.py -f query/queryCountCSVData.py
python3 ./test.py -f query/natualInterval.py
python3 ./test.py -f query/bug1471.py
+python3 ./test.py -f query/dataLossTest.py
#stream
python3 ./test.py -f stream/metric_1.py
python3 ./test.py -f stream/new.py
python3 ./test.py -f stream/stream1.py
python3 ./test.py -f stream/stream2.py
-python3 ./test.py -f stream/parser.py
+#python3 ./test.py -f stream/parser.py
python3 ./test.py -f stream/history.py
#alter table
@@ -186,7 +189,7 @@ python3 ./test.py -f functions/function_leastsquares.py -r 1
python3 ./test.py -f functions/function_max.py -r 1
python3 ./test.py -f functions/function_min.py -r 1
python3 ./test.py -f functions/function_operations.py -r 1
-python3 ./test.py -f functions/function_percentile.py
+python3 ./test.py -f functions/function_percentile.py -r 1
python3 ./test.py -f functions/function_spread.py -r 1
python3 ./test.py -f functions/function_stddev.py -r 1
python3 ./test.py -f functions/function_sum.py -r 1
@@ -204,3 +207,20 @@ python3 test.py -f tools/taosdemo.py
python3 test.py -f subscribe/singlemeter.py
#python3 test.py -f subscribe/stability.py
python3 test.py -f subscribe/supertable.py
+
+
+# update
+python3 ./test.py -f update/allow_update.py
+python3 ./test.py -f update/allow_update-0.py
+python3 ./test.py -f update/append_commit_data.py
+python3 ./test.py -f update/append_commit_last-0.py
+python3 ./test.py -f update/append_commit_last.py
+python3 ./test.py -f update/merge_commit_data.py
+python3 ./test.py -f update/merge_commit_data-0.py
+python3 ./test.py -f update/merge_commit_data2.py
+python3 ./test.py -f update/merge_commit_data2_update0.py
+python3 ./test.py -f update/merge_commit_last-0.py
+python3 ./test.py -f update/merge_commit_last.py
+
+# wal
+python3 ./test.py -f wal/addOldWalTest.py
\ No newline at end of file
diff --git a/tests/pytest/functions/function_percentile.py b/tests/pytest/functions/function_percentile.py
index aaeb94372e44282da0d83849c18e40857463b42c..e63d65f2e6a429015e2b4d7dcbe5e8c9884eea5e 100644
--- a/tests/pytest/functions/function_percentile.py
+++ b/tests/pytest/functions/function_percentile.py
@@ -130,8 +130,19 @@ class TDTestCase:
tdSql.query("select percentile(col6, 100) from test")
tdSql.checkData(0, 0, np.percentile(floatData, 100))
tdSql.query("select apercentile(col6, 100) from test")
- print("apercentile result: %s" % tdSql.getData(0, 0))
+ print("apercentile result: %s" % tdSql.getData(0, 0))
+ tdSql.execute("create table meters (ts timestamp, voltage int) tags(loc nchar(20))")
+ tdSql.execute("create table t0 using meters tags('beijing')")
+ tdSql.execute("create table t1 using meters tags('shanghai')")
+ for i in range(self.rowNum):
+ tdSql.execute("insert into t0 values(%d, %d)" % (self.ts + i, i + 1))
+ tdSql.execute("insert into t1 values(%d, %d)" % (self.ts + i, i + 1))
+
+ tdSql.error("select percentile(voltage, 20) from meters")
+ tdSql.query("select apercentile(voltage, 20) from meters")
+ print("apercentile result: %s" % tdSql.getData(0, 0))
+
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh
new file mode 100755
index 0000000000000000000000000000000000000000..2d48de65c9a16fdb6f5f906167baf772694e9a05
--- /dev/null
+++ b/tests/pytest/handle_crash_gen_val_log.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+nohup /root/TDinternal/debug/build/bin/taosd -c /root/TDinternal/community/sim/dnode1/cfg >/dev/null &
+./crash_gen.sh --valgrind -p -t 10 -s 100 -b 4
+pidof taosd|xargs kill
+grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log
+
+for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'`
+do
+if [ -n "$memError" ]; then
+ if [ "$memError" -gt 12 ]; then
+ echo -e "${RED} ## Memory errors number valgrind reports is $memError.\
+ More than our threshold! ## ${NC}"
+ fi
+fi
+done
+
+grep 'start to execute\|definitely lost:' valgrind.err|grep -v 'grep'|uniq|tee crash_gen-definitely-lost-out.log
+for defiMemError in `grep 'definitely lost:' crash_gen-definitely-lost-out.log | awk '{print $7}'`
+do
+
+if [ -n "$defiMemError" ]; then
+ if [ "$defiMemError" -gt 3 ]; then
+ echo -e "${RED} ## Memory errors number valgrind reports \
+ Definitely lost is $defiMemError. More than our threshold! ## ${NC}"
+ exit 8
+ fi
+fi
+done
\ No newline at end of file
diff --git a/tests/pytest/insert/ningsiInsert.py b/tests/pytest/insert/ningsiInsert.py
new file mode 100644
index 0000000000000000000000000000000000000000..bcad2b03ed52bc01cae8b8b800edcd7c213ca375
--- /dev/null
+++ b/tests/pytest/insert/ningsiInsert.py
@@ -0,0 +1,88 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import requests, json
+import threading
+import string
+import random
+import time
+
+class RestfulInsert:
+ def init(self):
+ self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='}
+ self.url = "http://ningsi60:6041/rest/sql"
+ self.ts = 1104508800000
+ self.numOfThreads = 10
+ self.numOfTables = 3000
+ self.dbName = 'netmonitortaos'
+ self.stbName = 'devinfomt'
+ self.prefix = 'dev'
+
+ def get_random_string(self, length):
+ letters = string.ascii_lowercase
+ result_str = ''.join(random.choice(letters) for i in range(length))
+ return result_str
+
+ def createTables(self, threadID):
+ print("create table: thread %d started" % threadID)
+ tablesPerThread = int (self.numOfTables / self.numOfThreads)
+ for i in range(tablesPerThread):
+ data = "create table '%s'.dev_%d using '%s'.'%s' tags('%s', '%s')" % (self.dbName, i + threadID * tablesPerThread, self.dbName, self.stbName, self.get_random_string(25), self.get_random_string(25))
+ response = requests.post(self.url, data, headers = self.header)
+ if response.status_code != 200:
+ print(response.content)
+
+ def insertData(self, threadID):
+ print("insert data: thread %d started" % threadID)
+ tablesPerThread = int (self.numOfTables / self.numOfThreads)
+ base_ts = self.ts
+ while True:
+ i = 0
+ for i in range(tablesPerThread):
+ data = "insert into %s.dev_%d values(%d, '%s', '%s', %d, %d, %d)" % (self.dbName, i + threadID * tablesPerThread, base_ts, self.get_random_string(25), self.get_random_string(30), random.randint(1, 10000), random.randint(1, 10000), random.randint(1, 10000))
+ response = requests.post(self.url, data, headers = self.header)
+ if response.status_code != 200:
+ print(response.content)
+
+ time.sleep(30)
+ base_ts = base_ts + 1
+
+ def run(self):
+ data = "create database if not exists %s keep 7300" % self.dbName
+ requests.post(self.url, data, headers = self.header)
+
+ data = "create table '%s'.'%s' (timeid timestamp, devdesc binary(50), devname binary(50), cpu bigint, temp bigint, ram bigint) tags(devid binary(50), modelid binary(50))" % (self.dbName, self.stbName)
+ requests.post(self.url, data, headers = self.header)
+
+ threads = []
+ for i in range(self.numOfThreads):
+ thread = threading.Thread(target=self.createTables, args=(i,))
+ thread.start()
+ threads.append(thread)
+
+ for i in range(self.numOfThreads):
+ threads[i].join()
+
+ threads = []
+ for i in range(self.numOfThreads):
+ thread = threading.Thread(target=self.insertData, args=(i,))
+ thread.start()
+ threads.append(thread)
+
+ for i in range(self.numOfThreads):
+ threads[i].join()
+
+ri = RestfulInsert()
+ri.init()
+ri.run()
diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py
new file mode 100644
index 0000000000000000000000000000000000000000..a6c9b074e12c571e7ef715a520e20295c936362c
--- /dev/null
+++ b/tests/pytest/insert/restfulInsert.py
@@ -0,0 +1,83 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import requests
+import threading
+import random
+import time
+
+class RestfulInsert:
+ def init(self):
+ self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='}
+ self.url = "http://127.0.0.1:6041/rest/sql"
+ self.ts = 1500000000000
+ self.numOfThreads = 20
+ self.numOfTables = 10000
+ self.recordsPerTable = 10000
+ self.batchSize = 1000
+ self.tableNamePerfix = 't'
+
+ def createTable(self, threadID):
+ tablesPerThread = int (self.numOfTables / self.numOfThreads)
+ print("create table %d to %d" % (tablesPerThread * threadID, tablesPerThread * (threadID + 1) - 1))
+ for i in range(tablesPerThread):
+ tableID = threadID * tablesPerThread
+ name = 'beijing' if tableID % 2 == 0 else 'shanghai'
+ data = "create table test.%s%d using test.meters tags(%d, '%s')" % (self.tableNamePerfix, tableID + i, tableID + i, name)
+ requests.post(self.url, data, headers = self.header)
+
+ def insertData(self, threadID):
+ print("thread %d started" % threadID)
+ tablesPerThread = int (self.numOfTables / self.numOfThreads)
+ for i in range(tablesPerThread):
+ tableID = i + threadID * tablesPerThread
+ start = self.ts
+ for j in range(int(self.recordsPerTable / self.batchSize)):
+ data = "insert into test.%s%d values" % (self.tableNamePerfix, tableID)
+ for k in range(self.batchSize):
+ data += "(%d, %d, %d, %d)" % (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100))
+ requests.post(self.url, data, headers = self.header)
+
+ def run(self):
+ data = "drop database if exists test"
+ requests.post(self.url, data, headers = self.header)
+ data = "create database test"
+ requests.post(self.url, data, headers = self.header)
+ data = "create table test.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))"
+ requests.post(self.url, data, headers = self.header)
+
+ threads = []
+ startTime = time.time()
+ for i in range(self.numOfThreads):
+ thread = threading.Thread(target=self.createTable, args=(i,))
+ thread.start()
+ threads.append(thread)
+ for i in range(self.numOfThreads):
+ threads[i].join()
+ print("createing %d tables takes %d seconds" % (self.numOfTables, (time.time() - startTime)))
+
+ print("inserting data =======")
+ threads = []
+ startTime = time.time()
+ for i in range(self.numOfThreads):
+ thread = threading.Thread(target=self.insertData, args=(i,))
+ thread.start()
+ threads.append(thread)
+
+ for i in range(self.numOfThreads):
+ threads[i].join()
+ print("inserting %d records takes %d seconds" % (self.numOfTables * self.recordsPerTable, (time.time() - startTime)))
+
+ri = RestfulInsert()
+ri.init()
+ri.run()
\ No newline at end of file
diff --git a/tests/pytest/query/bug1874.py b/tests/pytest/query/bug1874.py
new file mode 100644
index 0000000000000000000000000000000000000000..717748487089f229cc578882b91000a7d488fcda
--- /dev/null
+++ b/tests/pytest/query/bug1874.py
@@ -0,0 +1,59 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ def run(self):
+ tdSql.prepare()
+ print("==========step1")
+ print("create table && insert data")
+
+ tdSql.execute("create table join_mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))")
+ tdSql.execute("create table join_mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))")
+ stable=0
+ insertRows = 1000
+ tbnum = 3
+ t0 = 1604298064000
+ tdLog.info("insert %d rows" % (insertRows))
+ for i in range(tbnum):
+ tdSql.execute("create table join_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i))
+ for j in range(insertRows):
+ ret = tdSql.execute(
+ "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" %
+ (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100)))
+ stable=stable+1
+ for i in range(tbnum):
+ tdSql.execute("create table join_1_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i))
+ for j in range(insertRows):
+ ret = tdSql.execute(
+ "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" %
+ (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100)))
+ print("==========step2")
+ print("join query ")
+ tdLog.info("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;")
+ tdSql.error("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;")
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/pytest/query/bug1875.py b/tests/pytest/query/bug1875.py
new file mode 100644
index 0000000000000000000000000000000000000000..12e22a7a259e1612d8da4f11647a0adcde5dae02
--- /dev/null
+++ b/tests/pytest/query/bug1875.py
@@ -0,0 +1,60 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ def run(self):
+ tdSql.prepare()
+ print("==========step1")
+ print("create table && insert data")
+
+ tdSql.execute("create table join_mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))")
+ tdSql.execute("create table join_mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))")
+ stable=0
+ insertRows = 1000
+ tbnum = 3
+ t0 = 1604298064000
+ tdLog.info("insert %d rows" % (insertRows))
+ for i in range(tbnum):
+ tdSql.execute("create table join_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i))
+ for j in range(insertRows):
+ ret = tdSql.execute(
+ "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" %
+ (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100)))
+ stable=stable+1
+ for i in range(tbnum):
+ tdSql.execute("create table join_1_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i))
+ for j in range(insertRows):
+ ret = tdSql.execute(
+ "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" %
+ (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100)))
+ print("==========step2")
+ print("join query ")
+ tdLog.info("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;")
+ tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc")
+ tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc limit 3;")
+ tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc slimit 3;")
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/pytest/query/bug1876.py b/tests/pytest/query/bug1876.py
new file mode 100644
index 0000000000000000000000000000000000000000..a1cc4b6eb2dc3368fdb21989d3b8df27f74cc8f9
--- /dev/null
+++ b/tests/pytest/query/bug1876.py
@@ -0,0 +1,58 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ def run(self):
+ tdSql.prepare()
+ print("==========step1")
+ print("create table && insert data")
+
+ tdSql.execute("create table join_mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))")
+ tdSql.execute("create table join_mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))")
+ stable=0
+ insertRows = 1000
+ tbnum = 3
+ t0 = 1604298064000
+ tdLog.info("insert %d rows" % (insertRows))
+ for i in range(tbnum):
+ tdSql.execute("create table join_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i))
+ for j in range(insertRows):
+ ret = tdSql.execute(
+ "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" %
+ (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100)))
+ stable=stable+1
+ for i in range(tbnum):
+ tdSql.execute("create table join_1_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i))
+ for j in range(insertRows):
+ ret = tdSql.execute(
+ "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" %
+ (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100)))
+ print("==========step2")
+ print("join query ")
+ tdLog.info("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;")
+ tdSql.error("select count(join_mt0.c1), first(join_mt0.c1)-first(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;")
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/pytest/query/dataLossTest.py b/tests/pytest/query/dataLossTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..b29dc1fa9f977b1f3f68b3e8f7781c6ac5a1e646
--- /dev/null
+++ b/tests/pytest/query/dataLossTest.py
@@ -0,0 +1,76 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+import os
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+import inspect
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor())
+
+ self.numberOfTables = 240
+ self.numberOfRecords = 10000
+
+ def run(self):
+ tdSql.prepare()
+
+ os.system("yes | taosdemo -t %d -n %d" % (self.numberOfTables, self.numberOfRecords))
+ print("==============step1")
+
+ tdSql.execute("use test")
+ sql = "select count(*) from meters"
+ tdSql.query(sql)
+ rows = tdSql.getData(0, 0)
+ print ("number of records: %d" % rows)
+
+ newRows = rows
+ for i in range(10000):
+ print("kill taosd")
+ time.sleep(10)
+ os.system("sudo kill -9 $(pgrep taosd)")
+ tdDnodes.startWithoutSleep(1)
+ while True:
+ try:
+ tdSql.query(sql)
+ newRows = tdSql.getData(0, 0)
+ print("numer of records after kill taosd %d" % newRows)
+ time.sleep(10)
+ break
+ except Exception as e:
+ pass
+ continue
+
+ if newRows < rows:
+ caller = inspect.getframeinfo(inspect.stack()[1][0])
+ args = (caller.filename, caller.lineno, sql, newRows, rows)
+ tdLog.exit("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args)
+ break
+
+ tdSql.query(sql)
+ tdSql.checkData(0, 0, rows)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/query/kill_query.py b/tests/pytest/query/kill_query.py
new file mode 100644
index 0000000000000000000000000000000000000000..8975eea2685b93e551e0008fbf5013d5e16e934f
--- /dev/null
+++ b/tests/pytest/query/kill_query.py
@@ -0,0 +1,82 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+from util.dnodes import tdDnodes
+import os
+import threading
+import time
+
+
+class TDTestCase:
+ """
+ kill query
+ """
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+
+ def query(self):
+ conn = taos.connect(host='127.0.0.1', user='root', password='taosdata', config='/etc/config')
+ cursor = conn.cursor()
+ while True:
+ cursor.execute('show queries;')
+ print('show queries!')
+ temp = cursor.fetchall()
+ if temp:
+ print(temp[0][0])
+ cursor.execute('kill query %s ;' % temp[0][0])
+ print('kill query success')
+ break
+ time.sleep(0.5)
+ cursor.close()
+ conn.close()
+
+ def run(self):
+ tdSql.prepare()
+
+ print("==============step1")
+ os.system('yes | sudo taosdemo -n 100')
+ print('insert into test.meters 10000000 rows')
+
+
+ t1 = threading.Thread(target=self.query)
+ t1.setDaemon(True)
+ t1.start()
+
+ print("==============step2")
+ tdSql.execute('use test;')
+ try:
+ print('============begin select * from 10000000 rows')
+ tdSql.query('select * from test.meters;')
+ # print(tdSql.queryResult)
+ except Exception as e:
+ if not "ProgrammingError('Query terminated'" in str(e):
+ raise Exception('fail')
+
+ print('success')
+ print('kill query success')
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
+
diff --git a/tests/pytest/query/queryCountCSVData.py b/tests/pytest/query/queryCountCSVData.py
new file mode 100644
index 0000000000000000000000000000000000000000..6c73425faec24afeed0c8a5a168f575ec182c771
--- /dev/null
+++ b/tests/pytest/query/queryCountCSVData.py
@@ -0,0 +1,71 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+from util.dnodes import tdDnodes
+
+
+class TDTestCase:
+ """
+ create table and insert data from disordered.csv which timestamp is disordered and
+ ordered.csv which timestamp is ordered.
+ then execute 'select count(*) from table xx;'
+ """
+
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ def run(self):
+ tdSql.prepare()
+
+ print("==============step1")
+ tdSql.execute("create database if not exists demo;");
+ tdSql.execute("use demo;")
+ tdSql.execute("CREATE TABLE IF NOT EXISTS test1 (ts TIMESTAMP, ValueID int, "
+ "VariantValue float, Quality int, Flags int);")
+ tdSql.execute("CREATE TABLE IF NOT EXISTS test2 (ts TIMESTAMP, ValueID int, "
+ "VariantValue float, Quality int, Flags int);")
+ ordered_csv = __file__.split('query')[0] + 'test_data/ordered.csv'
+ disordered_csv = __file__.split('query')[0] + 'test_data/disordered.csv'
+
+ tdSql.execute(" insert into test1 file '{file}';".format(file=ordered_csv))
+ tdSql.execute(" insert into test2 file '{file}';".format(file=disordered_csv))
+ print("==============insert into test1 and test2 form test file")
+
+
+ print("==============step2")
+ tdSql.query('select * from test1;')
+ with open(ordered_csv) as f1:
+ num1 = len(f1.readlines())
+ tdSql.checkRows(num1)
+
+
+ tdSql.query('select * from test2;')
+ with open(disordered_csv) as f2:
+ num2 = len(f2.readlines())
+ tdSql.checkRows(num2)
+ print("=============execute select count(*) from xxx")
+
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py
index 99222016048f71ae66768665fb70522d466b248b..ec3255ba7f12302f6ee00aeda391c11972f1cf26 100644
--- a/tests/pytest/query/queryInterval.py
+++ b/tests/pytest/query/queryInterval.py
@@ -35,10 +35,11 @@ class TDTestCase:
% (self.ts, self.ts + 2000000000, self.ts + 4000000000, self.ts + 5000000000, self.ts + 7000000000))
tdSql.query("select avg(voltage) from st interval(1n)")
- tdSql.checkRows(3)
- tdSql.checkData(0, 1, 221.4)
- tdSql.checkData(1, 1, 227.0)
- tdSql.checkData(2, 1, 222.0)
+ tdSql.checkRows(4)
+ tdSql.checkData(0, 1, 220.0)
+ tdSql.checkData(1, 1, 222.33333333333334)
+ tdSql.checkData(2, 1, 227.0)
+ tdSql.checkData(3, 1, 222.0)
tdSql.query("select avg(voltage) from st interval(1n, 15d)")
tdSql.checkRows(4)
diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py
index 17027cf498ff9e87b558866cd4d1e6a8c865afc0..5ad49a265ea52279f7934476a4a1db2fbbdf5883 100644
--- a/tests/pytest/query/queryJoin.py
+++ b/tests/pytest/query/queryJoin.py
@@ -95,16 +95,18 @@ class TDTestCase:
tdSql.error(
"select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id group by stb_t.id")
tdSql.error(
- "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.name;")
- tdSql.error(
- "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.location = stb_t.name")
+ "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.name;")
tdSql.execute("alter table stb_t add tag pid int")
tdSql.execute("alter table tb_t1 set tag pid=2")
tdSql.execute("alter table tb_t2 set tag pid=1")
+ tdSql.query(
+ "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.location = stb_t.name")
+ tdSql.checkRows(0)
+
tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.pid")
- tdSql.checkRows(3)
+ tdSql.checkRows(6)
tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id")
tdSql.checkRows(6)
diff --git a/tests/pytest/query/queryLike.py b/tests/pytest/query/queryLike.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c3b030f8f718371867f323f2496fb17b6b962e1
--- /dev/null
+++ b/tests/pytest/query/queryLike.py
@@ -0,0 +1,45 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor())
+
+ def run(self):
+ tdSql.prepare()
+
+ tdSql.execute("create table cars(ts timestamp, c nchar(2)) tags(t1 nchar(2))")
+ tdSql.execute("insert into car0 using cars tags('aa') values(now, 'bb');")
+ tdSql.query("select count(*) from cars where t1 like '%50 90 30 04 00 00%'")
+ tdSql.checkRows(0)
+
+ tdSql.execute("create table test_cars(ts timestamp, c nchar(2)) tags(t1 nchar(20))")
+ tdSql.execute("insert into car1 using test_cars tags('150 90 30 04 00 002') values(now, 'bb');")
+ tdSql.query("select * from test_cars where t1 like '%50 90 30 04 00 00%'")
+ tdSql.checkRows(1)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/query/queryPerformance.py b/tests/pytest/query/queryPerformance.py
index a7fc08c5a39d7b9a59068463f521d83106865c2d..e09900acc4cde66b1f2505c0fb532889d2db81e4 100644
--- a/tests/pytest/query/queryPerformance.py
+++ b/tests/pytest/query/queryPerformance.py
@@ -11,6 +11,7 @@
# -*- coding: utf-8 -*-
+
import sys
import os
import taos
@@ -32,17 +33,23 @@ class taosdemoQueryPerformace:
def query(self):
cursor = self.conn.cursor()
- cursor.execute("use test")
+ cursor.execute("use test")
totalTime = 0
- for i in range(100):
- startTime = time.time()
+ for i in range(100):
+ if(sys.argv[1] == '1'):
+ # root permission is required
+ os.system("echo 3 > /proc/sys/vm/drop_caches")
+ startTime = time.time()
cursor.execute("select count(*) from test.meters")
totalTime += time.time() - startTime
print("query time for: select count(*) from test.meters %f seconds" % (totalTime / 100))
totalTime = 0
for i in range(100):
+ if(sys.argv[1] == '1'):
+ # root permission is required
+ os.system("echo 3 > /proc/sys/vm/drop_caches")
startTime = time.time()
cursor.execute("select avg(f1), max(f2), min(f3) from test.meters")
totalTime += time.time() - startTime
@@ -50,6 +57,9 @@ class taosdemoQueryPerformace:
totalTime = 0
for i in range(100):
+ if(sys.argv[1] == '1'):
+ # root permission is required
+ os.system("echo 3 > /proc/sys/vm/drop_caches")
startTime = time.time()
cursor.execute("select count(*) from test.meters where loc='beijing'")
totalTime += time.time() - startTime
@@ -57,6 +67,9 @@ class taosdemoQueryPerformace:
totalTime = 0
for i in range(100):
+ if(sys.argv[1] == '1'):
+ # root permission is required
+ os.system("echo 3 > /proc/sys/vm/drop_caches")
startTime = time.time()
cursor.execute("select avg(f1), max(f2), min(f3) from test.meters where areaid=10")
totalTime += time.time() - startTime
@@ -64,6 +77,9 @@ class taosdemoQueryPerformace:
totalTime = 0
for i in range(100):
+ if(sys.argv[1] == '1'):
+ # root permission is required
+ os.system("echo 3 > /proc/sys/vm/drop_caches")
startTime = time.time()
cursor.execute("select avg(f1), max(f2), min(f3) from test.t10 interval(10s)")
totalTime += time.time() - startTime
@@ -71,11 +87,34 @@ class taosdemoQueryPerformace:
totalTime = 0
for i in range(100):
+ if(sys.argv[1] == '1'):
+ # root permission is required
+ os.system("echo 3 > /proc/sys/vm/drop_caches")
startTime = time.time()
cursor.execute("select last_row(*) from meters")
totalTime += time.time() - startTime
print("query time for: select last_row(*) from meters %f seconds" % (totalTime / 100))
+ totalTime = 0
+ for i in range(100):
+ if(sys.argv[1] == '1'):
+ # root permission is required
+ os.system("echo 3 > /proc/sys/vm/drop_caches")
+ startTime = time.time()
+ cursor.execute("select * from meters")
+ totalTime += time.time() - startTime
+ print("query time for: select * from meters %f seconds" % (totalTime / 100))
+
+ totalTime = 0
+ for i in range(100):
+ if(sys.argv[1] == '1'):
+ # root permission is required
+ os.system("echo 3 > /proc/sys/vm/drop_caches")
+ startTime = time.time()
+ cursor.execute("select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'")
+ totalTime += time.time() - startTime
+ print("query time for: select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000' %f seconds" % (totalTime / 100))
+
if __name__ == '__main__':
perftest = taosdemoQueryPerformace()
perftest.initConnection()
diff --git a/tests/pytest/query/querySort.py b/tests/pytest/query/querySort.py
index e5d3c8ce1f4eb9c1d2003bd659771562c9ea14e5..649e0dc1cb3191ba08b3f2da0a5edee3afc66575 100644
--- a/tests/pytest/query/querySort.py
+++ b/tests/pytest/query/querySort.py
@@ -96,6 +96,12 @@ class TDTestCase:
tdSql.query("select * from st order by ts desc")
self.checkColumnSorted(0, "desc")
+ print("======= step 2: verify order for special column =========")
+
+ tdSql.query("select tbcol1 from st order by ts desc")
+
+ tdSql.query("select tbcol6 from st order by ts desc")
+
for i in range(1, 10):
tdSql.error("select * from st order by tbcol%d" % i)
tdSql.error("select * from st order by tbcol%d asc" % i)
diff --git a/tests/pytest/query/query_performance.py b/tests/pytest/query/query_performance.py
new file mode 100644
index 0000000000000000000000000000000000000000..c31569ac1382aca17d3d5b37fbef981620c0df83
--- /dev/null
+++ b/tests/pytest/query/query_performance.py
@@ -0,0 +1,219 @@
+import time
+import taos
+import csv
+import numpy as np
+import random
+import os
+import requests
+import json
+import sys
+
+"""
+需要第三方库: taos,requests,numpy
+当前机器已经启动taosd服务
+使用方法见底部示例
+"""
+
+
+
+class Ding:
+ """
+ 发送消息到钉钉,
+ urls: 钉钉群的token组成的list,可以发多个钉钉群,需要提前加白名单或其他放行策略
+ at_mobiles: 需要@的人的手机号组成的list
+ msg: 要发送的str
+ """
+ def __init__(self, url_list, at_mobiles):
+ self.urls = url_list
+ self.at_mobiles = at_mobiles
+
+ def send_message(self, msg):
+ data1 = {
+ "msgtype": "text",
+ "text": {
+ "content": msg
+ },
+ "at": {
+ "atMobiles": self.at_mobiles,
+ "isAtAll": False
+ }
+ }
+
+ header = {'Content-Type': 'application/json; charset=utf-8'}
+
+ for url in self.urls:
+ requests.post(url=url, data=json.dumps(data1), headers=header)
+
+
+
+
+class TDConn:
+ def __init__(self, config:dict):
+ self.host = config['host']
+ self.user = config['user']
+ self.password = config['password']
+ self.config = config['config']
+ self.conn = None
+ self.cursor = None
+
+ def connect(self):
+ conn = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config)
+ cursor = conn.cursor()
+ self.conn = conn
+ self.cursor = cursor
+ print('connect ...')
+ return self.cursor
+
+ def close(self):
+ self.cursor.close()
+ self.conn.close()
+ print('close ... ')
+
+
+class Tool:
+ """
+ 可能有用
+ """
+ @staticmethod
+ def str_gen(num):
+ return ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', num))
+
+ @staticmethod
+ def float_gen(n, m):
+ return random.uniform(n, m)
+
+ @staticmethod
+ def int_gen(n, m):
+ return random.randint(n, m)
+
+class Demo:
+ def __init__(self, engine):
+ self.engine = engine['engine'](engine['config'])
+ self.cursor = self.engine.connect()
+
+
+ def date_gen(self, db, number_per_table, type_of_cols, num_of_cols_per_record, num_of_tables):
+ """
+ :目前都是 taosdemo 的参数
+ :return:
+ """
+ sql = 'yes | sudo taosdemo -d {db} -n {number_per_table} -b {type_of_cols} -l {num_of_cols_per_record} ' \
+ '-t {num_of_tables}'.format(db=db, number_per_table=number_per_table, type_of_cols=type_of_cols,
+ num_of_cols_per_record=num_of_cols_per_record, num_of_tables=num_of_tables)
+ os.system(sql)
+ print('insert data completed')
+
+
+ # def main(self, db, circle, csv_name, case_func, result_csv, nums, ding_flag):
+ def main(self, every_num_per_table, result_csv, all_result_csv, values):
+ db = values['db_name']
+ number_per_table = every_num_per_table
+ type_of_cols = values['col_type']
+ num_of_cols_per_record = values['col_num']
+ num_of_tables = values['table_num']
+ self.date_gen(db=db, number_per_table=number_per_table, type_of_cols=type_of_cols,
+ num_of_cols_per_record=num_of_cols_per_record, num_of_tables=num_of_tables)
+
+ circle = values['circle']
+ # print(every_num_per_table, result_csv, values)
+ csv_name = result_csv
+ case_func = values['sql_func']
+ nums = num_of_tables * number_per_table
+ ding_flag = values['ding_flag']
+
+ _data = []
+ f = open(csv_name,'w',encoding='utf-8')
+ f1 = open(all_result_csv,'a',encoding='utf-8')
+ csv_writer = csv.writer(f)
+ csv_writer1 = csv.writer(f1)
+ csv_writer.writerow(["number", "elapse", 'sql'])
+ self.cursor.execute('use {db};'.format(db=db))
+
+
+ for i in range(circle):
+ self.cursor.execute('reset query cache;')
+ sql = case_func()
+ start = time.time()
+ self.cursor.execute(sql)
+ self.cursor.fetchall()
+ end = time.time()
+ _data.append(end-start)
+ elapse = '%.4f' %(end -start)
+ print(sql, i, elapse, '\n')
+ csv_writer.writerow([i+1, elapse, sql])
+
+ # time.sleep(1)
+ _list = [nums, np.mean(_data)]
+ _str = '总数据: %s 条 , table数: %s , 每个table数据数: %s , 数据类型: %s \n' % \
+ (nums, num_of_tables, number_per_table, type_of_cols)
+ # print('avg : ', np.mean(_data), '\n')
+ _str += '平均值 : %.4f 秒\n' % np.mean(_data)
+ for each in (50, 80, 90, 95):
+ _list.append(np.percentile(_data,each))
+ _str += ' %d 分位数 : %.4f 秒\n' % (each , np.percentile(_data,each))
+
+ print(_str)
+ if ding_flag:
+ ding = Ding(values['ding_config']['urls'], values['ding_config']['at_mobiles'])
+ ding.send_message(_str)
+ csv_writer1.writerow(_list)
+ f.close()
+ f1.close()
+ self.engine.close()
+
+
+def run(engine, test_cases: dict, result_dir):
+ for each_case, values in test_cases.items():
+ for every_num_per_table in values['number_per_table']:
+ result_csv = result_dir + '{case}_table{table_num}_{number_per_table}.csv'.\
+ format(case=each_case, table_num=values['table_num'], number_per_table=every_num_per_table)
+ all_result_csv = result_dir + '{case_all}_result.csv'.format(case_all=each_case)
+ d = Demo(engine)
+ # print(each_case, result_csv)
+ d.main(every_num_per_table, result_csv, all_result_csv, values)
+
+
+
+if __name__ == '__main__':
+ """
+ 测试用例在test_cases中添加。
+ result_dir: 报告生成目录,会生成每次测试结果,和具体某一用例的统计结果.需注意目录权限需要执行用户可写。
+ case1、case2 : 具体用例名称
+ engine: 数据库引擎,目前只有taosd。使用时需开启taosd服务。
+ table_num: 造数据时的table数目
+ circle: 循环测试次数,求平均值
+ number_per_table:需要传list,多个数值代表会按照list内的数值逐个测试
+ col_num:table col的数目
+ col_type: 表中数据类型
+ db_name: 造数据的db名,默认用test
+ sql_func: 当前测试的sql方法,需要自己定义
+ ding_flag: 如果需要钉钉发送数据,flag设置真值,
+ ding_config: 如ding_flag 设置为真值,此项才有意义。ding_flag为假时此项可以为空。urls传入一list,内容为要发送的群的token,
+ 需提前设置白名单,at_mobiles传入一list,内容为在群内需要@的人的手机号
+ """
+ engine_dict = {
+ 'taosd': {'engine': TDConn, 'config':
+ {'host': '127.0.0.1', 'user': 'root', 'password': 'taosdata', 'config':'/etc/taos'}}
+ }
+
+ def case1():
+ return 'select * from meters where f1 = {n};'.format(n=random.randint(1,30))
+
+ def case2():
+ return 'select * from meters where f1 = %.4f;' %random.uniform(1,30)
+
+
+ result_dir = '/usr/local/demo/benchmarktestdata/'
+ test_cases = {
+ 'case1': {'engine':'taosd', 'table_num': 10, 'circle': 100, 'number_per_table':[10, 100], 'col_num': 5,
+ 'col_type': 'INT', 'db_name': 'test', 'sql_func': case1, 'ding_flag': True,
+ 'ding_config':
+ {'urls': [r'https://oapi.dingtalk.com/robot/send?access_token=xxxxxxx0cd93'],
+ 'at_mobiles':[17000000000,],}},
+ 'case2': {'engine':'taosd', 'table_num': 10, 'circle': 50, 'number_per_table':[10, 100], 'col_num': 5,
+ 'col_type': 'FLOAT', 'db_name': 'test', 'sql_func': case2, 'ding_flag': False,
+ 'ding_config': None
+ }
+ }
+
+ run(engine_dict['taosd'], test_cases, result_dir)
diff --git a/tests/pytest/query/sliding.py b/tests/pytest/query/sliding.py
new file mode 100644
index 0000000000000000000000000000000000000000..810d90117a73bba81ea010f81aa605196a496dc4
--- /dev/null
+++ b/tests/pytest/query/sliding.py
@@ -0,0 +1,63 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+import random
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ self.ts = 1500000000000
+
+ def run(self):
+ tdSql.prepare()
+
+ tdSql.execute("create table meters(ts timestamp, col1 int) tags(id int, loc nchar(20))")
+ sql = "insert into t0 using meters tags(1, 'beijing') values"
+ for i in range(100):
+ sql += "(%d, %d)" % (self.ts + i * 1000, random.randint(1, 100))
+ tdSql.execute(sql)
+
+ sql = "insert into t1 using meters tags(2, 'shanghai') values"
+ for i in range(100):
+ sql += "(%d, %d)" % (self.ts + i * 1000, random.randint(1, 100))
+ tdSql.execute(sql)
+
+ tdSql.query("select count(*) from meters interval(10s) sliding(5s)")
+ tdSql.checkRows(21)
+
+ tdSql.error("select count(*) from meters sliding(5s)")
+
+ tdSql.error("select count(*) from meters sliding(5s) interval(10s)")
+
+ tdSql.error("select * from meters sliding(5s) order by ts desc")
+
+ tdSql.query("select count(*) from meters group by loc")
+ tdSql.checkRows(2)
+
+ tdSql.error("select * from meters group by loc sliding(5s)")
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/stream/new.py b/tests/pytest/stream/new.py
index eac93dc0e649f5d48481079d75851a27be270567..12ec6d4507710869632eac77d719217b3b0ed7b3 100644
--- a/tests/pytest/stream/new.py
+++ b/tests/pytest/stream/new.py
@@ -26,7 +26,6 @@ class TDTestCase:
def run(self):
rowNum = 200
- totalNum = 200
tdSql.prepare()
tdLog.info("=============== step1")
@@ -42,7 +41,9 @@ class TDTestCase:
tdSql.execute("create table st as select count(*), count(tbcol), count(tbcol2) from mt interval(10s)")
tdLog.info("=============== step3")
+ start = time.time()
tdSql.waitedQuery("select * from st", 1, 120)
+ delay = int(time.time() - start) + 20
v = tdSql.getData(0, 3)
if v >= 51:
tdLog.exit("value is %d, which is larger than 51" % v)
@@ -54,11 +55,18 @@ class TDTestCase:
tdSql.execute("insert into tb%d values(now + %ds, %d, %d)" % (i, j, j, j))
tdLog.info("=============== step5")
- tdLog.sleep(40)
- tdSql.waitedQuery("select * from st order by ts desc", 1, 120)
- v = tdSql.getData(0, 3)
- if v <= 51:
- tdLog.exit("value is %d, which is smaller than 51" % v)
+ maxValue = 0
+ for i in range(delay):
+ time.sleep(1)
+ tdSql.query("select * from st order by ts desc")
+ v = tdSql.getData(0, 3)
+ if v > maxValue:
+ maxValue = v
+ if v > 51:
+ break
+
+ if maxValue <= 51:
+ tdLog.exit("value is %d, which is smaller than 51" % maxValue)
def stop(self):
tdSql.close()
diff --git a/tests/pytest/table/create_sensitive.py b/tests/pytest/table/create_sensitive.py
new file mode 100644
index 0000000000000000000000000000000000000000..1934b662c7a57c13c2a1b8e8dfd65ab6ddbe13a4
--- /dev/null
+++ b/tests/pytest/table/create_sensitive.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+
+import sys
+import string
+import random
+import subprocess
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ def run(self):
+ tdSql.prepare()
+
+ tdLog.info('=============== step1')
+ tdLog.info('create table TestSensitiveT(ts timestamp, i int)')
+ tdSql.execute('create table TestSensitiveT(ts timestamp, i int)')
+ tdLog.info('create table TestSensitiveSt(ts timestamp,i int) tags(j int)')
+ tdSql.execute('create table TestSensitiveSt(ts timestamp,i int) tags(j int)')
+ tdLog.info('create table Abcde using TestSensitiveSt tags(1)')
+ tdSql.execute('create table AbcdeFgh using TestSensitiveSt tags(1)')
+ tdLog.info('=============== step2')
+ tdLog.info('test normal table ')
+ tdSql.error('create table testsensitivet(ts timestamp, i int)')
+ tdSql.error('create table testsensitivet(ts timestamp, j int)')
+ tdSql.error('create table testsensItivet(ts timestamp, j int)')
+ tdSql.error('create table TESTSENSITIVET(ts timestamp, i int)')
+ tdLog.info('=============== step3')
+ tdLog.info('test super table ')
+ tdSql.error('create table testsensitivest(ts timestamp,i int) tags(j int)')
+ tdSql.error('create table testsensitivest(ts timestamp,i int) tags(k int)')
+ tdSql.error('create table TESTSENSITIVEST(ts timestamp,i int) tags(j int)')
+ tdSql.error('create table Testsensitivest(ts timestamp,i int) tags(j int)')
+ tdLog.info('=============== step4')
+ tdLog.info('test subtable ')
+ tdSql.error('create table abcdefgh using TestSensitiveSt tags(1)')
+ tdSql.error('create table ABCDEFGH using TestSensitiveSt tags(1)')
+ tdSql.error('create table Abcdefgh using TestSensitiveSt tags(1)')
+ tdSql.error('create table abcdeFgh using TestSensitiveSt tags(1)')
+ tdSql.error('insert into table abcdefgh using TestSensitiveSt tags(1) values(now,1)')
+ tdSql.error('insert into table ABCDEFGH using TestSensitiveSt tags(1) values(now,1)')
+ tdSql.error('insert into table Abcdefgh using TestSensitiveSt tags(1) values(now,1)')
+ tdSql.error('insert into table abcdeFgH using TestSensitiveSt tags(1) values(now,1)')
+ tdSql.query('show tables')
+ tdLog.info('tdSql.checkRow(0)')
+ tdSql.checkRows(2)
+
+
+
+
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/test_data/__init__.py b/tests/pytest/test_data/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..1bd7af5b9db25bf22f960bfca6bf18b1518cc86f
--- /dev/null
+++ b/tests/pytest/test_data/__init__.py
@@ -0,0 +1,15 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+
+"""
+this directory contains test data files
+"""
\ No newline at end of file
diff --git a/tests/pytest/test_data/disordered.csv b/tests/pytest/test_data/disordered.csv
new file mode 100644
index 0000000000000000000000000000000000000000..0e6fd75e5b1a43da360541f57377236e0984093a
--- /dev/null
+++ b/tests/pytest/test_data/disordered.csv
@@ -0,0 +1,500 @@
+"2020-03-01 20:01:49.493","130","7.595","128","8392704"
+"2020-03-01 20:01:50.493","130","7.598","128","8392704"
+"2020-03-01 20:01:51.493","130","7.602","128","8392704"
+"2020-03-01 20:01:52.493","130","7.604","128","8392704"
+"2020-03-01 20:01:53.493","130","7.604","128","8392704"
+"2020-03-01 20:01:54.493","130","7.606","128","8392704"
+"2020-03-01 20:01:55.493","130","7.607","128","8392704"
+"2020-03-01 20:01:56.493","130","7.607","128","8392704"
+"2020-03-01 20:01:57.493","130","7.607","128","8392704"
+"2020-03-01 20:01:58.493","130","7.607","128","8392704"
+"2020-03-01 20:01:59.493","130","7.606","128","8392704"
+"2020-03-01 20:02:00.493","130","7.606","128","8392704"
+"2020-03-01 20:02:01.493","130","7.606","128","8392704"
+"2020-03-01 20:02:02.493","130","7.607","128","8392704"
+"2020-03-01 20:02:03.493","130","7.608","128","8392704"
+"2020-03-01 20:02:04.493","130","7.609","128","8392704"
+"2020-03-01 20:02:05.493","130","7.609","128","8392704"
+"2020-03-01 20:02:06.493","130","7.608","128","8392704"
+"2020-03-01 20:02:07.493","130","7.606","128","8392704"
+"2020-03-01 20:02:08.493","130","7.606","128","8392704"
+"2020-03-01 20:02:09.493","130","7.607","128","8392704"
+"2020-03-01 20:02:10.493","130","7.609","128","8392704"
+"2020-03-01 20:02:11.493","130","7.61","128","8392704"
+"2020-03-01 20:02:12.493","130","7.611","128","8392704"
+"2020-03-01 20:02:13.493","130","7.61","128","8392704"
+"2020-03-01 20:02:14.493","130","7.607","128","8392704"
+"2020-03-01 20:02:15.493","130","7.605","128","8392704"
+"2020-03-01 20:02:16.493","130","7.604","128","8392704"
+"2020-03-01 20:02:17.493","130","7.603","128","8392704"
+"2020-03-01 20:02:18.493","130","7.604","128","8392704"
+"2020-03-01 20:02:19.493","130","7.604","128","8392704"
+"2020-03-01 20:02:20.493","130","7.603","128","8392704"
+"2020-03-01 20:02:21.493","130","7.601","128","8392704"
+"2020-03-01 20:02:22.493","130","7.598","128","8392704"
+"2020-03-01 20:02:23.493","130","7.595","128","8392704"
+"2020-03-01 20:02:24.493","130","7.594","128","8392704"
+"2020-03-01 20:02:25.493","130","7.594","128","8392704"
+"2020-03-01 20:02:26.493","130","7.595","128","8392704"
+"2020-03-01 20:02:27.493","130","7.596","128","8392704"
+"2020-03-01 20:02:28.493","130","7.596","128","8392704"
+"2020-03-01 20:02:29.493","130","7.598","128","8392704"
+"2020-03-01 20:02:30.493","130","7.6","128","8392704"
+"2020-03-01 20:02:31.493","130","7.6","128","8392704"
+"2020-03-01 20:02:32.493","130","7.6","128","8392704"
+"2020-03-01 20:02:33.493","130","7.601","128","8392704"
+"2020-03-01 20:02:34.493","130","7.603","128","8392704"
+"2020-03-01 20:02:35.493","130","7.604","128","8392704"
+"2020-03-01 20:02:36.493","130","7.605","128","8392704"
+"2020-03-01 20:02:37.493","130","7.606","128","8392704"
+"2020-03-01 20:02:38.493","130","7.61","128","8392704"
+"2020-03-01 20:02:39.493","130","7.611","128","8392704"
+"2020-03-01 20:02:40.493","130","7.61","128","8392704"
+"2020-03-01 20:02:41.493","130","7.609","128","8392704"
+"2020-03-01 20:02:42.493","130","7.611","128","8392704"
+"2020-03-01 20:02:43.493","130","7.61","128","8392704"
+"2020-03-01 20:02:44.493","130","7.607","128","8392704"
+"2020-03-01 20:02:45.493","130","7.605","128","8392704"
+"2020-03-01 20:02:46.493","130","7.606","128","8392704"
+"2020-03-01 20:02:47.493","130","7.604","128","8392704"
+"2020-03-01 20:02:48.493","130","7.599","128","8392704"
+"2020-03-01 20:02:49.493","130","7.595","128","8392704"
+"2020-03-01 06:41:17.493","130","6.742","128","8392704"
+"2020-03-01 06:41:18.493","130","6.741","128","8392704"
+"2020-03-01 06:41:19.493","130","6.737","128","8392704"
+"2020-03-01 06:41:20.493","130","6.734","128","8392704"
+"2020-03-01 06:41:21.493","130","6.734","128","8392704"
+"2020-03-01 06:41:22.493","130","6.733","128","8392704"
+"2020-03-01 06:41:23.493","130","6.736","128","8392704"
+"2020-03-01 06:41:24.493","130","6.739","128","8392704"
+"2020-03-01 06:41:25.493","130","6.738","128","8392704"
+"2020-03-01 06:41:26.493","130","6.74","128","8392704"
+"2020-03-01 06:41:27.493","130","6.745","128","8392704"
+"2020-03-01 06:41:28.493","130","6.749","128","8392704"
+"2020-03-01 06:41:29.493","130","6.753","128","8392704"
+"2020-03-01 06:41:30.493","130","6.753","128","8392704"
+"2020-03-01 06:41:31.493","130","6.757","128","8392704"
+"2020-03-01 06:41:32.493","130","6.763","128","8392704"
+"2020-03-01 06:41:33.493","130","6.765","128","8392704"
+"2020-03-01 06:41:34.493","130","6.764","128","8392704"
+"2020-03-01 06:41:35.493","130","6.762","128","8392704"
+"2020-03-01 06:41:36.493","130","6.758","128","8392704"
+"2020-03-01 06:41:37.493","130","6.756","128","8392704"
+"2020-03-01 06:41:38.493","130","6.755","128","8392704"
+"2020-03-01 06:41:39.493","130","6.754","128","8392704"
+"2020-03-01 06:41:40.493","130","6.755","128","8392704"
+"2020-03-01 06:41:41.493","130","6.756","128","8392704"
+"2020-03-01 06:41:42.493","130","6.757","128","8392704"
+"2020-03-01 06:41:43.493","130","6.756","128","8392704"
+"2020-03-01 06:41:44.493","130","6.756","128","8392704"
+"2020-03-01 06:41:45.493","130","6.756","128","8392704"
+"2020-03-01 06:41:46.493","130","6.759","128","8392704"
+"2020-03-01 06:41:47.493","130","6.759","128","8392704"
+"2020-03-01 06:41:48.493","130","6.758","128","8392704"
+"2020-03-01 06:41:49.493","130","6.758","128","8392704"
+"2020-03-01 06:41:50.493","130","6.756","128","8392704"
+"2020-03-01 06:41:51.493","130","6.755","128","8392704"
+"2020-03-01 06:41:52.493","130","6.755","128","8392704"
+"2020-03-01 06:41:53.493","130","6.754","128","8392704"
+"2020-03-01 06:41:54.493","130","6.751","128","8392704"
+"2020-03-01 06:41:55.493","130","6.752","128","8392704"
+"2020-03-01 06:41:56.493","130","6.753","128","8392704"
+"2020-03-01 06:41:57.493","130","6.753","128","8392704"
+"2020-03-01 06:41:58.493","130","6.753","128","8392704"
+"2020-03-01 06:41:59.493","130","6.755","128","8392704"
+"2020-03-01 06:42:00.493","130","6.752","128","8392704"
+"2020-03-01 06:42:01.493","130","6.75","128","8392704"
+"2020-03-01 06:42:02.493","130","6.75","128","8392704"
+"2020-03-01 06:42:03.493","130","6.753","128","8392704"
+"2020-03-01 06:42:04.493","130","6.755","128","8392704"
+"2020-03-01 06:42:05.493","130","6.754","128","8392704"
+"2020-03-01 06:42:06.493","130","6.754","128","8392704"
+"2020-03-01 06:42:07.493","130","6.752","128","8392704"
+"2020-03-01 06:42:08.493","130","6.748","128","8392704"
+"2020-03-01 06:42:09.493","130","6.747","128","8392704"
+"2020-03-01 06:42:10.493","130","6.747","128","8392704"
+"2020-03-01 06:42:11.493","130","6.748","128","8392704"
+"2020-03-01 06:42:12.493","130","6.748","128","8392704"
+"2020-03-01 06:42:13.493","130","6.75","128","8392704"
+"2020-03-01 06:42:14.493","130","6.754","128","8392704"
+"2020-03-01 06:42:15.493","130","6.754","128","8392704"
+"2020-03-01 06:42:16.493","130","6.756","128","8392704"
+"2020-03-01 06:42:17.493","130","6.756","128","8392704"
+"2020-03-01 06:42:18.493","130","6.757","128","8392704"
+"2020-03-01 06:42:19.493","130","6.757","128","8392704"
+"2020-03-01 06:42:20.493","130","6.76","128","8392704"
+"2020-03-01 06:42:21.493","130","6.761","128","8392704"
+"2020-03-01 06:42:22.493","130","6.76","128","8392704"
+"2020-03-01 06:42:23.493","130","6.76","128","8392704"
+"2020-03-01 06:42:24.493","130","6.76","128","8392704"
+"2020-03-01 06:42:25.493","130","6.76","128","8392704"
+"2020-03-01 06:42:26.493","130","6.758","128","8392704"
+"2020-03-01 06:42:27.493","130","6.757","128","8392704"
+"2020-03-01 06:42:28.493","130","6.752","128","8392704"
+"2020-03-01 06:42:29.493","130","6.746","128","8392704"
+"2020-03-01 06:42:30.493","130","6.742","128","8392704"
+"2020-03-01 06:42:31.493","130","6.741","128","8392704"
+"2020-03-01 06:42:32.493","130","6.739","128","8392704"
+"2020-03-01 06:42:33.493","130","6.739","128","8392704"
+"2020-03-01 06:42:34.493","130","6.737","128","8392704"
+"2020-03-01 06:42:35.493","130","6.737","128","8392704"
+"2020-03-01 06:42:36.493","130","6.738","128","8392704"
+"2020-03-01 06:42:37.493","130","6.739","128","8392704"
+"2020-03-01 06:42:38.493","130","6.743","128","8392704"
+"2020-03-01 06:42:39.493","130","6.747","128","8392704"
+"2020-03-01 06:42:40.493","130","6.748","128","8392704"
+"2020-03-01 06:42:41.493","130","6.746","128","8392704"
+"2020-03-01 06:42:42.493","130","6.746","128","8392704"
+"2020-03-01 06:42:43.493","130","6.745","128","8392704"
+"2020-03-01 06:42:44.493","130","6.742","128","8392704"
+"2020-03-01 06:42:45.493","130","6.741","128","8392704"
+"2020-03-01 06:42:46.493","130","6.74","128","8392704"
+"2020-03-01 06:42:47.493","130","6.742","128","8392704"
+"2020-03-01 06:42:48.493","130","6.743","128","8392704"
+"2020-03-01 06:42:49.493","130","6.742","128","8392704"
+"2020-03-01 06:42:50.493","130","6.742","128","8392704"
+"2020-03-01 06:42:51.493","130","6.741","128","8392704"
+"2020-03-01 06:42:52.493","130","6.741","128","8392704"
+"2020-03-01 06:42:53.493","130","6.742","128","8392704"
+"2020-03-01 06:42:54.493","130","6.742","128","8392704"
+"2020-03-01 06:42:55.493","130","6.745","128","8392704"
+"2020-03-01 06:42:56.493","130","6.747","128","8392704"
+"2020-03-01 06:42:57.493","130","6.748","128","8392704"
+"2020-03-01 06:42:58.493","130","6.75","128","8392704"
+"2020-03-01 06:42:59.493","130","6.75","128","8392704"
+"2020-03-01 06:43:00.493","130","6.748","128","8392704"
+"2020-03-01 06:43:01.493","130","6.748","128","8392704"
+"2020-03-01 06:43:02.493","130","6.746","128","8392704"
+"2020-03-01 06:43:03.493","130","6.745","128","8392704"
+"2020-03-01 06:43:04.493","130","6.745","128","8392704"
+"2020-03-01 06:43:05.493","130","6.745","128","8392704"
+"2020-03-01 06:43:06.493","130","6.744","128","8392704"
+"2020-03-01 06:43:07.493","130","6.749","128","8392704"
+"2020-03-01 06:43:08.493","130","6.756","128","8392704"
+"2020-03-01 06:43:09.493","130","6.759","128","8392704"
+"2020-03-01 06:43:10.493","130","6.759","128","8392704"
+"2020-03-01 06:43:11.493","130","6.758","128","8392704"
+"2020-03-01 06:43:12.493","130","6.758","128","8392704"
+"2020-03-01 06:43:13.493","130","6.759","128","8392704"
+"2020-03-01 06:43:14.493","130","6.759","128","8392704"
+"2020-03-01 06:43:15.493","130","6.753","128","8392704"
+"2020-03-01 06:43:16.493","130","6.751","128","8392704"
+"2020-03-01 20:02:50.493","130","7.596","128","8392704"
+"2020-03-01 20:02:51.493","130","7.597","128","8392704"
+"2020-03-01 20:02:52.493","130","7.599","128","8392704"
+"2020-03-01 20:02:53.493","130","7.6","128","8392704"
+"2020-03-01 20:02:54.493","130","7.601","128","8392704"
+"2020-03-01 20:02:55.493","130","7.603","128","8392704"
+"2020-03-01 20:02:56.493","130","7.602","128","8392704"
+"2020-03-01 20:02:57.493","130","7.601","128","8392704"
+"2020-03-01 20:02:58.493","130","7.601","128","8392704"
+"2020-03-01 20:02:59.493","130","7.6","128","8392704"
+"2020-03-01 20:03:00.493","130","7.599","128","8392704"
+"2020-03-01 20:03:01.493","130","7.599","128","8392704"
+"2020-03-01 20:03:02.493","130","7.6","128","8392704"
+"2020-03-01 20:03:03.493","130","7.601","128","8392704"
+"2020-03-01 20:03:04.493","130","7.601","128","8392704"
+"2020-03-01 20:03:05.493","130","7.601","128","8392704"
+"2020-03-01 20:03:06.493","130","7.6","128","8392704"
+"2020-03-01 20:03:07.493","130","7.602","128","8392704"
+"2020-03-01 20:03:08.493","130","7.606","128","8392704"
+"2020-03-01 20:03:09.493","130","7.609","128","8392704"
+"2020-03-01 20:03:10.493","130","7.612","128","8392704"
+"2020-03-01 20:03:11.493","130","7.614","128","8392704"
+"2020-03-01 20:03:12.493","130","7.615","128","8392704"
+"2020-03-01 20:03:13.493","130","7.614","128","8392704"
+"2020-03-01 20:03:14.493","130","7.613","128","8392704"
+"2020-03-01 20:03:15.493","130","7.614","128","8392704"
+"2020-03-01 20:03:16.493","130","7.612","128","8392704"
+"2020-03-01 20:03:17.493","130","7.609","128","8392704"
+"2020-03-01 20:03:18.493","130","7.606","128","8392704"
+"2020-03-01 20:03:19.493","130","7.604","128","8392704"
+"2020-03-01 20:03:20.493","130","7.604","128","8392704"
+"2020-03-01 20:03:21.493","130","7.605","128","8392704"
+"2020-03-01 20:03:22.493","130","7.605","128","8392704"
+"2020-03-01 20:03:23.493","130","7.605","128","8392704"
+"2020-03-01 20:03:24.493","130","7.605","128","8392704"
+"2020-03-01 20:03:25.493","130","7.604","128","8392704"
+"2020-03-01 20:03:26.493","130","7.603","128","8392704"
+"2020-03-01 20:03:27.493","130","7.604","128","8392704"
+"2020-03-01 20:03:28.493","130","7.605","128","8392704"
+"2020-03-01 20:03:29.493","130","7.607","128","8392704"
+"2020-03-01 20:03:30.493","130","7.609","128","8392704"
+"2020-03-01 20:03:31.493","130","7.609","128","8392704"
+"2020-03-01 20:03:32.493","130","7.607","128","8392704"
+"2020-03-01 20:03:33.493","130","7.606","128","8392704"
+"2020-03-01 20:03:34.493","130","7.607","128","8392704"
+"2020-03-01 20:03:35.493","130","7.608","128","8392704"
+"2020-03-01 20:03:36.493","130","7.609","128","8392704"
+"2020-03-01 20:03:37.493","130","7.609","128","8392704"
+"2020-03-01 20:03:38.493","130","7.607","128","8392704"
+"2020-03-01 20:03:39.493","130","7.602","128","8392704"
+"2020-03-01 20:03:40.493","130","7.599","128","8392704"
+"2020-03-01 20:03:41.493","130","7.598","128","8392704"
+"2020-03-01 20:03:42.493","130","7.596","128","8392704"
+"2020-03-01 20:03:43.493","130","7.595","128","8392704"
+"2020-03-01 20:03:44.493","130","7.594","128","8392704"
+"2020-03-01 20:03:45.493","130","7.595","128","8392704"
+"2020-03-01 20:03:46.493","130","7.597","128","8392704"
+"2020-03-01 20:03:47.493","130","7.596","128","8392704"
+"2020-03-01 20:03:48.493","130","7.595","128","8392704"
+"2020-03-01 20:03:49.493","130","7.596","128","8392704"
+"2020-03-01 20:03:50.493","130","7.596","128","8392704"
+"2020-03-01 20:03:51.493","130","7.595","128","8392704"
+"2020-03-01 20:03:52.493","130","7.596","128","8392704"
+"2020-03-01 20:03:53.493","130","7.597","128","8392704"
+"2020-03-01 20:03:54.493","130","7.598","128","8392704"
+"2020-03-01 20:03:55.493","130","7.596","128","8392704"
+"2020-03-01 20:03:56.493","130","7.596","128","8392704"
+"2020-03-01 20:03:57.493","130","7.599","128","8392704"
+"2020-03-01 20:03:58.493","130","7.602","128","8392704"
+"2020-03-01 20:03:59.493","130","7.603","128","8392704"
+"2020-03-01 20:04:00.493","130","7.602","128","8392704"
+"2020-03-01 20:04:01.493","130","7.6","128","8392704"
+"2020-03-01 20:04:02.493","130","7.598","128","8392704"
+"2020-03-01 20:04:03.493","130","7.595","128","8392704"
+"2020-03-01 20:04:04.493","130","7.593","128","8392704"
+"2020-03-01 20:04:05.493","130","7.592","128","8392704"
+"2020-03-01 20:04:06.493","130","7.591","128","8392704"
+"2020-03-01 20:04:07.493","130","7.591","128","8392704"
+"2020-03-01 20:04:08.493","130","7.591","128","8392704"
+"2020-03-01 20:04:09.493","130","7.592","128","8392704"
+"2020-03-01 20:04:10.493","130","7.59","128","8392704"
+"2020-03-01 20:04:11.493","130","7.587","128","8392704"
+"2020-03-01 20:04:12.493","130","7.584","128","8392704"
+"2020-03-01 20:04:13.493","130","7.583","128","8392704"
+"2020-03-01 20:04:14.493","130","7.581","128","8392704"
+"2020-03-01 20:04:15.493","130","7.578","128","8392704"
+"2020-03-01 20:04:16.493","130","7.576","128","8392704"
+"2020-03-01 20:04:17.493","130","7.577","128","8392704"
+"2020-03-01 20:04:18.493","130","7.579","128","8392704"
+"2020-03-01 20:04:19.493","130","7.583","128","8392704"
+"2020-03-01 20:04:20.493","130","7.587","128","8392704"
+"2020-03-01 20:04:21.493","130","7.588","128","8392704"
+"2020-03-01 20:04:22.493","130","7.589","128","8392704"
+"2020-03-01 20:04:23.493","130","7.59","128","8392704"
+"2020-03-01 20:04:24.493","130","7.593","128","8392704"
+"2020-03-01 20:04:25.493","130","7.597","128","8392704"
+"2020-03-01 20:04:26.493","130","7.6","128","8392704"
+"2020-03-01 20:04:27.493","130","7.603","128","8392704"
+"2020-03-01 20:04:28.493","130","7.606","128","8392704"
+"2020-03-01 20:04:29.493","130","7.608","128","8392704"
+"2020-03-01 20:04:30.493","130","7.609","128","8392704"
+"2020-03-01 20:04:31.493","130","7.607","128","8392704"
+"2020-03-01 20:04:32.493","130","7.607","128","8392704"
+"2020-03-01 20:04:33.493","130","7.607","128","8392704"
+"2020-03-01 20:04:34.493","130","7.602","128","8392704"
+"2020-03-01 20:04:35.493","130","7.599","128","8392704"
+"2020-03-01 20:04:36.493","130","7.599","128","8392704"
+"2020-03-01 20:04:37.493","130","7.599","128","8392704"
+"2020-03-01 20:04:38.493","130","7.598","128","8392704"
+"2020-03-01 20:04:39.493","130","7.596","128","8392704"
+"2020-03-01 20:04:40.493","130","7.595","128","8392704"
+"2020-03-01 20:04:41.493","130","7.592","128","8392704"
+"2020-03-01 20:04:42.493","130","7.586","128","8392704"
+"2020-03-01 20:04:43.493","130","7.582","128","8392704"
+"2020-03-01 20:04:44.493","130","7.582","128","8392704"
+"2020-03-01 20:04:45.493","130","7.584","128","8392704"
+"2020-03-01 20:04:46.493","130","7.583","128","8392704"
+"2020-03-01 20:04:47.493","130","7.582","128","8392704"
+"2020-03-01 20:04:48.493","130","7.582","128","8392704"
+"2020-03-01 20:04:49.493","130","7.585","128","8392704"
+"2020-03-01 06:43:17.493","130","6.751","128","8392704"
+"2020-03-01 06:43:18.493","130","6.75","128","8392704"
+"2020-03-01 06:43:19.493","130","6.748","128","8392704"
+"2020-03-01 06:43:20.493","130","6.751","128","8392704"
+"2020-03-01 06:43:21.493","130","6.752","128","8392704"
+"2020-03-01 06:43:22.493","130","6.751","128","8392704"
+"2020-03-01 06:43:23.493","130","6.746","128","8392704"
+"2020-03-01 06:43:24.493","130","6.739","128","8392704"
+"2020-03-01 06:43:25.493","130","6.737","128","8392704"
+"2020-03-01 06:43:26.493","130","6.735","128","8392704"
+"2020-03-01 06:43:27.493","130","6.735","128","8392704"
+"2020-03-01 06:43:28.493","130","6.734","128","8392704"
+"2020-03-01 06:43:29.493","130","6.731","128","8392704"
+"2020-03-01 06:43:30.493","130","6.729","128","8392704"
+"2020-03-01 06:43:31.493","130","6.73","128","8392704"
+"2020-03-01 06:43:32.493","130","6.736","128","8392704"
+"2020-03-01 06:43:33.493","130","6.74","128","8392704"
+"2020-03-01 06:43:34.493","130","6.741","128","8392704"
+"2020-03-01 06:43:35.493","130","6.743","128","8392704"
+"2020-03-01 06:43:36.493","130","6.743","128","8392704"
+"2020-03-01 06:43:37.493","130","6.745","128","8392704"
+"2020-03-01 06:43:38.493","130","6.747","128","8392704"
+"2020-03-01 06:43:39.493","130","6.747","128","8392704"
+"2020-03-01 06:43:40.493","130","6.746","128","8392704"
+"2020-03-01 06:43:41.493","130","6.745","128","8392704"
+"2020-03-01 06:43:42.493","130","6.743","128","8392704"
+"2020-03-01 06:43:43.493","130","6.741","128","8392704"
+"2020-03-01 06:43:44.493","130","6.737","128","8392704"
+"2020-03-01 06:43:45.493","130","6.737","128","8392704"
+"2020-03-01 06:43:46.493","130","6.74","128","8392704"
+"2020-03-01 06:43:47.493","130","6.744","128","8392704"
+"2020-03-01 06:43:48.493","130","6.746","128","8392704"
+"2020-03-01 06:43:49.493","130","6.745","128","8392704"
+"2020-03-01 06:43:50.493","130","6.743","128","8392704"
+"2020-03-01 06:43:51.493","130","6.745","128","8392704"
+"2020-03-01 06:43:52.493","130","6.747","128","8392704"
+"2020-03-01 06:43:53.493","130","6.748","128","8392704"
+"2020-03-01 06:43:54.493","130","6.748","128","8392704"
+"2020-03-01 06:43:55.493","130","6.747","128","8392704"
+"2020-03-01 06:43:56.493","130","6.746","128","8392704"
+"2020-03-01 06:43:57.493","130","6.744","128","8392704"
+"2020-03-01 06:43:58.493","130","6.742","128","8392704"
+"2020-03-01 06:43:59.493","130","6.74","128","8392704"
+"2020-03-01 06:44:00.493","130","6.739","128","8392704"
+"2020-03-01 06:44:01.493","130","6.739","128","8392704"
+"2020-03-01 06:44:02.493","130","6.742","128","8392704"
+"2020-03-01 06:44:03.493","130","6.742","128","8392704"
+"2020-03-01 06:44:04.493","130","6.756","128","8392704"
+"2020-03-01 06:44:05.493","130","6.757","128","8392704"
+"2020-03-01 06:44:06.493","130","6.757","128","8392704"
+"2020-03-01 06:44:07.493","130","6.757","128","8392704"
+"2020-03-01 06:44:08.493","130","6.759","128","8392704"
+"2020-03-01 06:44:09.493","130","6.759","128","8392704"
+"2020-03-01 06:44:10.493","130","6.75","128","8392704"
+"2020-03-01 06:44:11.493","130","6.744","128","8392704"
+"2020-03-01 06:44:12.493","130","6.739","128","8392704"
+"2020-03-01 06:44:13.493","130","6.739","128","8392704"
+"2020-03-01 06:44:14.493","130","6.736","128","8392704"
+"2020-03-01 06:44:15.493","130","6.734","128","8392704"
+"2020-03-01 06:44:16.493","130","6.735","128","8392704"
+"2020-03-01 06:44:17.493","130","6.734","128","8392704"
+"2020-03-01 06:44:18.493","130","6.736","128","8392704"
+"2020-03-01 06:44:19.493","130","6.741","128","8392704"
+"2020-03-01 06:44:20.493","130","6.744","128","8392704"
+"2020-03-01 06:44:21.493","130","6.746","128","8392704"
+"2020-03-01 06:44:22.493","130","6.746","128","8392704"
+"2020-03-01 06:44:23.493","130","6.748","128","8392704"
+"2020-03-01 06:44:24.493","130","6.751","128","8392704"
+"2020-03-01 06:44:25.493","130","6.752","128","8392704"
+"2020-03-01 06:44:26.493","130","6.752","128","8392704"
+"2020-03-01 06:44:27.493","130","6.752","128","8392704"
+"2020-03-01 06:44:28.493","130","6.753","128","8392704"
+"2020-03-01 06:44:29.493","130","6.751","128","8392704"
+"2020-03-01 06:44:30.493","130","6.751","128","8392704"
+"2020-03-01 06:44:31.493","130","6.749","128","8392704"
+"2020-03-01 06:44:32.493","130","6.747","128","8392704"
+"2020-03-01 06:44:33.493","130","6.748","128","8392704"
+"2020-03-01 06:44:34.493","130","6.749","128","8392704"
+"2020-03-01 06:44:35.493","130","6.746","128","8392704"
+"2020-03-01 06:44:36.493","130","6.742","128","8392704"
+"2020-03-01 06:44:37.493","130","6.742","128","8392704"
+"2020-03-01 06:44:38.493","130","6.743","128","8392704"
+"2020-03-01 06:44:39.493","130","6.743","128","8392704"
+"2020-03-01 06:44:40.493","130","6.743","128","8392704"
+"2020-03-01 06:44:41.493","130","6.741","128","8392704"
+"2020-03-01 06:44:42.493","130","6.741","128","8392704"
+"2020-03-01 06:44:43.493","130","6.741","128","8392704"
+"2020-03-01 06:44:44.493","130","6.74","128","8392704"
+"2020-03-01 06:44:45.493","130","6.74","128","8392704"
+"2020-03-01 06:44:46.493","130","6.739","128","8392704"
+"2020-03-01 06:44:47.493","130","6.738","128","8392704"
+"2020-03-01 06:44:48.493","130","6.738","128","8392704"
+"2020-03-01 06:44:49.493","130","6.741","128","8392704"
+"2020-03-01 06:44:50.493","130","6.749","128","8392704"
+"2020-03-01 06:44:51.493","130","6.756","128","8392704"
+"2020-03-01 06:44:52.493","130","6.763","128","8392704"
+"2020-03-01 06:44:53.493","130","6.768","128","8392704"
+"2020-03-01 06:44:54.493","130","6.771","128","8392704"
+"2020-03-01 06:44:55.493","130","6.774","128","8392704"
+"2020-03-01 06:44:56.493","130","6.774","128","8392704"
+"2020-03-01 06:44:57.493","130","6.774","128","8392704"
+"2020-03-01 06:44:58.493","130","6.765","128","8392704"
+"2020-03-01 06:44:59.493","130","6.763","128","8392704"
+"2020-03-01 06:45:00.493","130","6.761","128","8392704"
+"2020-03-01 06:45:01.493","130","6.758","128","8392704"
+"2020-03-01 06:45:02.493","130","6.756","128","8392704"
+"2020-03-01 06:45:03.493","130","6.756","128","8392704"
+"2020-03-01 06:45:04.493","130","6.756","128","8392704"
+"2020-03-01 06:45:05.493","130","6.763","128","8392704"
+"2020-03-01 06:45:06.493","130","6.763","128","8392704"
+"2020-03-01 06:45:07.493","130","6.764","128","8392704"
+"2020-03-01 06:45:08.493","130","6.762","128","8392704"
+"2020-03-01 06:45:09.493","130","6.763","128","8392704"
+"2020-03-01 06:45:10.493","130","6.764","128","8392704"
+"2020-03-01 06:45:11.493","130","6.763","128","8392704"
+"2020-03-01 06:45:12.493","130","6.76","128","8392704"
+"2020-03-01 06:45:13.493","130","6.759","128","8392704"
+"2020-03-01 06:45:14.493","130","6.758","128","8392704"
+"2020-03-01 06:45:15.493","130","6.758","128","8392704"
+"2020-03-01 06:45:16.493","130","6.755","128","8392704"
+"2020-03-01 20:04:50.493","130","7.59","128","8392704"
+"2020-03-01 20:04:51.493","130","7.592","128","8392704"
+"2020-03-01 20:04:52.493","130","7.592","128","8392704"
+"2020-03-01 20:04:53.493","130","7.593","128","8392704"
+"2020-03-01 20:04:54.493","130","7.592","128","8392704"
+"2020-03-01 20:04:55.493","130","7.592","128","8392704"
+"2020-03-01 20:04:56.493","130","7.593","128","8392704"
+"2020-03-01 20:04:57.493","130","7.593","128","8392704"
+"2020-03-01 20:04:58.493","130","7.593","128","8392704"
+"2020-03-01 20:04:59.493","130","7.594","128","8392704"
+"2020-03-01 20:05:00.493","130","7.595","128","8392704"
+"2020-03-01 20:05:01.493","130","7.596","128","8392704"
+"2020-03-01 20:05:02.493","130","7.595","128","8392704"
+"2020-03-01 20:05:03.493","130","7.595","128","8392704"
+"2020-03-01 20:05:04.493","130","7.594","128","8392704"
+"2020-03-01 20:05:05.493","130","7.595","128","8392704"
+"2020-03-01 20:05:06.493","130","7.598","128","8392704"
+"2020-03-01 20:05:07.493","130","7.597","128","8392704"
+"2020-03-01 20:05:08.493","130","7.595","128","8392704"
+"2020-03-01 20:05:09.493","130","7.597","128","8392704"
+"2020-03-01 20:05:10.493","130","7.598","128","8392704"
+"2020-03-01 20:05:11.493","130","7.598","128","8392704"
+"2020-03-01 20:05:12.493","130","7.597","128","8392704"
+"2020-03-01 20:05:13.493","130","7.595","128","8392704"
+"2020-03-01 20:05:14.493","130","7.591","128","8392704"
+"2020-03-01 20:05:15.493","130","7.589","128","8392704"
+"2020-03-01 20:05:16.493","130","7.588","128","8392704"
+"2020-03-01 20:05:17.493","130","7.589","128","8392704"
+"2020-03-01 20:05:18.493","130","7.589","128","8392704"
+"2020-03-01 20:05:19.493","130","7.589","128","8392704"
+"2020-03-01 20:05:20.493","130","7.587","128","8392704"
+"2020-03-01 20:05:21.493","130","7.584","128","8392704"
+"2020-03-01 20:05:22.493","130","7.583","128","8392704"
+"2020-03-01 20:05:23.493","130","7.585","128","8392704"
+"2020-03-01 20:05:24.493","130","7.586","128","8392704"
+"2020-03-01 20:05:25.493","130","7.586","128","8392704"
+"2020-03-01 20:05:26.493","130","7.586","128","8392704"
+"2020-03-01 20:05:27.493","130","7.586","128","8392704"
+"2020-03-01 20:05:28.493","130","7.587","128","8392704"
+"2020-03-01 20:05:29.493","130","7.585","128","8392704"
+"2020-03-01 20:05:30.493","130","7.584","128","8392704"
+"2020-03-01 20:05:31.493","130","7.586","128","8392704"
+"2020-03-01 20:05:32.493","130","7.589","128","8392704"
+"2020-03-01 20:05:33.493","130","7.59","128","8392704"
+"2020-03-01 20:05:34.493","130","7.591","128","8392704"
+"2020-03-01 20:05:35.493","130","7.591","128","8392704"
+"2020-03-01 20:05:36.493","130","7.594","128","8392704"
+"2020-03-01 20:05:37.493","130","7.599","128","8392704"
+"2020-03-01 20:05:38.493","130","7.602","128","8392704"
+"2020-03-01 20:05:39.493","130","7.604","128","8392704"
+"2020-03-01 20:05:40.493","130","7.605","128","8392704"
+"2020-03-01 20:05:41.493","130","7.607","128","8392704"
+"2020-03-01 20:05:42.493","130","7.607","128","8392704"
+"2020-03-01 20:05:43.493","130","7.604","128","8392704"
+"2020-03-01 20:05:44.493","130","7.597","128","8392704"
+"2020-03-01 20:05:45.493","130","7.592","128","8392704"
+"2020-03-01 20:05:46.493","130","7.59","128","8392704"
+"2020-03-01 20:05:47.493","130","7.59","128","8392704"
+"2020-03-01 20:05:48.493","130","7.591","128","8392704"
+"2020-03-01 20:05:49.493","130","7.591","128","8392704"
+"2020-03-01 20:05:50.493","130","7.591","128","8392704"
+"2020-03-01 20:05:51.493","130","7.594","128","8392704"
+"2020-03-01 20:05:52.493","130","7.599","128","8392704"
+"2020-03-01 20:05:53.493","130","7.601","128","8392704"
+"2020-03-01 20:05:54.493","130","7.602","128","8392704"
+"2020-03-01 20:05:55.493","130","7.602","128","8392704"
+"2020-03-01 20:05:56.493","130","7.602","128","8392704"
+"2020-03-01 20:05:57.493","130","7.603","128","8392704"
+"2020-03-01 20:05:58.493","130","7.604","128","8392704"
+"2020-03-01 20:05:59.493","130","7.604","128","8392704"
+"2020-03-01 20:06:00.493","130","7.605","128","8392704"
+"2020-03-01 20:06:01.493","130","7.606","128","8392704"
+"2020-03-01 20:06:02.493","130","7.607","128","8392704"
+"2020-03-01 20:06:03.493","130","7.605","128","8392704"
+"2020-03-01 20:06:04.493","130","7.604","128","8392704"
+"2020-03-01 20:06:05.493","130","7.603","128","8392704"
+"2020-03-01 20:06:06.493","130","7.602","128","8392704"
+"2020-03-01 20:06:07.493","130","7.603","128","8392704"
+"2020-03-01 20:06:08.493","130","7.604","128","8392704"
\ No newline at end of file
diff --git a/tests/pytest/test_data/ordered.csv b/tests/pytest/test_data/ordered.csv
new file mode 100644
index 0000000000000000000000000000000000000000..14da572d75e3c9bef32d6de7696ce65485b06d23
--- /dev/null
+++ b/tests/pytest/test_data/ordered.csv
@@ -0,0 +1,500 @@
+"2020-03-01 19:46:50.493","130","7.617","128","8392704"
+"2020-03-01 19:46:51.493","130","7.615","128","8392704"
+"2020-03-01 19:46:52.493","130","7.613","128","8392704"
+"2020-03-01 19:46:53.493","130","7.612","128","8392704"
+"2020-03-01 19:46:54.493","130","7.611","128","8392704"
+"2020-03-01 19:46:55.493","130","7.612","128","8392704"
+"2020-03-01 19:46:56.493","130","7.611","128","8392704"
+"2020-03-01 19:46:57.493","130","7.61","128","8392704"
+"2020-03-01 19:46:58.493","130","7.61","128","8392704"
+"2020-03-01 19:46:59.493","130","7.613","128","8392704"
+"2020-03-01 19:47:00.493","130","7.617","128","8392704"
+"2020-03-01 19:47:01.493","130","7.618","128","8392704"
+"2020-03-01 19:47:02.493","130","7.619","128","8392704"
+"2020-03-01 19:47:03.493","130","7.62","128","8392704"
+"2020-03-01 19:47:04.493","130","7.619","128","8392704"
+"2020-03-01 19:47:05.493","130","7.62","128","8392704"
+"2020-03-01 19:47:06.493","130","7.62","128","8392704"
+"2020-03-01 19:47:07.493","130","7.618","128","8392704"
+"2020-03-01 19:47:08.493","130","7.618","128","8392704"
+"2020-03-01 19:47:09.493","130","7.616","128","8392704"
+"2020-03-01 19:47:10.493","130","7.615","128","8392704"
+"2020-03-01 19:47:11.493","130","7.614","128","8392704"
+"2020-03-01 19:47:12.493","130","7.614","128","8392704"
+"2020-03-01 19:47:13.493","130","7.615","128","8392704"
+"2020-03-01 19:47:14.493","130","7.617","128","8392704"
+"2020-03-01 19:47:15.493","130","7.617","128","8392704"
+"2020-03-01 19:47:16.493","130","7.612","128","8392704"
+"2020-03-01 19:47:17.493","130","7.609","128","8392704"
+"2020-03-01 19:47:18.493","130","7.609","128","8392704"
+"2020-03-01 19:47:19.493","130","7.609","128","8392704"
+"2020-03-01 19:47:20.493","130","7.611","128","8392704"
+"2020-03-01 19:47:21.493","130","7.613","128","8392704"
+"2020-03-01 19:47:22.493","130","7.612","128","8392704"
+"2020-03-01 19:47:23.493","130","7.612","128","8392704"
+"2020-03-01 19:47:24.493","130","7.612","128","8392704"
+"2020-03-01 19:47:25.493","130","7.613","128","8392704"
+"2020-03-01 19:47:26.493","130","7.617","128","8392704"
+"2020-03-01 19:47:27.493","130","7.62","128","8392704"
+"2020-03-01 19:47:28.493","130","7.621","128","8392704"
+"2020-03-01 19:47:29.493","130","7.621","128","8392704"
+"2020-03-01 19:47:30.493","130","7.623","128","8392704"
+"2020-03-01 19:47:31.493","130","7.624","128","8392704"
+"2020-03-01 19:47:32.493","130","7.621","128","8392704"
+"2020-03-01 19:47:33.493","130","7.619","128","8392704"
+"2020-03-01 19:47:34.493","130","7.618","128","8392704"
+"2020-03-01 19:47:35.493","130","7.616","128","8392704"
+"2020-03-01 19:47:36.493","130","7.618","128","8392704"
+"2020-03-01 19:47:37.493","130","7.618","128","8392704"
+"2020-03-01 19:47:38.493","130","7.616","128","8392704"
+"2020-03-01 19:47:39.493","130","7.615","128","8392704"
+"2020-03-01 19:47:40.493","130","7.615","128","8392704"
+"2020-03-01 19:47:41.493","130","7.614","128","8392704"
+"2020-03-01 19:47:42.493","130","7.613","128","8392704"
+"2020-03-01 19:47:43.493","130","7.612","128","8392704"
+"2020-03-01 19:47:44.493","130","7.611","128","8392704"
+"2020-03-01 19:47:45.493","130","7.612","128","8392704"
+"2020-03-01 19:47:46.493","130","7.614","128","8392704"
+"2020-03-01 19:47:47.493","130","7.618","128","8392704"
+"2020-03-01 19:47:48.493","130","7.62","128","8392704"
+"2020-03-01 19:47:49.493","130","7.62","128","8392704"
+"2020-03-01 19:47:50.493","130","7.621","128","8392704"
+"2020-03-01 19:47:51.493","130","7.62","128","8392704"
+"2020-03-01 19:47:52.493","130","7.619","128","8392704"
+"2020-03-01 19:47:53.493","130","7.621","128","8392704"
+"2020-03-01 19:47:54.493","130","7.622","128","8392704"
+"2020-03-01 19:47:55.493","130","7.622","128","8392704"
+"2020-03-01 19:47:56.493","130","7.62","128","8392704"
+"2020-03-01 19:47:57.493","130","7.617","128","8392704"
+"2020-03-01 19:47:58.493","130","7.616","128","8392704"
+"2020-03-01 19:47:59.493","130","7.618","128","8392704"
+"2020-03-01 19:48:00.493","130","7.62","128","8392704"
+"2020-03-01 19:48:01.493","130","7.62","128","8392704"
+"2020-03-01 19:48:02.493","130","7.616","128","8392704"
+"2020-03-01 19:48:03.493","130","7.612","128","8392704"
+"2020-03-01 19:48:04.493","130","7.609","128","8392704"
+"2020-03-01 19:48:05.493","130","7.608","128","8392704"
+"2020-03-01 19:48:06.493","130","7.605","128","8392704"
+"2020-03-01 19:48:07.493","130","7.604","128","8392704"
+"2020-03-01 19:48:08.493","130","7.605","128","8392704"
+"2020-03-01 19:48:09.493","130","7.604","128","8392704"
+"2020-03-01 19:48:10.493","130","7.604","128","8392704"
+"2020-03-01 19:48:11.493","130","7.608","128","8392704"
+"2020-03-01 19:48:12.493","130","7.611","128","8392704"
+"2020-03-01 19:48:13.493","130","7.614","128","8392704"
+"2020-03-01 19:48:14.493","130","7.616","128","8392704"
+"2020-03-01 19:48:15.493","130","7.618","128","8392704"
+"2020-03-01 19:48:16.493","130","7.62","128","8392704"
+"2020-03-01 19:48:17.493","130","7.617","128","8392704"
+"2020-03-01 19:48:18.493","130","7.61","128","8392704"
+"2020-03-01 19:48:19.493","130","7.607","128","8392704"
+"2020-03-01 19:48:20.493","130","7.604","128","8392704"
+"2020-03-01 19:48:21.493","130","7.601","128","8392704"
+"2020-03-01 19:48:22.493","130","7.601","128","8392704"
+"2020-03-01 19:48:23.493","130","7.601","128","8392704"
+"2020-03-01 19:48:24.493","130","7.598","128","8392704"
+"2020-03-01 19:48:25.493","130","7.598","128","8392704"
+"2020-03-01 19:48:26.493","130","7.604","128","8392704"
+"2020-03-01 19:48:27.493","130","7.608","128","8392704"
+"2020-03-01 19:48:28.493","130","7.609","128","8392704"
+"2020-03-01 19:48:29.493","130","7.61","128","8392704"
+"2020-03-01 19:48:30.493","130","7.611","128","8392704"
+"2020-03-01 19:48:31.493","130","7.614","128","8392704"
+"2020-03-01 19:48:32.493","130","7.614","128","8392704"
+"2020-03-01 19:48:33.493","130","7.611","128","8392704"
+"2020-03-01 19:48:34.493","130","7.607","128","8392704"
+"2020-03-01 19:48:35.493","130","7.601","128","8392704"
+"2020-03-01 19:48:36.493","130","7.596","128","8392704"
+"2020-03-01 19:48:37.493","130","7.593","128","8392704"
+"2020-03-01 19:48:38.493","130","7.593","128","8392704"
+"2020-03-01 19:48:39.493","130","7.593","128","8392704"
+"2020-03-01 19:48:40.493","130","7.595","128","8392704"
+"2020-03-01 19:48:41.493","130","7.596","128","8392704"
+"2020-03-01 19:48:42.493","130","7.599","128","8392704"
+"2020-03-01 19:48:43.493","130","7.603","128","8392704"
+"2020-03-01 19:48:44.493","130","7.605","128","8392704"
+"2020-03-01 19:48:45.493","130","7.607","128","8392704"
+"2020-03-01 19:48:46.493","130","7.608","128","8392704"
+"2020-03-01 19:48:47.493","130","7.609","128","8392704"
+"2020-03-01 19:48:48.493","130","7.61","128","8392704"
+"2020-03-01 19:48:49.493","130","7.608","128","8392704"
+"2020-03-01 19:48:50.493","130","7.605","128","8392704"
+"2020-03-01 19:48:51.493","130","7.605","128","8392704"
+"2020-03-01 19:48:52.493","130","7.607","128","8392704"
+"2020-03-01 19:48:53.493","130","7.608","128","8392704"
+"2020-03-01 19:48:54.493","130","7.608","128","8392704"
+"2020-03-01 19:48:55.493","130","7.608","128","8392704"
+"2020-03-01 19:48:56.493","130","7.61","128","8392704"
+"2020-03-01 19:48:57.493","130","7.613","128","8392704"
+"2020-03-01 19:48:58.493","130","7.612","128","8392704"
+"2020-03-01 19:48:59.493","130","7.61","128","8392704"
+"2020-03-01 19:49:00.493","130","7.609","128","8392704"
+"2020-03-01 19:49:01.493","130","7.61","128","8392704"
+"2020-03-01 19:49:02.493","130","7.611","128","8392704"
+"2020-03-01 19:49:03.493","130","7.61","128","8392704"
+"2020-03-01 19:49:04.493","130","7.61","128","8392704"
+"2020-03-01 19:49:05.493","130","7.613","128","8392704"
+"2020-03-01 19:49:06.493","130","7.615","128","8392704"
+"2020-03-01 19:49:07.493","130","7.614","128","8392704"
+"2020-03-01 19:49:08.493","130","7.613","128","8392704"
+"2020-03-01 19:49:09.493","130","7.613","128","8392704"
+"2020-03-01 19:49:10.493","130","7.615","128","8392704"
+"2020-03-01 19:49:11.493","130","7.619","128","8392704"
+"2020-03-01 19:49:12.493","130","7.62","128","8392704"
+"2020-03-01 19:49:13.493","130","7.618","128","8392704"
+"2020-03-01 19:49:14.493","130","7.619","128","8392704"
+"2020-03-01 19:49:15.493","130","7.618","128","8392704"
+"2020-03-01 19:49:16.493","130","7.617","128","8392704"
+"2020-03-01 19:49:17.493","130","7.617","128","8392704"
+"2020-03-01 19:49:18.493","130","7.618","128","8392704"
+"2020-03-01 19:49:19.493","130","7.617","128","8392704"
+"2020-03-01 19:49:20.493","130","7.616","128","8392704"
+"2020-03-01 19:49:21.493","130","7.615","128","8392704"
+"2020-03-01 19:49:22.493","130","7.616","128","8392704"
+"2020-03-01 19:49:23.493","130","7.617","128","8392704"
+"2020-03-01 19:49:24.493","130","7.615","128","8392704"
+"2020-03-01 19:49:25.493","130","7.613","128","8392704"
+"2020-03-01 19:49:26.493","130","7.612","128","8392704"
+"2020-03-01 19:49:27.493","130","7.613","128","8392704"
+"2020-03-01 19:49:28.493","130","7.614","128","8392704"
+"2020-03-01 19:49:29.493","130","7.612","128","8392704"
+"2020-03-01 19:49:30.493","130","7.611","128","8392704"
+"2020-03-01 19:49:31.493","130","7.611","128","8392704"
+"2020-03-01 19:49:32.493","130","7.612","128","8392704"
+"2020-03-01 19:49:33.493","130","7.613","128","8392704"
+"2020-03-01 19:49:34.493","130","7.614","128","8392704"
+"2020-03-01 19:49:35.493","130","7.612","128","8392704"
+"2020-03-01 19:49:36.493","130","7.607","128","8392704"
+"2020-03-01 19:49:37.493","130","7.603","128","8392704"
+"2020-03-01 19:49:38.493","130","7.599","128","8392704"
+"2020-03-01 19:49:39.493","130","7.599","128","8392704"
+"2020-03-01 19:49:40.493","130","7.599","128","8392704"
+"2020-03-01 19:49:41.493","130","7.599","128","8392704"
+"2020-03-01 19:49:42.493","130","7.601","128","8392704"
+"2020-03-01 19:49:43.493","130","7.605","128","8392704"
+"2020-03-01 19:49:44.493","130","7.606","128","8392704"
+"2020-03-01 19:49:45.493","130","7.606","128","8392704"
+"2020-03-01 19:49:46.493","130","7.606","128","8392704"
+"2020-03-01 19:49:47.493","130","7.604","128","8392704"
+"2020-03-01 19:49:48.493","130","7.604","128","8392704"
+"2020-03-01 19:49:49.493","130","7.603","128","8392704"
+"2020-03-01 19:49:50.493","130","7.604","128","8392704"
+"2020-03-01 19:49:51.493","130","7.608","128","8392704"
+"2020-03-01 19:49:52.493","130","7.614","128","8392704"
+"2020-03-01 19:49:53.493","130","7.618","128","8392704"
+"2020-03-01 19:49:54.493","130","7.621","128","8392704"
+"2020-03-01 19:49:55.493","130","7.623","128","8392704"
+"2020-03-01 19:49:56.493","130","7.623","128","8392704"
+"2020-03-01 19:49:57.493","130","7.624","128","8392704"
+"2020-03-01 19:49:58.493","130","7.626","128","8392704"
+"2020-03-01 19:49:59.493","130","7.628","128","8392704"
+"2020-03-01 19:50:00.493","130","7.627","128","8392704"
+"2020-03-01 19:50:01.493","130","7.625","128","8392704"
+"2020-03-01 19:50:02.493","130","7.627","128","8392704"
+"2020-03-01 19:50:03.493","130","7.63","128","8392704"
+"2020-03-01 19:50:04.493","130","7.633","128","8392704"
+"2020-03-01 19:50:05.493","130","7.635","128","8392704"
+"2020-03-01 19:50:06.493","130","7.634","128","8392704"
+"2020-03-01 19:50:07.493","130","7.632","128","8392704"
+"2020-03-01 19:50:08.493","130","7.628","128","8392704"
+"2020-03-01 19:50:09.493","130","7.625","128","8392704"
+"2020-03-01 19:50:10.493","130","7.625","128","8392704"
+"2020-03-01 19:50:11.493","130","7.623","128","8392704"
+"2020-03-01 19:50:12.493","130","7.623","128","8392704"
+"2020-03-01 19:50:13.493","130","7.623","128","8392704"
+"2020-03-01 19:50:14.493","130","7.622","128","8392704"
+"2020-03-01 19:50:15.493","130","7.621","128","8392704"
+"2020-03-01 19:50:16.493","130","7.618","128","8392704"
+"2020-03-01 19:50:17.493","130","7.618","128","8392704"
+"2020-03-01 19:50:18.493","130","7.617","128","8392704"
+"2020-03-01 19:50:19.493","130","7.616","128","8392704"
+"2020-03-01 19:50:20.493","130","7.615","128","8392704"
+"2020-03-01 19:50:21.493","130","7.615","128","8392704"
+"2020-03-01 19:50:22.493","130","7.616","128","8392704"
+"2020-03-01 19:50:23.493","130","7.619","128","8392704"
+"2020-03-01 19:50:24.493","130","7.622","128","8392704"
+"2020-03-01 19:50:25.493","130","7.624","128","8392704"
+"2020-03-01 19:50:26.493","130","7.627","128","8392704"
+"2020-03-01 19:50:27.493","130","7.627","128","8392704"
+"2020-03-01 19:50:28.493","130","7.625","128","8392704"
+"2020-03-01 19:50:29.493","130","7.625","128","8392704"
+"2020-03-01 19:50:30.493","130","7.625","128","8392704"
+"2020-03-01 19:50:31.493","130","7.624","128","8392704"
+"2020-03-01 19:50:32.493","130","7.624","128","8392704"
+"2020-03-01 19:50:33.493","130","7.624","128","8392704"
+"2020-03-01 19:50:34.493","130","7.626","128","8392704"
+"2020-03-01 19:50:35.493","130","7.627","128","8392704"
+"2020-03-01 19:50:36.493","130","7.627","128","8392704"
+"2020-03-01 19:50:37.493","130","7.626","128","8392704"
+"2020-03-01 19:50:38.493","130","7.623","128","8392704"
+"2020-03-01 19:50:39.493","130","7.619","128","8392704"
+"2020-03-01 19:50:40.493","130","7.616","128","8392704"
+"2020-03-01 19:50:41.493","130","7.616","128","8392704"
+"2020-03-01 19:50:42.493","130","7.615","128","8392704"
+"2020-03-01 19:50:43.493","130","7.613","128","8392704"
+"2020-03-01 19:50:44.493","130","7.614","128","8392704"
+"2020-03-01 19:50:45.493","130","7.614","128","8392704"
+"2020-03-01 19:50:46.493","130","7.612","128","8392704"
+"2020-03-01 19:50:47.493","130","7.611","128","8392704"
+"2020-03-01 19:50:48.493","130","7.611","128","8392704"
+"2020-03-01 19:50:49.493","130","7.611","128","8392704"
+"2020-03-01 19:50:50.493","130","7.612","128","8392704"
+"2020-03-01 19:50:51.493","130","7.613","128","8392704"
+"2020-03-01 19:50:52.493","130","7.613","128","8392704"
+"2020-03-01 19:50:53.493","130","7.615","128","8392704"
+"2020-03-01 19:50:54.493","130","7.617","128","8392704"
+"2020-03-01 19:50:55.493","130","7.617","128","8392704"
+"2020-03-01 19:50:56.493","130","7.619","128","8392704"
+"2020-03-01 19:50:57.493","130","7.622","128","8392704"
+"2020-03-01 19:50:58.493","130","7.624","128","8392704"
+"2020-03-01 19:50:59.493","130","7.625","128","8392704"
+"2020-03-01 19:51:00.493","130","7.624","128","8392704"
+"2020-03-01 19:51:01.493","130","7.624","128","8392704"
+"2020-03-01 19:51:02.493","130","7.622","128","8392704"
+"2020-03-01 19:51:03.493","130","7.62","128","8392704"
+"2020-03-01 19:51:04.493","130","7.617","128","8392704"
+"2020-03-01 19:51:05.493","130","7.617","128","8392704"
+"2020-03-01 19:51:06.493","130","7.618","128","8392704"
+"2020-03-01 19:51:07.493","130","7.618","128","8392704"
+"2020-03-01 19:51:08.493","130","7.618","128","8392704"
+"2020-03-01 19:51:09.493","130","7.62","128","8392704"
+"2020-03-01 19:51:10.493","130","7.622","128","8392704"
+"2020-03-01 19:51:11.493","130","7.623","128","8392704"
+"2020-03-01 19:51:12.493","130","7.624","128","8392704"
+"2020-03-01 19:51:13.493","130","7.625","128","8392704"
+"2020-03-01 19:51:14.493","130","7.626","128","8392704"
+"2020-03-01 19:51:15.493","130","7.626","128","8392704"
+"2020-03-01 19:51:16.493","130","7.626","128","8392704"
+"2020-03-01 19:51:17.493","130","7.627","128","8392704"
+"2020-03-01 19:51:18.493","130","7.627","128","8392704"
+"2020-03-01 19:51:19.493","130","7.629","128","8392704"
+"2020-03-01 19:51:20.493","130","7.629","128","8392704"
+"2020-03-01 19:51:21.493","130","7.626","128","8392704"
+"2020-03-01 19:51:22.493","130","7.625","128","8392704"
+"2020-03-01 19:51:23.493","130","7.625","128","8392704"
+"2020-03-01 19:51:24.493","130","7.626","128","8392704"
+"2020-03-01 19:51:25.493","130","7.626","128","8392704"
+"2020-03-01 19:51:26.493","130","7.624","128","8392704"
+"2020-03-01 19:51:27.493","130","7.623","128","8392704"
+"2020-03-01 19:51:28.493","130","7.624","128","8392704"
+"2020-03-01 19:51:29.493","130","7.624","128","8392704"
+"2020-03-01 19:51:30.493","130","7.624","128","8392704"
+"2020-03-01 19:51:31.493","130","7.624","128","8392704"
+"2020-03-01 19:51:32.493","130","7.626","128","8392704"
+"2020-03-01 19:51:33.493","130","7.626","128","8392704"
+"2020-03-01 19:51:34.493","130","7.626","128","8392704"
+"2020-03-01 19:51:35.493","130","7.625","128","8392704"
+"2020-03-01 19:51:36.493","130","7.624","128","8392704"
+"2020-03-01 19:51:37.493","130","7.623","128","8392704"
+"2020-03-01 19:51:38.493","130","7.622","128","8392704"
+"2020-03-01 19:51:39.493","130","7.62","128","8392704"
+"2020-03-01 19:51:40.493","130","7.62","128","8392704"
+"2020-03-01 19:51:41.493","130","7.62","128","8392704"
+"2020-03-01 19:51:42.493","130","7.621","128","8392704"
+"2020-03-01 19:51:43.493","130","7.62","128","8392704"
+"2020-03-01 19:51:44.493","130","7.619","128","8392704"
+"2020-03-01 19:51:45.493","130","7.62","128","8392704"
+"2020-03-01 19:51:46.493","130","7.62","128","8392704"
+"2020-03-01 19:51:47.493","130","7.618","128","8392704"
+"2020-03-01 19:51:48.493","130","7.619","128","8392704"
+"2020-03-01 19:51:49.493","130","7.62","128","8392704"
+"2020-03-01 19:51:50.493","130","7.622","128","8392704"
+"2020-03-01 19:51:51.493","130","7.622","128","8392704"
+"2020-03-01 19:51:52.493","130","7.62","128","8392704"
+"2020-03-01 19:51:53.493","130","7.62","128","8392704"
+"2020-03-01 19:51:54.493","130","7.622","128","8392704"
+"2020-03-01 19:51:55.493","130","7.624","128","8392704"
+"2020-03-01 19:51:56.493","130","7.622","128","8392704"
+"2020-03-01 19:51:57.493","130","7.616","128","8392704"
+"2020-03-01 19:51:58.493","130","7.611","128","8392704"
+"2020-03-01 19:51:59.493","130","7.61","128","8392704"
+"2020-03-01 19:52:00.493","130","7.608","128","8392704"
+"2020-03-01 19:52:01.493","130","7.606","128","8392704"
+"2020-03-01 19:52:02.493","130","7.607","128","8392704"
+"2020-03-01 19:52:03.493","130","7.608","128","8392704"
+"2020-03-01 19:52:04.493","130","7.61","128","8392704"
+"2020-03-01 19:52:05.493","130","7.612","128","8392704"
+"2020-03-01 19:52:06.493","130","7.615","128","8392704"
+"2020-03-01 19:52:07.493","130","7.62","128","8392704"
+"2020-03-01 19:52:08.493","130","7.623","128","8392704"
+"2020-03-01 19:52:09.493","130","7.624","128","8392704"
+"2020-03-01 19:52:10.493","130","7.623","128","8392704"
+"2020-03-01 19:52:11.493","130","7.623","128","8392704"
+"2020-03-01 19:52:12.493","130","7.624","128","8392704"
+"2020-03-01 19:52:13.493","130","7.622","128","8392704"
+"2020-03-01 19:52:14.493","130","7.62","128","8392704"
+"2020-03-01 19:52:15.493","130","7.621","128","8392704"
+"2020-03-01 19:52:16.493","130","7.62","128","8392704"
+"2020-03-01 19:52:17.493","130","7.622","128","8392704"
+"2020-03-01 19:52:18.493","130","7.625","128","8392704"
+"2020-03-01 19:52:19.493","130","7.627","128","8392704"
+"2020-03-01 19:52:20.493","130","7.625","128","8392704"
+"2020-03-01 19:52:21.493","130","7.621","128","8392704"
+"2020-03-01 19:52:22.493","130","7.617","128","8392704"
+"2020-03-01 19:52:23.493","130","7.617","128","8392704"
+"2020-03-01 19:52:24.493","130","7.617","128","8392704"
+"2020-03-01 19:52:25.493","130","7.616","128","8392704"
+"2020-03-01 19:52:26.493","130","7.615","128","8392704"
+"2020-03-01 19:52:27.493","130","7.616","128","8392704"
+"2020-03-01 19:52:28.493","130","7.619","128","8392704"
+"2020-03-01 19:52:29.493","130","7.621","128","8392704"
+"2020-03-01 19:52:30.493","130","7.621","128","8392704"
+"2020-03-01 19:52:31.493","130","7.621","128","8392704"
+"2020-03-01 19:52:32.493","130","7.621","128","8392704"
+"2020-03-01 19:52:33.493","130","7.621","128","8392704"
+"2020-03-01 19:52:34.493","130","7.623","128","8392704"
+"2020-03-01 19:52:35.493","130","7.621","128","8392704"
+"2020-03-01 19:52:36.493","130","7.617","128","8392704"
+"2020-03-01 19:52:37.493","130","7.615","128","8392704"
+"2020-03-01 19:52:38.493","130","7.612","128","8392704"
+"2020-03-01 19:52:39.493","130","7.609","128","8392704"
+"2020-03-01 19:52:40.493","130","7.606","128","8392704"
+"2020-03-01 19:52:41.493","130","7.606","128","8392704"
+"2020-03-01 19:52:42.493","130","7.609","128","8392704"
+"2020-03-01 19:52:43.493","130","7.612","128","8392704"
+"2020-03-01 19:52:44.493","130","7.616","128","8392704"
+"2020-03-01 19:52:45.493","130","7.619","128","8392704"
+"2020-03-01 19:52:46.493","130","7.62","128","8392704"
+"2020-03-01 19:52:47.493","130","7.622","128","8392704"
+"2020-03-01 19:52:48.493","130","7.622","128","8392704"
+"2020-03-01 19:52:49.493","130","7.621","128","8392704"
+"2020-03-01 19:52:50.493","130","7.618","128","8392704"
+"2020-03-01 19:52:51.493","130","7.616","128","8392704"
+"2020-03-01 19:52:52.493","130","7.613","128","8392704"
+"2020-03-01 19:52:53.493","130","7.612","128","8392704"
+"2020-03-01 19:52:54.493","130","7.612","128","8392704"
+"2020-03-01 19:52:55.493","130","7.611","128","8392704"
+"2020-03-01 19:52:56.493","130","7.609","128","8392704"
+"2020-03-01 19:52:57.493","130","7.608","128","8392704"
+"2020-03-01 19:52:58.493","130","7.609","128","8392704"
+"2020-03-01 19:52:59.493","130","7.611","128","8392704"
+"2020-03-01 19:53:00.493","130","7.612","128","8392704"
+"2020-03-01 19:53:01.493","130","7.614","128","8392704"
+"2020-03-01 19:53:02.493","130","7.618","128","8392704"
+"2020-03-01 19:53:03.493","130","7.62","128","8392704"
+"2020-03-01 19:53:04.493","130","7.62","128","8392704"
+"2020-03-01 19:53:05.493","130","7.62","128","8392704"
+"2020-03-01 19:53:06.493","130","7.619","128","8392704"
+"2020-03-01 19:53:07.493","130","7.617","128","8392704"
+"2020-03-01 19:53:08.493","130","7.615","128","8392704"
+"2020-03-01 19:53:09.493","130","7.612","128","8392704"
+"2020-03-01 19:53:10.493","130","7.61","128","8392704"
+"2020-03-01 19:53:11.493","130","7.609","128","8392704"
+"2020-03-01 19:53:12.493","130","7.608","128","8392704"
+"2020-03-01 19:53:13.493","130","7.61","128","8392704"
+"2020-03-01 19:53:14.493","130","7.611","128","8392704"
+"2020-03-01 19:53:15.493","130","7.609","128","8392704"
+"2020-03-01 19:53:16.493","130","7.608","128","8392704"
+"2020-03-01 19:53:17.493","130","7.608","128","8392704"
+"2020-03-01 19:53:18.493","130","7.607","128","8392704"
+"2020-03-01 19:53:19.493","130","7.607","128","8392704"
+"2020-03-01 19:53:20.493","130","7.605","128","8392704"
+"2020-03-01 19:53:21.493","130","7.603","128","8392704"
+"2020-03-01 19:53:22.493","130","7.606","128","8392704"
+"2020-03-01 19:53:23.493","130","7.611","128","8392704"
+"2020-03-01 19:53:24.493","130","7.615","128","8392704"
+"2020-03-01 19:53:25.493","130","7.618","128","8392704"
+"2020-03-01 19:53:26.493","130","7.62","128","8392704"
+"2020-03-01 19:53:27.493","130","7.622","128","8392704"
+"2020-03-01 19:53:28.493","130","7.624","128","8392704"
+"2020-03-01 19:53:29.493","130","7.626","128","8392704"
+"2020-03-01 19:53:30.493","130","7.624","128","8392704"
+"2020-03-01 19:53:31.493","130","7.617","128","8392704"
+"2020-03-01 19:53:32.493","130","7.613","128","8392704"
+"2020-03-01 19:53:33.493","130","7.613","128","8392704"
+"2020-03-01 19:53:34.493","130","7.613","128","8392704"
+"2020-03-01 19:53:35.493","130","7.61","128","8392704"
+"2020-03-01 19:53:36.493","130","7.609","128","8392704"
+"2020-03-01 19:53:37.493","130","7.611","128","8392704"
+"2020-03-01 19:53:38.493","130","7.61","128","8392704"
+"2020-03-01 19:53:39.493","130","7.609","128","8392704"
+"2020-03-01 19:53:40.493","130","7.608","128","8392704"
+"2020-03-01 19:53:41.493","130","7.605","128","8392704"
+"2020-03-01 19:53:42.493","130","7.601","128","8392704"
+"2020-03-01 19:53:43.493","130","7.6","128","8392704"
+"2020-03-01 19:53:44.493","130","7.602","128","8392704"
+"2020-03-01 19:53:45.493","130","7.604","128","8392704"
+"2020-03-01 19:53:46.493","130","7.605","128","8392704"
+"2020-03-01 19:53:47.493","130","7.606","128","8392704"
+"2020-03-01 19:53:48.493","130","7.605","128","8392704"
+"2020-03-01 19:53:49.493","130","7.605","128","8392704"
+"2020-03-01 19:53:50.493","130","7.606","128","8392704"
+"2020-03-01 19:53:51.493","130","7.606","128","8392704"
+"2020-03-01 19:53:52.493","130","7.604","128","8392704"
+"2020-03-01 19:53:53.493","130","7.606","128","8392704"
+"2020-03-01 19:53:54.493","130","7.61","128","8392704"
+"2020-03-01 19:53:55.493","130","7.612","128","8392704"
+"2020-03-01 19:53:56.493","130","7.613","128","8392704"
+"2020-03-01 19:53:57.493","130","7.613","128","8392704"
+"2020-03-01 19:53:58.493","130","7.613","128","8392704"
+"2020-03-01 19:53:59.493","130","7.615","128","8392704"
+"2020-03-01 19:54:00.493","130","7.616","128","8392704"
+"2020-03-01 19:54:01.493","130","7.616","128","8392704"
+"2020-03-01 19:54:02.493","130","7.618","128","8392704"
+"2020-03-01 19:54:03.493","130","7.621","128","8392704"
+"2020-03-01 19:54:04.493","130","7.623","128","8392704"
+"2020-03-01 19:54:05.493","130","7.623","128","8392704"
+"2020-03-01 19:54:06.493","130","7.624","128","8392704"
+"2020-03-01 19:54:07.493","130","7.624","128","8392704"
+"2020-03-01 19:54:08.493","130","7.622","128","8392704"
+"2020-03-01 19:54:09.493","130","7.62","128","8392704"
+"2020-03-01 19:54:10.493","130","7.621","128","8392704"
+"2020-03-01 19:54:11.493","130","7.62","128","8392704"
+"2020-03-01 19:54:12.493","130","7.617","128","8392704"
+"2020-03-01 19:54:13.493","130","7.617","128","8392704"
+"2020-03-01 19:54:14.493","130","7.618","128","8392704"
+"2020-03-01 19:54:15.493","130","7.617","128","8392704"
+"2020-03-01 19:54:16.493","130","7.615","128","8392704"
+"2020-03-01 19:54:17.493","130","7.613","128","8392704"
+"2020-03-01 19:54:18.493","130","7.612","128","8392704"
+"2020-03-01 19:54:19.493","130","7.609","128","8392704"
+"2020-03-01 19:54:20.493","130","7.605","128","8392704"
+"2020-03-01 19:54:21.493","130","7.604","128","8392704"
+"2020-03-01 19:54:22.493","130","7.605","128","8392704"
+"2020-03-01 19:54:23.493","130","7.608","128","8392704"
+"2020-03-01 19:54:24.493","130","7.611","128","8392704"
+"2020-03-01 19:54:25.493","130","7.613","128","8392704"
+"2020-03-01 19:54:26.493","130","7.613","128","8392704"
+"2020-03-01 19:54:27.493","130","7.611","128","8392704"
+"2020-03-01 19:54:28.493","130","7.611","128","8392704"
+"2020-03-01 19:54:29.493","130","7.611","128","8392704"
+"2020-03-01 19:54:30.493","130","7.612","128","8392704"
+"2020-03-01 19:54:31.493","130","7.614","128","8392704"
+"2020-03-01 19:54:32.493","130","7.61","128","8392704"
+"2020-03-01 19:54:33.493","130","7.603","128","8392704"
+"2020-03-01 19:54:34.493","130","7.598","128","8392704"
+"2020-03-01 19:54:35.493","130","7.594","128","8392704"
+"2020-03-01 19:54:36.493","130","7.591","128","8392704"
+"2020-03-01 19:54:37.493","130","7.591","128","8392704"
+"2020-03-01 19:54:38.493","130","7.59","128","8392704"
+"2020-03-01 19:54:39.493","130","7.588","128","8392704"
+"2020-03-01 19:54:40.493","130","7.593","128","8392704"
+"2020-03-01 19:54:41.493","130","7.599","128","8392704"
+"2020-03-01 19:54:42.493","130","7.602","128","8392704"
+"2020-03-01 19:54:43.493","130","7.604","128","8392704"
+"2020-03-01 19:54:44.493","130","7.606","128","8392704"
+"2020-03-01 19:54:45.493","130","7.609","128","8392704"
+"2020-03-01 19:54:46.493","130","7.612","128","8392704"
+"2020-03-01 19:54:47.493","130","7.614","128","8392704"
+"2020-03-01 19:54:48.493","130","7.616","128","8392704"
+"2020-03-01 19:54:49.493","130","7.617","128","8392704"
+"2020-03-01 19:54:50.493","130","7.619","128","8392704"
+"2020-03-01 19:54:51.493","130","7.623","128","8392704"
+"2020-03-01 19:54:52.493","130","7.626","128","8392704"
+"2020-03-01 19:54:53.493","130","7.626","128","8392704"
+"2020-03-01 19:54:54.493","130","7.624","128","8392704"
+"2020-03-01 19:54:55.493","130","7.623","128","8392704"
+"2020-03-01 19:54:56.493","130","7.618","128","8392704"
+"2020-03-01 19:54:57.493","130","7.613","128","8392704"
+"2020-03-01 19:54:58.493","130","7.61","128","8392704"
+"2020-03-01 19:54:59.493","130","7.605","128","8392704"
+"2020-03-01 19:55:00.493","130","7.604","128","8392704"
+"2020-03-01 19:55:01.493","130","7.603","128","8392704"
+"2020-03-01 19:55:02.493","130","7.602","128","8392704"
+"2020-03-01 19:55:03.493","130","7.602","128","8392704"
+"2020-03-01 19:55:04.493","130","7.602","128","8392704"
+"2020-03-01 19:55:05.493","130","7.603","128","8392704"
+"2020-03-01 19:55:06.493","130","7.608","128","8392704"
+"2020-03-01 19:55:07.493","130","7.609","128","8392704"
+"2020-03-01 19:55:08.493","130","7.608","128","8392704"
+"2020-03-01 19:55:09.493","130","7.609","128","8392704"
\ No newline at end of file
diff --git a/tests/pytest/update/allow_update-0.py b/tests/pytest/update/allow_update-0.py
new file mode 100644
index 0000000000000000000000000000000000000000..61295ec4b7680b42519bf7c9e0a01e0e523aa38b
--- /dev/null
+++ b/tests/pytest/update/allow_update-0.py
@@ -0,0 +1,170 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ self.numOfRecords = 10
+ self.ts = 1604295582000
+
+ def restartTaosd(self):
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.execute("use udb")
+
+ def run(self):
+ tdSql.prepare()
+ startTs = self.ts
+
+ print("==============step1")
+ tdSql.execute("create database udb update 0")
+ tdSql.execute("use udb")
+ tdSql.execute("create table t (ts timestamp, a int)")
+ tdSql.execute("insert into t values (%d, 1)" % (startTs))
+ tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
+ tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(3)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 1)
+ tdSql.checkData(1, 0, 1)
+ tdSql.checkData(2, 0, 1)
+
+ print("==============step2")
+ tdSql.execute("insert into t values (%d, 2)" % (startTs))
+ tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
+ tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(3)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 1)
+ tdSql.checkData(1, 0, 1)
+ tdSql.checkData(2, 0, 1)
+
+ print("==============step3")
+ tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
+ tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
+ tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
+ tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(7)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 3)
+ tdSql.checkData(1, 0, 1)
+ tdSql.checkData(2, 0, 3)
+ tdSql.checkData(3, 0, 1)
+ tdSql.checkData(4, 0, 3)
+ tdSql.checkData(5, 0, 1)
+ tdSql.checkData(6, 0, 3)
+
+ print("==============step4")
+ tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
+ tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
+ tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
+ tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(7)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 3)
+ tdSql.checkData(1, 0, 1)
+ tdSql.checkData(2, 0, 3)
+ tdSql.checkData(3, 0, 1)
+ tdSql.checkData(4, 0, 3)
+ tdSql.checkData(5, 0, 1)
+ tdSql.checkData(6, 0, 3)
+
+ print("==============step5")
+ tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
+ tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(9)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 3)
+ tdSql.checkData(1, 0, 1)
+ tdSql.checkData(2, 0, 3)
+ tdSql.checkData(3, 0, 5)
+ tdSql.checkData(4, 0, 1)
+ tdSql.checkData(5, 0, 5)
+ tdSql.checkData(6, 0, 3)
+ tdSql.checkData(7, 0, 1)
+ tdSql.checkData(8, 0, 3)
+
+ print("==============step6")
+ tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(9)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 3)
+ tdSql.checkData(1, 0, 1)
+ tdSql.checkData(2, 0, 3)
+ tdSql.checkData(3, 0, 5)
+ tdSql.checkData(4, 0, 1)
+ tdSql.checkData(5, 0, 5)
+ tdSql.checkData(6, 0, 3)
+ tdSql.checkData(7, 0, 1)
+ tdSql.checkData(8, 0, 3)
+
+ # restart taosd to commit, and check
+ self.restartTaosd();
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(9)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 3)
+ tdSql.checkData(1, 0, 1)
+ tdSql.checkData(2, 0, 3)
+ tdSql.checkData(3, 0, 5)
+ tdSql.checkData(4, 0, 1)
+ tdSql.checkData(5, 0, 5)
+ tdSql.checkData(6, 0, 3)
+ tdSql.checkData(7, 0, 1)
+ tdSql.checkData(8, 0, 3)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/update/allow_update.py b/tests/pytest/update/allow_update.py
new file mode 100644
index 0000000000000000000000000000000000000000..5871197f0213dbcf0611b25685510a1d2e97610f
--- /dev/null
+++ b/tests/pytest/update/allow_update.py
@@ -0,0 +1,266 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ self.numOfRecords = 10
+ self.ts = 1604295582000
+
+ def restartTaosd(self):
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.execute("use udb")
+
+ def run(self):
+ tdSql.prepare()
+ startTs = self.ts
+
+ tdSql.execute("create database udb update 1")
+ tdSql.execute("use udb")
+ tdSql.execute("create table t (ts timestamp, a int)")
+
+ print("==============step1")
+ tdSql.execute("insert into t values (%d, 1)" % (startTs))
+ tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
+ tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(3)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 1)
+ tdSql.checkData(1, 0, 1)
+ tdSql.checkData(2, 0, 1)
+
+ print("==============step2")
+ tdSql.execute("insert into t values (%d, 2)" % (startTs))
+ tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
+ tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(3)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 2)
+ tdSql.checkData(1, 0, 2)
+ tdSql.checkData(2, 0, 2)
+
+ print("==============step3")
+ tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
+ tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
+ tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
+ tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(7)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 3)
+ tdSql.checkData(1, 0, 2)
+ tdSql.checkData(2, 0, 3)
+ tdSql.checkData(3, 0, 2)
+ tdSql.checkData(4, 0, 3)
+ tdSql.checkData(5, 0, 2)
+ tdSql.checkData(6, 0, 3)
+
+ print("==============step4")
+ tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
+ tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
+ tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
+ tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(7)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 4)
+ tdSql.checkData(1, 0, 2)
+ tdSql.checkData(2, 0, 4)
+ tdSql.checkData(3, 0, 2)
+ tdSql.checkData(4, 0, 4)
+ tdSql.checkData(5, 0, 2)
+ tdSql.checkData(6, 0, 4)
+
+ print("==============step5")
+ tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
+ tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(9)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 4)
+ tdSql.checkData(1, 0, 2)
+ tdSql.checkData(2, 0, 4)
+ tdSql.checkData(3, 0, 5)
+ tdSql.checkData(4, 0, 2)
+ tdSql.checkData(5, 0, 5)
+ tdSql.checkData(6, 0, 4)
+ tdSql.checkData(7, 0, 2)
+ tdSql.checkData(8, 0, 4)
+
+ print("==============step6")
+ tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
+ tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(9)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 6)
+ tdSql.checkData(1, 0, 6)
+ tdSql.checkData(2, 0, 6)
+ tdSql.checkData(3, 0, 6)
+ tdSql.checkData(4, 0, 6)
+ tdSql.checkData(5, 0, 6)
+ tdSql.checkData(6, 0, 6)
+ tdSql.checkData(7, 0, 6)
+ tdSql.checkData(8, 0, 6)
+
+ # restart taosd to commit, and check
+ self.restartTaosd();
+
+ tdSql.query("select * from t")
+ tdSql.checkRows(9)
+
+ tdSql.query("select a from t")
+ tdSql.checkData(0, 0, 6)
+ tdSql.checkData(1, 0, 6)
+ tdSql.checkData(2, 0, 6)
+ tdSql.checkData(3, 0, 6)
+ tdSql.checkData(4, 0, 6)
+ tdSql.checkData(5, 0, 6)
+ tdSql.checkData(6, 0, 6)
+ tdSql.checkData(7, 0, 6)
+ tdSql.checkData(8, 0, 6)
+
+ tdSql.execute("create table subt (ts timestamp, a int, b float, c binary(16), d bool)")
+
+ print("==============step7")
+ tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+0')" % (startTs))
+ tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c-3')" % (startTs - 3))
+ tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+3')" % (startTs + 3))
+
+ tdSql.query("select * from subt")
+ tdSql.checkRows(3)
+
+ tdSql.query("select a,b,c,d from subt")
+ tdSql.checkData(0, 0, 1)
+ tdSql.checkData(1, 0, 1)
+ tdSql.checkData(2, 0, 1)
+ tdSql.checkData(0, 1, None)
+ tdSql.checkData(1, 1, None)
+ tdSql.checkData(2, 1, None)
+ tdSql.checkData(0, 2, 'c-3')
+ tdSql.checkData(1, 2, 'c+0')
+ tdSql.checkData(2, 2, 'c+3')
+ tdSql.checkData(0, 3, None)
+ tdSql.checkData(1, 3, None)
+ tdSql.checkData(2, 3, None)
+
+ print("==============step8")
+ tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs))
+ tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs - 3))
+ tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, false)" % (startTs + 3))
+
+ tdSql.query("select * from subt")
+ tdSql.checkRows(3)
+
+ tdSql.query("select a,b,c,d from subt")
+ tdSql.checkData(0, 0, None)
+ tdSql.checkData(1, 0, None)
+ tdSql.checkData(2, 0, None)
+ tdSql.checkData(0, 1, 2.0)
+ tdSql.checkData(1, 1, 2.0)
+ tdSql.checkData(2, 1, 2.0)
+ tdSql.checkData(0, 2, None)
+ tdSql.checkData(1, 2, None)
+ tdSql.checkData(2, 2, None)
+ tdSql.checkData(0, 3, 1)
+ tdSql.checkData(1, 3, 1)
+ tdSql.checkData(2, 3, 0)
+
+ # restart taosd to commit, and check
+ self.restartTaosd();
+
+ tdSql.query("select * from subt")
+ tdSql.checkRows(3)
+
+ tdSql.query("select a,b,c,d from subt")
+ tdSql.checkData(0, 0, None)
+ tdSql.checkData(1, 0, None)
+ tdSql.checkData(2, 0, None)
+ tdSql.checkData(0, 1, 2.0)
+ tdSql.checkData(1, 1, 2.0)
+ tdSql.checkData(2, 1, 2.0)
+ tdSql.checkData(0, 2, None)
+ tdSql.checkData(1, 2, None)
+ tdSql.checkData(2, 2, None)
+ tdSql.checkData(0, 3, 1)
+ tdSql.checkData(1, 3, 1)
+ tdSql.checkData(2, 3, 0)
+
+
+
+ tdSql.execute("create table ct (ts timestamp, a int, b float, c binary(128))")
+
+ print("==============step9")
+ insertRows = 20000
+ for i in range(0, insertRows):
+ tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i, i))
+
+ tdSql.query("select * from ct")
+ tdSql.checkRows(insertRows)
+
+ for i in range(0, insertRows):
+ tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i+insertRows, i+insertRows))
+
+ tdSql.query("select * from ct")
+ tdSql.checkRows(insertRows)
+
+ tdSql.query("select a,b from ct limit 3")
+ tdSql.checkData(0, 0, insertRows+0)
+ tdSql.checkData(1, 0, insertRows+1)
+ tdSql.checkData(2, 0, insertRows+2)
+
+ tdSql.checkData(0, 1, insertRows+0)
+ tdSql.checkData(1, 1, insertRows+1)
+ tdSql.checkData(2, 1, insertRows+2)
+
+
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/update/append_commit_data-0.py b/tests/pytest/update/append_commit_data-0.py
new file mode 100644
index 0000000000000000000000000000000000000000..b844a50a086dc52d7edb5250801ee87cf68ee28f
--- /dev/null
+++ b/tests/pytest/update/append_commit_data-0.py
@@ -0,0 +1,84 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ def run(self):
+ print("==========step1")
+ print("create table && insert data")
+ s = 'reset query cache'
+ tdSql.execute(s)
+ s = 'drop database if exists db'
+ tdSql.execute(s)
+ s = 'create database db'
+ tdSql.execute(s)
+ s = 'use db'
+ tdSql.execute(s)
+ ret = tdSql.execute('create table t1 (ts timestamp, a int)')
+
+ insertRows = 200
+ t0 = 1604298064000
+ tdLog.info("insert %d rows" % (insertRows))
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t1 values (%d , 1)' %
+ (t0+i))
+ print("==========step2")
+ print("restart to commit ")
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from db.t1")
+ tdSql.checkRows(insertRows)
+ for k in range(0,100):
+ tdLog.info("insert %d rows" % (insertRows))
+ for i in range (0,insertRows):
+ ret = tdSql.execute(
+ 'insert into db.t1 values(%d,1)' %
+ (t0+k*200+i)
+ )
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from db.t1")
+ tdSql.checkRows(insertRows+200*k)
+ print("==========step2")
+ print("insert into another table ")
+ s = 'use db'
+ tdSql.execute(s)
+ ret = tdSql.execute('create table t2 (ts timestamp, a int)')
+ insertRows = 20000
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t2 values (%d, 1)' %
+ (t0+i))
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t2")
+ tdSql.checkRows(insertRows)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py
new file mode 100644
index 0000000000000000000000000000000000000000..3169b748e0843c720beb54eced28a14f1ca747a6
--- /dev/null
+++ b/tests/pytest/update/append_commit_data.py
@@ -0,0 +1,84 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ def run(self):
+ print("==========step1")
+ print("create table && insert data")
+ s = 'reset query cache'
+ tdSql.execute(s)
+ s = 'drop database if exists db'
+ tdSql.execute(s)
+ s = 'create database db update 1'
+ tdSql.execute(s)
+ s = 'use db'
+ tdSql.execute(s)
+ ret = tdSql.execute('create table t1 (ts timestamp, a int)')
+
+ insertRows = 200
+ t0 = 1604298064000
+ tdLog.info("insert %d rows" % (insertRows))
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t1 values (%d , 1)' %
+ (t0+i))
+ print("==========step2")
+ print("restart to commit ")
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from db.t1")
+ tdSql.checkRows(insertRows)
+ for k in range(0,100):
+ tdLog.info("insert %d rows" % (insertRows))
+ for i in range (0,insertRows):
+ ret = tdSql.execute(
+ 'insert into db.t1 values(%d,1)' %
+ (t0+k*200+i)
+ )
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from db.t1")
+ tdSql.checkRows(insertRows+200*k)
+ print("==========step2")
+ print("insert into another table ")
+ s = 'use db'
+ tdSql.execute(s)
+ ret = tdSql.execute('create table t2 (ts timestamp, a int)')
+ insertRows = 20000
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t2 values (%d, 1)' %
+ (t0+i))
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t2")
+ tdSql.checkRows(insertRows)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/update/append_commit_last-0.py b/tests/pytest/update/append_commit_last-0.py
new file mode 100644
index 0000000000000000000000000000000000000000..c884207f2bba5dd0da09cf4aae501d27caef7aab
--- /dev/null
+++ b/tests/pytest/update/append_commit_last-0.py
@@ -0,0 +1,90 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor())
+
+ self.ts = 1604298064000
+
+ def restartTaosd(self):
+ tdDnodes.stop(1)
+ tdDnodes.startWithoutSleep(1)
+ tdSql.execute("use db")
+
+ def run(self):
+ tdSql.prepare()
+
+ print("==============step1")
+ tdSql.execute("create table t1 (ts timestamp, a int)")
+
+ for i in range(10):
+ tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i))
+ self.restartTaosd()
+ tdSql.query("select * from t1")
+ tdSql.checkRows(i + 1)
+ tdSql.query("select sum(a) from t1")
+ tdSql.checkData(0, 0, i + 1)
+
+ print("==============step2")
+ tdSql.execute("create table t2 (ts timestamp, a int)")
+ tdSql.execute("insert into t2 values(%d, 1)" % self.ts)
+ self.restartTaosd()
+ tdSql.query("select * from t2")
+ tdSql.checkRows(1)
+ tdSql.checkData(0, 1, 1)
+
+ for i in range(1, 151):
+ tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t2")
+ tdSql.checkRows(151)
+ tdSql.query("select sum(a) from t2")
+ tdSql.checkData(0, 0, 151)
+
+
+ print("==============step3")
+ tdSql.execute("create table t3 (ts timestamp, a int)")
+ tdSql.execute("insert into t3 values(%d, 1)" % self.ts)
+ self.restartTaosd()
+ tdSql.query("select * from t3")
+ tdSql.checkRows(1)
+ tdSql.checkData(0, 1, 1)
+
+ for i in range(8):
+ for j in range(1, 11):
+ tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j))
+
+ self.restartTaosd()
+ tdSql.query("select * from t3")
+ tdSql.checkRows(81)
+ tdSql.query("select sum(a) from t3")
+ tdSql.checkData(0, 0, 81)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/update/append_commit_last.py b/tests/pytest/update/append_commit_last.py
new file mode 100644
index 0000000000000000000000000000000000000000..013983f9402292d03d26bd998c96eaf39b26a8fd
--- /dev/null
+++ b/tests/pytest/update/append_commit_last.py
@@ -0,0 +1,85 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor())
+
+ self.ts = 1604298064000
+
+ def restartTaosd(self):
+ tdDnodes.stop(1)
+ tdDnodes.startWithoutSleep(1)
+ tdSql.execute("use udb")
+
+ def run(self):
+ tdSql.prepare()
+
+ print("==============step1")
+ tdSql.execute("create database udb update 1")
+ tdSql.execute("use udb")
+ tdSql.execute("create table t1 (ts timestamp, a int)")
+
+ for i in range(10):
+ tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i))
+ self.restartTaosd()
+ tdSql.query("select * from t1")
+ tdSql.checkRows(i + 1)
+
+
+ print("==============step2")
+ tdSql.execute("create table t2 (ts timestamp, a int)")
+ tdSql.execute("insert into t2 values(%d, 1)" % self.ts)
+ self.restartTaosd()
+ tdSql.query("select * from t2")
+ tdSql.checkRows(1)
+
+ for i in range(1, 151):
+ tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t2")
+ tdSql.checkRows(151)
+
+
+ print("==============step3")
+ tdSql.execute("create table t3 (ts timestamp, a int)")
+ tdSql.execute("insert into t3 values(%d, 1)" % self.ts)
+ self.restartTaosd()
+ tdSql.query("select * from t3")
+ tdSql.checkRows(1)
+
+ for i in range(8):
+ for j in range(1, 11):
+ tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j))
+
+ self.restartTaosd()
+ tdSql.query("select * from t3")
+ tdSql.checkRows(81)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/update/merge_commit_data-0.py b/tests/pytest/update/merge_commit_data-0.py
new file mode 100644
index 0000000000000000000000000000000000000000..14d435f7f20d9e04565fdb7036da043d948b1dcf
--- /dev/null
+++ b/tests/pytest/update/merge_commit_data-0.py
@@ -0,0 +1,351 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ def run(self):
+ print("==========step1")
+ print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY")
+ s = 'reset query cache'
+ tdSql.execute(s)
+ s = 'drop database if exists db'
+ tdSql.execute(s)
+ s = 'create database db days 30'
+ tdSql.execute(s)
+ s = 'use db'
+ tdSql.execute(s)
+ ret = tdSql.execute('create table t1 (ts timestamp, a int)')
+
+ insertRows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % (insertRows))
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t1 values (%d , 1)' %
+ (t0 + i))
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t1")
+ tdSql.checkRows(insertRows)
+
+ for k in range(0,10):
+ for i in range (0,insertRows):
+ ret = tdSql.execute(
+ 'insert into t1 values(%d,1)' %
+ (t0+i)
+ )
+ tdSql.query("select * from t1")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t1")
+ tdSql.checkRows(insertRows)
+ print("==========step2")
+ print("PREPEND DATA ")
+ ret = tdSql.execute('create table t2 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t2 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t2")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t2")
+ tdSql.checkRows(insertRows)
+ for i in range(-100,0):
+ ret = tdSql.execute(
+ 'insert into t2 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t2")
+ tdSql.checkRows(insertRows+100)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t2")
+ tdSql.checkRows(insertRows+100)
+ print("==========step3")
+ print("PREPEND MASSIVE DATA ")
+ ret = tdSql.execute('create table t3 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t3 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t3")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t3")
+ tdSql.checkRows(insertRows)
+ for i in range(-6000,0):
+ ret = tdSql.execute(
+ 'insert into t3 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t3")
+ tdSql.checkRows(insertRows+6000)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t3")
+ tdSql.checkRows(insertRows+6000)
+ print("==========step4")
+ print("APPEND DATA")
+ ret = tdSql.execute('create table t4 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t4 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t4")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t4")
+ tdSql.checkRows(insertRows)
+ for i in range(0,100):
+ ret = tdSql.execute(
+ 'insert into t4 values (%d , 1)' %
+ (t0+200+i))
+ tdSql.query("select * from t4")
+ tdSql.checkRows(insertRows+100)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t4")
+ tdSql.checkRows(insertRows+100)
+ print("==========step5")
+ print("APPEND DATA")
+ ret = tdSql.execute('create table t5 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t5 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t5")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t5")
+ tdSql.checkRows(insertRows)
+ for i in range(0,6000):
+ ret = tdSql.execute(
+ 'insert into t5 values (%d , 1)' %
+ (t0+200+i))
+ tdSql.query("select * from t5")
+ tdSql.checkRows(insertRows+6000)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t5")
+ tdSql.checkRows(insertRows+6000)
+ print("==========step6")
+ print("UPDATE BLOCK IN TWO STEP")
+ ret = tdSql.execute('create table t6 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t6 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t6")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t6")
+ tdSql.checkRows(insertRows)
+ for i in range(0,100):
+ ret = tdSql.execute(
+ 'insert into t6 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t6")
+ tdSql.checkRows(insertRows)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0,0,'200')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t6")
+ tdSql.checkRows(insertRows)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0,0,'200')
+ for i in range(0,200):
+ ret = tdSql.execute(
+ 'insert into t6 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t6")
+ tdSql.checkRows(insertRows)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0,0,'200')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t6")
+ tdSql.checkRows(insertRows)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0,0,'200')
+ print("==========step7")
+ print("UPDATE LAST HALF AND INSERT LITTLE DATA")
+ ret = tdSql.execute('create table t7 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t7 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t7")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t7")
+ tdSql.checkRows(insertRows)
+ for i in range(100,300):
+ ret = tdSql.execute(
+ 'insert into t7 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t7")
+ tdSql.checkRows(300)
+ tdSql.query("select sum(a) from t7")
+ tdSql.checkData(0,0,'400')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t7")
+ tdSql.checkRows(300)
+ tdSql.query("select sum(a) from t7")
+ tdSql.checkData(0,0,'400')
+ print("==========step8")
+ print("UPDATE LAST HALF AND INSERT MASSIVE DATA")
+ ret = tdSql.execute('create table t8 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t8 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t8")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t8")
+ tdSql.checkRows(insertRows)
+ for i in range(6000):
+ ret = tdSql.execute(
+ 'insert into t8 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t8")
+ tdSql.checkRows(6000)
+ tdSql.query("select sum(a) from t8")
+ tdSql.checkData(0,0,'11800')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t8")
+ tdSql.checkRows(6000)
+ tdSql.query("select sum(a) from t8")
+ tdSql.checkData(0,0,'11800')
+ print("==========step9")
+ print("UPDATE FIRST HALF AND PREPEND LITTLE DATA")
+ ret = tdSql.execute('create table t9 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t9 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t9")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t9")
+ tdSql.checkRows(insertRows)
+ for i in range(-100,100):
+ ret = tdSql.execute(
+ 'insert into t9 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t9")
+ tdSql.checkRows(300)
+ tdSql.query("select sum(a) from t9")
+ tdSql.checkData(0,0,'400')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t9")
+ tdSql.checkRows(300)
+ tdSql.query("select sum(a) from t9")
+ tdSql.checkData(0,0,'400')
+ print("==========step10")
+ print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA")
+ ret = tdSql.execute('create table t10 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t10 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t10")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t10")
+ tdSql.checkRows(insertRows)
+ for i in range(-6000,100):
+ ret = tdSql.execute(
+ 'insert into t10 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t10")
+ tdSql.checkRows(6200)
+ tdSql.query("select sum(a) from t10")
+ tdSql.checkData(0,0,'12200')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t10")
+ tdSql.checkRows(6200)
+ tdSql.query("select sum(a) from t10")
+ tdSql.checkData(0,0,'12200')
+ print("==========step11")
+ print("UPDATE FIRST HALF AND APPEND MASSIVE DATA")
+ ret = tdSql.execute('create table t11 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t11 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t11")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t11")
+ tdSql.checkRows(insertRows)
+ for i in range(100):
+ ret = tdSql.execute(
+ 'insert into t11 values (%d , 2)' %
+ (t0+i))
+ for i in range(200,6000):
+ ret = tdSql.execute(
+ 'insert into t11 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t11")
+ tdSql.checkRows(6000)
+ tdSql.query("select sum(a) from t11")
+ tdSql.checkData(0,0,'11800')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t11")
+ tdSql.checkRows(6000)
+ tdSql.query("select sum(a) from t11")
+ tdSql.checkData(0,0,'11800')
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/update/merge_commit_data.py b/tests/pytest/update/merge_commit_data.py
new file mode 100644
index 0000000000000000000000000000000000000000..4fb6765361e8099acb0f1f861623a88fd6b8e466
--- /dev/null
+++ b/tests/pytest/update/merge_commit_data.py
@@ -0,0 +1,351 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+ def run(self):
+ print("==========step1")
+ print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY")
+ s = 'reset query cache'
+ tdSql.execute(s)
+ s = 'drop database if exists db'
+ tdSql.execute(s)
+ s = 'create database db update 1 days 30'
+ tdSql.execute(s)
+ s = 'use db'
+ tdSql.execute(s)
+ ret = tdSql.execute('create table t1 (ts timestamp, a int)')
+
+ insertRows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % (insertRows))
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t1 values (%d , 1)' %
+ (t0 + i))
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t1")
+ tdSql.checkRows(insertRows)
+
+ for k in range(0,10):
+ for i in range (0,insertRows):
+ ret = tdSql.execute(
+ 'insert into t1 values(%d,1)' %
+ (t0+i)
+ )
+ tdSql.query("select * from t1")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t1")
+ tdSql.checkRows(insertRows)
+ print("==========step2")
+ print("PREPEND DATA ")
+ ret = tdSql.execute('create table t2 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t2 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t2")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t2")
+ tdSql.checkRows(insertRows)
+ for i in range(-100,0):
+ ret = tdSql.execute(
+ 'insert into t2 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t2")
+ tdSql.checkRows(insertRows+100)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t2")
+ tdSql.checkRows(insertRows+100)
+ print("==========step3")
+ print("PREPEND MASSIVE DATA ")
+ ret = tdSql.execute('create table t3 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t3 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t3")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t3")
+ tdSql.checkRows(insertRows)
+ for i in range(-6000,0):
+ ret = tdSql.execute(
+ 'insert into t3 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t3")
+ tdSql.checkRows(insertRows+6000)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t3")
+ tdSql.checkRows(insertRows+6000)
+ print("==========step4")
+ print("APPEND DATA")
+ ret = tdSql.execute('create table t4 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t4 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t4")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t4")
+ tdSql.checkRows(insertRows)
+ for i in range(0,100):
+ ret = tdSql.execute(
+ 'insert into t4 values (%d , 1)' %
+ (t0+200+i))
+ tdSql.query("select * from t4")
+ tdSql.checkRows(insertRows+100)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t4")
+ tdSql.checkRows(insertRows+100)
+ print("==========step5")
+ print("APPEND DATA")
+ ret = tdSql.execute('create table t5 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t5 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t5")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t5")
+ tdSql.checkRows(insertRows)
+ for i in range(0,6000):
+ ret = tdSql.execute(
+ 'insert into t5 values (%d , 1)' %
+ (t0+200+i))
+ tdSql.query("select * from t5")
+ tdSql.checkRows(insertRows+6000)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t5")
+ tdSql.checkRows(insertRows+6000)
+ print("==========step6")
+ print("UPDATE BLOCK IN TWO STEP")
+ ret = tdSql.execute('create table t6 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t6 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t6")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t6")
+ tdSql.checkRows(insertRows)
+ for i in range(0,100):
+ ret = tdSql.execute(
+ 'insert into t6 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t6")
+ tdSql.checkRows(insertRows)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0,0,'300')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t6")
+ tdSql.checkRows(insertRows)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0,0,'300')
+ for i in range(0,200):
+ ret = tdSql.execute(
+ 'insert into t6 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t6")
+ tdSql.checkRows(insertRows)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0,0,'400')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t6")
+ tdSql.checkRows(insertRows)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0,0,'400')
+ print("==========step7")
+ print("UPDATE LAST HALF AND INSERT LITTLE DATA")
+ ret = tdSql.execute('create table t7 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t7 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t7")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t7")
+ tdSql.checkRows(insertRows)
+ for i in range(100,300):
+ ret = tdSql.execute(
+ 'insert into t7 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t7")
+ tdSql.checkRows(300)
+ tdSql.query("select sum(a) from t7")
+ tdSql.checkData(0,0,'500')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t7")
+ tdSql.checkRows(300)
+ tdSql.query("select sum(a) from t7")
+ tdSql.checkData(0,0,'500')
+ print("==========step8")
+ print("UPDATE LAST HALF AND INSERT MASSIVE DATA")
+ ret = tdSql.execute('create table t8 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t8 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t8")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t8")
+ tdSql.checkRows(insertRows)
+ for i in range(6000):
+ ret = tdSql.execute(
+ 'insert into t8 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t8")
+ tdSql.checkRows(6000)
+ tdSql.query("select sum(a) from t8")
+ tdSql.checkData(0,0,'12000')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t8")
+ tdSql.checkRows(6000)
+ tdSql.query("select sum(a) from t8")
+ tdSql.checkData(0,0,'12000')
+ print("==========step9")
+ print("UPDATE FIRST HALF AND PREPEND LITTLE DATA")
+ ret = tdSql.execute('create table t9 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t9 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t9")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t9")
+ tdSql.checkRows(insertRows)
+ for i in range(-100,100):
+ ret = tdSql.execute(
+ 'insert into t9 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t9")
+ tdSql.checkRows(300)
+ tdSql.query("select sum(a) from t9")
+ tdSql.checkData(0,0,'500')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t9")
+ tdSql.checkRows(300)
+ tdSql.query("select sum(a) from t9")
+ tdSql.checkData(0,0,'500')
+ print("==========step10")
+ print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA")
+ ret = tdSql.execute('create table t10 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t10 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t10")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t10")
+ tdSql.checkRows(insertRows)
+ for i in range(-6000,100):
+ ret = tdSql.execute(
+ 'insert into t10 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t10")
+ tdSql.checkRows(6200)
+ tdSql.query("select sum(a) from t10")
+ tdSql.checkData(0,0,'12300')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t10")
+ tdSql.checkRows(6200)
+ tdSql.query("select sum(a) from t10")
+ tdSql.checkData(0,0,'12300')
+ print("==========step11")
+ print("UPDATE FIRST HALF AND APPEND MASSIVE DATA")
+ ret = tdSql.execute('create table t11 (ts timestamp, a int)')
+ insertRows = 200
+ for i in range(0, insertRows):
+ ret = tdSql.execute(
+ 'insert into t11 values (%d , 1)' %
+ (t0+i))
+ tdSql.query("select * from t11")
+ tdSql.checkRows(insertRows)
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t11")
+ tdSql.checkRows(insertRows)
+ for i in range(100):
+ ret = tdSql.execute(
+ 'insert into t11 values (%d , 2)' %
+ (t0+i))
+ for i in range(200,6000):
+ ret = tdSql.execute(
+ 'insert into t11 values (%d , 2)' %
+ (t0+i))
+ tdSql.query("select * from t11")
+ tdSql.checkRows(6000)
+ tdSql.query("select sum(a) from t11")
+ tdSql.checkData(0,0,'11900')
+ tdDnodes.stop(1)
+ tdDnodes.start(1)
+ tdSql.query("select * from t11")
+ tdSql.checkRows(6000)
+ tdSql.query("select sum(a) from t11")
+ tdSql.checkData(0,0,'11900')
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/update/merge_commit_data2.py b/tests/pytest/update/merge_commit_data2.py
new file mode 100644
index 0000000000000000000000000000000000000000..3f0fc718ad83244353bf88da905e6ac0ff800cb5
--- /dev/null
+++ b/tests/pytest/update/merge_commit_data2.py
@@ -0,0 +1,352 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+import time
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+
+ def restart_taosd(self,db):
+ tdDnodes.stop(1)
+ tdDnodes.startWithoutSleep(1)
+ tdSql.execute("use %s;" % db)
+
+ def date_to_timestamp_microseconds(self, date):
+ datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f")
+ obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0)
+ return obj_stamp
+
+ def timestamp_microseconds_to_date(self, timestamp):
+ d = datetime.datetime.fromtimestamp(timestamp/1000)
+ str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f")
+ return str1
+
+
+
+ def run(self):
+ print("==========step1")
+ print("create table && insert data")
+ sql = 'reset query cache'
+ tdSql.execute(sql)
+ sql = 'drop database if exists db'
+ tdSql.execute(sql)
+ sql = 'create database db update 1 days 30;'
+ tdSql.execute(sql)
+ sql = 'use db;'
+ tdSql.execute(sql)
+ tdSql.execute('create table t1 (ts timestamp, a int)')
+
+
+ print("==================================1 start")
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t1 values (%d , 1)' %(t0+i))
+ print("==========step2")
+ print("restart to commit ")
+ self.restart_taosd('db')
+
+ print('check query result after restart')
+ tdSql.query('select * from db.t1;')
+ for i in range(insert_rows):
+ tdSql.checkData(i, 1, 1)
+
+ print("==========step3")
+ print('insert data')
+ for i in range(insert_rows):
+ tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
+ print('check query result before restart')
+ tdSql.query('select * from db.t1;')
+ for i in range(insert_rows, insert_rows*2):
+ tdSql.checkData(i, 1, 1)
+
+ self.restart_taosd('db')
+ print('check query result after restart')
+ tdSql.query('select * from db.t1;')
+ for i in range(insert_rows, insert_rows*2):
+ tdSql.checkData(i, 1, 1)
+
+ print("==========step4")
+ print('insert data')
+ for i in range(insert_rows):
+ tdSql.execute('insert into t1 values (%d , 2)' %(t0+i))
+ for i in range(insert_rows):
+ tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
+
+ print('check query result before restart')
+ tdSql.query('select * from db.t1;')
+ print(tdSql.queryResult)
+ for i in range(insert_rows):
+ tdSql.checkData(i, 1, 2)
+ for i in range(insert_rows, insert_rows*2):
+ tdSql.checkData(i, 1, 1)
+
+ print('check query result after restart')
+ self.restart_taosd('db')
+ tdSql.query('select * from db.t1;')
+ # print(tdSql.queryResult)
+ for i in range(insert_rows):
+ tdSql.checkData(i, 1, 2)
+ for i in range(insert_rows, insert_rows*2):
+ tdSql.checkData(i, 1, 1)
+
+ print("==================================2 start")
+ print("==========step1")
+ print("create table && insert data")
+ tdSql.execute('create table t2 (ts timestamp, a int)')
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t2 values (%d , 1)' %(t0+i))
+ print('restart to commit')
+ self.restart_taosd('db')
+ for i in range(insert_rows):
+ tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for k in range(10):
+ for i in range(10):
+ tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
+ print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
+
+
+ print("==========step2")
+ print('check query result before restart')
+ tdSql.query('select * from db.t2;')
+ for i in range(insert_rows*2+100):
+ tdSql.checkData(i, 1, 1)
+ # print(tdSql.queryResult)
+ print('restart to commit')
+ self.restart_taosd('db')
+ print('check query result after restart')
+ tdSql.query('select * from db.t2;')
+ for i in range(insert_rows*2+100):
+ tdSql.checkData(i, 1, 1)
+
+
+ print("==================================3 start")
+ print("==========step1")
+ print("create table && insert data")
+ tdSql.execute('create table t3 (ts timestamp, a int)')
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t3 values (%d , 1)' %(t0+i))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(insert_rows):
+ tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(5200):
+ tdSql.execute('insert into t3 values (%d , 2)' %(t0+i))
+
+ print("==========step2")
+ print('check query result before restart')
+ tdSql.query('select * from db.t3;')
+ for i in range(5200):
+ tdSql.checkData(i, 1, 2)
+ # print(tdSql.queryResult)
+ print('restart to commit')
+ self.restart_taosd('db')
+ print('check query result after restart')
+ tdSql.query('select * from db.t3;')
+ for i in range(5200):
+ tdSql.checkData(i, 1, 2)
+
+ print("==================================4 start")
+ print("==========step1")
+ print("create table && insert data")
+ tdSql.execute('create table t4 (ts timestamp, a int)')
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t4 values (%d , 1)' %(t0+i))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(insert_rows):
+ tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(100):
+ tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
+
+ for i in range(200, 5000):
+ tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
+
+ for i in range(100):
+ tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
+
+ print('check query result before restart')
+ tdSql.query('select * from db.t4;')
+ for i in range(100):
+ tdSql.checkData(i, 1, 2)
+ for i in range(100, 200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(200, 5000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(5000, 5200):
+ tdSql.checkData(i, 1, 1)
+
+ print('check query result after restart')
+ self.restart_taosd('db')
+ tdSql.query('select * from db.t4;')
+ for i in range(100):
+ tdSql.checkData(i, 1, 2)
+ for i in range(100, 200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(200, 5000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(5000, 5200):
+ tdSql.checkData(i, 1, 1)
+
+ print("==================================5 start")
+ print("==========step1")
+ print("create table && insert data")
+ tdSql.execute('create table t5 (ts timestamp, a int)')
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t5 values (%d , 1)' %(t0+i))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(insert_rows):
+ tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(100, 200):
+ tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
+
+ for i in range(200, 5000):
+ tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
+
+ for i in range(100, 200):
+ tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000))
+
+ print('check query result before restart')
+ tdSql.query('select * from db.t5;')
+ for i in range(100):
+ tdSql.checkData(i, 1, 1)
+ for i in range(100, 5000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(5000, 5100):
+ tdSql.checkData(i, 1, 1)
+ for i in range(5100, 5200):
+ tdSql.checkData(i, 1, 2)
+
+ print('check query result after restart')
+ self.restart_taosd('db')
+ tdSql.query('select * from db.t5;')
+ for i in range(100):
+ tdSql.checkData(i, 1, 1)
+ for i in range(100, 5000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(5000, 5100):
+ tdSql.checkData(i, 1, 1)
+ for i in range(5100, 5200):
+ tdSql.checkData(i, 1, 2)
+
+ print("==================================6 start")
+ print("==========step1")
+ print("create table && insert data")
+ tdSql.execute('create table t6 (ts timestamp, a int)')
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t6 values (%d , 1)' %(t0+i))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(insert_rows):
+ tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(-1000, 10000):
+ tdSql.execute('insert into t6 values (%d , 2)' %(t0+i))
+
+ print('check query result before restart')
+ tdSql.query('select * from db.t6;')
+ tdSql.checkRows(11000)
+ for i in range(11000):
+ tdSql.checkData(i, 1, 2)
+
+ print('check query result after restart')
+ self.restart_taosd('db')
+ tdSql.query('select * from db.t6;')
+ tdSql.checkRows(11000)
+ for i in range(11000):
+ tdSql.checkData(i, 1, 2)
+
+
+ print("==================================7 start")
+ print("==========step1")
+ print("create table && insert data")
+ tdSql.execute('create table t7 (ts timestamp, a int)')
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t7 values (%d , 1)' %(t0+i))
+
+ for i in range(insert_rows):
+ tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(-1000, 10000):
+ tdSql.execute('insert into t7 values (%d , 2)' %(t0+i))
+
+ print('check query result before restart')
+ tdSql.query('select * from db.t7;')
+ tdSql.checkRows(11000)
+ for i in range(11000):
+ tdSql.checkData(i, 1, 2)
+
+ print('check query result after restart')
+ self.restart_taosd('db')
+ tdSql.query('select * from db.t7;')
+ tdSql.checkRows(11000)
+ for i in range(11000):
+ tdSql.checkData(i, 1, 2)
+
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/update/merge_commit_data2_update0.py b/tests/pytest/update/merge_commit_data2_update0.py
new file mode 100644
index 0000000000000000000000000000000000000000..def50e04661b1752668202359eec7dd89df9b6f0
--- /dev/null
+++ b/tests/pytest/update/merge_commit_data2_update0.py
@@ -0,0 +1,384 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+import time
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), logSql)
+
+
+ def restart_taosd(self,db):
+ tdDnodes.stop(1)
+ tdDnodes.startWithoutSleep(1)
+ tdSql.execute("use %s;" % db)
+
+ def date_to_timestamp_microseconds(self, date):
+ datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f")
+ obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0)
+ return obj_stamp
+
+ def timestamp_microseconds_to_date(self, timestamp):
+ d = datetime.datetime.fromtimestamp(timestamp/1000)
+ str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f")
+ return str1
+
+
+
+ def run(self):
+ print("==========step1")
+ print("create table && insert data")
+ sql = 'reset query cache'
+ tdSql.execute(sql)
+ sql = 'drop database if exists db'
+ tdSql.execute(sql)
+ sql = 'create database db update 0 days 30;'
+ tdSql.execute(sql)
+ sql = 'use db;'
+ tdSql.execute(sql)
+ tdSql.execute('create table t1 (ts timestamp, a int)')
+
+
+ print("==================================1 start")
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t1 values (%d , 1)' %(t0+i))
+ print("==========step2")
+ print("restart to commit ")
+ self.restart_taosd('db')
+
+ print('check query result after restart')
+ tdSql.query('select * from db.t1;')
+ for i in range(insert_rows):
+ tdSql.checkData(i, 1, 1)
+
+ print("==========step3")
+ print('insert data')
+ for i in range(insert_rows):
+ tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
+ print('check query result before restart')
+ tdSql.query('select * from db.t1;')
+ for i in range(insert_rows, insert_rows*2):
+ tdSql.checkData(i, 1, 1)
+
+ self.restart_taosd('db')
+ print('check query result after restart')
+ tdSql.query('select * from db.t1;')
+ for i in range(insert_rows, insert_rows*2):
+ tdSql.checkData(i, 1, 1)
+
+ print("==========step4")
+ print('insert data')
+ for i in range(insert_rows):
+ tdSql.execute('insert into t1 values (%d , 2)' %(t0+i))
+ for i in range(insert_rows):
+ tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
+
+ print('check query result before restart')
+ tdSql.query('select * from db.t1;')
+ print(tdSql.queryResult)
+ for i in range(insert_rows):
+ tdSql.checkData(i, 1, 1)
+ for i in range(insert_rows, insert_rows*2):
+ tdSql.checkData(i, 1, 1)
+
+ print('check query result after restart')
+ self.restart_taosd('db')
+ tdSql.query('select * from db.t1;')
+ # print(tdSql.queryResult)
+ for i in range(insert_rows):
+ tdSql.checkData(i, 1, 1)
+ for i in range(insert_rows, insert_rows*2):
+ tdSql.checkData(i, 1, 1)
+
+ print("==================================2 start")
+ print("==========step1")
+ print("create table && insert data")
+ tdSql.execute('create table t2 (ts timestamp, a int)')
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t2 values (%d , 1)' %(t0+i))
+ print('restart to commit')
+ self.restart_taosd('db')
+ for i in range(insert_rows):
+ tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for k in range(10):
+ for i in range(10):
+ tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
+ # print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
+
+
+ print("==========step2")
+ print('check query result before restart')
+ tdSql.query('select * from db.t2;')
+ for i in range(insert_rows*2+100):
+ tdSql.checkData(i, 1, 1)
+ # print(tdSql.queryResult)
+ print('restart to commit')
+ self.restart_taosd('db')
+ print('check query result after restart')
+ tdSql.query('select * from db.t2;')
+ for i in range(insert_rows*2+100):
+ tdSql.checkData(i, 1, 1)
+
+
+ print("==================================3 start")
+ print("==========step1")
+ print("create table && insert data")
+ tdSql.execute('create table t3 (ts timestamp, a int)')
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t3 values (%d , 1)' %(t0+i))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(insert_rows):
+ tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(5200):
+ tdSql.execute('insert into t3 values (%d , 2)' %(t0+i))
+
+ print("==========step2")
+ print('check query result before restart')
+ tdSql.query('select * from db.t3;')
+ for i in range(200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(200, 5000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(5000, 5200):
+ tdSql.checkData(i, 1, 1)
+ # print(tdSql.queryResult)
+ print('restart to commit')
+ self.restart_taosd('db')
+ print('check query result after restart')
+ tdSql.query('select * from db.t3;')
+ for i in range(200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(200, 5000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(5000, 5200):
+ tdSql.checkData(i, 1, 1)
+
+ print("==================================4 start")
+ print("==========step1")
+ print("create table && insert data")
+ tdSql.execute('create table t4 (ts timestamp, a int)')
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t4 values (%d , 1)' %(t0+i))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(insert_rows):
+ tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(100):
+ tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
+
+ for i in range(200, 5000):
+ tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
+
+ for i in range(100):
+ tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
+
+ print('check query result before restart')
+ tdSql.query('select * from db.t4;')
+ for i in range(200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(200, 5000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(5000, 5200):
+ tdSql.checkData(i, 1, 1)
+
+ print('check query result after restart')
+ self.restart_taosd('db')
+ tdSql.query('select * from db.t4;')
+ for i in range(200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(200, 5000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(5000, 5200):
+ tdSql.checkData(i, 1, 1)
+ #
+ print("==================================5 start")
+ print("==========step1")
+ print("create table && insert data")
+ tdSql.execute('create table t5 (ts timestamp, a int)')
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t5 values (%d , 1)' %(t0+i))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(insert_rows):
+ tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(100, 200):
+ tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
+
+ for i in range(200, 5000):
+ tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
+
+ for i in range(100, 200):
+ tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000))
+
+ print('check query result before restart')
+ tdSql.query('select * from db.t5;')
+ for i in range(200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(200, 5000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(5000, 5200):
+ tdSql.checkData(i, 1, 1)
+
+ print('check query result after restart')
+ self.restart_taosd('db')
+ tdSql.query('select * from db.t5;')
+ for i in range(200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(200, 5000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(5000, 5200):
+ tdSql.checkData(i, 1, 1)
+
+ print("==================================6 start")
+ print("==========step1")
+ print("create table && insert data")
+ tdSql.execute('create table t6 (ts timestamp, a int)')
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t6 values (%d , 1)' %(t0+i))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(insert_rows):
+ tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(-1000, 10000):
+ tdSql.execute('insert into t6 values (%d , 2)' %(t0+i))
+
+ print('check query result before restart')
+ tdSql.query('select * from db.t6;')
+ tdSql.checkRows(11000)
+ for i in range(1000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(1000,1200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(1200,6000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(6000,6200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(6200, 11000):
+ tdSql.checkData(i, 1, 2)
+
+ print('check query result after restart')
+ self.restart_taosd('db')
+ tdSql.query('select * from db.t6;')
+ tdSql.checkRows(11000)
+ for i in range(1000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(1000,1200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(1200,6000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(6000,6200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(6200, 11000):
+ tdSql.checkData(i, 1, 2)
+
+
+ print("==================================7 start")
+ print("==========step1")
+ print("create table && insert data")
+ tdSql.execute('create table t7 (ts timestamp, a int)')
+ insert_rows = 200
+ t0 = 1603152000000
+ tdLog.info("insert %d rows" % insert_rows)
+ for i in range(insert_rows):
+ tdSql.execute('insert into t7 values (%d , 1)' %(t0+i))
+
+ for i in range(insert_rows):
+ tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000))
+ print('restart to commit')
+ self.restart_taosd('db')
+
+ for i in range(-1000, 10000):
+ tdSql.execute('insert into t7 values (%d , 2)' %(t0+i))
+
+ print('check query result before restart')
+ tdSql.query('select * from db.t7;')
+ tdSql.checkRows(11000)
+ for i in range(1000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(1000,1200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(1200,6000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(6000,6200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(6200, 11000):
+ tdSql.checkData(i, 1, 2)
+
+ print('check query result after restart')
+ self.restart_taosd('db')
+ tdSql.query('select * from db.t7;')
+ tdSql.checkRows(11000)
+ for i in range(1000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(1000,1200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(1200,6000):
+ tdSql.checkData(i, 1, 2)
+ for i in range(6000,6200):
+ tdSql.checkData(i, 1, 1)
+ for i in range(6200, 11000):
+ tdSql.checkData(i, 1, 2)
+
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/update/merge_commit_last-0.py b/tests/pytest/update/merge_commit_last-0.py
new file mode 100644
index 0000000000000000000000000000000000000000..8a247f38091467f69c74c57f00341adde0e15992
--- /dev/null
+++ b/tests/pytest/update/merge_commit_last-0.py
@@ -0,0 +1,309 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor())
+
+ self.ts = 1603152000000
+
+ def restartTaosd(self):
+ tdDnodes.stop(1)
+ tdDnodes.startWithoutSleep(1)
+ tdSql.execute("use db")
+
+ def run(self):
+ tdSql.prepare()
+
+ print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY")
+ tdSql.execute("create table t1 (ts timestamp, a int)")
+
+ for i in range(5):
+ tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i))
+ self.restartTaosd()
+ tdSql.query("select * from t1")
+ tdSql.checkRows(1)
+ tdSql.checkData(0, 1, 0)
+
+ print("==============step 2: UPDATE THE WHOLE LAST BLOCK")
+ tdSql.execute("create table t2 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t2")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t2")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(50):
+ tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i))
+ tdSql.query("select * from t2")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t2")
+ tdSql.checkData(0, 0, 50)
+
+ self.restartTaosd()
+ tdSql.query("select * from t2")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t2")
+ tdSql.checkData(0, 0, 50)
+
+ print("==============step 3: UPDATE PART OF THE LAST BLOCK")
+ tdSql.execute("create table t3 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i))
+ self.restartTaosd()
+ tdSql.query("select * from t3")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t3")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(25):
+ tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t3")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t3")
+ tdSql.checkData(0, 0, 50)
+
+ self.restartTaosd()
+ tdSql.query("select * from t3")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t3")
+ tdSql.checkData(0, 0, 50)
+
+ print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA")
+ tdSql.execute("create table t4 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t4")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t4")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(25):
+ tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
+
+ for i in range(50, 60):
+ tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t4")
+ tdSql.checkRows(60)
+ tdSql.query("select sum(a) from t4")
+ tdSql.checkData(0, 0, 70)
+
+ self.restartTaosd()
+ tdSql.query("select * from t4")
+ tdSql.checkRows(60)
+ tdSql.query("select sum(a) from t4")
+ tdSql.checkData(0, 0, 70)
+
+ print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA")
+ tdSql.execute("create table t5 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t5")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t5")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(-10, 0):
+ tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
+
+ for i in range(25):
+ tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t5")
+ tdSql.checkRows(60)
+ tdSql.query("select sum(a) from t5")
+ tdSql.checkData(0, 0, 70)
+
+ self.restartTaosd()
+ tdSql.query("select * from t5")
+ tdSql.checkRows(60)
+ tdSql.query("select sum(a) from t5")
+ tdSql.checkData(0, 0, 70)
+
+ for i in range(-10, 0):
+ tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
+
+ for i in range(25, 50):
+ tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
+
+ tdSql.query("select * from t5")
+ tdSql.checkRows(60)
+ tdSql.query("select sum(a) from t5")
+ tdSql.checkData(0, 0, 70)
+
+ self.restartTaosd()
+ tdSql.query("select * from t5")
+ tdSql.checkRows(60)
+ tdSql.query("select sum(a) from t5")
+ tdSql.checkData(0, 0, 70)
+
+
+ print("==============step 6: INSERT AHEAD A LOT OF DATA")
+ tdSql.execute("create table t6 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t6")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(-1000, 0):
+ tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t6")
+ tdSql.checkRows(1050)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0, 0, 2050)
+
+ self.restartTaosd()
+ tdSql.query("select * from t6")
+ tdSql.checkRows(1050)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0, 0, 2050)
+
+ print("==============step 7: INSERT AHEAD A LOT AND UPDATE")
+ tdSql.execute("create table t7 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t7")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t7")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(-1000, 25):
+ tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t7")
+ tdSql.checkRows(1050)
+ tdSql.query("select sum(a) from t7")
+ tdSql.checkData(0, 0, 2050)
+
+ self.restartTaosd()
+ tdSql.query("select * from t7")
+ tdSql.checkRows(1050)
+ tdSql.query("select sum(a) from t7")
+ tdSql.checkData(0, 0, 2050)
+
+ print("==============step 8: INSERT AFTER A LOT AND UPDATE")
+ tdSql.execute("create table t8 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t8")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t8")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(25, 6000):
+ tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t8")
+ tdSql.checkRows(6000)
+ tdSql.query("select sum(a) from t8")
+ tdSql.checkData(0, 0, 11950)
+
+ self.restartTaosd()
+ tdSql.query("select * from t8")
+ tdSql.checkRows(6000)
+ tdSql.query("select sum(a) from t8")
+ tdSql.checkData(0, 0, 11950)
+
+ print("==============step 9: UPDATE ONLY MIDDLE")
+ tdSql.execute("create table t9 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t9")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t9")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(20, 30):
+ tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t9")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t9")
+ tdSql.checkData(0, 0, 50)
+
+ self.restartTaosd()
+ tdSql.query("select * from t9")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t9")
+ tdSql.checkData(0, 0, 50)
+
+ print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK")
+ tdSql.execute("create table t10 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t10")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t10")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(-4000, 4000):
+ tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t10")
+ tdSql.checkRows(8000)
+ tdSql.query("select sum(a) from t10")
+ tdSql.checkData(0, 0, 15950)
+
+ self.restartTaosd()
+ tdSql.query("select * from t10")
+ tdSql.checkRows(8000)
+ tdSql.query("select sum(a) from t10")
+ tdSql.checkData(0, 0, 15950)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/update/merge_commit_last.py b/tests/pytest/update/merge_commit_last.py
new file mode 100644
index 0000000000000000000000000000000000000000..183cca0a1e40fd995daaed0f271bb5083838a78f
--- /dev/null
+++ b/tests/pytest/update/merge_commit_last.py
@@ -0,0 +1,321 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor())
+
+ self.ts = 1603152000000
+
+ def restartTaosd(self):
+ tdDnodes.stop(1)
+ tdDnodes.startWithoutSleep(1)
+ tdSql.execute("use udb")
+
+ def run(self):
+ tdSql.prepare()
+
+ tdSql.execute("create database udb update 1 days 30")
+ tdSql.execute("use udb")
+
+ print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY")
+ tdSql.execute("create table t1 (ts timestamp, a int)")
+
+ for i in range(5):
+ tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i))
+ self.restartTaosd()
+ tdSql.query("select * from t1")
+ tdSql.checkRows(1)
+ tdSql.checkData(0, 1, i)
+
+ print("==============step 2: UPDATE THE WHOLE LAST BLOCK")
+ tdSql.execute("create table t2 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
+ self.restartTaosd()
+ tdSql.query("select * from t2")
+ tdSql.checkRows(50)
+ for i in range(50):
+ tdSql.checkData(i, 1, 1)
+
+ for i in range(50):
+ tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i))
+ tdSql.query("select * from t2")
+ for i in range(50):
+ tdSql.checkData(i, 1, 2)
+
+ self.restartTaosd()
+ tdSql.query("select * from t2")
+ tdSql.checkRows(50)
+ for i in range(50):
+ tdSql.checkData(i, 1, 2)
+
+ print("==============step 3: UPDATE PART OF THE LAST BLOCK")
+ tdSql.execute("create table t3 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i))
+ self.restartTaosd()
+ tdSql.query("select * from t3")
+ tdSql.checkRows(50)
+ for i in range(50):
+ tdSql.checkData(i, 1, 1)
+
+ for i in range(25):
+ tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t3")
+ for i in range(25):
+ tdSql.checkData(i, 1, 2)
+ for i in range(25, 50):
+ tdSql.checkData(i, 1, 1)
+
+ self.restartTaosd()
+ tdSql.query("select * from t3")
+ tdSql.checkRows(50)
+ for i in range(25):
+ tdSql.checkData(i, 1, 2)
+ for i in range(25, 50):
+ tdSql.checkData(i, 1, 1)
+
+ print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA")
+ tdSql.execute("create table t4 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t4")
+ tdSql.checkRows(50)
+ for i in range(50):
+ tdSql.checkData(i, 1, 1)
+
+ for i in range(25):
+ tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
+
+ for i in range(50, 60):
+ tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t4")
+ tdSql.checkRows(60)
+ for i in range(25):
+ tdSql.checkData(i, 1, 2)
+ for i in range(25, 50):
+ tdSql.checkData(i, 1, 1)
+ for i in range(50, 60):
+ tdSql.checkData(i, 1, 2)
+
+ self.restartTaosd()
+ tdSql.query("select * from t4")
+ tdSql.checkRows(60)
+ for i in range(25):
+ tdSql.checkData(i, 1, 2)
+ for i in range(25, 50):
+ tdSql.checkData(i, 1, 1)
+ for i in range(50, 60):
+ tdSql.checkData(i, 1, 2)
+
+ print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA")
+ tdSql.execute("create table t5 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t5")
+ tdSql.checkRows(50)
+ for i in range(50):
+ tdSql.checkData(i, 1, 1)
+
+ for i in range(-10, 0):
+ tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
+
+ for i in range(25):
+ tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t5")
+ tdSql.checkRows(60)
+ tdSql.query("select sum(a) from t5")
+ tdSql.checkData(0, 0, 95)
+
+ self.restartTaosd()
+ tdSql.query("select * from t5")
+ tdSql.checkRows(60)
+ tdSql.query("select sum(a) from t5")
+ tdSql.checkData(0, 0, 95)
+
+ for i in range(-10, 0):
+ tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
+
+ for i in range(25, 50):
+ tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
+
+ tdSql.query("select * from t5")
+ tdSql.checkRows(60)
+ tdSql.query("select sum(a) from t5")
+ tdSql.checkData(0, 0, 155)
+
+ self.restartTaosd()
+ tdSql.query("select * from t5")
+ tdSql.checkRows(60)
+ tdSql.query("select sum(a) from t5")
+ tdSql.checkData(0, 0, 155)
+
+
+ print("==============step 6: INSERT AHEAD A LOT OF DATA")
+ tdSql.execute("create table t6 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t6")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(-1000, 0):
+ tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t6")
+ tdSql.checkRows(1050)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0, 0, 2050)
+
+ self.restartTaosd()
+ tdSql.query("select * from t6")
+ tdSql.checkRows(1050)
+ tdSql.query("select sum(a) from t6")
+ tdSql.checkData(0, 0, 2050)
+
+ print("==============step 7: INSERT AHEAD A LOT AND UPDATE")
+ tdSql.execute("create table t7 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t7")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t7")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(-1000, 25):
+ tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t7")
+ tdSql.checkRows(1050)
+ tdSql.query("select sum(a) from t7")
+ tdSql.checkData(0, 0, 2075)
+
+ self.restartTaosd()
+ tdSql.query("select * from t7")
+ tdSql.checkRows(1050)
+ tdSql.query("select sum(a) from t7")
+ tdSql.checkData(0, 0, 2075)
+
+ print("==============step 8: INSERT AFTER A LOT AND UPDATE")
+ tdSql.execute("create table t8 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t8")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t8")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(25, 6000):
+ tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t8")
+ tdSql.checkRows(6000)
+ tdSql.query("select sum(a) from t8")
+ tdSql.checkData(0, 0, 11975)
+
+ self.restartTaosd()
+ tdSql.query("select * from t8")
+ tdSql.checkRows(6000)
+ tdSql.query("select sum(a) from t8")
+ tdSql.checkData(0, 0, 11975)
+
+ print("==============step 9: UPDATE ONLY MIDDLE")
+ tdSql.execute("create table t9 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t9")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t9")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(20, 30):
+ tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t9")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t9")
+ tdSql.checkData(0, 0, 60)
+
+ self.restartTaosd()
+ tdSql.query("select * from t9")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t9")
+ tdSql.checkData(0, 0, 60)
+
+ print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK")
+ tdSql.execute("create table t10 (ts timestamp, a int)")
+
+ for i in range(50):
+ tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i))
+
+ self.restartTaosd()
+ tdSql.query("select * from t10")
+ tdSql.checkRows(50)
+ tdSql.query("select sum(a) from t10")
+ tdSql.checkData(0, 0, 50)
+
+ for i in range(-4000, 4000):
+ tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i))
+
+ tdSql.query("select * from t10")
+ tdSql.checkRows(8000)
+ tdSql.query("select sum(a) from t10")
+ tdSql.checkData(0, 0, 16000)
+
+ self.restartTaosd()
+ tdSql.query("select * from t10")
+ tdSql.checkRows(8000)
+ tdSql.query("select sum(a) from t10")
+ tdSql.checkData(0, 0, 16000)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py
index 1ac492bb3ad2733c7b2ebb46560edbb7204e8951..757399b4a262dff7b11619791d3c82686fb293e8 100644
--- a/tests/pytest/util/dnodes.py
+++ b/tests/pytest/util/dnodes.py
@@ -15,6 +15,7 @@ import sys
import os
import os.path
import subprocess
+from time import sleep
from util.log import *
@@ -210,6 +211,7 @@ class TDDnode:
(self.index, self.cfgPath))
def getBuildPath(self):
+ buildPath = ""
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
@@ -256,6 +258,35 @@ class TDDnode:
tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index))
time.sleep(5)
+
+ def startWithoutSleep(self):
+ buildPath = self.getBuildPath()
+
+ if (buildPath == ""):
+ tdLog.exit("taosd not found!")
+ else:
+ tdLog.info("taosd found in %s" % buildPath)
+
+ binPath = buildPath + "/build/bin/taosd"
+
+ if self.deployed == 0:
+ tdLog.exit("dnode:%d is not deployed" % (self.index))
+
+ if self.valgrind == 0:
+ cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
+ binPath, self.cfgDir)
+ else:
+ valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"
+
+ cmd = "nohup %s %s -c %s 2>&1 & " % (
+ valgrindCmdline, binPath, self.cfgDir)
+
+ print(cmd)
+
+ if os.system(cmd) != 0:
+ tdLog.exit(cmd)
+ self.running = 1
+ tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
def stop(self):
if self.valgrind == 0:
@@ -425,6 +456,10 @@ class TDDnodes:
def start(self, index):
self.check(index)
self.dnodes[index - 1].start()
+
+ def startWithoutSleep(self, index):
+ self.check(index)
+ self.dnodes[index - 1].startWithoutSleep()
def stop(self, index):
self.check(index)
diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py
index 9abec354c6d58507ff3bcc74d1c0dc03f691440c..b2ed6212fd643c158f7ed6f4cc6cb2449a512a2e 100644
--- a/tests/pytest/util/sql.py
+++ b/tests/pytest/util/sql.py
@@ -25,7 +25,7 @@ class TDSql:
self.queryCols = 0
self.affectedRows = 0
- def init(self, cursor, log=True):
+ def init(self, cursor, log=False):
self.cursor = cursor
if (log):
diff --git a/tests/pytest/wal/addOldWalTest.py b/tests/pytest/wal/addOldWalTest.py
new file mode 100644
index 0000000000000000000000000000000000000000..2f4dcd5ce807cf7bbadfa480af6ed6342058a78a
--- /dev/null
+++ b/tests/pytest/wal/addOldWalTest.py
@@ -0,0 +1,70 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor())
+
+ def createOldDir(self):
+ oldDir = tdDnodes.getDnodesRootDir() + "dnode1/data/vnode/vnode2/wal/old"
+ os.system("sudo mkdir -p %s" % oldDir)
+
+ def createOldDirAndAddWal(self):
+ oldDir = tdDnodes.getDnodesRootDir() + "dnode1/data/vnode/vnode2/wal/old"
+ os.system("sudo echo 'test' >> %s/wal" % oldDir)
+
+
+ def run(self):
+ tdSql.prepare()
+
+ tdSql.execute("create table t1(ts timestamp, a int)")
+ tdSql.execute("insert into t1 values(now, 1)")
+
+ # create old dir only
+ self.createOldDir()
+ os.system("sudo kill -9 $(pgrep taosd)")
+ tdDnodes.start(1)
+
+ tdSql.execute("use db")
+ tdSql.query("select * from t1")
+ tdSql.checkRows(1)
+ tdSql.checkData(0, 1, 1)
+
+ # create old dir and add wal under old dir
+ self.createOldDir()
+ self.createOldDirAndAddWal()
+ os.system("sudo kill -9 $(pgrep taosd)")
+ tdDnodes.start(1)
+
+ tdSql.query("select * from t1")
+ tdSql.checkRows(1)
+ tdSql.checkData(0, 1, 1)
+
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/script/general/db/nosuchfile.sim b/tests/script/general/db/nosuchfile.sim
new file mode 100644
index 0000000000000000000000000000000000000000..98ac4ec012dc694357878a61ca0dbc11259f0a9e
--- /dev/null
+++ b/tests/script/general/db/nosuchfile.sim
@@ -0,0 +1,66 @@
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c wallevel -v 2
+
+print ========== step1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+sleep 3000
+
+print ========== step3
+sql create database d1
+sql create table d1.t1 (t timestamp, i int)
+sql insert into d1.t1 values(now+1s, 35)
+sql insert into d1.t1 values(now+2s, 34)
+sql insert into d1.t1 values(now+3s, 33)
+sql insert into d1.t1 values(now+4s, 32)
+sql insert into d1.t1 values(now+5s, 31)
+
+print ========== step4
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+
+print ========== step5
+sql select * from d1.t1 order by t desc
+print $data01 $data11 $data21 $data31 $data41
+if $data01 != 31 then
+ return -1
+endi
+if $data11 != 32 then
+ return -1
+endi
+if $data21 != 33 then
+ return -1
+endi
+if $data31 != 34 then
+ return -1
+endi
+if $data41 != 35 then
+ return -1
+endi
+
+print ========== step6
+system_content rm -rf ../../../sim/dnode1/data/vnode/vnode2/tsdb/data
+
+print ========== step7
+sql select * from d1.t1 order by t desc
+print $data01 $data11 $data21 $data31 $data41
+if $data01 != null then
+ return -1
+endi
+if $data11 != null then
+ return -1
+endi
+if $data21 != null then
+ return -1
+endi
+if $data31 != null then
+ return -1
+endi
+if $data41 != null then
+ return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim
index 8d2f1a7c00304c42c91311ae703bcef97aa6ace0..94ecb59f75304d99f48ebdb644be432370f86f2a 100644
--- a/tests/script/general/http/restful_full.sim
+++ b/tests/script/general/http/restful_full.sim
@@ -81,7 +81,7 @@ print =============== step2 - no db
#11
system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql
print 11-> $system_content
-if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0}@ then
+if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","update","status"],"data":[],"rows":0}@ then
return -1
endi
diff --git a/tests/script/general/insert/basic.sim b/tests/script/general/insert/basic.sim
index ba8cff83fa8b470af1b1f7fccf74d0e1d042049e..3f0f25a95ba68c8099c02e6daab933411089a632 100644
--- a/tests/script/general/insert/basic.sim
+++ b/tests/script/general/insert/basic.sim
@@ -8,8 +8,8 @@ sleep 3000
sql connect
$i = 0
-$dbPrefix = tb_in_db
-$tbPrefix = tb_in_tb
+$dbPrefix = d
+$tbPrefix = t
$db = $dbPrefix . $i
$tb = $tbPrefix . $i
@@ -22,28 +22,27 @@ sql create table $tb (ts timestamp, speed int)
$x = 0
while $x < 10
- $ms = $x . m
- sql insert into $tb values (now + $ms , $x )
+ $cc = $x * 60000
+ $ms = 1601481600000 + $cc
+
+ sql insert into $tb values ($ms , $x )
$x = $x + 1
endw
print =============== step 2
-sql insert into $tb values (now - 5m , 10)
-sql insert into $tb values (now - 6m , 10)
-sql insert into $tb values (now - 7m , 10)
-sql insert into $tb values (now - 8m , 10)
+$x = 0
+while $x < 5
+ $cc = $x * 60000
+ $ms = 1551481600000 + $cc
+
+ sql insert into $tb values ($ms , $x )
+ $x = $x + 1
+endw
sql select * from $tb
print $rows points data are retrieved
-if $rows != 14 then
- return -1
-endi
-
-sql drop database $db
-sleep 1000
-sql show databases
-if $rows != 0 then
+if $rows != 15 then
return -1
endi
diff --git a/tests/script/general/parser/fill.sim b/tests/script/general/parser/fill.sim
index f89c27d71fd8ba02105f502afd4df26a1870ffb5..92aa6a922cc901c88ad68f87f61e8befbed5e87c 100644
--- a/tests/script/general/parser/fill.sim
+++ b/tests/script/general/parser/fill.sim
@@ -116,7 +116,7 @@ if $data81 != 1 then
endi
# avg_with_fill
-print avg_witt_constant_fill
+print avg_with_constant_fill
sql select avg(c1), avg(c2), avg(c3), avg(c4), avg(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6)
if $rows != 9 then
return -1
@@ -371,12 +371,10 @@ if $data11 != 99 then
endi
sql select * from $tb
-#print data08 = $data08
if $data08 != NCHAR then
+ print expect NCHAR, actual:$data08
return -1
endi
-#return -1
-
# fill_into_nonarithmetic_fieds
sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000)
diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim
index bd0d3c1a12c77570c19ea1ef061395912ad9f93a..145c4a008f1ae26378baeabdfd5103cc807266f3 100644
--- a/tests/script/general/parser/groupby.sim
+++ b/tests/script/general/parser/groupby.sim
@@ -27,7 +27,7 @@ $mt = $mtPrefix . $i
$tstart = 100000
-sql drop database if exits $db -x step1
+sql drop database if exists $db -x step1
step1:
sql create database if not exists $db keep 36500
sql use $db
@@ -435,53 +435,53 @@ sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.00
sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ;
sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ;
-sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2;
-if $rows != 40 then
- return -1
-endi
-
-if $data01 != 1.000000000 then
- return -1
-endi
-if $data02 != t1 then
- return -1
-endi
-if $data03 != 1 then
- return -1
-endi
-if $data04 != 1 then
- return -1
-endi
-
-if $data11 != 1.000000000 then
- return -1
-endi
-if $data12 != t1 then
- return -1
-endi
-if $data13 != 1 then
- return -1
-endi
-if $data14 != 1 then
- return -1
-endi
-
-sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2 limit 1;
-if $rows != 2 then
- return -1
-endi
-
-if $data11 != 1.000000000 then
- return -1
-endi
-if $data12 != t2 then
- return -1
-endi
-if $data13 != 1 then
- return -1
-endi
-if $data14 != 2 then
- return -1
-endi
+#sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2;
+#if $rows != 40 then
+# return -1
+#endi
+#
+#if $data01 != 1.000000000 then
+# return -1
+#endi
+#if $data02 != t1 then
+# return -1
+#endi
+#if $data03 != 1 then
+# return -1
+#endi
+#if $data04 != 1 then
+# return -1
+#endi
+#
+#if $data11 != 1.000000000 then
+# return -1
+#endi
+#if $data12 != t1 then
+# return -1
+#endi
+#if $data13 != 1 then
+# return -1
+#endi
+#if $data14 != 1 then
+# return -1
+#endi
+#
+#sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2 limit 1;
+#if $rows != 2 then
+# return -1
+#endi
+#
+#if $data11 != 1.000000000 then
+# return -1
+#endi
+#if $data12 != t2 then
+# return -1
+#endi
+#if $data13 != 1 then
+# return -1
+#endi
+#if $data14 != 2 then
+# return -1
+#endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim
index 254571bda103957fbaeaf0e311e7be03b4dcfc35..624c2e72c40bfee35f8b0b17646c97149eda65c1 100644
--- a/tests/script/general/parser/join.sim
+++ b/tests/script/general/parser/join.sim
@@ -24,7 +24,7 @@ $mt = $mtPrefix . $i
$tstart = 100000
-sql drop database if exits $db -x step1
+sql drop database if exists $db -x step1
step1:
sql create database if not exists $db keep 36500
sql use $db
@@ -444,6 +444,9 @@ if $rows != $val then
return -1
endi
+#===============================================================
+sql select first(join_tb0.c8),first(join_tb0.c9) from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts <= 100002 and join_tb0.c7 = true
+
#====================group by=========================================
diff --git a/tests/script/general/parser/join_multivnode.sim b/tests/script/general/parser/join_multivnode.sim
index 51f1ef11c7fc9f8cfff60ebe86ad00104266e7ad..76230f79f0136975006e81cee4236a2fa9521214 100644
--- a/tests/script/general/parser/join_multivnode.sim
+++ b/tests/script/general/parser/join_multivnode.sim
@@ -22,7 +22,7 @@ $mt = $mtPrefix . $i
$tstart = 100000
-sql drop database if exits $db -x step1
+sql drop database if exists $db -x step1
step1:
sql create database if not exists $db keep 36500
sql use $db
@@ -132,4 +132,84 @@ sql select join_mt0.ts, join_mt1.t1, join_mt0.t1, join_mt1.tbname, join_mt0.tbna
sql select join_mt0.ts, join_mt1.t1, join_mt0.t1, join_mt1.tbname, join_mt0.tbname from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1 limit 1
+#1970-01-01 08:01:40.800 | 10 | 45.000000000 | 0 | true | false | 0 |
+#1970-01-01 08:01:40.790 | 10 | 945.000000000 | 90 | true | true | 0 |
+sql_error select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc limit 20 offset 19;
+
+
+sql select count(join_mt0.c1), sum(join_mt0.c2)/count(*), avg(c2), first(join_mt0.c5), last(c7) from join_mt0 interval(10a) group by join_mt0.t1 order by join_mt0.ts desc;
+if $rows != 100 then
+ return -1
+endi
+
+if $data00 != @70-01-01 08:01:40.990@ then
+ print expect 0, actual: $data00
+ return -1
+endi
+
+if $data01 != 30 then
+ return -1
+endi
+
+if $data02 != 94.500000000 then
+ print expect 94.500000000, actual $data02
+ return -1
+endi
+
+if $data03 != 94.500000000 then
+ return -1
+endi
+
+if $data04 != 90 then
+ return -1
+endi
+
+if $data05 != 1 then
+ return -1
+endi
+
+if $data06 != 2 then
+ return -1
+endi
+
+if $data10 != @70-01-01 08:01:40.980@ then
+ print expect 70-01-01 08:01:40.980, actual: $data10
+ return -1
+endi
+
+if $data11 != 30 then
+ return -1
+endi
+
+if $data12 != 84.500000000 then
+ print expect 84.500000000, actual $data12
+ return -1
+endi
+
+if $data13 != 84.500000000 then
+ return -1
+endi
+
+if $data14 != 80 then
+ return -1
+endi
+
+if $data15 != 1 then
+ return -1
+endi
+
+if $data16 != 2 then
+ return -1
+endi
+
+# this function will cause shell crash
+sql_error select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc;
+
+sql_error select last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10m) group by join_mt0.t1 order by join_mt0.ts asc;
+
+sql_error select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL);
+sql select count(join_mt0.c1), first(join_mt0.c1)/count(*), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL);
+sql select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL);
+sql select last(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL);
+
system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/script/general/parser/projection_limit_offset.sim b/tests/script/general/parser/projection_limit_offset.sim
index fbff99d58f5b6355863b90172c2fb14c1f2ba393..7c83ca0c2ff3d44bf487f8a94750736767834cfa 100644
--- a/tests/script/general/parser/projection_limit_offset.sim
+++ b/tests/script/general/parser/projection_limit_offset.sim
@@ -21,7 +21,7 @@ $mt = $mtPrefix . $i
$tstart = 100000
-sql drop database if exits $db -x step1
+sql drop database if exists $db -x step1
step1:
sql create database if not exists $db keep 36500
sql use $db
@@ -324,8 +324,22 @@ sql create table tm0 using m1 tags(1);
sql create table tm1 using m1 tags(2);
sql insert into tm0 values(10000, 1) (20000, 2)(30000, 3) (40000, NULL) (50000, 2) tm1 values(10001, 2)(20000,4)(90000,9);
-sql select count(*),first(k),last(k) from m1 where tbname in ('tm0') interval(1s) order by ts desc;
+#=============================tbase-1205
+sql select count(*) from tm1 where ts= now -1d interval(1h) fill(NULL);
+
+print ===================>TD-1834
+sql select * from tm0 where ts>11000 and ts< 20000 order by ts asc
+if $rows != 0 then
+ return -1
+endi
+
+sql select * from tm0 where ts>11000 and ts< 20000 order by ts desc
+if $rows != 0 then
+ return -1
+endi
+
+sql select count(*),first(k),last(k) from m1 where tbname in ('tm0') interval(1s) order by ts desc;
if $row != 5 then
return -1
endi
@@ -386,7 +400,25 @@ sql_error select k+1,sum(k) from tm0;
sql_error select k, sum(k) from tm0;
sql_error select k, sum(k)+1 from tm0;
+print ================== restart server to commit data into disk
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+sleep 5000
+system sh/exec.sh -n dnode1 -s start
+print ================== server restart completed
+
#=============================tbase-1205
sql select count(*) from tm1 where ts= now -1d interval(1h) fill(NULL);
+print ===================>TD-1834
+sql select * from tm0 where ts>11000 and ts< 20000 order by ts asc
+if $rows != 0 then
+ return -1
+endi
+
+sql select * from tm0 where ts>11000 and ts< 20000 order by ts desc
+if $rows != 0 then
+ return -1
+endi
+
+
diff --git a/tests/script/general/parser/sliding.sim b/tests/script/general/parser/sliding.sim
index f85211beb83e575e2a73518d89f4b7d989486f83..ec0e31311afe1a08644aa28515071bced71ae0f0 100644
--- a/tests/script/general/parser/sliding.sim
+++ b/tests/script/general/parser/sliding.sim
@@ -26,7 +26,7 @@ $i = 0
$db = $dbPrefix . $i
$mt = $mtPrefix . $i
-sql drop database if exits $db -x step1
+sql drop database if exists $db -x step1
step1:
sql create database if not exists $db maxtables 4 keep 36500
sql use $db
diff --git a/tests/script/general/parser/stream_on_sys.sim b/tests/script/general/parser/stream_on_sys.sim
index 5507b4db484006fc8ae7887a47bcd83df5208c50..845a484488137e509a79bbb057f104378843cb8f 100644
--- a/tests/script/general/parser/stream_on_sys.sim
+++ b/tests/script/general/parser/stream_on_sys.sim
@@ -22,12 +22,12 @@ $i = 0
sql use $db
-sql create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s) sliding(2s)
-sql create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s) sliding(2s)
-sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s) sliding(2s)
-sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s) sliding(2s)
-sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s) sliding(2s)
-sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s) sliding(2s)
+sql create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s)
+sql create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s)
+sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s)
+sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s)
+sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s)
+sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s)
sleep 120000
sql select * from cpustrm
if $rows <= 0 then
diff --git a/tests/script/general/parser/union.sim b/tests/script/general/parser/union.sim
index 4af482bde073000d9f2ba098b469f8ec33d7f419..024b9c76efe6b8ddd69c723ffc0e1e0b0d82e075 100644
--- a/tests/script/general/parser/union.sim
+++ b/tests/script/general/parser/union.sim
@@ -27,7 +27,7 @@ $j = 1
$mt1 = $mtPrefix . $j
-sql drop database if exits $db -x step1
+sql drop database if exists $db -x step1
step1:
sql create database if not exists $db
sql use $db
diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim
index 5cac3f47235c1321d740e373d0f5828f076c31c7..066fac43ad9ee617a2cb9a91a62a6ebf984be6f0 100644
--- a/tests/script/general/parser/where.sim
+++ b/tests/script/general/parser/where.sim
@@ -20,7 +20,7 @@ $i = 0
$db = $dbPrefix . $i
$mt = $mtPrefix . $i
-sql drop database if exits $db -x step1
+sql drop database if exists $db -x step1
step1:
sql create database if not exists $db
sql use $db
diff --git a/tests/script/general/wal/kill.sim b/tests/script/general/wal/kill.sim
new file mode 100644
index 0000000000000000000000000000000000000000..7f103874a561c2bb2534996ab30d30ab0e8907a3
--- /dev/null
+++ b/tests/script/general/wal/kill.sim
@@ -0,0 +1,77 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+
+print ============== deploy
+system sh/exec.sh -n dnode1 -s start
+sleep 3001
+sql connect
+
+sql create database d1
+sql use d1
+
+sql create table t1 (ts timestamp, i int)
+sql insert into t1 values(now, 1);
+
+print =============== step3
+sleep 3000
+sql select * from t1;
+print rows: $rows
+if $rows != 1 then
+ return -1
+endi
+system sh/exec.sh -n dnode1 -s stop -x SIGKILL
+sleep 3000
+
+print =============== step4
+system sh/exec.sh -n dnode1 -s start -x SIGKILL
+sleep 3000
+sql select * from t1;
+print rows: $rows
+if $rows != 1 then
+ return -1
+endi
+system sh/exec.sh -n dnode1 -s stop -x SIGKILL
+sleep 3000
+
+print =============== step5
+system sh/exec.sh -n dnode1 -s start -x SIGKILL
+sleep 3000
+sql select * from t1;
+print rows: $rows
+if $rows != 1 then
+ return -1
+endi
+system sh/exec.sh -n dnode1 -s stop -x SIGKILL
+sleep 3000
+
+print =============== step6
+system sh/exec.sh -n dnode1 -s start -x SIGKILL
+sleep 3000
+sql select * from t1;
+print rows: $rows
+if $rows != 1 then
+ return -1
+endi
+system sh/exec.sh -n dnode1 -s stop -x SIGKILL
+sleep 3000
+
+print =============== step7
+system sh/exec.sh -n dnode1 -s start -x SIGKILL
+sleep 3000
+sql select * from t1;
+print rows: $rows
+if $rows != 1 then
+ return -1
+endi
+system sh/exec.sh -n dnode1 -s stop -x SIGKILL
+sleep 3000
+
+print =============== step8
+system sh/exec.sh -n dnode1 -s start -x SIGKILL
+sleep 3000
+sql select * from t1;
+print rows: $rows
+if $rows != 1 then
+ return -1
+endi
+system sh/exec.sh -n dnode1 -s stop -x SIGKILL
diff --git a/tests/script/general/wal/maxtables.sim b/tests/script/general/wal/maxtables.sim
new file mode 100644
index 0000000000000000000000000000000000000000..e504c7e92e3447f7d29dbb2dc03456c3775ced2c
--- /dev/null
+++ b/tests/script/general/wal/maxtables.sim
@@ -0,0 +1,46 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100
+system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1
+system sh/cfg.sh -n dnode1 -c tableIncStepPerVnode -v 2
+
+
+print ============== deploy
+system sh/exec.sh -n dnode1 -s start
+sleep 3001
+sql connect
+
+sql create database d1
+sql use d1
+sql create table st (ts timestamp, tbcol int) TAGS(tgcol int)
+
+$i = 0
+while $i < 100
+ $tb = t . $i
+ sql create table $tb using st tags( $i )
+ sql insert into $tb values (now , $i )
+ $i = $i + 1
+endw
+
+sql_error sql create table tt (ts timestamp, i int)
+
+print =============== step3
+sql select * from st;
+if $rows != 100 then
+ return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4
+sleep 3000
+
+print =============== step4
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+
+sql select * from st;
+if $rows != 100 then
+ return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/general/wal/sync.sim b/tests/script/general/wal/sync.sim
new file mode 100644
index 0000000000000000000000000000000000000000..abaf22f91921e5bcc8effbe6fc1c66766ff92a3f
--- /dev/null
+++ b/tests/script/general/wal/sync.sim
@@ -0,0 +1,124 @@
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+
+system sh/cfg.sh -n dnode1 -c http -v 1
+system sh/cfg.sh -n dnode2 -c http -v 1
+system sh/cfg.sh -n dnode3 -c http -v 1
+
+system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000
+system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000
+system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000
+
+system sh/cfg.sh -n dnode1 -c replica -v 3
+system sh/cfg.sh -n dnode2 -c replica -v 3
+system sh/cfg.sh -n dnode3 -c replica -v 3
+
+system sh/cfg.sh -n dnode1 -c maxSQLLength -v 940032
+system sh/cfg.sh -n dnode2 -c maxSQLLength -v 940032
+system sh/cfg.sh -n dnode3 -c maxSQLLength -v 940032
+
+print ============== deploy
+
+system sh/exec.sh -n dnode1 -s start
+sleep 5001
+sql connect
+
+sql create dnode $hostname2
+sql create dnode $hostname3
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+
+print =============== step1
+$x = 0
+show1:
+ $x = $x + 1
+ sleep 2000
+ if $x == 5 then
+ return -1
+ endi
+sql show mnodes -x show1
+$mnode1Role = $data2_1
+print mnode1Role $mnode1Role
+$mnode2Role = $data2_2
+print mnode2Role $mnode2Role
+$mnode3Role = $data2_3
+print mnode3Role $mnode3Role
+
+if $mnode1Role != master then
+ goto show1
+endi
+if $mnode2Role != slave then
+ goto show1
+endi
+if $mnode3Role != slave then
+ goto show1
+endi
+
+print =============== step2
+sql create database d1 replica 3
+sql use d1
+
+sql create table table_rest (ts timestamp, i int)
+print sql length is 870KB
+restful d1 table_rest 1591072800 30000
+restful d1 table_rest 1591172800 30000
+restful d1 table_rest 1591272800 30000
+restful d1 table_rest 1591372800 30000
+restful d1 table_rest 1591472800 30000
+restful d1 table_rest 1591572800 30000
+restful d1 table_rest 1591672800 30000
+restful d1 table_rest 1591772800 30000
+restful d1 table_rest 1591872800 30000
+restful d1 table_rest 1591972800 30000
+
+sql select * from table_rest;
+print rows: $rows
+if $rows != 300000 then
+ return -1
+endi
+
+print =============== step3
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+sleep 5000
+sql select * from table_rest;
+print rows: $rows
+if $rows != 300000 then
+ return -1
+endi
+system sh/exec.sh -n dnode1 -s start -x SIGINT
+sleep 5000
+
+print =============== step4
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+sleep 5000
+sql select * from table_rest;
+print rows: $rows
+if $rows != 300000 then
+ return -1
+endi
+system sh/exec.sh -n dnode2 -s start -x SIGINT
+sleep 5000
+
+print =============== step5
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sleep 5000
+sql select * from table_rest;
+print rows: $rows
+if $rows != 300000 then
+ return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 4e68d1566a76468b06115a80aace70871665c9cb..daf92679bd61d8317b648b0285e46a4c956a9d94 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -64,6 +64,7 @@ cd ../../../debug; make
./test.sh -f general/db/repeat.sim
./test.sh -f general/db/tables.sim
./test.sh -f general/db/vnodes.sim
+./test.sh -f general/db/nosuchfile.sim
./test.sh -f general/field/2.sim
./test.sh -f general/field/3.sim
@@ -235,6 +236,10 @@ cd ../../../debug; make
./test.sh -f general/vector/table_query.sim
./test.sh -f general/vector/table_time.sim
+./test.sh -f general/wal/sync.sim
+./test.sh -f general/wal/kill.sim
+./test.sh -f general/wal/maxtables.sim
+
./test.sh -f unique/account/account_create.sim
./test.sh -f unique/account/account_delete.sim
./test.sh -f unique/account/account_len.sim
@@ -277,6 +282,7 @@ cd ../../../debug; make
./test.sh -f unique/dnode/balance2.sim
./test.sh -f unique/dnode/balance3.sim
./test.sh -f unique/dnode/balancex.sim
+./test.sh -f unique/dnode/data1.sim
./test.sh -f unique/dnode/offline1.sim
./test.sh -f unique/dnode/offline2.sim
./test.sh -f unique/dnode/reason.sim
diff --git a/tests/script/unique/dnode/data1.sim b/tests/script/unique/dnode/data1.sim
new file mode 100644
index 0000000000000000000000000000000000000000..61a991148b21b471aef906223a5f600f7db38f5f
--- /dev/null
+++ b/tests/script/unique/dnode/data1.sim
@@ -0,0 +1,137 @@
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
+
+system sh/cfg.sh -n dnode1 -c wallevel -v 2
+system sh/cfg.sh -n dnode2 -c wallevel -v 2
+system sh/cfg.sh -n dnode3 -c wallevel -v 2
+system sh/cfg.sh -n dnode4 -c wallevel -v 2
+
+print ========== step1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+sleep 3000
+
+print ========== step2
+sql create dnode $hostname2
+system sh/exec.sh -n dnode2 -s start
+sql create dnode $hostname3
+system sh/exec.sh -n dnode3 -s start
+sql create dnode $hostname4
+system sh/exec.sh -n dnode4 -s start
+
+$x = 0
+show2:
+ $x = $x + 1
+ sleep 3000
+ if $x == 10 then
+ return -1
+ endi
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+if $data2_1 != 0 then
+ goto show2
+endi
+if $data2_2 != 0 then
+ goto show2
+endi
+if $data2_3 != 0 then
+ goto show2
+endi
+if $data2_4 != 0 then
+ goto show2
+endi
+
+print ========== step3
+sql create database d1 replica 3
+sql create table d1.t1 (t timestamp, i int)
+sql insert into d1.t1 values(now+1s, 35)
+sql insert into d1.t1 values(now+2s, 34)
+sql insert into d1.t1 values(now+3s, 33)
+sql insert into d1.t1 values(now+4s, 32)
+sql insert into d1.t1 values(now+5s, 31)
+
+$x = 0
+show3:
+ $x = $x + 1
+ sleep 3000
+ if $x == 10 then
+ return -1
+ endi
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+if $data2_1 != 0 then
+ goto show3
+endi
+if $data2_2 != 1 then
+ goto show3
+endi
+if $data2_3 != 1 then
+ goto show3
+endi
+if $data2_4 != 1 then
+ goto show3
+endi
+
+print ========== step4
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+
+print ========== step5
+system_content rm -rf ../../../sim/dnode4/data/vnode/vnode2/tsdb/data
+
+print ========== step6
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+sleep 10000
+
+print ========== step7
+sql select * from d1.t1 order by t desc
+print $data01 $data11 $data21 $data31 $data41
+if $data01 != 31 then
+ return -1
+endi
+if $data11 != 32 then
+ return -1
+endi
+if $data21 != 33 then
+ return -1
+endi
+if $data31 != 34 then
+ return -1
+endi
+if $data41 != 35 then
+ return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+system sh/exec.sh -n dnode5 -s stop -x SIGINT
+system sh/exec.sh -n dnode6 -s stop -x SIGINT
+system sh/exec.sh -n dnode7 -s stop -x SIGINT
+system sh/exec.sh -n dnode8 -s stop -x SIGINT
\ No newline at end of file
diff --git a/tests/script/unique/dnode/offline1.sim b/tests/script/unique/dnode/offline1.sim
index 02d03dee97be0f2de62b3f8eb18194c595e7b050..beebbfda60c03d24a774347c8ecf8c2f9a4c6c9e 100644
--- a/tests/script/unique/dnode/offline1.sim
+++ b/tests/script/unique/dnode/offline1.sim
@@ -49,7 +49,7 @@ print dnode1 $data4_2
if $data4_1 != ready then
return -1
endi
-if $data4_2 != offline then
+if $data4_2 == ready then
return -1
endi
diff --git a/tests/test-all.sh b/tests/test-all.sh
index e45dd15fedc999c08037544254f13f78607ee638..214360f3b8d438198bbeadedf63296cbfbaa05a9 100755
--- a/tests/test-all.sh
+++ b/tests/test-all.sh
@@ -9,7 +9,7 @@ NC='\033[0m'
function runSimCaseOneByOne {
while read -r line; do
- if [[ $line =~ ^./test.sh* ]]; then
+ if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then
case=`echo $line | grep sim$ |awk '{print $NF}'`
start_time=`date +%s`
@@ -26,7 +26,12 @@ function runPyCaseOneByOne {
while read -r line; do
if [[ $line =~ ^python.* ]]; then
if [[ $line != *sleep* ]]; then
- case=`echo $line|awk '{print $NF}'`
+
+ if [[ $line =~ '-r' ]];then
+ case=`echo $line|awk '{print $4}'`
+ else
+ case=`echo $line|awk '{print $NF}'`
+ fi
start_time=`date +%s`
$line > /dev/null 2>&1 && \
echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log || \
diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h
index 58e58a442cedca564864578604faa6cea28ede68..01e5016557f2987741fdec6c6fd54facd885ea8a 100644
--- a/tests/tsim/inc/sim.h
+++ b/tests/tsim/inc/sim.h
@@ -100,7 +100,7 @@ typedef struct _cmd_t {
int16_t cmdno;
int16_t nlen;
char name[MAX_SIM_CMD_NAME_LEN];
- bool (*parseCmd)(char *, struct _cmd_t *, int);
+ bool (*parseCmd)(char *, struct _cmd_t *, int32_t);
bool (*executeCmd)(struct _script_t *script, char *option);
struct _cmd_t *next;
} SCommand;
@@ -111,7 +111,7 @@ typedef struct {
int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag
// will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */
int16_t lineNum; // correspodning line number in original file
- int optionOffset; // relative option offset
+ int32_t optionOffset;// relative option offset
} SCmdLine;
typedef struct _var_t {
@@ -121,59 +121,56 @@ typedef struct _var_t {
} SVariable;
typedef struct _script_t {
- int type;
- bool killed;
-
- void *taos;
- char rows[12]; // number of rows data retrieved
- char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM]
- [MAX_QUERY_VALUE_LEN]; // query results
- char system_exit_code[12];
- char system_ret_content[MAX_SYSTEM_RESULT_LEN];
-
- int varLen;
- int linePos; // current cmd position
- int numOfLines; // number of lines in the script
- int bgScriptLen;
- char fileName[MAX_FILE_NAME_LEN]; // script file name
- char error[MAX_ERROR_LEN];
- char *optionBuffer;
+ int32_t type;
+ bool killed;
+ void * taos;
+ char rows[12]; // number of rows data retrieved
+ char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results
+ char system_exit_code[12];
+ char system_ret_content[MAX_SYSTEM_RESULT_LEN];
+ int32_t varLen;
+ int32_t linePos; // current cmd position
+ int32_t numOfLines; // number of lines in the script
+ int32_t bgScriptLen;
+ char fileName[MAX_FILE_NAME_LEN]; // script file name
+ char error[MAX_ERROR_LEN];
+ char * optionBuffer;
SCmdLine *lines; // command list
SVariable variables[MAX_VAR_LEN];
+ pthread_t bgPid;
+ char auth[128];
struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM];
- char auth[128];
} SScript;
extern SScript *simScriptList[MAX_MAIN_SCRIPT_NUM];
extern SCommand simCmdList[];
-extern int simScriptPos;
-extern int simScriptSucced;
-extern int simDebugFlag;
-extern char tsScriptDir[];
-extern bool simAsyncQuery;
+extern int32_t simScriptPos;
+extern int32_t simScriptSucced;
+extern int32_t simDebugFlag;
+extern char tsScriptDir[];
+extern bool simAsyncQuery;
SScript *simParseScript(char *fileName);
-
SScript *simProcessCallOver(SScript *script);
-void *simExecuteScript(void *script);
-void simInitsimCmdList();
-bool simSystemInit();
-void simSystemCleanUp();
-char *simGetVariable(SScript *script, char *varName, int varLen);
-bool simExecuteExpCmd(SScript *script, char *option);
-bool simExecuteTestCmd(SScript *script, char *option);
-bool simExecuteGotoCmd(SScript *script, char *option);
-bool simExecuteRunCmd(SScript *script, char *option);
-bool simExecuteRunBackCmd(SScript *script, char *option);
-bool simExecuteSystemCmd(SScript *script, char *option);
-bool simExecuteSystemContentCmd(SScript *script, char *option);
-bool simExecutePrintCmd(SScript *script, char *option);
-bool simExecuteSleepCmd(SScript *script, char *option);
-bool simExecuteReturnCmd(SScript *script, char *option);
-bool simExecuteSqlCmd(SScript *script, char *option);
-bool simExecuteSqlErrorCmd(SScript *script, char *rest);
-bool simExecuteSqlSlowCmd(SScript *script, char *option);
-bool simExecuteRestfulCmd(SScript *script, char *rest);
-void simVisuallizeOption(SScript *script, char *src, char *dst);
+void * simExecuteScript(void *script);
+void simInitsimCmdList();
+bool simSystemInit();
+void simSystemCleanUp();
+char * simGetVariable(SScript *script, char *varName, int32_t varLen);
+bool simExecuteExpCmd(SScript *script, char *option);
+bool simExecuteTestCmd(SScript *script, char *option);
+bool simExecuteGotoCmd(SScript *script, char *option);
+bool simExecuteRunCmd(SScript *script, char *option);
+bool simExecuteRunBackCmd(SScript *script, char *option);
+bool simExecuteSystemCmd(SScript *script, char *option);
+bool simExecuteSystemContentCmd(SScript *script, char *option);
+bool simExecutePrintCmd(SScript *script, char *option);
+bool simExecuteSleepCmd(SScript *script, char *option);
+bool simExecuteReturnCmd(SScript *script, char *option);
+bool simExecuteSqlCmd(SScript *script, char *option);
+bool simExecuteSqlErrorCmd(SScript *script, char *rest);
+bool simExecuteSqlSlowCmd(SScript *script, char *option);
+bool simExecuteRestfulCmd(SScript *script, char *rest);
+void simVisuallizeOption(SScript *script, char *src, char *dst);
#endif
\ No newline at end of file
diff --git a/tests/tsim/inc/simParse.h b/tests/tsim/inc/simParse.h
index d3f92add71f3dc6ddb205c1810e1a3caace47e36..ef7d8e5ce72cc9bf0ae52380089d36576e84bd28 100644
--- a/tests/tsim/inc/simParse.h
+++ b/tests/tsim/inc/simParse.h
@@ -50,6 +50,6 @@ typedef struct {
char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */
} SBlock;
-bool simParseExpression(char *token, int lineNum);
+bool simParseExpression(char *token, int32_t lineNum);
#endif
\ No newline at end of file
diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c
index 7f786dfaa9de5f6a93d3a181ddd53e37e8694310..2db750cdd302c63522e008e3ea324230dd87b6e5 100644
--- a/tests/tsim/src/simExe.c
+++ b/tests/tsim/src/simExe.c
@@ -13,6 +13,7 @@
* along with this program. If not, see .
*/
+#define _DEFAULT_SOURCE
#include "os.h"
#include "sim.h"
#include "taos.h"
@@ -38,30 +39,28 @@ void simLogSql(char *sql, bool useSharp) {
} else {
fprintf(fp, "%s;\n", sql);
}
-
+
fflush(fp);
}
char *simParseArbitratorName(char *varName);
char *simParseHostName(char *varName);
-char *simGetVariable(SScript *script, char *varName, int varLen) {
+char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
if (strncmp(varName, "hostname", 8) == 0) {
return simParseHostName(varName);
}
if (strncmp(varName, "arbitrator", 10) == 0) {
- return simParseArbitratorName(varName);
+ return simParseArbitratorName(varName);
}
if (strncmp(varName, "error", varLen) == 0) return script->error;
if (strncmp(varName, "rows", varLen) == 0) return script->rows;
- if (strncmp(varName, "system_exit", varLen) == 0)
- return script->system_exit_code;
+ if (strncmp(varName, "system_exit", varLen) == 0) return script->system_exit_code;
- if (strncmp(varName, "system_content", varLen) == 0)
- return script->system_ret_content;
+ if (strncmp(varName, "system_content", varLen) == 0) return script->system_ret_content;
// variable like data2_192.168.0.1
if (strncmp(varName, "data", 4) == 0) {
@@ -70,16 +69,16 @@ char *simGetVariable(SScript *script, char *varName, int varLen) {
}
if (varName[5] == '_') {
- int col = varName[4] - '0';
+ int32_t col = varName[4] - '0';
if (col < 0 || col >= MAX_QUERY_COL_NUM) {
return "null";
}
- char *keyName;
- int keyLen;
+ char * keyName;
+ int32_t keyLen;
paGetToken(varName + 6, &keyName, &keyLen);
- for (int i = 0; i < MAX_QUERY_ROW_NUM; ++i) {
+ for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) {
if (strncmp(keyName, script->data[i][0], keyLen) == 0) {
simDebug("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]);
return script->data[i][col];
@@ -87,16 +86,16 @@ char *simGetVariable(SScript *script, char *varName, int varLen) {
}
return "null";
} else if (varName[6] == '_') {
- int col = (varName[4] - '0') * 10 + (varName[5] - '0');
+ int32_t col = (varName[4] - '0') * 10 + (varName[5] - '0');
if (col < 0 || col >= MAX_QUERY_COL_NUM) {
return "null";
}
- char *keyName;
- int keyLen;
+ char * keyName;
+ int32_t keyLen;
paGetToken(varName + 7, &keyName, &keyLen);
- for (int i = 0; i < MAX_QUERY_ROW_NUM; ++i) {
+ for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) {
if (strncmp(keyName, script->data[i][0], keyLen) == 0) {
simTrace("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]);
return script->data[i][col];
@@ -104,8 +103,8 @@ char *simGetVariable(SScript *script, char *varName, int varLen) {
}
return "null";
} else {
- int row = varName[4] - '0';
- int col = varName[5] - '0';
+ int32_t row = varName[4] - '0';
+ int32_t col = varName[5] - '0';
if (row < 0 || row >= MAX_QUERY_ROW_NUM) {
return "null";
}
@@ -118,7 +117,7 @@ char *simGetVariable(SScript *script, char *varName, int varLen) {
}
}
- for (int i = 0; i < script->varLen; ++i) {
+ for (int32_t i = 0; i < script->varLen; ++i) {
SVariable *var = &script->variables[i];
if (var->varNameLen != varLen) {
continue;
@@ -144,11 +143,11 @@ char *simGetVariable(SScript *script, char *varName, int varLen) {
return var->varValue;
}
-int simExecuteExpression(SScript *script, char *exp) {
- char *op1, *op2, *var1, *var2, *var3, *rest;
- int op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1;
- char t0[512], t1[512], t2[512], t3[1024];
- int result;
+int32_t simExecuteExpression(SScript *script, char *exp) {
+ char * op1, *op2, *var1, *var2, *var3, *rest;
+ int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1;
+ char t0[512], t1[512], t2[512], t3[1024];
+ int32_t result;
rest = paGetToken(exp, &var1, &var1Len);
rest = paGetToken(rest, &op1, &op1Len);
@@ -234,7 +233,7 @@ bool simExecuteExpCmd(SScript *script, char *option) {
}
bool simExecuteTestCmd(SScript *script, char *option) {
- int result;
+ int32_t result;
result = simExecuteExpression(script, option);
if (result >= 0)
@@ -285,13 +284,12 @@ bool simExecuteRunBackCmd(SScript *script, char *option) {
sprintf(script->error, "lineNum:%d. parse file:%s error", script->lines[script->linePos].lineNum, fileName);
return false;
}
- simInfo("script:%s, start to execute in background", newScript->fileName);
newScript->type = SIM_SCRIPT_TYPE_BACKGROUND;
script->bgScripts[script->bgScriptLen++] = newScript;
+ simInfo("script:%s, start to execute in background,", newScript->fileName);
- pthread_t pid;
- if (pthread_create(&pid, NULL, simExecuteScript, (void *)newScript) != 0) {
+ if (pthread_create(&newScript->bgPid, NULL, simExecuteScript, (void *)newScript) != 0) {
sprintf(script->error, "lineNum:%d. create background thread failed", script->lines[script->linePos].lineNum);
return false;
}
@@ -307,13 +305,13 @@ bool simExecuteSystemCmd(SScript *script, char *option) {
simVisuallizeOption(script, option, buf + strlen(buf));
simLogSql(buf, true);
- int code = system(buf);
- int repeatTimes = 0;
+ int32_t code = system(buf);
+ int32_t repeatTimes = 0;
while (code < 0) {
- simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d",
- script->fileName, buf, code, errno, strerror(errno), repeatTimes);
+ simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", script->fileName, buf, code,
+ errno, strerror(errno), repeatTimes);
taosMsleep(1000);
-#ifdef LINUX
+#ifdef LINUX
signal(SIGCHLD, SIG_DFL);
#endif
if (repeatTimes++ >= 10) {
@@ -368,8 +366,8 @@ bool simExecutePrintCmd(SScript *script, char *rest) {
}
bool simExecuteSleepCmd(SScript *script, char *option) {
- int delta;
- char buf[1024];
+ int32_t delta;
+ char buf[1024];
simVisuallizeOption(script, option, buf);
option = buf;
@@ -395,7 +393,7 @@ bool simExecuteReturnCmd(SScript *script, char *option) {
simVisuallizeOption(script, option, buf);
option = buf;
- int ret = 1;
+ int32_t ret = 1;
if (option && option[0] != 0) ret = atoi(option);
if (ret < 0) {
@@ -411,8 +409,8 @@ bool simExecuteReturnCmd(SScript *script, char *option) {
}
void simVisuallizeOption(SScript *script, char *src, char *dst) {
- char *var, *token, *value;
- int dstLen, srcLen, tokenLen;
+ char * var, *token, *value;
+ int32_t dstLen, srcLen, tokenLen;
dst[0] = 0, dstLen = 0;
@@ -420,14 +418,14 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) {
var = strchr(src, '$');
if (var == NULL) break;
if (var && ((var - src - 1) > 0) && *(var - 1) == '\\') {
- srcLen = (int)(var - src - 1);
+ srcLen = (int32_t)(var - src - 1);
memcpy(dst + dstLen, src, srcLen);
dstLen += srcLen;
src = var;
break;
}
- srcLen = (int)(var - src);
+ srcLen = (int32_t)(var - src);
memcpy(dst + dstLen, src, srcLen);
dstLen += srcLen;
@@ -435,13 +433,13 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) {
value = simGetVariable(script, token, tokenLen);
strcpy(dst + dstLen, value);
- dstLen += (int)strlen(value);
+ dstLen += (int32_t)strlen(value);
}
strcpy(dst + dstLen, src);
}
-void simCloseRestFulConnect(SScript *script) {
+void simCloseRestFulConnect(SScript *script) {
memset(script->auth, 0, sizeof(script->auth));
}
@@ -465,7 +463,7 @@ void simCloseTaosdConnect(SScript *script) {
// {"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"}
// {"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}
// {"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10}
-int simParseHttpCommandResult(SScript *script, char *command) {
+int32_t simParseHttpCommandResult(SScript *script, char *command) {
cJSON* root = cJSON_Parse(command);
if (root == NULL) {
simError("script:%s, failed to parse json, response:%s", script->fileName, command);
@@ -492,14 +490,15 @@ int simParseHttpCommandResult(SScript *script, char *command) {
cJSON_Delete(root);
return -1;
}
- int retcode = (int)code->valueint;
+ int32_t retcode = (int32_t)code->valueint;
if (retcode != 1017) {
- simError("script:%s, json:status:%s not equal to succ, response:%s", script->fileName, status->valuestring, command);
+ simError("script:%s, json:status:%s not equal to succ, response:%s", script->fileName, status->valuestring,
+ command);
cJSON_Delete(root);
return retcode;
} else {
simDebug("script:%s, json:status:%s not equal to succ, but code is %d, response:%s", script->fileName,
- status->valuestring, retcode, command);
+ status->valuestring, retcode, command);
cJSON_Delete(root);
return 0;
}
@@ -524,27 +523,27 @@ int simParseHttpCommandResult(SScript *script, char *command) {
return -1;
}
- int rowsize = cJSON_GetArraySize(data);
+ int32_t rowsize = cJSON_GetArraySize(data);
if (rowsize < 0) {
simError("script:%s, failed to parse json:data, data size %d, response:%s", script->fileName, rowsize, command);
cJSON_Delete(root);
return -1;
}
- int rowIndex = 0;
+ int32_t rowIndex = 0;
sprintf(script->rows, "%d", rowsize);
- for (int r = 0; r < rowsize; ++r) {
+ for (int32_t r = 0; r < rowsize; ++r) {
cJSON *row = cJSON_GetArrayItem(data, r);
if (row == NULL) continue;
if (rowIndex++ >= 10) break;
- int colsize = cJSON_GetArraySize(row);
+ int32_t colsize = cJSON_GetArraySize(row);
if (colsize < 0) {
break;
}
colsize = MIN(10, colsize);
- for (int c = 0; c < colsize; ++c) {
+ for (int32_t c = 0; c < colsize; ++c) {
cJSON *col = cJSON_GetArrayItem(row, c);
if (col->valuestring != NULL) {
strcpy(script->data[r][c], col->valuestring);
@@ -561,7 +560,7 @@ int simParseHttpCommandResult(SScript *script, char *command) {
return 0;
}
-int simExecuteRestFulCommand(SScript *script, char *command) {
+int32_t simExecuteRestFulCommand(SScript *script, char *command) {
char buf[5000] = {0};
sprintf(buf, "%s 2>/dev/null", command);
@@ -571,13 +570,13 @@ int simExecuteRestFulCommand(SScript *script, char *command) {
return -1;
}
- int mallocSize = 2000;
- int alreadyReadSize = 0;
- char* content = malloc(mallocSize);
+ int32_t mallocSize = 2000;
+ int32_t alreadyReadSize = 0;
+ char * content = malloc(mallocSize);
while (!feof(fp)) {
- int availSize = mallocSize - alreadyReadSize;
- int len = (int)fread(content + alreadyReadSize, 1, availSize, fp);
+ int32_t availSize = mallocSize - alreadyReadSize;
+ int32_t len = (int32_t)fread(content + alreadyReadSize, 1, availSize, fp);
if (len >= availSize) {
alreadyReadSize += len;
mallocSize *= 2;
@@ -595,10 +594,11 @@ bool simCreateRestFulConnect(SScript *script, char *user, char *pass) {
sprintf(command, "curl 127.0.0.1:6041/rest/login/%s/%s", user, pass);
bool success = false;
- for (int attempt = 0; attempt < 10; ++attempt) {
+ for (int32_t attempt = 0; attempt < 10; ++attempt) {
success = simExecuteRestFulCommand(script, command) == 0;
if (!success) {
- simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), attempt);
+ simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL),
+ attempt);
taosMsleep(1000);
} else {
simDebug("script:%s, user:%s connect taosd successed, attempt:%d", script->fileName, user, attempt);
@@ -607,7 +607,8 @@ bool simCreateRestFulConnect(SScript *script, char *user, char *pass) {
}
if (!success) {
- sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, taos_errstr(NULL));
+ sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum,
+ taos_errstr(NULL));
return false;
}
@@ -619,10 +620,11 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) {
simCloseTaosdConnect(script);
void *taos = NULL;
taosMsleep(2000);
- for (int attempt = 0; attempt < 10; ++attempt) {
+ for (int32_t attempt = 0; attempt < 10; ++attempt) {
taos = taos_connect(NULL, user, pass, NULL, tsDnodeShellPort);
if (taos == NULL) {
- simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), attempt);
+ simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL),
+ attempt);
taosMsleep(1000);
} else {
simDebug("script:%s, user:%s connect taosd successed, attempt:%d", script->fileName, user, attempt);
@@ -631,7 +633,8 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) {
}
if (taos == NULL) {
- sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, taos_errstr(NULL));
+ sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum,
+ taos_errstr(NULL));
return false;
}
@@ -642,9 +645,9 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) {
}
bool simCreateTaosdConnect(SScript *script, char *rest) {
- char *user = TSDB_DEFAULT_USER;
- char *token;
- int tokenLen;
+ char * user = TSDB_DEFAULT_USER;
+ char * token;
+ int32_t tokenLen;
rest = paGetToken(rest, &token, &tokenLen);
rest = paGetToken(rest, &token, &tokenLen);
if (tokenLen != 0) {
@@ -659,26 +662,27 @@ bool simCreateTaosdConnect(SScript *script, char *rest) {
}
bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
- char timeStr[30] = {0};
- time_t tt;
+ char timeStr[30] = {0};
+ time_t tt;
struct tm *tp;
- SCmdLine *line = &script->lines[script->linePos];
- int ret = -1;
+ SCmdLine * line = &script->lines[script->linePos];
+ int32_t ret = -1;
+
+ TAOS_RES *pSql = NULL;
- TAOS_RES* pSql = NULL;
-
- for (int attempt = 0; attempt < 10; ++attempt) {
+ for (int32_t attempt = 0; attempt < 10; ++attempt) {
simLogSql(rest, false);
pSql = taos_query(script->taos, rest);
ret = taos_errno(pSql);
-
+
if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) {
- simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret));
+ simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF,
+ tstrerror(ret));
ret = 0;
break;
} else if (ret != 0) {
- simDebug("script:%s, taos:%p, %s failed, ret:%d:%s, error:%s",
- script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret), taos_errstr(pSql));
+ simDebug("script:%s, taos:%p, %s failed, ret:%d:%s, error:%s", script->fileName, script->taos, rest, ret & 0XFFFF,
+ tstrerror(ret), taos_errstr(pSql));
if (line->errorJump == SQL_JUMP_TRUE) {
script->linePos = line->jump;
@@ -689,7 +693,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
} else {
break;
}
-
+
taos_free_result(pSql);
}
@@ -698,8 +702,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
return false;
}
- int numOfRows = 0;
- int num_fields = taos_field_count(pSql);
+ int32_t numOfRows = 0;
+ int32_t num_fields = taos_field_count(pSql);
if (num_fields != 0) {
if (pSql == NULL) {
simDebug("script:%s, taos:%p, %s failed, result is null", script->fileName, script->taos, rest);
@@ -717,9 +721,9 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
while ((row = taos_fetch_row(pSql))) {
if (numOfRows < MAX_QUERY_ROW_NUM) {
TAOS_FIELD *fields = taos_fetch_fields(pSql);
- int* length = taos_fetch_lengths(pSql);
-
- for (int i = 0; i < num_fields; i++) {
+ int32_t * length = taos_fetch_lengths(pSql);
+
+ for (int32_t i = 0; i < num_fields; i++) {
char *value = NULL;
if (i < MAX_QUERY_COL_NUM) {
value = script->data[numOfRows][i];
@@ -735,8 +739,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
switch (fields[i].type) {
case TSDB_DATA_TYPE_BOOL:
- sprintf(value, "%s",
- ((((int)(*((char *)row[i]))) == 1) ? "1" : "0"));
+ sprintf(value, "%s", ((((int32_t)(*((char *)row[i]))) == 1) ? "1" : "0"));
break;
case TSDB_DATA_TYPE_TINYINT:
sprintf(value, "%d", *((int8_t *)row[i]));
@@ -779,9 +782,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
tp = localtime(&tt);
strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", tp);
- sprintf(value, "%s.%03d", timeStr,
- (int)(*((int64_t *)row[i]) % 1000));
-
+ sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000));
+
break;
default:
break;
@@ -814,17 +816,16 @@ bool simExecuteRestFulSqlCommand(SScript *script, char *rest) {
char command[4096];
sprintf(command, "curl -H 'Authorization: Taosd %s' -d \"%s\" 127.0.0.1:6041/rest/sql", script->auth, rest);
- int ret = -1;
- for (int attempt = 0; attempt < 10; ++attempt) {
+ int32_t ret = -1;
+ for (int32_t attempt = 0; attempt < 10; ++attempt) {
ret = simExecuteRestFulCommand(script, command);
- if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST ||
- ret == TSDB_CODE_MND_DB_ALREADY_EXIST) {
- simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret));
+ if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) {
+ simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF,
+ tstrerror(ret));
ret = 0;
break;
} else if (ret != 0) {
- simDebug("script:%s, taos:%p, %s failed, ret:%d",
- script->fileName, script->taos, rest, ret);
+ simDebug("script:%s, taos:%p, %s failed, ret:%d", script->fileName, script->taos, rest, ret);
if (line->errorJump == SQL_JUMP_TRUE) {
script->linePos = line->jump;
@@ -854,8 +855,8 @@ bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) {
simDebug("script:%s, exec:%s", script->fileName, rest);
strcpy(script->rows, "-1");
- for (int row = 0; row < MAX_QUERY_ROW_NUM; ++row) {
- for (int col = 0; col < MAX_QUERY_COL_NUM; ++col) {
+ for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) {
+ for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) {
strcpy(script->data[row][col], "null");
}
}
@@ -903,23 +904,23 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) {
bool simExecuteRestfulCmd(SScript *script, char *rest) {
FILE *fp = NULL;
- char filename[256];
- sprintf(filename, "%s/tmp.sql", tsScriptDir);
+ char filename[256];
+ sprintf(filename, "%s/tmp.sql", tsScriptDir);
fp = fopen(filename, "w");
if (fp == NULL) {
fprintf(stderr, "ERROR: failed to open file: %s\n", filename);
return false;
}
- char db[64] = {0};
- char tb[64] = {0};
- char gzip[32] = {0};
+ char db[64] = {0};
+ char tb[64] = {0};
+ char gzip[32] = {0};
int32_t ts;
int32_t times;
sscanf(rest, "%s %s %d %d %s", db, tb, &ts, ×, gzip);
-
+
fprintf(fp, "insert into %s.%s values ", db, tb);
- for (int i = 0; i < times; ++i) {
+ for (int32_t i = 0; i < times; ++i) {
fprintf(fp, "(%d000, %d)", ts + i, ts);
}
fprintf(fp, " \n");
@@ -951,8 +952,8 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) {
simDebug("script:%s, exec:%s", script->fileName, rest);
strcpy(script->rows, "-1");
- for (int row = 0; row < MAX_QUERY_ROW_NUM; ++row) {
- for (int col = 0; col < MAX_QUERY_COL_NUM; ++col) {
+ for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) {
+ for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) {
strcpy(script->data[row][col], "null");
}
}
@@ -981,27 +982,27 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) {
return true;
}
- int ret;
- TAOS_RES* pSql = NULL;
+ int32_t ret;
+ TAOS_RES *pSql = NULL;
if (simAsyncQuery) {
char command[4096];
sprintf(command, "curl -H 'Authorization: Taosd %s' -d '%s' 127.0.0.1:6041/rest/sql", script->auth, rest);
ret = simExecuteRestFulCommand(script, command);
- }
- else {
+ } else {
pSql = taos_query(script->taos, rest);
ret = taos_errno(pSql);
taos_free_result(pSql);
}
if (ret != TSDB_CODE_SUCCESS) {
- simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s",
- script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret));
+ simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s", script->fileName, script->taos,
+ rest, ret & 0XFFFF, tstrerror(ret));
script->linePos++;
return true;
}
-
- sprintf(script->error, "lineNum:%d. sql:%s expect failed, but success, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, tstrerror(ret));
+
+ sprintf(script->error, "lineNum:%d. sql:%s expect failed, but success, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF,
+ tstrerror(ret));
return false;
}
diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c
index ef1a488f602959b740cbe863cc22654a24b2d41b..33fd24dd5823b8ec7bf818cec7c36ddac783bc24 100644
--- a/tests/tsim/src/simMain.c
+++ b/tests/tsim/src/simMain.c
@@ -13,6 +13,7 @@
* along with this program. If not, see .
*/
+#define _DEFAULT_SOURCE
#include "os.h"
#include "tglobal.h"
#include "sim.h"
@@ -20,15 +21,15 @@
bool simAsyncQuery = false;
-void simHandleSignal(int signo) {
+void simHandleSignal(int32_t signo) {
simSystemCleanUp();
exit(1);
}
-int main(int argc, char *argv[]) {
+int32_t main(int32_t argc, char *argv[]) {
char scriptFile[MAX_FILE_NAME_LEN] = "sim_main_test.sim";
- for (int i = 1; i < argc; ++i) {
+ for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0 && i < argc - 1) {
tstrncpy(configDir, argv[++i], MAX_FILE_NAME_LEN);
} else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) {
@@ -37,8 +38,7 @@ int main(int argc, char *argv[]) {
simAsyncQuery = true;
} else {
printf("usage: %s [options] \n", argv[0]);
- printf(" [-c config]: config directory, default is: %s\n",
- configDir);
+ printf(" [-c config]: config directory, default is: %s\n", configDir);
printf(" [-f script]: script filename\n");
exit(0);
}
diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c
index 2e6121304f18022a74b9e4f09a8043b5b4beb909..b909f5bd8fc10bea09afd65dc504ae35d6de3505 100644
--- a/tests/tsim/src/simParse.c
+++ b/tests/tsim/src/simParse.c
@@ -57,6 +57,7 @@
*
*/
+#define _DEFAULT_SOURCE
#include "os.h"
#include "sim.h"
#include "simParse.h"
@@ -64,16 +65,16 @@
#undef TAOS_MEM_CHECK
static SCommand *cmdHashList[MAX_NUM_CMD];
-static SCmdLine cmdLine[MAX_CMD_LINES];
-static char parseErr[MAX_ERROR_LEN];
-static char optionBuffer[MAX_OPTION_BUFFER];
-static int numOfLines, optionOffset;
-static SLabel label, dest;
-static SBlock block;
+static SCmdLine cmdLine[MAX_CMD_LINES];
+static char parseErr[MAX_ERROR_LEN];
+static char optionBuffer[MAX_OPTION_BUFFER];
+static int32_t numOfLines, optionOffset;
+static SLabel label, dest;
+static SBlock block;
-int simHashCmd(char *token, int tokenLen) {
- int i;
- int hash = 0;
+int32_t simHashCmd(char *token, int32_t tokenLen) {
+ int32_t i;
+ int32_t hash = 0;
for (i = 0; i < tokenLen; ++i) hash += token[i];
@@ -82,8 +83,8 @@ int simHashCmd(char *token, int tokenLen) {
return hash;
}
-SCommand *simCheckCmd(char *token, int tokenLen) {
- int hash;
+SCommand *simCheckCmd(char *token, int32_t tokenLen) {
+ int32_t hash;
SCommand *node;
hash = simHashCmd(token, tokenLen);
@@ -102,10 +103,10 @@ SCommand *simCheckCmd(char *token, int tokenLen) {
}
void simAddCmdIntoHash(SCommand *pCmd) {
- int hash;
+ int32_t hash;
SCommand *node;
- hash = simHashCmd(pCmd->name, (int)strlen(pCmd->name));
+ hash = simHashCmd(pCmd->name, (int32_t)strlen(pCmd->name));
node = cmdHashList[hash];
pCmd->next = node;
cmdHashList[hash] = pCmd;
@@ -122,7 +123,7 @@ void simResetParser() {
}
SScript *simBuildScriptObj(char *fileName) {
- int i, destPos;
+ int32_t i, destPos;
/* process labels */
@@ -176,11 +177,11 @@ SScript *simBuildScriptObj(char *fileName) {
}
SScript *simParseScript(char *fileName) {
- FILE *fd;
- int tokenLen, lineNum = 0;
- char buffer[MAX_LINE_LEN], name[128], *token, *rest;
+ FILE * fd;
+ int32_t tokenLen, lineNum = 0;
+ char buffer[MAX_LINE_LEN], name[128], *token, *rest;
SCommand *pCmd;
- SScript *script;
+ SScript * script;
if ((fileName[0] == '.') || (fileName[0] == '/')) {
strcpy(name, fileName);
@@ -199,12 +200,13 @@ SScript *simParseScript(char *fileName) {
if (fgets(buffer, sizeof(buffer), fd) == NULL) continue;
lineNum++;
- int cmdlen = (int)strlen(buffer);
- if (buffer[cmdlen - 1] == '\r' || buffer[cmdlen - 1] == '\n')
+ int32_t cmdlen = (int32_t)strlen(buffer);
+ if (buffer[cmdlen - 1] == '\r' || buffer[cmdlen - 1] == '\n') {
buffer[cmdlen - 1] = 0;
+ }
rest = buffer;
- for (int i = 0; i < cmdlen; ++i) {
+ for (int32_t i = 0; i < cmdlen; ++i) {
if (buffer[i] == '\r' || buffer[i] == '\n') {
buffer[i] = ' ';
}
@@ -249,9 +251,9 @@ SScript *simParseScript(char *fileName) {
return script;
}
-int simCheckExpression(char *exp) {
- char *op1, *op2, *op, *rest;
- int op1Len, op2Len, opLen;
+int32_t simCheckExpression(char *exp) {
+ char * op1, *op2, *op, *rest;
+ int32_t op1Len, op2Len, opLen;
rest = paGetToken(exp, &op1, &op1Len);
if (op1Len == 0) {
@@ -282,8 +284,7 @@ int simCheckExpression(char *exp) {
return -1;
}
} else if (opLen == 2) {
- if (op[1] != '=' ||
- (op[0] != '=' && op[0] != '<' && op[0] != '>' && op[0] != '!')) {
+ if (op[1] != '=' || (op[0] != '=' && op[0] != '<' && op[0] != '>' && op[0] != '!')) {
sprintf(parseErr, "left side of assignment must be variable");
return -1;
}
@@ -294,10 +295,10 @@ int simCheckExpression(char *exp) {
rest = paGetToken(rest, &op, &opLen);
- if (opLen == 0) return (int)(rest - exp);
+ if (opLen == 0) return (int32_t)(rest - exp);
/* if it is key word "then" */
- if (strncmp(op, "then", 4) == 0) return (int)(op - exp);
+ if (strncmp(op, "then", 4) == 0) return (int32_t)(op - exp);
rest = paGetToken(rest, &op2, &op2Len);
if (op2Len == 0) {
@@ -310,16 +311,15 @@ int simCheckExpression(char *exp) {
return -1;
}
- if (op[0] == '+' || op[0] == '-' || op[0] == '*' || op[0] == '/' ||
- op[0] == '.') {
- return (int)(rest - exp);
+ if (op[0] == '+' || op[0] == '-' || op[0] == '*' || op[0] == '/' || op[0] == '.') {
+ return (int32_t)(rest - exp);
}
return -1;
}
-bool simParseExpression(char *token, int lineNum) {
- int expLen;
+bool simParseExpression(char *token, int32_t lineNum) {
+ int32_t expLen;
expLen = simCheckExpression(token);
if (expLen <= 0) return -1;
@@ -335,9 +335,9 @@ bool simParseExpression(char *token, int lineNum) {
return true;
}
-bool simParseIfCmd(char *rest, SCommand *pCmd, int lineNum) {
- char *ret;
- int expLen;
+bool simParseIfCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ char * ret;
+ int32_t expLen;
expLen = simCheckExpression(rest);
@@ -364,8 +364,8 @@ bool simParseIfCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseElifCmd(char *rest, SCommand *pCmd, int lineNum) {
- int expLen;
+bool simParseElifCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ int32_t expLen;
expLen = simCheckExpression(rest);
@@ -382,8 +382,7 @@ bool simParseElifCmd(char *rest, SCommand *pCmd, int lineNum) {
}
cmdLine[numOfLines].cmdno = SIM_CMD_GOTO;
- block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] =
- &(cmdLine[numOfLines].jump);
+ block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump);
block.numJump[block.top - 1]++;
numOfLines++;
@@ -402,7 +401,7 @@ bool simParseElifCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseElseCmd(char *rest, SCommand *pCmd, int lineNum) {
+bool simParseElseCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
if (block.top < 1) {
sprintf(parseErr, "no matching if");
return false;
@@ -414,8 +413,7 @@ bool simParseElseCmd(char *rest, SCommand *pCmd, int lineNum) {
}
cmdLine[numOfLines].cmdno = SIM_CMD_GOTO;
- block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] =
- &(cmdLine[numOfLines].jump);
+ block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump);
block.numJump[block.top - 1]++;
numOfLines++;
@@ -426,8 +424,8 @@ bool simParseElseCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseEndiCmd(char *rest, SCommand *pCmd, int lineNum) {
- int i;
+bool simParseEndiCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ int32_t i;
if (block.top < 1) {
sprintf(parseErr, "no matching if");
@@ -441,8 +439,9 @@ bool simParseEndiCmd(char *rest, SCommand *pCmd, int lineNum) {
if (block.pos[block.top - 1]) *(block.pos[block.top - 1]) = numOfLines;
- for (i = 0; i < block.numJump[block.top - 1]; ++i)
+ for (i = 0; i < block.numJump[block.top - 1]; ++i) {
*(block.jump[block.top - 1][i]) = numOfLines;
+ }
block.numJump[block.top - 1] = 0;
block.top--;
@@ -450,8 +449,8 @@ bool simParseEndiCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseWhileCmd(char *rest, SCommand *pCmd, int lineNum) {
- int expLen;
+bool simParseWhileCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ int32_t expLen;
expLen = simCheckExpression(rest);
@@ -473,8 +472,8 @@ bool simParseWhileCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseEndwCmd(char *rest, SCommand *pCmd, int lineNum) {
- int i;
+bool simParseEndwCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ int32_t i;
if (block.top < 1) {
sprintf(parseErr, "no matching while");
@@ -493,17 +492,18 @@ bool simParseEndwCmd(char *rest, SCommand *pCmd, int lineNum) {
*(block.pos[block.top - 1]) = numOfLines;
- for (i = 0; i < block.numJump[block.top - 1]; ++i)
+ for (i = 0; i < block.numJump[block.top - 1]; ++i) {
*(block.jump[block.top - 1][i]) = numOfLines;
+ }
block.top--;
return true;
}
-bool simParseSwitchCmd(char *rest, SCommand *pCmd, int lineNum) {
- char *token;
- int tokenLen;
+bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ char * token;
+ int32_t tokenLen;
rest = paGetToken(rest, &token, &tokenLen);
if (tokenLen == 0) {
@@ -524,9 +524,9 @@ bool simParseSwitchCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseCaseCmd(char *rest, SCommand *pCmd, int lineNum) {
- char *token;
- int tokenLen;
+bool simParseCaseCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ char * token;
+ int32_t tokenLen;
rest = paGetToken(rest, &token, &tokenLen);
if (tokenLen == 0) {
@@ -544,16 +544,16 @@ bool simParseCaseCmd(char *rest, SCommand *pCmd, int lineNum) {
return false;
}
- if (block.pos[block.top - 1] != NULL)
+ if (block.pos[block.top - 1] != NULL) {
*(block.pos[block.top - 1]) = numOfLines;
+ }
block.pos[block.top - 1] = &(cmdLine[numOfLines].jump);
cmdLine[numOfLines].cmdno = SIM_CMD_TEST;
cmdLine[numOfLines].lineNum = lineNum;
cmdLine[numOfLines].optionOffset = optionOffset;
- memcpy(optionBuffer + optionOffset, block.sexp[block.top - 1],
- block.sexpLen[block.top - 1]);
+ memcpy(optionBuffer + optionOffset, block.sexp[block.top - 1], block.sexpLen[block.top - 1]);
optionOffset += block.sexpLen[block.top - 1];
*(optionBuffer + optionOffset++) = ' ';
*(optionBuffer + optionOffset++) = '=';
@@ -567,20 +567,18 @@ bool simParseCaseCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseBreakCmd(char *rest, SCommand *pCmd, int lineNum) {
+bool simParseBreakCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
if (block.top < 1) {
sprintf(parseErr, "no blcok exists");
return false;
}
- if (block.type[block.top - 1] != BLOCK_SWITCH &&
- block.type[block.top - 1] != BLOCK_WHILE) {
+ if (block.type[block.top - 1] != BLOCK_SWITCH && block.type[block.top - 1] != BLOCK_WHILE) {
sprintf(parseErr, "not in switch or while block");
return false;
}
- block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] =
- &(cmdLine[numOfLines].jump);
+ block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump);
block.numJump[block.top - 1]++;
cmdLine[numOfLines].cmdno = SIM_CMD_GOTO;
@@ -590,7 +588,7 @@ bool simParseBreakCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseDefaultCmd(char *rest, SCommand *pCmd, int lineNum) {
+bool simParseDefaultCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
if (block.top < 1) {
sprintf(parseErr, "no matching switch");
return false;
@@ -601,14 +599,15 @@ bool simParseDefaultCmd(char *rest, SCommand *pCmd, int lineNum) {
return false;
}
- if (block.pos[block.top - 1] != NULL)
+ if (block.pos[block.top - 1] != NULL) {
*(block.pos[block.top - 1]) = numOfLines;
+ }
return true;
}
-bool simParseEndsCmd(char *rest, SCommand *pCmd, int lineNum) {
- int i;
+bool simParseEndsCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ int32_t i;
if (block.top < 1) {
sprintf(parseErr, "no matching switch");
@@ -620,8 +619,9 @@ bool simParseEndsCmd(char *rest, SCommand *pCmd, int lineNum) {
return false;
}
- for (i = 0; i < block.numJump[block.top - 1]; ++i)
+ for (i = 0; i < block.numJump[block.top - 1]; ++i) {
*(block.jump[block.top - 1][i]) = numOfLines;
+ }
block.numJump[block.top - 1] = 0;
block.top--;
@@ -629,7 +629,7 @@ bool simParseEndsCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseContinueCmd(char *rest, SCommand *pCmd, int lineNum) {
+bool simParseContinueCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
if (block.top < 1) {
sprintf(parseErr, "no matching while");
return false;
@@ -648,14 +648,14 @@ bool simParseContinueCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParsePrintCmd(char *rest, SCommand *pCmd, int lineNum) {
- int expLen;
+bool simParsePrintCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ int32_t expLen;
rest++;
cmdLine[numOfLines].cmdno = SIM_CMD_PRINT;
cmdLine[numOfLines].lineNum = lineNum;
cmdLine[numOfLines].optionOffset = optionOffset;
- expLen = (int)strlen(rest);
+ expLen = (int32_t)strlen(rest);
memcpy(optionBuffer + optionOffset, rest, expLen);
optionOffset += expLen + 1;
*(optionBuffer + optionOffset - 1) = 0;
@@ -665,8 +665,8 @@ bool simParsePrintCmd(char *rest, SCommand *pCmd, int lineNum) {
}
void simCheckSqlOption(char *rest) {
- int valueLen;
- char *value, *xpos;
+ int32_t valueLen;
+ char * value, *xpos;
xpos = strstr(rest, " -x"); // need a blank
if (xpos) {
@@ -682,15 +682,15 @@ void simCheckSqlOption(char *rest) {
}
}
-bool simParseSqlCmd(char *rest, SCommand *pCmd, int lineNum) {
- int expLen;
+bool simParseSqlCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ int32_t expLen;
rest++;
simCheckSqlOption(rest);
cmdLine[numOfLines].cmdno = SIM_CMD_SQL;
cmdLine[numOfLines].lineNum = lineNum;
cmdLine[numOfLines].optionOffset = optionOffset;
- expLen = (int)strlen(rest);
+ expLen = (int32_t)strlen(rest);
memcpy(optionBuffer + optionOffset, rest, expLen);
optionOffset += expLen + 1;
*(optionBuffer + optionOffset - 1) = 0;
@@ -699,14 +699,14 @@ bool simParseSqlCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int lineNum) {
- int expLen;
+bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ int32_t expLen;
rest++;
cmdLine[numOfLines].cmdno = SIM_CMD_SQL_ERROR;
cmdLine[numOfLines].lineNum = lineNum;
cmdLine[numOfLines].optionOffset = optionOffset;
- expLen = (int)strlen(rest);
+ expLen = (int32_t)strlen(rest);
memcpy(optionBuffer + optionOffset, rest, expLen);
optionOffset += expLen + 1;
*(optionBuffer + optionOffset - 1) = 0;
@@ -715,26 +715,26 @@ bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int lineNum) {
+bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
simParseSqlCmd(rest, pCmd, lineNum);
cmdLine[numOfLines - 1].cmdno = SIM_CMD_SQL_SLOW;
return true;
}
-bool simParseRestfulCmd(char *rest, SCommand *pCmd, int lineNum) {
+bool simParseRestfulCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
simParseSqlCmd(rest, pCmd, lineNum);
cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL;
return true;
}
-bool simParseSystemCmd(char *rest, SCommand *pCmd, int lineNum) {
- int expLen;
+bool simParseSystemCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ int32_t expLen;
rest++;
cmdLine[numOfLines].cmdno = SIM_CMD_SYSTEM;
cmdLine[numOfLines].lineNum = lineNum;
cmdLine[numOfLines].optionOffset = optionOffset;
- expLen = (int)strlen(rest);
+ expLen = (int32_t)strlen(rest);
memcpy(optionBuffer + optionOffset, rest, expLen);
optionOffset += expLen + 1;
*(optionBuffer + optionOffset - 1) = 0;
@@ -743,15 +743,15 @@ bool simParseSystemCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseSystemContentCmd(char *rest, SCommand *pCmd, int lineNum) {
+bool simParseSystemContentCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
simParseSystemCmd(rest, pCmd, lineNum);
cmdLine[numOfLines - 1].cmdno = SIM_CMD_SYSTEM_CONTENT;
return true;
}
-bool simParseSleepCmd(char *rest, SCommand *pCmd, int lineNum) {
- char *token;
- int tokenLen;
+bool simParseSleepCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ char * token;
+ int32_t tokenLen;
cmdLine[numOfLines].cmdno = SIM_CMD_SLEEP;
cmdLine[numOfLines].lineNum = lineNum;
@@ -768,9 +768,9 @@ bool simParseSleepCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseReturnCmd(char *rest, SCommand *pCmd, int lineNum) {
- char *token;
- int tokenLen;
+bool simParseReturnCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ char * token;
+ int32_t tokenLen;
cmdLine[numOfLines].cmdno = SIM_CMD_RETURN;
cmdLine[numOfLines].lineNum = lineNum;
@@ -787,9 +787,9 @@ bool simParseReturnCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseGotoCmd(char *rest, SCommand *pCmd, int lineNum) {
- char *token;
- int tokenLen;
+bool simParseGotoCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ char * token;
+ int32_t tokenLen;
rest = paGetToken(rest, &token, &tokenLen);
@@ -810,9 +810,9 @@ bool simParseGotoCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseRunCmd(char *rest, SCommand *pCmd, int lineNum) {
- char *token;
- int tokenLen;
+bool simParseRunCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
+ char * token;
+ int32_t tokenLen;
rest = paGetToken(rest, &token, &tokenLen);
@@ -832,14 +832,14 @@ bool simParseRunCmd(char *rest, SCommand *pCmd, int lineNum) {
return true;
}
-bool simParseRunBackCmd(char *rest, SCommand *pCmd, int lineNum) {
+bool simParseRunBackCmd(char *rest, SCommand *pCmd, int32_t lineNum) {
simParseRunCmd(rest, pCmd, lineNum);
cmdLine[numOfLines - 1].cmdno = SIM_CMD_RUN_BACK;
return true;
}
void simInitsimCmdList() {
- int cmdno;
+ int32_t cmdno;
memset(simCmdList, 0, SIM_CMD_END * sizeof(SCommand));
/* internal command */
diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c
index 17df7f306a47545f7283df1848f98ce18b87db79..693ade7b35a095b499198db9dcc27335f73e14bd 100644
--- a/tests/tsim/src/simSystem.c
+++ b/tests/tsim/src/simSystem.c
@@ -13,6 +13,7 @@
* along with this program. If not, see .
*/
+#define _DEFAULT_SOURCE
#include "os.h"
#include "sim.h"
#include "taos.h"
@@ -24,11 +25,11 @@
SScript *simScriptList[MAX_MAIN_SCRIPT_NUM];
SCommand simCmdList[SIM_CMD_END];
-int simScriptPos = -1;
-int simScriptSucced = 0;
-int simDebugFlag = 135;
-void simCloseTaosdConnect(SScript *script);
-char simHostName[128];
+int32_t simScriptPos = -1;
+int32_t simScriptSucced = 0;
+int32_t simDebugFlag = 135;
+void simCloseTaosdConnect(SScript *script);
+char simHostName[128];
char *simParseArbitratorName(char *varName) {
static char hostName[140];
@@ -39,8 +40,8 @@ char *simParseArbitratorName(char *varName) {
char *simParseHostName(char *varName) {
static char hostName[140];
- int index = atoi(varName + 8);
- int port = 7100;
+ int32_t index = atoi(varName + 8);
+ int32_t port = 7100;
switch (index) {
case 1:
port = 7100;
@@ -70,9 +71,9 @@ char *simParseHostName(char *varName) {
port = 7900;
break;
}
-
+
sprintf(hostName, "'%s:%d'", simHostName, port);
- //simInfo("hostName:%s", hostName);
+ // simInfo("hostName:%s", hostName);
return hostName;
}
@@ -88,39 +89,45 @@ void simSystemCleanUp() {}
void simFreeScript(SScript *script) {
if (script->type == SIM_SCRIPT_TYPE_MAIN) {
- for (int i = 0; i < script->bgScriptLen; ++i) {
+ simInfo("script:%s, background script num:%d, stop them", script->fileName, script->bgScriptLen);
+
+ for (int32_t i = 0; i < script->bgScriptLen; ++i) {
SScript *bgScript = script->bgScripts[i];
+ simInfo("script:%s, set stop flag", script->fileName);
bgScript->killed = true;
+ if (taosCheckPthreadValid(bgScript->bgPid)) {
+ pthread_join(bgScript->bgPid, NULL);
+ }
}
}
+ simDebug("script:%s, is freed", script->fileName);
taos_close(script->taos);
- taosTFree(script->lines);
- taosTFree(script->optionBuffer);
- taosTFree(script);
+ tfree(script->lines);
+ tfree(script->optionBuffer);
+ tfree(script);
}
SScript *simProcessCallOver(SScript *script) {
if (script->type == SIM_SCRIPT_TYPE_MAIN) {
if (script->killed) {
- simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX
- "failed" FAILED_POSTFIX ", error:%s",
- script->fileName, script->error);
+ simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX "failed" FAILED_POSTFIX ", error:%s",
+ script->fileName, script->error);
exit(-1);
} else {
- simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX
- "success" SUCCESS_POSTFIX,
- script->fileName);
+ simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX "success" SUCCESS_POSTFIX,
+ script->fileName);
simCloseTaosdConnect(script);
simScriptSucced++;
simScriptPos--;
+
+ simFreeScript(script);
if (simScriptPos == -1) {
simInfo("----------------------------------------------------------------------");
simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced);
exit(0);
}
- simFreeScript(script);
return simScriptList[simScriptPos];
}
} else {
@@ -143,11 +150,11 @@ void *simExecuteScript(void *inputScript) {
if (script == NULL) break;
} else {
SCmdLine *line = &script->lines[script->linePos];
- char *option = script->optionBuffer + line->optionOffset;
+ char * option = script->optionBuffer + line->optionOffset;
simDebug("script:%s, line:%d with option \"%s\"", script->fileName, line->lineNum, option);
SCommand *cmd = &simCmdList[line->cmdno];
- int ret = (*(cmd->executeCmd))(script, option);
+ int32_t ret = (*(cmd->executeCmd))(script, option);
if (!ret) {
script->killed = true;
}