diff --git a/.gitmodules b/.gitmodules index 7c84eac8a4ee7529005855bc836387561c49ae2d..049b39abfb2cf5f31abe10f194e7a09c4dc932f0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "src/connector/grafanaplugin"] path = src/connector/grafanaplugin url = https://github.com/taosdata/grafanaplugin +[submodule "src/connector/hivemq-tdengine-extension"] + path = src/connector/hivemq-tdengine-extension + url = https://github.com/huskar-t/hivemq-tdengine-extension.git diff --git a/Jenkinsfile b/Jenkinsfile index ea50d6ef5a13fed2114868f8a36cd0acace78dd5..dc7836c3daacaa457f721f9278687b99770fc394 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,6 +27,7 @@ pipeline { cd debug cmake .. > /dev/null make > /dev/null + make install > /dev/null cd ${WKC}/tests #./test-all.sh smoke ./test-all.sh pytest @@ -79,7 +80,20 @@ pipeline { cmake .. > /dev/null make > /dev/null cd ${WKC}/tests/pytest - ./crash_gen.sh -a -p -t 4 -s 2000 + ''' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + ./crash_gen.sh -a -p -t 4 -s 2000 + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + ./handle_crash_gen_val_log.sh + ''' + } + sh ''' date cd ${WKC}/tests ./test-all.sh b2 @@ -124,14 +138,33 @@ pipeline { sh''' cd ${WORKSPACE} git checkout develop - cd tests/gotest - bash batchtest.sh - cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/ - mvn clean package assembly:single >/dev/null - java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 - cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker - python3 PythonChecker.py ''' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WORKSPACE}/tests/gotest + bash batchtest.sh + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker + python3 PythonChecker.py + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/ + mvn clean package assembly:single >/dev/null + java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC# + dotnet run + ''' + } + } } @@ -139,5 +172,82 @@ pipeline { } } - + post { + success { + emailext ( + subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支:${PROJECT_NAME}
  • +
  • 构建结果: Successful
  • +
  • 构建编号:${BUILD_NUMBER}
  • +
  • 触发用户:${CAUSE}
  • +
  • 变更概要:${CHANGES}
  • +
  • 构建地址:${BUILD_URL}
  • +
  • 构建日志:${BUILD_URL}console
  • +
  • 变更集:${JELLY_SCRIPT}
  • +
    +
+
+ + ''', + to: "yqliu@taosdata.com,pxiao@taosdata.com", + from: "support@taosdata.com" + ) + } + failure { + emailext ( + subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支:${PROJECT_NAME}
  • +
  • 构建结果: Successful
  • +
  • 构建编号:${BUILD_NUMBER}
  • +
  • 触发用户:${CAUSE}
  • +
  • 变更概要:${CHANGES}
  • +
  • 构建地址:${BUILD_URL}
  • +
  • 构建日志:${BUILD_URL}console
  • +
  • 变更集:${JELLY_SCRIPT}
  • +
    +
+
+ + ''', + to: "yqliu@taosdata.com,pxiao@taosdata.com", + from: "support@taosdata.com" + ) + } + } } \ No newline at end of file diff --git a/cmake/install.inc b/cmake/install.inc index dfca758b9362c96bec0ce45aa385d54a4e75a9e5..9bbcc2cf40520a9ab5e7aa4d19f46b47b6b42192 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -13,6 +13,7 @@ ELSEIF (TD_WINDOWS) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/nodejs DESTINATION connector) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/C\# DESTINATION connector) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include) diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md index 44b3ad46712019870be6fefb5234611f55f6e03e..79388a2edb9404a0f7b31b9182eb5ce2cb0d52be 100644 --- a/documentation/webdocs/markdowndocs/administrator-ch.md +++ b/documentation/webdocs/markdowndocs/administrator-ch.md @@ -87,6 +87,7 @@ TDengine系统后台服务由taosd提供,可以在配置文件taos.cfg里修 - httpPort: RESTful服务使用的端口号,所有的HTTP请求(TCP)都需要向该接口发起查询/写入请求。 - dataDir: 数据文件目录,所有的数据文件都将写入该目录。默认值:/var/lib/taos。 - logDir:日志文件目录,客户端和服务器的运行日志文件将写入该目录。默认值:/var/log/taos。 +- tempDir:临时文件目录,客户端和服务器的临时文件(主要是查询时用于保存中间结果的问题)将写入该目录。 默认值:Linux下为 /tmp/,Windows下为环境变量 tmp 或 temp 指向的目录。 - arbitrator:系统中裁决器的end point, 缺省值为空。 - role:dnode的可选角色。0-any; 既可作为mnode,也可分配vnode;1-mgmt;只能作为mnode,不能分配vnode;2-dnode;不能作为mnode,只能分配vnode - debugFlag:运行日志开关。131(输出错误和警告日志),135( 输出错误、警告和调试日志),143( 输出错误、警告、调试和跟踪日志)。默认值:131或135(不同模块有不同的默认值)。 diff --git a/documentation20/webdocs/markdowndocs/Documentation-ch.md b/documentation20/webdocs/markdowndocs/Documentation-ch.md index 077a0431383da8067b6bd33c4b7218627b7adbd0..f1f2d58f0562b831395e5177eb34ec3534aaa715 100644 --- a/documentation20/webdocs/markdowndocs/Documentation-ch.md +++ b/documentation20/webdocs/markdowndocs/Documentation-ch.md @@ -34,7 +34,8 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台,专 - [SQL写入](https://www.taosdata.com/cn/documentation20/insert/#SQL写入):使用SQL insert命令向一张或多张表写入单条或多条记录 - [Telegraf写入](https://www.taosdata.com/cn/documentation20/insert/#Telegraf直接写入):配置Telegraf, 不用任何代码,将采集数据直接写入 - [Prometheus写入](https://www.taosdata.com/cn/documentation20/insert/#Prometheus直接写入):配置Prometheus, 不用任何代码,将数据直接写入 -- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入):配置EMQ X,不用任何代码,就可将MQTT数据直接写入 +- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入):配置EMQ X,不用任何代码,就可将 MQTT 数据直接写入 +- [HiveMQ Broker](https://www.taosdata.com/cn/documentation20/insert/#HiveMQ-Broker直接写入):通过 HiveMQ Extension,不用任何代码,就可将 MQTT 数据直接写入 ## [高效查询数据](https://www.taosdata.com/cn/documentation20/queries) diff --git a/documentation20/webdocs/markdowndocs/Evaluation-ch.md b/documentation20/webdocs/markdowndocs/Evaluation-ch.md index 9e7e0ec6aa8b60bfa9dbe603a45a265a1d1eba00..a92f97a8d9dfc9a47d5554daa076f4d2f0774c92 100644 --- a/documentation20/webdocs/markdowndocs/Evaluation-ch.md +++ b/documentation20/webdocs/markdowndocs/Evaluation-ch.md @@ -10,7 +10,7 @@ TDengine的模块之一是时序数据库。但除此之外,为减少研发的 * __硬件或云服务成本降至1/5__:由于超强性能,计算资源不到通用大数据方案的1/5;通过列式存储和先进的压缩算法,存储空间不到通用数据库的1/10。 * __全栈时序数据处理引擎__:将数据库、消息队列、缓存、流式计算等功能融为一体,应用无需再集成Kafka/Redis/HBase/Spark/HDFS等软件,大幅降低应用开发和维护的复杂度成本。 * __强大的分析功能__:无论是十年前还是一秒钟前的数据,指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell, Python, R, Matlab随时进行。 -* __与第三方工具无缝连接__:不用一行代码,即可与Telegraf, Grafana, EMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。 +* __与第三方工具无缝连接__:不用一行代码,即可与Telegraf, Grafana, EMQ, HiveMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。 * __零运维成本、零学习成本__:安装集群简单快捷,无需分库分表,实时备份。类似标准SQL,支持RESTful, 支持Python/Java/C/C++/C#/Go/Node.js, 与MySQL相似,零学习成本。 采用TDengine,可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是,因充分利用了物联网时序数据的特点,它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。 diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md index 4082d72f112e8ff62d2e8b2c8b15391dd6f39d8a..905d3b2cd72044e621c125641ca053d1cda809d9 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md @@ -844,7 +844,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 - **PERCENTILE** ```mysql - SELECT PERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause]; + SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; ``` 功能说明:统计表中某列的值百分比分位数。 返回结果数据类型: 双精度浮点数Double。 @@ -1016,9 +1016,9 @@ SELECT AVG(current),MAX(current),LEASTSQUARES(current, start_val, step_val), PER ``` ## TAOS SQL 边界限制 -- 数据库名最大长度为33 -- 表名最大长度为193,每行数据最大长度16k个字符 -- 列名最大长度为65,最多允许1024列,最少需要2列,第一列必须是时间戳 +- 数据库名最大长度为32 +- 表名最大长度为192,每行数据最大长度16k个字符 +- 列名最大长度为64,最多允许1024列,最少需要2列,第一列必须是时间戳 - 标签最多允许128个,可以0个,标签总长度不超过16k个字符 - SQL语句最大长度65480个字符,但可通过系统配置参数maxSQLLength修改,最长可配置为1M - 库的数目,超级表的数目、表的数目,系统不做限制,仅受系统资源限制 diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 4b274e05e6e392879fc887ec1137968270a8e4b8..36466d2b7ea29307b5c17ad21b0db47a98598fa4 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -35,7 +35,7 @@ TDengine相对于通用数据库,有超高的压缩比,在绝大多数场景 Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable ``` -示例:1000万台智能电表,每台电表每15分钟采集一次数据,每次采集的数据128字节,那么一年的原始数据量是:10000000\*128\*24\*60/15*365 = 44851T。TDengine大概需要消耗44851/5=8970T, 8.9P空间。 +示例:1000万台智能电表,每台电表每15分钟采集一次数据,每次采集的数据128字节,那么一年的原始数据量是:10000000\*128\*24\*60/15*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。 用户可以通过参数keep,设置数据在磁盘中的最大保存时长。为进一步减少存储成本,TDengine还提供多级存储,最冷的数据可以存放在最廉价的存储介质上,应用的访问不用做任何调整,只是读取速度降低了。 @@ -253,7 +253,7 @@ ALTER USER PASS <'password'>; 修改用户密码, 为避免被转换为小写,密码需要用单引号引用,单引号为英文半角 ``` -ALTER USER PRIVILEDGE ; +ALTER USER PRIVILEGE ; ``` 修改用户权限为:super/write/read,不需要添加单引号 diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index d4705ccb05c092d8da38072368a167466bd78968..3721d3c2cd7d28327887c8fe90b53b8c2f63e564 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -221,7 +221,7 @@ TDengine采用时间驱动缓存管理策略(First-In-First-Out,FIFO), TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中,可以更加快速地响应用户针对最近一条或一批数据的查询分析,整体上提供更快的数据库查询响应能力。从这个意义上来说,**可通过设置合适的配置参数将TDengine作为数据缓存来使用,而不需要再部署Redis或其他额外的缓存系统**,可有效地简化系统架构,降低运维的成本。需要注意的是,TDengine重启以后系统的缓存将被清空,之前缓存的数据均会被批量写入磁盘,缓存的数据将不会像专门的Key-value缓存系统再将之前缓存的数据重新加载到缓存中。 -每个vnode有自己独立的内存,而且由多个固定大小的内存块组成,不同vnode之间完全隔离。数据写入时,类似于日志的写法,数据被顺序追加写入内存,但每个vnode维护有自己的skip list,便于迅速查找。当一半以上的内存块写满时,启动落盘操作,而且后续写的操作在新的内存块进行。这样,一个vnode里有一半内存块是保留有最近的数据的,以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定,内存块的大小由配置参数cache决定。 +每个vnode有自己独立的内存,而且由多个固定大小的内存块组成,不同vnode之间完全隔离。数据写入时,类似于日志的写法,数据被顺序追加写入内存,但每个vnode维护有自己的skip list,便于迅速查找。当三分之一以上的内存块写满时,启动落盘操作,而且后续写的操作在新的内存块进行。这样,一个vnode里有三分之一内存块是保留有最近的数据的,以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定,内存块的大小由配置参数cache决定。 ### 持久化存储 TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久化存储。当vnode中缓存的数据达到一定规模时,为了不阻塞后续数据的写入,TDengine也会拉起落盘线程将缓存的数据写入持久化存储。TDengine在数据落盘时会打开新的数据库日志文件,在落盘成功后则会删除老的数据库日志文件,避免日志文件无限制的增长。 diff --git a/documentation20/webdocs/markdowndocs/connector-ch.md b/documentation20/webdocs/markdowndocs/connector-ch.md index 0e29b324871e61d184fde17710c863f06b60bdcc..c5a955f43fcc30e70d8aac3919433c2e767f76ba 100644 --- a/documentation20/webdocs/markdowndocs/connector-ch.md +++ b/documentation20/webdocs/markdowndocs/connector-ch.md @@ -616,6 +616,43 @@ HTTP请求URL采用`sqlutc`时,返回结果集的时间戳将采用UTC时间 - httpEnableCompress: 是否支持压缩,默认不支持,目前TDengine仅支持gzip压缩格式 - httpDebugFlag: 日志开关,131:仅错误和报警信息,135:调试信息,143:非常详细的调试信息,默认131 +## CSharp Connector + +在Windows系统上,C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作,后续版本将提供ORM(dapper)框架驱动。 + +#### 安装TDengine客户端 + +C#连接器需要使用`libtaos.so`和`taos.h`。因此,在使用C#连接器之前,需在程序运行的Windows环境安装TDengine的Windows客户端,以便获得相关驱动文件。 + +安装完成后,在文件夹`C:/TDengine/examples/C#`中,将会看到两个文件 + +- TDengineDriver.cs 调用taos.dll文件的Native C方法 +- TDengineTest.cs 参考程序示例 + +在文件夹`C:\Windows\System32`,将会看到`taos.dll`文件 + +#### 使用方法 + +- 将C#接口文件TDengineDriver.cs加入到应用程序所在.NET项目中 +- 参考TDengineTest.cs来定义数据库连接参数,及执行数据插入、查询等操作的方法 +- 因为C#接口需要用到`taos.dll`文件,用户可以将`taos.dll`文件加入.NET解决方案中 + +#### 注意事项 + +- `taos.dll`文件使用x64平台编译,所以.NET项目在生成.exe文件时,“解决方案”/“项目”的“平台”请均选择“x64”。 +- 此.NET接口目前已经在Visual Studio 2013/2015/2017中验证过,其它VS版本尚待验证。 + +#### 第三方驱动 + +Maikebing.Data.Taos是一个TDengine的ADO.Net提供器,支持linux,windows。该开发包由热心贡献者`麦壳饼@@maikebing`提供,具体请参考 + +``` +//接口下载 +https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos +//用法说明 +https://www.taosdata.com/blog/2020/11/02/1901.html +``` + ## Go Connector diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 80deb889ef0eee8e9b47f86b5e58a76c6c070d5b..757b6d9929b68a90e7bc558b553233f24b09ba17 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -38,9 +38,9 @@ 6. 检查防火墙设置,确认TCP/UDP 端口6030-6042 是打开的 -7. 对于Linux上的JDBC(ODBC, Python, Go等接口类似)连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 +7. 对于Linux上的JDBC(ODBC, Python, Go等接口类似)连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 -8. 对于windows上的JDBC, ODBC, Python, Go等连接,确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) +8. 对于windows上的JDBC, ODBC, Python, Go等连接,确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) 9. 如果仍不能排除连接故障,请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅 检查UDP端口连接是否工作:`nc -vuz {hostIP} {port} ` diff --git a/documentation20/webdocs/markdowndocs/insert-ch.md b/documentation20/webdocs/markdowndocs/insert-ch.md index fa53cbd62b17169c0f54877a62da8c48ac21edcf..77ba596d4ec6de76a39b878e57ea255b081fea45 100644 --- a/documentation20/webdocs/markdowndocs/insert-ch.md +++ b/documentation20/webdocs/markdowndocs/insert-ch.md @@ -1,6 +1,6 @@ # 高效写入数据 -TDengine支持多种接口写入数据,包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, CSV文件等,后续还将提供Kafka, OPC等接口。数据可以单条插入,也可以批量插入,可以插入一个数据采集点的数据,也可以同时插入多个数据采集点的数据。支持多线程插入,支持时间乱序数据插入,也支持历史数据插入。 +TDengine支持多种接口写入数据,包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, HiveMQ Broker, CSV文件等,后续还将提供Kafka, OPC等接口。数据可以单条插入,也可以批量插入,可以插入一个数据采集点的数据,也可以同时插入多个数据采集点的数据。支持多线程插入,支持时间乱序数据插入,也支持历史数据插入。 ## SQL写入 @@ -218,7 +218,15 @@ use telegraf; select * from cpu; ``` -## EMQ X Broker直接写入 -MQTT是一流行的物联网数据传输协议,[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件,无需任何代码,只需要在EMQ Dashboard里使用“规则”做简单配置,即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDEngine,也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。 + +MQTT是一流行的物联网数据传输协议,TDengine 可以很方便的接入 MQTT Broker 接受的数据并写入到 TDengine。 + +## EMQ Broker 直接写入 + +[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件,无需任何代码,只需要在EMQ Dashboard里使用“规则”做简单配置,即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine,也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。 + +## HiveMQ Broker 直接写入 + +[HiveMQ](https://www.hivemq.com/) 是一个提供免费个人版和企业版的 MQTT 代理,主要用于企业和新兴的机器到机器M2M通讯和内部传输,满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 [HiveMQ extension - TDengine 说明文档](https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md)。 diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 974b2b05c12f9fd5a482ef0072a9dacc2d1f09dc..ca88bca3c863dcfd4a95497fc572499dc93f31f9 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -20,6 +20,9 @@ # data file's directory # dataDir /var/lib/taos +# temporary file's directory +# tempDir /tmp/ + # the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only # arbitrator arbitrator_hostname:6042 @@ -256,3 +259,5 @@ # maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden # maxBinaryDisplayWidth 30 +# enable/disable telemetry reporting +# telemetryReporting 1 \ No newline at end of file diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 450c6a8f551fe7dd8dff5e53f5dad639aec4656e..516d91258a759545842a6086a2966e02e32826a0 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -48,6 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_pat cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin +#cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver @@ -58,7 +59,7 @@ cp -r ${top_dir}/src/connector/grafanaplugin ${pkg_dir}${install_home_pat cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector -cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.* ${pkg_dir}${install_home_path}/connector +cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.* ${pkg_dir}${install_home_path}/connector ||: cp -r ${compile_dir}/../packaging/deb/DEBIAN ${pkg_dir}/ chmod 755 ${pkg_dir}/DEBIAN/* diff --git a/packaging/release.sh b/packaging/release.sh index 7542a5b4cafb69d5cee16bddfc9a5651eb717b92..68f947ccab3ef18a1b351b91a58db64a8f465c8e 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -156,9 +156,15 @@ build_time=$(date +"%F %R") # get commint id from git gitinfo=$(git rev-parse --verify HEAD) -enterprise_dir="${top_dir}/../enterprise" -cd ${enterprise_dir} -gitinfoOfInternal=$(git rev-parse --verify HEAD) + +if [[ "$verMode" == "cluster" ]]; then + enterprise_dir="${top_dir}/../enterprise" + cd ${enterprise_dir} + gitinfoOfInternal=$(git rev-parse --verify HEAD) +else + gitinfoOfInternal=NULL +fi + cd ${curr_dir} # 2. cmake executable file @@ -193,23 +199,35 @@ cd ${curr_dir} # 3. Call the corresponding script for packaging if [ "$osType" != "Darwin" ]; then if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]] && [[ "$dbName" == "taos" ]]; then - echo "====do deb package for the ubuntu system====" - output_dir="${top_dir}/debs" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} + ret='0' + command -v dpkg >/dev/null 2>&1 || { ret='1'; } + if [ "$ret" -eq 0 ]; then + echo "====do deb package for the ubuntu system====" + output_dir="${top_dir}/debs" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} + fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/deb + ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + else + echo "==========dpkg command not exist, so not release deb package!!!" fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/deb - ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} - - echo "====do rpm package for the centos system====" - output_dir="${top_dir}/rpms" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} + + ret='0' + command -v rpmbuild >/dev/null 2>&1 || { ret='1'; } + if [ "$ret" -eq 0 ]; then + echo "====do rpm package for the centos system====" + output_dir="${top_dir}/rpms" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} + fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/rpm + ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + else + echo "==========rpmbuild command not exist, so not release rpm package!!!" fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/rpm - ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} fi echo "====do tar.gz package for all systems====" diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 4e40263dc4ebaf6b566d20890ecd97f64e160340..19d932e0745e4f3106f10bc360fb110a0db12c3b 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -2,7 +2,7 @@ %define cfg_install_dir /etc/taos %define __strip /bin/true -Name: TDengine +Name: tdengine Version: %{_version} Release: 3%{?dist} Summary: tdengine from taosdata @@ -58,6 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/scri cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin +#cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include @@ -65,7 +66,7 @@ cp -r %{_compiledir}/../src/connector/grafanaplugin %{buildroot}%{homepath}/conn cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector -cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.* %{buildroot}%{homepath}/connector +cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.* %{buildroot}%{homepath}/connector ||: cp -r %{_compiledir}/../tests/examples/* %{buildroot}%{homepath}/examples #Scripts executed before installation @@ -134,6 +135,7 @@ if [ $1 -eq 0 ];then ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + #${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : ${csudo} rm -f ${inc_link_dir}/taoserror.h || : diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index eff70d8035af0291f6dc7040ec13632fec4fa3be..831012851ad70d05080bfae161c0b925d5215ae9 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -278,11 +278,11 @@ function install_service_on_sysvinit() { # Install taosd service if ((${os_type}==1)); then - ${csudo} cp -f ${script_dir}/../deb/init.d/taosd ${install_main_dir}/init.d - ${csudo} cp ${script_dir}/../deb/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd + ${csudo} cp -f ${script_dir}/../deb/taosd ${install_main_dir}/init.d + ${csudo} cp ${script_dir}/../deb/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd elif ((${os_type}==2)); then - ${csudo} cp -f ${script_dir}/../rpm/init.d/taosd ${install_main_dir}/init.d - ${csudo} cp ${script_dir}/../rpm/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd + ${csudo} cp -f ${script_dir}/../rpm/taosd ${install_main_dir}/init.d + ${csudo} cp ${script_dir}/../rpm/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd fi #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index e17c678f263cb6b7a0ccbc32250265b9bc5cbd0e..4f4c6dee1dfc82dfaa486efb6c0447274a9eb4bd 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -45,6 +45,7 @@ if [ "$osType" != "Darwin" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" else + #bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" @@ -110,7 +111,7 @@ mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then - cp ${build_dir}/lib/*.jar ${install_dir}/connector + cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ cp -r ${connector_dir}/python ${install_dir}/connector/ diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index faa5a03f52b4c9b56981a6b1c0918e543262b3bb..7e8bef0dffffa3028c82ed00ab5695970222770d 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -135,7 +135,7 @@ mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then - cp ${build_dir}/lib/*.jar ${install_dir}/connector + cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ cp -r ${connector_dir}/python ${install_dir}/connector/ diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 75b45b544e0a4abbf709cc4c5b3a3b55dc315f0f..2be60709aa2ca3191542fd24d03599cd7cbb2895 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -36,6 +36,7 @@ if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh" else + #bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh" bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh" fi @@ -124,7 +125,7 @@ cp ${lib_files} ${install_dir}/driver connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then - cp ${build_dir}/lib/*.jar ${install_dir}/connector + cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ cp -r ${connector_dir}/python ${install_dir}/connector/ cp -r ${connector_dir}/go ${install_dir}/connector diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index 2c02b99787c6d5ad6234de2319bf78b0b09d7e8a..0ffcb63e3f93c300ce41d627ccf43b7f089d6c2e 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -156,7 +156,7 @@ cp ${lib_files} ${install_dir}/driver connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then - cp ${build_dir}/lib/*.jar ${install_dir}/connector + cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ cp -r ${connector_dir}/python ${install_dir}/connector/ cp -r ${connector_dir}/go ${install_dir}/connector diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index d91daaa5c44488e34dea7ec2ddec0863699446f2..00705fad778c065eddbd7cb65a2f9c5583a9997b 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -81,8 +81,10 @@ function install_lib() { ${csudo} ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - ${csudo} ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : - ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then + ${csudo} ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi } function install_bin() { @@ -121,8 +123,11 @@ function install_config() { echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join${NC}" echo echo -e -n "${GREEN}OR leave it blank to build one${NC}:" - read firstEp - while true; do + #read firstEp + if exec < /dev/tty; then + read firstEp; + fi + while true; do if [ ! -z "$firstEp" ]; then # check the format of the firstEp #if [[ $firstEp == $FQDN_PATTERN ]]; then diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c index 0e9bb85b25defd169fea8711d3e0b40304500de4..df78f4fe270d34d17e436e80322f6c81b68d0d2c 100644 --- a/src/balance/src/balance.c +++ b/src/balance/src/balance.c @@ -490,7 +490,7 @@ static bool balanceMontiorDropping() { if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { if (pDnode->lastAccess + tsOfflineThreshold > tsAccessSquence) continue; - if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) continue; + if (dnodeIsMasterEp(pDnode->dnodeEp)) continue; if (mnodeGetDnodesNum() <= 1) continue; mLInfo("dnode:%d, set to removing state for it offline:%d seconds", pDnode->dnodeId, @@ -571,8 +571,8 @@ static void balanceCheckDnodeAccess() { if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) { pDnode->status = TAOS_DN_STATUS_OFFLINE; pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT; - mInfo("dnode:%d, set to offline state, access seq:%d, last seq:%d", pDnode->dnodeId, tsAccessSquence, - pDnode->lastAccess); + mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence, + pDnode->lastAccess, pDnode->status); balanceSetVgroupOffline(pDnode); } } diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index 5baa66a9e0229f35c431cea7a0d2dbb9e2ffb0e2..0af8c8b5768c844954a072521e67a3cb6e914ac3 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -64,9 +64,8 @@ typedef struct SLocalReducer { SColumnModel * resColModel; tExtMemBuffer ** pExtMemBuffer; // disk-based buffer SFillInfo* pFillInfo; // interpolation support structure - char * pFinalRes; // result data after interpo - tFilePage * discardData; - SResultInfo * pResInfo; + char* pFinalRes; // result data after interpo + tFilePage* discardData; bool discard; int32_t offset; // limit offset value bool orderPrjOnSTable; // projection query on stable diff --git a/src/client/inc/tscSubquery.h b/src/client/inc/tscSubquery.h index bc01de110345e4c90cf5c15d3d7f6b010cb7308d..3406dcd8589e149f2a62040d1271fbfd40caaac7 100644 --- a/src/client/inc/tscSubquery.h +++ b/src/client/inc/tscSubquery.h @@ -23,7 +23,7 @@ extern "C" { #include "tscUtil.h" #include "tsclient.h" -void tscFetchDatablockFromSubquery(SSqlObj* pSql); +void tscFetchDatablockForSubquery(SSqlObj* pSql); void tscSetupOutputColumnIndex(SSqlObj* pSql); void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code); diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 3df493349e387c1dd33a945886389218f7f7cf0f..6628e3087456c52b2daf1b9ef84efa9577ad4f36 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -75,6 +75,7 @@ typedef struct SJoinSupporter { SArray* exprList; SFieldInfo fieldsInfo; STagCond tagCond; + SSqlGroupbyExpr groupInfo; // group by info struct STSBuf* pTSBuf; // the TSBuf struct that holds the compressed timestamp array FILE* f; // temporary file in order to create TSBuf char path[PATH_MAX]; // temporary file path, todo dynamic allocate memory @@ -82,11 +83,12 @@ typedef struct SJoinSupporter { char* pIdTagList; // result of first stage tags int32_t totalLen; int32_t num; + SArray* pVgroupTables; } SJoinSupporter; typedef struct SVgroupTableInfo { - SCMVgroupInfo vgInfo; - SArray* itemList; //SArray + SVgroupInfo vgInfo; + SArray* itemList; //SArray } SVgroupTableInfo; static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) { @@ -215,7 +217,7 @@ SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex); void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, SArray* pTagCols); + SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables); STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); @@ -224,6 +226,9 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo); void tscClearSubqueryInfo(SSqlCmd* pCmd); void tscFreeVgroupTableInfo(SArray* pVgroupTables); +SArray* tscVgroupTableInfoClone(SArray* pVgroupTables); +void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index); +void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); @@ -234,7 +239,7 @@ void tscDoQuery(SSqlObj* pSql); SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *pInfo); void* tscVgroupInfoClear(SVgroupsInfo *pInfo); -void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src); +void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src); /** * The create object function must be successful expect for the out of memory issue. * @@ -262,6 +267,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t sub void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex); int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid); +int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId); void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex); diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h index 67942ad42a0942756efe18e44eff711df59ba1d9..f6dc45398f35c38598f3f3132b2f6f5601a4ed68 100644 --- a/src/client/inc/tschemautil.h +++ b/src/client/inc/tschemautil.h @@ -77,7 +77,7 @@ SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex); * @param colId * @return */ -SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId); +SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId); /** * check if the schema is valid or not, including following aspects: @@ -107,9 +107,6 @@ SSchema tscGetTbnameColumnSchema(); */ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size); -//todo tags value as well as the table id structure needs refactor -char *tsGetTagsValue(STableMeta *pMeta); - #ifdef __cplusplus } #endif diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index fa215db2702f63aff4c704f8393255a94987d681..4c85af4919c74de5f41272272bd1bdbf4f6c6da3 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -30,6 +30,7 @@ extern "C" { #include "tsqlfunction.h" #include "tutil.h" #include "tcache.h" +#include "tref.h" #include "qExecutor.h" #include "qSqlparser.h" @@ -89,12 +90,12 @@ typedef struct STableComInfo { int32_t rowSize; } STableComInfo; -typedef struct SCMCorVgroupInfo { - int32_t version; - int8_t inUse; - int8_t numOfEps; - SEpAddr1 epAddr[TSDB_MAX_REPLICA]; -} SCMCorVgroupInfo; +typedef struct SCorVgroupInfo { + int32_t version; + int8_t inUse; + int8_t numOfEps; + SEpAddr1 epAddr[TSDB_MAX_REPLICA]; +} SCorVgroupInfo; typedef struct STableMeta { STableComInfo tableInfo; @@ -102,8 +103,8 @@ typedef struct STableMeta { int16_t sversion; int16_t tversion; char sTableId[TSDB_TABLE_FNAME_LEN]; - SCMVgroupInfo vgroupInfo; - SCMCorVgroupInfo corVgroupInfo; + SVgroupInfo vgroupInfo; + SCorVgroupInfo corVgroupInfo; STableId id; SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info } STableMeta; @@ -127,7 +128,7 @@ typedef struct STableMetaInfo { typedef struct SSqlExpr { char aliasName[TSDB_COL_NAME_LEN]; // as aliasName SColIndex colInfo; - int64_t uid; // refactor use the pointer + uint64_t uid; // refactor use the pointer int16_t functionId; // function id in aAgg array int16_t resType; // return value type int16_t resBytes; // length of return value @@ -329,6 +330,7 @@ typedef struct STscObj { char writeAuth : 1; char superAuth : 1; uint32_t connId; + uint64_t rid; // ref ID returned by taosAddRef struct SSqlObj * pHb; struct SSqlObj * sqlList; struct SSqlStream *streamList; @@ -338,16 +340,16 @@ typedef struct STscObj { } STscObj; typedef struct SSubqueryState { - int32_t numOfRemain; // the number of remain unfinished subquery - int32_t numOfSub; // the number of total sub-queries - uint64_t numOfRetrievedRows; // total number of points in this query + int32_t numOfRemain; // the number of remain unfinished subquery + int32_t numOfSub; // the number of total sub-queries + uint64_t numOfRetrievedRows; // total number of points in this query } SSubqueryState; typedef struct SSqlObj { void *signature; pthread_t owner; // owner of sql object, by which it is executed STscObj *pTscObj; - void *pRpcCtx; + int64_t rpcRid; void (*fp)(); void (*fetchFp)(); void *param; @@ -430,14 +432,6 @@ void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache); */ void tscFreeSqlResult(SSqlObj *pSql); -/** - * only free part of resources allocated during query. - * TODO remove it later - * Note: this function is multi-thread safe. - * @param pObj - */ -void tscPartiallyFreeSqlObj(SSqlObj *pSql); - /** * free sql object, release allocated resource * @param pObj @@ -446,7 +440,7 @@ void tscFreeSqlObj(SSqlObj *pSql); void tscFreeRegisteredSqlObj(void *pSql); void tscFreeTableMetaHelper(void *pTableMeta); -void tscCloseTscObj(STscObj *pObj); +void tscCloseTscObj(void *pObj); // todo move to taos? or create a new file: taos_internal.h TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), @@ -516,12 +510,12 @@ extern void * tscQhandle; extern int tscKeepConn[]; extern int tsInsertHeadSize; extern int tscNumOfThreads; +extern int tscRefId; extern SRpcCorEpSet tscMgmtEpSet; extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); -int32_t tscCompareTidTags(const void* p1, const void* p2); void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables); #ifdef __cplusplus diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index c996bb2a76ff505fca4e09cd8763b324e5c4cb8d..99c03c65801a64eb6b913373603cfa9c48d36911 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -176,7 +176,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo } if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscFetchDatablockFromSubquery(pSql); + tscFetchDatablockForSubquery(pSql); } else { tscProcessSql(pSql); } @@ -226,7 +226,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) { // handle the sub queries of join query if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscFetchDatablockFromSubquery(pSql); + tscFetchDatablockForSubquery(pSql); } else if (pRes->completed) { if(pCmd->command == TSDB_SQL_FETCH || (pCmd->command >= TSDB_SQL_SERV_STATUS && pCmd->command <= TSDB_SQL_CURRENT_USER)) { if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes. @@ -405,7 +405,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { SSqlRes *pRes = &pSql->res; pRes->code = code; - const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta"; + SSqlObj *sub = (SSqlObj*) res; + const char* msg = (sub->cmd.command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta"; if (code != TSDB_CODE_SUCCESS) { tscError("%p get %s failed, code:%s", pSql, msg, tstrerror(code)); goto _error; @@ -427,8 +428,12 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { } else { assert(code == TSDB_CODE_SUCCESS); } - - assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pSql->param != NULL); + + // param already freed by other routine and pSql in tscCache when ctrl + c + if (atomic_load_ptr(&pSql->param) == NULL) { + return; + } + assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0)); SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param; SSqlObj * pParObj = trs->pParentSql; @@ -437,6 +442,20 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex && pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL); + // tscProcessSql can add error into async res + tscProcessSql(pSql); + return; + } else if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) { + tscDebug("%p update table meta in local cache, continue to process sql and send corresponding tid_tag query", pSql); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + code = tscGetTableMeta(pSql, pTableMetaInfo); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + return; + } else { + assert(code == TSDB_CODE_SUCCESS); + } + + assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0)); // tscProcessSql can add error into async res tscProcessSql(pSql); return; @@ -461,7 +480,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { tscResetSqlCmdObj(pCmd, false); code = tsParseSql(pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { return; } else if (code != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 1b4f92d3fc9fc78951e23eaec4c438d7522bf7f1..39cc753d404925fdf74efc26e48d54df61af664e 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -99,7 +99,7 @@ typedef struct SSumInfo { // the attribute of hasResult is not needed since the num attribute would server as this purpose typedef struct SAvgInfo { double sum; - int64_t num; // num servers as the hasResult attribute in other struct + int64_t num; } SAvgInfo; typedef struct SStddevInfo { @@ -167,7 +167,13 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - *interBytes = *bytes + sizeof(SResultInfo); + + if (functionId == TSDB_FUNC_INTERP) { + *interBytes = sizeof(SInterpInfoDetail); + } else { + *interBytes = 0; + } + return TSDB_CODE_SUCCESS; } @@ -175,21 +181,21 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI if (functionId == TSDB_FUNC_TID_TAG) { // todo use struct *type = TSDB_DATA_TYPE_BINARY; *bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE); - *interBytes = *bytes; + *interBytes = 0; return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_COUNT) { *type = TSDB_DATA_TYPE_BIGINT; *bytes = sizeof(int64_t); - *interBytes = *bytes; + *interBytes = 0; return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_ARITHM) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); - *interBytes = *bytes; + *interBytes = 0; return TSDB_CODE_SUCCESS; } @@ -298,7 +304,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - *interBytes = dataBytes + sizeof(SResultInfo); + *interBytes = dataBytes; } else if (functionId == TSDB_FUNC_SPREAD) { *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); @@ -310,7 +316,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else if (functionId == TSDB_FUNC_LEASTSQR) { *type = TSDB_DATA_TYPE_BINARY; *bytes = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE; // string - *interBytes = *bytes + sizeof(SResultInfo); + *interBytes = *bytes; } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { *type = TSDB_DATA_TYPE_BINARY; *bytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); @@ -334,28 +340,20 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } -void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf) { - assert(pResInfo->interResultBuf == NULL); - - pResInfo->bufLen = size; - pResInfo->superTableQ = superTable; - pResInfo->interResultBuf = buf; -} - // set the query flag to denote that query is completed static void no_next_step(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->complete = true; } static bool function_setup(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->initialized) { return false; } memset(pCtx->aOutputBuf, 0, (size_t)pCtx->outputBytes); - initResultInfo(pResInfo); + initResultInfo(pResInfo, pCtx->interBufBytes); return true; } @@ -367,7 +365,7 @@ static bool function_setup(SQLFunctionCtx *pCtx) { * @param pCtx */ static void function_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { setVardataNull(pCtx->aOutputBuf, pCtx->outputType); @@ -431,7 +429,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { *((int64_t *)pCtx->aOutputBuf) += 1; // do not need it actually - SResultInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; } @@ -592,8 +590,8 @@ static void sum_function(SQLFunctionCtx *pCtx) { do_sum(pCtx); // keep the result data in output buffer, not in the intermediate buffer - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { // set the flag for super table query SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf; pSum->hasResult = DATA_SET_FLAG; @@ -604,8 +602,8 @@ static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { do_sum_f(pCtx, index); // keep the result data in output buffer, not in the intermediate buffer - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf; pSum->hasResult = DATA_SET_FLAG; } @@ -615,8 +613,7 @@ static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; GET_TRUE_DATA_TYPE(); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + assert(pCtx->stableQuery); for (int32_t i = 0; i < pCtx->size; ++i) { char * input = GET_INPUT_CHAR_INDEX(pCtx, i); @@ -661,7 +658,7 @@ static void sum_func_second_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = sum_merge_impl(pCtx); SET_VAL(pCtx, notNullElems, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; @@ -755,9 +752,9 @@ static void avg_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; // NOTE: keep the intermediate result into the interResultBuf - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); double * pVal = &pAvgInfo->sum; if (pCtx->preAggVals.isSet) { @@ -800,8 +797,8 @@ static void avg_function(SQLFunctionCtx *pCtx) { } // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); } } @@ -814,9 +811,9 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); // NOTE: keep the intermediate result into the interResultBuf - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { pAvgInfo->sum += GET_INT8_VAL(pData); @@ -839,16 +836,16 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->hasResult = DATA_SET_FLAG; // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); } } static void avg_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->stableQuery); - SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); char * input = GET_INPUT_CHAR(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { @@ -864,12 +861,12 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) { // if the data set hasResult is not set, the result is null if (pAvgInfo->num > 0) { pResInfo->hasResult = DATA_SET_FLAG; - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); } } static void avg_func_second_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); double *sum = (double*) pCtx->aOutputBuf; char * input = GET_INPUT_CHAR(pCtx); @@ -883,7 +880,7 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) { *sum += pInput->sum; // keep the number of data into the temp buffer - *(int64_t *)pResInfo->interResultBuf += pInput->num; + *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo) += pInput->num; } } @@ -891,21 +888,21 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) { * the average value is calculated in finalize routine, since current routine does not know the exact number of points */ static void avg_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - if (GET_INT64_VAL(pResInfo->interResultBuf) <= 0) { + if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; // empty table } - *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)pResInfo->interResultBuf; + *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo); } else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY assert(pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE); - SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pAvgInfo->num == 0) { // all data are NULL or empty table setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); @@ -1116,11 +1113,11 @@ static void min_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the flag for super table query - if (pResInfo->superTableQ) { + if (pCtx->stableQuery) { *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; } } @@ -1133,11 +1130,11 @@ static void max_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the flag for super table query - if (pResInfo->superTableQ) { + if (pCtx->stableQuery) { *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; } } @@ -1148,8 +1145,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp GET_TRUE_DATA_TYPE(); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + assert(pCtx->stableQuery); for (int32_t i = 0; i < pCtx->size; ++i) { char *input = GET_INPUT_CHAR_INDEX(pCtx, i); @@ -1210,7 +1206,7 @@ static void min_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); - if (notNullElems > 0) { // for super table query, SResultInfo is not used + if (notNullElems > 0) { // for super table query, SResultRowCellInfo is not used char *flag = pCtx->aOutputBuf + pCtx->inputBytes; *flag = DATA_SET_FLAG; } @@ -1221,7 +1217,7 @@ static void min_func_second_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -1242,7 +1238,7 @@ static void max_func_second_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, numOfElem, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (numOfElem > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -1297,8 +1293,8 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); minMax_function_f(pCtx, index, 0); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { char *flag = pCtx->aOutputBuf + pCtx->inputBytes; *flag = DATA_SET_FLAG; } @@ -1313,8 +1309,8 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); minMax_function_f(pCtx, index, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { char *flag = pCtx->aOutputBuf + pCtx->inputBytes; *flag = DATA_SET_FLAG; } @@ -1330,7 +1326,7 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void stddev_function(SQLFunctionCtx *pCtx) { // the second stage to calculate standard deviation - SStddevInfo *pStd = GET_RES_INFO(pCtx)->interResultBuf; + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->stage == 0) { // the first stage is to calculate average value avg_function(pCtx); @@ -1381,8 +1377,8 @@ static void stddev_function(SQLFunctionCtx *pCtx) { static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { // the second stage to calculate standard deviation - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevInfo *pStd = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); /* the first stage is to calculate average value */ if (pStd->stage == 0) { @@ -1433,8 +1429,8 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { * the stddevInfo and the average info struct share the same buffer area * And the position of each element in their struct is exactly the same matched */ - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevInfo *pStd = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); if (pStd->stage == 0) { /* @@ -1449,7 +1445,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { pResInfo->initialized = true; // set it initialized to avoid re-initialization // save average value into tmpBuf, for second stage scan - SAvgInfo *pAvg = pResInfo->interResultBuf; + SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); pStd->avg = GET_DOUBLE_VAL(pCtx->aOutputBuf); assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); @@ -1459,7 +1455,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { } static void stddev_finalizer(SQLFunctionCtx *pCtx) { - SStddevInfo *pStd = (SStddevInfo *)GET_RES_INFO(pCtx)->interResultBuf; + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->num <= 0) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); @@ -1505,7 +1501,7 @@ static void first_function(SQLFunctionCtx *pCtx) { TSKEY k = pCtx->ptsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, k); - SResultInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; @@ -1532,7 +1528,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { TSKEY ts = pCtx->ptsList[index]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - SResultInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; // get the first not-null data, completed } @@ -1576,7 +1572,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { first_data_assign_impl(pCtx, data, i); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; notNullElems++; @@ -1604,8 +1600,7 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void first_dist_func_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->size == 1 && pResInfo->superTableQ); + assert(pCtx->size == 1 && pCtx->stableQuery); SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); if (pInput->hasResult != DATA_SET_FLAG) { @@ -1620,8 +1615,8 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { } static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { - assert(pCtx->resultInfo->superTableQ); - + assert(pCtx->stableQuery); + char * pData = GET_INPUT_CHAR(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { @@ -1668,7 +1663,7 @@ static void last_function(SQLFunctionCtx *pCtx) { TSKEY ts = pCtx->ptsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - SResultInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; // set query completed on this column @@ -1691,7 +1686,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { TSKEY ts = pCtx->ptsList[index]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; pResInfo->complete = true; // set query completed } @@ -1740,7 +1735,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { last_data_assign_impl(pCtx, data, i); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; notNullElems++; @@ -1776,8 +1771,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void last_dist_func_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->size == 1 && pResInfo->superTableQ); + assert(pCtx->size == 1 && pCtx->stableQuery); // the input data is null SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); @@ -1833,11 +1827,11 @@ static void last_row_function(SQLFunctionCtx *pCtx) { // assign the last element in current data block assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the result to final result buffer in case of super table query - if (pResInfo->superTableQ) { + if (pCtx->stableQuery) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); pInfo1->ts = pCtx->ptsList[pCtx->size - 1]; pInfo1->hasResult = DATA_SET_FLAG; @@ -1852,7 +1846,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) { static void last_row_finalizer(SQLFunctionCtx *pCtx) { // do nothing at the first stage - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { setVardataNull(pCtx->aOutputBuf, pCtx->outputType); @@ -2044,8 +2038,8 @@ static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STopBotInfo *pRes = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); tValuePair **tvp = pRes->res; @@ -2123,7 +2117,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { } } - taosTFree(pData); + tfree(pData); } /* @@ -2135,18 +2129,18 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { * top/bottom use the intermediate result buffer to keep the intermediate result */ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); // only the first_stage_merge is directly written data into final output buffer - if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { + if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { return (STopBotInfo*) pCtx->aOutputBuf; } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer - return pResInfo->interResultBuf; + return GET_ROWCELL_INTERBUF(pResInfo); } } bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo == NULL) { return true; } @@ -2252,7 +2246,7 @@ static void top_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2270,7 +2264,7 @@ static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, 0); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } @@ -2285,8 +2279,7 @@ static void top_func_merge(SQLFunctionCtx *pCtx) { // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary buildTopBotStruct(pInput, pCtx); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); + assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); @@ -2314,7 +2307,7 @@ static void top_func_second_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pInput->num, pOutput->num); if (pOutput->num > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2343,7 +2336,7 @@ static void bottom_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2359,7 +2352,7 @@ static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, 0); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } @@ -2374,8 +2367,7 @@ static void bottom_func_merge(SQLFunctionCtx *pCtx) { // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary buildTopBotStruct(pInput, pCtx); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); + assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); @@ -2403,16 +2395,16 @@ static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pInput->num, pOutput->num); if (pOutput->num > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); // data in temporary list is less than the required number of results, not enough qualified number of results - STopBotInfo *pRes = pResInfo->interResultBuf; + STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); if (pRes->num == 0) { // no result assert(pResInfo->hasResult != DATA_SET_FLAG); // TODO: @@ -2443,8 +2435,8 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) { } // in the first round, get the min-max value of all involved data - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX); SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX); pInfo->numOfElems = 0; @@ -2455,18 +2447,28 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) { static void percentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); // the first stage, only acquire the min/max value if (pInfo->stage == 0) { if (pCtx->preAggVals.isSet) { - if (GET_DOUBLE_VAL(&pInfo->minval) > pCtx->preAggVals.statis.min) { - SET_DOUBLE_VAL(&pInfo->minval, (double)pCtx->preAggVals.statis.min); + double tmin = 0.0, tmax = 0.0; + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { + tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min); + tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max); + } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { + tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min); + tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max); + } else { + assert(true); + } + if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) { + SET_DOUBLE_VAL(&pInfo->minval, tmin); } - if (GET_DOUBLE_VAL(&pInfo->maxval) < pCtx->preAggVals.statis.max) { - SET_DOUBLE_VAL(&pInfo->maxval, (double)pCtx->preAggVals.statis.max); + if (GET_DOUBLE_VAL(&pInfo->maxval) < tmax) { + SET_DOUBLE_VAL(&pInfo->maxval, tmax); } pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull); @@ -2536,9 +2538,9 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = (SPercentileInfo *)pResInfo->interResultBuf; + SPercentileInfo *pInfo = (SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->stage == 0) { // TODO extract functions @@ -2585,8 +2587,8 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void percentile_finalizer(SQLFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - tMemBucket * pMemBucket = ((SPercentileInfo *)pResInfo->interResultBuf)->pMemBucket; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + tMemBucket * pMemBucket = ((SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo))->pMemBucket; if (pMemBucket->total > 0) { // check for null *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v); @@ -2599,8 +2601,8 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { } static void percentile_next_step(SQLFunctionCtx *pCtx) { - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->stage == 0) { // all data are null, set it completed @@ -2617,12 +2619,12 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) { ////////////////////////////////////////////////////////////////////////////////// static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { + if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { return (SAPercentileInfo*) pCtx->aOutputBuf; } else { - return pResInfo->interResultBuf; + return GET_ROWCELL_INTERBUF(pResInfo); } } @@ -2641,7 +2643,7 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx) { static void apercentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo * pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pInfo = getAPerctInfo(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { @@ -2694,8 +2696,8 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); // pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SAPercentileInfo *pInfo = getAPerctInfo(pCtx); double v = 0; switch (pCtx->inputType) { @@ -2726,8 +2728,8 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void apercentile_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->stableQuery); SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); @@ -2784,7 +2786,7 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { pOutput->pHisto = pRes; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; SET_VAL(pCtx, 1, 1); } @@ -2792,8 +2794,8 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { static void apercentile_finalizer(SQLFunctionCtx *pCtx) { double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i64Key : pCtx->param[0].dKey; - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pOutput = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null @@ -2830,8 +2832,8 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) { return false; } - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); // 2*3 matrix pInfo->startVal = pCtx->param[0].dKey; @@ -2857,8 +2859,8 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) { } static void leastsquares_function(SQLFunctionCtx *pCtx) { - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); double(*param)[3] = pInfo->mat; double x = pInfo->startVal; @@ -2928,8 +2930,8 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); double(*param)[3] = pInfo->mat; @@ -2978,8 +2980,8 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { // no data in query - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->num == 0) { if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { @@ -3044,7 +3046,7 @@ static void col_project_function(SQLFunctionCtx *pCtx) { } static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 return; } @@ -3476,7 +3478,7 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { return false; } - SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf; + SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); // this is the server-side setup function in client-side, the secondary merge do not need this procedure if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { @@ -3491,8 +3493,8 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { } static void spread_function(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SSpreadInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t numOfElems = 0; @@ -3558,8 +3560,8 @@ static void spread_function(SQLFunctionCtx *pCtx) { } // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); } } @@ -3571,8 +3573,8 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SSpreadInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); double val = 0.0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { @@ -3601,16 +3603,16 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG; - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); } } void spread_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->stableQuery); - SSpreadInfo *pResData = pResInfo->interResultBuf; + SSpreadInfo *pResData = GET_ROWCELL_INTERBUF(pResInfo); int32_t notNullElems = 0; for (int32_t i = 0; i < pCtx->size; ++i) { @@ -3634,7 +3636,7 @@ void spread_func_merge(SQLFunctionCtx *pCtx) { } if (notNullElems > 0) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -3665,7 +3667,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { * here we do not check the input data types, because in case of metric query, * the type of intermediate data is binary */ - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); @@ -3680,7 +3682,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { assert((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); - SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf; + SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; @@ -3704,8 +3706,8 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) { return false; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; - STwaInfo * pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; + STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); pInfo->lastKey = INT64_MIN; pInfo->type = pCtx->inputType; @@ -3744,8 +3746,8 @@ static void twa_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo * pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t i = 0; @@ -3798,7 +3800,7 @@ static void twa_function(SQLFunctionCtx *pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } - if (pResInfo->superTableQ) { + if (pCtx->stableQuery) { memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo)); } @@ -3815,8 +3817,8 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { TSKEY *primaryKey = pCtx->ptsList; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->lastKey == INT64_MIN) { pInfo->lastKey = pCtx->nStartQueryTimestamp; @@ -3838,14 +3840,13 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { // pCtx->numOfIteratedElems += 1; pResInfo->hasResult = DATA_SET_FLAG; - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(STwaInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo)); } } static void twa_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + assert(pCtx->stableQuery); STwaInfo *pBuf = (STwaInfo *)pCtx->aOutputBuf; char * indicator = pCtx->aInputElemBuf; @@ -3885,16 +3886,16 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) { */ void twa_function_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); + memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((STwaInfo *)pCtx->aInputElemBuf)->hasResult; } void twa_function_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = (STwaInfo *)pResInfo->interResultBuf; + STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); assert(pInfo->EKey >= pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult); if (pInfo->hasResult != DATA_SET_FLAG) { @@ -3922,8 +3923,8 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) { */ static void interp_function(SQLFunctionCtx *pCtx) { // at this point, the value is existed, return directly - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SInterpInfoDetail* pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SInterpInfoDetail* pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->size == 1) { char *pData = GET_INPUT_CHAR(pCtx); @@ -3977,7 +3978,7 @@ static void interp_function(SQLFunctionCtx *pCtx) { if (isNull(data1, srcType) || isNull(data2, srcType)) { setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); } else { - taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point); + taosGetLinearInterpolationVal(pCtx->outputType, &point1, &point2, &point); } } else if (srcType == TSDB_DATA_TYPE_FLOAT) { point1.val = data1; @@ -3986,7 +3987,7 @@ static void interp_function(SQLFunctionCtx *pCtx) { if (isNull(data1, srcType) || isNull(data2, srcType)) { setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); } else { - taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point); + taosGetLinearInterpolationVal(pCtx->outputType, &point1, &point2, &point); } } else { @@ -3998,7 +3999,6 @@ static void interp_function(SQLFunctionCtx *pCtx) { } } } - } SET_VAL(pCtx, pCtx->size, 1); @@ -4009,8 +4009,8 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) { return false; // not initialized since it has been initialized } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STSCompInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); pInfo->pTSBuf = tsBufCreate(false, pCtx->order); pInfo->pTSBuf->tsOrder = pCtx->order; @@ -4018,18 +4018,18 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) { } static void ts_comp_function(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STSBuf * pTSbuf = ((STSCompInfo *)(pResInfo->interResultBuf))->pTSBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; const char *input = GET_INPUT_CHAR(pCtx); // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { - tsBufAppend(pTSbuf, 0, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); + tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *d = GET_INPUT_CHAR_INDEX(pCtx, i); - tsBufAppend(pTSbuf, 0, &pCtx->tag, d, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); } } @@ -4043,21 +4043,21 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STSCompInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); STSBuf *pTSbuf = pInfo->pTSBuf; - tsBufAppend(pTSbuf, 0, &pCtx->tag, pData, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, pData, TSDB_KEYSIZE); SET_VAL(pCtx, pCtx->size, 1); pResInfo->hasResult = DATA_SET_FLAG; } static void ts_comp_finalize(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STSCompInfo *pInfo = pResInfo->interResultBuf; + STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); STSBuf * pTSbuf = pInfo->pTSBuf; tsBufFlush(pTSbuf); @@ -4106,8 +4106,8 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { return false; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; - SRateInfo * pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; + SRateInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); pInfo->CorrectionValue = 0; pInfo->firstKey = INT64_MIN; @@ -4126,8 +4126,8 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { static void rate_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = pCtx->ptsList; tscDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); @@ -4190,8 +4190,8 @@ static void rate_function(SQLFunctionCtx *pCtx) { } // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4202,8 +4202,8 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { } // NOTE: keep the intermediate result into the interResultBuf - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = pCtx->ptsList; int64_t v = 0; @@ -4247,20 +4247,18 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->hasResult = DATA_SET_FLAG; // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } static void rate_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + assert(pCtx->stableQuery); tscDebug("rate_func_merge() size:%d", pCtx->size); - //SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; SRateInfo *pBuf = (SRateInfo *)pCtx->aOutputBuf; char *indicator = pCtx->aInputElemBuf; @@ -4293,8 +4291,8 @@ static void rate_func_merge(SQLFunctionCtx *pCtx) { static void rate_func_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((SRateInfo*)pCtx->aInputElemBuf)->hasResult; SRateInfo* pRateInfo = (SRateInfo*)pCtx->aInputElemBuf; @@ -4305,8 +4303,8 @@ static void rate_func_copy(SQLFunctionCtx *pCtx) { static void rate_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); tscDebug("%p isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", pCtx, pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); @@ -4331,8 +4329,8 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { static void irate_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = pCtx->ptsList; tscDebug("%p irate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); @@ -4394,8 +4392,8 @@ static void irate_function(SQLFunctionCtx *pCtx) { } // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4406,8 +4404,8 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { } // NOTE: keep the intermediate result into the interResultBuf - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = pCtx->ptsList; int64_t v = 0; @@ -4443,16 +4441,16 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->hasResult = DATA_SET_FLAG; // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } static void do_sumrate_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->stableQuery); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); char * input = GET_INPUT_CHAR(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { @@ -4476,7 +4474,7 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) { if (DATA_SET_FLAG == pRateInfo->hasResult) { pResInfo->hasResult = DATA_SET_FLAG; SET_VAL(pCtx, pRateInfo->num, 1); - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4491,10 +4489,10 @@ static void sumrate_func_second_merge(SQLFunctionCtx *pCtx) { } static void sumrate_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - tscDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pResInfo->superTableQ, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult); + tscDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pCtx->stableQuery, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult); if (pRateInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index b4c3f3549b1576b3ff83300d4f10d9e07778530d..44dffab56f90e2d88604ee96a6cc1ddc8a5cfbef 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -49,82 +49,6 @@ typedef struct SCreateBuilder { } SCreateBuilder; static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength); -static int32_t getToStringLength(const char *pData, int32_t length, int32_t type) { - char buf[512] = {0}; - - int32_t len = 0; - int32_t MAX_BOOL_TYPE_LENGTH = 5; // max(strlen("true"), strlen("false")); - switch (type) { - case TSDB_DATA_TYPE_BINARY: - return length; - case TSDB_DATA_TYPE_NCHAR: - return length; - case TSDB_DATA_TYPE_DOUBLE: { - double dv = 0; - dv = GET_DOUBLE_VAL(pData); - len = sprintf(buf, "%lf", dv); - if (strncasecmp("nan", buf, 3) == 0) { - len = 4; - } - } break; - case TSDB_DATA_TYPE_FLOAT: { - float fv = 0; - fv = GET_FLOAT_VAL(pData); - len = sprintf(buf, "%f", fv); - if (strncasecmp("nan", buf, 3) == 0) { - len = 4; - } - } break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - len = sprintf(buf, "%" PRId64, *(int64_t *)pData); - break; - case TSDB_DATA_TYPE_BOOL: - len = MAX_BOOL_TYPE_LENGTH; - break; - default: - len = sprintf(buf, "%d", *(int32_t *)pData); - break; - }; - return len; -} - -/* - * we need to convert all data into string, so we need to sprintf all kinds of - * non-string data into string, and record its length to get the right - * maximum length. The length may be less or greater than its original binary length: - * For example: - * length((short) 1) == 1, less than sizeof(short) - * length((uint64_t) 123456789011) > 12, greater than sizsof(uint64_t) - */ -static int32_t tscMaxLengthOfTagsFields(SSqlObj *pSql) { - STableMeta *pMeta = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->pTableMeta; - - if (pMeta->tableType == TSDB_SUPER_TABLE || pMeta->tableType == TSDB_NORMAL_TABLE || - pMeta->tableType == TSDB_STREAM_TABLE) { - return 0; - } - - char * pTagValue = tsGetTagsValue(pMeta); - SSchema *pTagsSchema = tscGetTableTagSchema(pMeta); - - int32_t len = getToStringLength(pTagValue, pTagsSchema[0].bytes, pTagsSchema[0].type); - - pTagValue += pTagsSchema[0].bytes; - int32_t numOfTags = tscGetNumOfTags(pMeta); - - for (int32_t i = 1; i < numOfTags; ++i) { - int32_t tLen = getToStringLength(pTagValue, pTagsSchema[i].bytes, pTagsSchema[i].type); - if (len < tLen) { - len = tLen; - } - - pTagValue += pTagsSchema[i].bytes; - } - - return len; -} - static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { SSqlRes *pRes = &pSql->res; @@ -186,8 +110,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { return 0; } - // the following is handle display tags value for meters created according to metric - char *pTagValue = tsGetTagsValue(pMeta); + // the following is handle display tags for table created according to super table for (int32_t i = numOfRows; i < totalNumOfRows; ++i) { // field name TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); @@ -219,8 +142,6 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i; const char *src = "TAG"; STR_WITH_MAXSIZE_TO_VARSTR(target, src, pField->bytes); - - pTagValue += pSchema[i].bytes; } return 0; @@ -286,10 +207,10 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { const int32_t TYPE_COLUMN_LENGTH = 16; const int32_t NOTE_COLUMN_MIN_LENGTH = 8; - int32_t noteFieldLen = tscMaxLengthOfTagsFields(pSql); - if (noteFieldLen == 0) { - noteFieldLen = NOTE_COLUMN_MIN_LENGTH; - } + int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH;//tscMaxLengthOfTagsFields(pSql); +// if (noteFieldLen == 0) { +// noteFieldLen = NOTE_COLUMN_MIN_LENGTH; +// } int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen); tscFieldInfoUpdateOffset(pQueryInfo); diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 44ccb2471aa980d789c8557aca7981b6981df02f..e18ad0b4b0133c7cd01050f03559449303566be2 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -97,14 +97,14 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc pCtx->param[2].i64Key = pQueryInfo->order.order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; pCtx->param[1].i64Key = pQueryInfo->order.orderColId; + } else if (functionId == TSDB_FUNC_APERCT) { + pCtx->param[0].i64Key = pExpr->param[0].i64Key; + pCtx->param[0].nType = pExpr->param[0].nType; } - SResultInfo *pResInfo = &pReducer->pResInfo[i]; - pResInfo->bufLen = pExpr->interBytes; - pResInfo->interResultBuf = calloc(1, (size_t) pResInfo->bufLen); - - pCtx->resultInfo = &pReducer->pResInfo[i]; - pCtx->resultInfo->superTableQ = true; + pCtx->interBufBytes = pExpr->interBytes; + pCtx->resultInfo = calloc(1, pCtx->interBufBytes + sizeof(SResultRowCellInfo)); + pCtx->stableQuery = true; } int16_t n = 0; @@ -227,7 +227,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (ds == NULL) { tscError("%p failed to create merge structure", pSql); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - taosTFree(pReducer); + tfree(pReducer); return; } @@ -254,7 +254,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (ds->filePage.num == 0) { // no data in this flush, the index does not increase tscDebug("%p flush data is empty, ignore %d flush record", pSql, idx); - taosTFree(ds); + tfree(ds); continue; } @@ -264,7 +264,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd // no data actually, no need to merge result. if (idx == 0) { - taosTFree(pReducer); + tfree(pReducer); return; } @@ -272,7 +272,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd SCompareParam *param = malloc(sizeof(SCompareParam)); if (param == NULL) { - taosTFree(pReducer); + tfree(pReducer); return; } @@ -286,8 +286,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator); if (pReducer->pLoserTree == NULL || pRes->code != 0) { - taosTFree(param); - taosTFree(pReducer); + tfree(param); + tfree(pReducer); return; } @@ -330,14 +330,14 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL || /*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) { - taosTFree(pReducer->pTempBuffer); - taosTFree(pReducer->discardData); - taosTFree(pReducer->pResultBuf); - taosTFree(pReducer->pFinalRes); - taosTFree(pReducer->prevRowOfInput); - taosTFree(pReducer->pLoserTree); - taosTFree(param); - taosTFree(pReducer); + tfree(pReducer->pTempBuffer); + tfree(pReducer->discardData); + tfree(pReducer->pResultBuf); + tfree(pReducer->pFinalRes); + tfree(pReducer->prevRowOfInput); + tfree(pReducer->pLoserTree); + tfree(param); + tfree(pReducer); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; return; } @@ -345,7 +345,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd size_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo); pReducer->pTempBuffer->num = 0; - pReducer->pResInfo = calloc(numOfCols, sizeof(SResultInfo)); tscCreateResPointerInfo(pRes, pQueryInfo); tscInitSqlContext(pCmd, pReducer, pDesc); @@ -489,47 +488,40 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { tscDebug("%p waiting for delete procedure, status: %d", pSql, status); } - pLocalReducer->pFillInfo = taosDestoryFillInfo(pLocalReducer->pFillInfo); + pLocalReducer->pFillInfo = taosDestroyFillInfo(pLocalReducer->pFillInfo); if (pLocalReducer->pCtx != NULL) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i]; tVariantDestroy(&pCtx->tag); + tfree(pCtx->resultInfo); + if (pCtx->tagInfo.pTagCtxList != NULL) { - taosTFree(pCtx->tagInfo.pTagCtxList); + tfree(pCtx->tagInfo.pTagCtxList); } } - taosTFree(pLocalReducer->pCtx); + tfree(pLocalReducer->pCtx); } - taosTFree(pLocalReducer->prevRowOfInput); - - taosTFree(pLocalReducer->pTempBuffer); - taosTFree(pLocalReducer->pResultBuf); - - if (pLocalReducer->pResInfo != NULL) { - size_t num = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < num; ++i) { - taosTFree(pLocalReducer->pResInfo[i].interResultBuf); - } + tfree(pLocalReducer->prevRowOfInput); - taosTFree(pLocalReducer->pResInfo); - } + tfree(pLocalReducer->pTempBuffer); + tfree(pLocalReducer->pResultBuf); if (pLocalReducer->pLoserTree) { - taosTFree(pLocalReducer->pLoserTree->param); - taosTFree(pLocalReducer->pLoserTree); + tfree(pLocalReducer->pLoserTree->param); + tfree(pLocalReducer->pLoserTree); } - taosTFree(pLocalReducer->pFinalRes); - taosTFree(pLocalReducer->discardData); + tfree(pLocalReducer->pFinalRes); + tfree(pLocalReducer->discardData); tscLocalReducerEnvDestroy(pLocalReducer->pExtMemBuffer, pLocalReducer->pDesc, pLocalReducer->resColModel, pLocalReducer->numOfVnode); for (int32_t i = 0; i < pLocalReducer->numOfBuffer; ++i) { - taosTFree(pLocalReducer->pLocalDataSrc[i]); + tfree(pLocalReducer->pLocalDataSrc[i]); } pLocalReducer->numOfBuffer = 0; @@ -596,7 +588,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm } *pOrderDesc = tOrderDesCreate(orderColIndexList, numOfGroupByCols, pModel, pQueryInfo->order.order); - taosTFree(orderColIndexList); + tfree(orderColIndexList); if (*pOrderDesc == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -698,7 +690,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pg *= 2; } - size_t numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups; + size_t numOfSubs = pSql->subState.numOfSub; + assert(numOfSubs <= pTableMetaInfo->vgroupList->numOfVgroups); for (int32_t i = 0; i < numOfSubs; ++i) { (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pg, pModel); (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; @@ -706,7 +699,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - taosTFree(pSchema); + tfree(pSchema); return pRes->code; } @@ -743,7 +736,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr } *pFinalModel = createColumnModel(pSchema, (int32_t)size, capacity); - taosTFree(pSchema); + tfree(pSchema); return TSDB_CODE_SUCCESS; } @@ -763,7 +756,7 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]); } - taosTFree(pMemBuffer); + tfree(pMemBuffer); } /** @@ -985,10 +978,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO pBeforeFillData->num = 0; for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - taosTFree(pResPages[i]); + tfree(pResPages[i]); } - taosTFree(pResPages); + tfree(pResPages); } static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { @@ -1071,7 +1064,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) continue; } - SResultInfo* pResInfo = GET_RES_INFO(&pCtx[j]); + SResultRowCellInfo* pResInfo = GET_RES_INFO(&pCtx[j]); if (maxOutput < pResInfo->numOfRes) { maxOutput = pResInfo->numOfRes; } @@ -1252,10 +1245,11 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur return true; } -void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { // reset output buffer to the beginning - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - pLocalReducer->pCtx[i].aOutputBuf = - pLocalReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pLocalReducer->resColModel->capacity; +void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {// reset output buffer to the beginning + size_t t = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < t; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + pExpr->offset * pLocalReducer->resColModel->capacity; } memset(pLocalReducer->pResultBuf, 0, pLocalReducer->nResultBufSize + sizeof(tFilePage)); @@ -1500,8 +1494,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { if (pLocalReducer->discard && sameGroup) { pLocalReducer->hasUnprocessedRow = false; tmpBuffer->num = 0; - } else { - // current row does not belongs to the previous group, so it is not be handled yet. + } else { // current row does not belongs to the previous group, so it is not be handled yet. pLocalReducer->hasUnprocessedRow = true; } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index ac34d26d2fe747773a550dd76ac7e967a592199e..c9115a8324721c0262a0ccaf54572631d89e423a 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -702,7 +702,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st } int32_t code = TSDB_CODE_TSC_INVALID_SQL; - char * tmpTokenBuf = calloc(1, 4096); // used for deleting Escape character: \\, \', \" + char * tmpTokenBuf = calloc(1, 16*1024); // used for deleting Escape character: \\, \', \" if (NULL == tmpTokenBuf) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1309,7 +1309,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) { if ((!pCmd->parseFinished) && (!initial)) { tscDebug("%p resume to parse sql: %s", pSql, pCmd->curSql); } - + ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); if (TSDB_CODE_SUCCESS != ret) { return ret; @@ -1406,7 +1406,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { assert(taos_errno(pSql) == code); taos_free_result(pSql); - taosTFree(pSupporter); + tfree(pSupporter); fclose(fp); pParentSql->res.code = code; @@ -1445,7 +1445,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { char *tokenBuf = calloc(1, 4096); - while ((readLen = taosGetline(&line, &n, fp)) != -1) { + while ((readLen = tgetline(&line, &n, fp)) != -1) { if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { line[--readLen] = 0; } @@ -1470,7 +1470,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { } } - taosTFree(tokenBuf); + tfree(tokenBuf); free(line); if (count > 0) { @@ -1483,7 +1483,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { } else { taos_free_result(pSql); - taosTFree(pSupporter); + tfree(pSupporter); fclose(fp); pParentSql->fp = pParentSql->fetchFp; @@ -1513,7 +1513,7 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) { pSql->res.code = TAOS_SYSTEM_ERROR(errno); tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code)); - taosTFree(pSupporter) + tfree(pSupporter) tscQueueAsyncRes(pSql); return; diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index eb6843b0e4fd844c9d13da7f55dca47008f62a79..acc5acd786bfe00036e539a104da04af1485a263 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -222,7 +222,7 @@ void tscKillStream(STscObj *pObj, uint32_t killId) { } int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { - SCMHeartBeatMsg *pHeartbeat = pMsg; + SHeartBeatMsg *pHeartbeat = pMsg; int allocedQueriesNum = pHeartbeat->numOfQueries; int allocedStreamsNum = pHeartbeat->numOfStreams; @@ -277,7 +277,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { } int32_t msgLen = pHeartbeat->numOfQueries * sizeof(SQueryDesc) + pHeartbeat->numOfStreams * sizeof(SStreamDesc) + - sizeof(SCMHeartBeatMsg); + sizeof(SHeartBeatMsg); pHeartbeat->connId = htonl(pObj->connId); pHeartbeat->numOfQueries = htonl(pHeartbeat->numOfQueries); pHeartbeat->numOfStreams = htonl(pHeartbeat->numOfStreams); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e2573f7e19303966e9f17a2731444bf9fca7ab87..579e5f541023de677264eb6d2fb9c66701d8b3c2 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -16,6 +16,7 @@ #define _BSD_SOURCE #define _XOPEN_SOURCE 500 #define _DEFAULT_SOURCE +#define _GNU_SOURCE #include "os.h" #include "qAst.h" @@ -65,7 +66,6 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength); -static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName); static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult); static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, @@ -80,9 +80,9 @@ static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd); -static int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); +static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); -static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); +static int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem); @@ -114,7 +114,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString); static int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); -static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate); +static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate); static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex); @@ -417,7 +417,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { char* pMsg = pCmd->payload; - SCMCfgDnodeMsg* pCfg = (SCMCfgDnodeMsg*)pMsg; + SCfgDnodeMsg* pCfg = (SCfgDnodeMsg*)pMsg; pDCL->a[0].n = strdequote(pDCL->a[0].z); strncpy(pCfg->ep, pDCL->a[0].z, pDCL->a[0].n); @@ -616,14 +616,20 @@ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) { return false; } -int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { +int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { const char* msg1 = "invalid query expression"; const char* msg2 = "interval cannot be less than 10 ms"; + const char* msg3 = "sliding cannot be used without interval"; + + SSqlCmd* pCmd = &pSql->cmd; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); if (pQuerySql->interval.type == 0 || pQuerySql->interval.n == 0) { + if (pQuerySql->sliding.n > 0) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } return TSDB_CODE_SUCCESS; } @@ -656,7 +662,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ return TSDB_CODE_TSC_INVALID_SQL; } - if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -709,7 +715,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ return TSDB_CODE_TSC_INVALID_SQL; } - if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -767,13 +773,15 @@ int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQue return TSDB_CODE_SUCCESS; } -int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { +int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { const char* msg0 = "sliding value too small"; const char* msg1 = "sliding value no larger than the interval value"; const char* msg2 = "sliding value can not less than 1% of interval value"; - const char* msg3 = "does not support sliding when interval is natual month/year"; + const char* msg3 = "does not support sliding when interval is natural month/year"; + const char* msg4 = "sliding not support yet in ordinary query"; const static int32_t INTERVAL_SLIDING_FACTOR = 100; + SSqlCmd* pCmd = &pSql->cmd; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -806,6 +814,10 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } + if (pQueryInfo->interval.sliding != pQueryInfo->interval.interval && pSql->pStream == NULL) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + return TSDB_CODE_SUCCESS; } @@ -877,22 +889,13 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) { int32_t nLen = 0; for (int32_t i = 0; i < pFieldList->nField; ++i) { - if (pFieldList->p[i].bytes == 0) { + TAOS_FIELD* pField = &pFieldList->p[i]; + + if (pField->bytes == 0) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); return false; } - nLen += pFieldList->p[i].bytes; - } - - // max row length must be less than TSDB_MAX_BYTES_PER_ROW - if (nLen > TSDB_MAX_BYTES_PER_ROW) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; - } - // field name must be unique - for (int32_t i = 0; i < pFieldList->nField; ++i) { - TAOS_FIELD* pField = &pFieldList->p[i]; if (pField->type < TSDB_DATA_TYPE_BOOL || pField->type > TSDB_DATA_TYPE_NCHAR) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); return false; @@ -909,10 +912,19 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) { return false; } + // field name must be unique if (has(pFieldList, i + 1, pFieldList->p[i].name) == true) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); return false; } + + nLen += pField->bytes; + } + + // max row length must be less than TSDB_MAX_BYTES_PER_ROW + if (nLen > TSDB_MAX_BYTES_PER_ROW) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return false; } return true; @@ -1211,7 +1223,7 @@ static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* tscColumnListInsert(pQueryInfo->colList, &tsCol); } static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) { - const char* msg1 = "invalid column name, or illegal column type"; + const char* msg1 = "invalid column name, illegal column type, or columns in arithmetic expression from two tables"; const char* msg2 = "invalid arithmetic expression in select clause"; const char* msg3 = "tag columns can not be used in arithmetic expression"; const char* msg4 = "columns from different table mixed up in arithmetic expression"; @@ -1625,16 +1637,15 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, - char* aliasName, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) { + const char* name, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) { const char* msg1 = "not support column types"; int16_t type = 0; int16_t bytes = 0; - char columnName[TSDB_COL_NAME_LEN] = {0}; int32_t functionID = cvtFunc.execFuncId; if (functionID == TSDB_FUNC_SPREAD) { - int32_t t1 = pSchema[pColIndex->columnIndex].type; + int32_t t1 = pSchema->type; if (t1 == TSDB_DATA_TYPE_BINARY || t1 == TSDB_DATA_TYPE_NCHAR || t1 == TSDB_DATA_TYPE_BOOL) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); return -1; @@ -1643,18 +1654,12 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS bytes = tDataTypeDesc[type].nSize; } } else { - type = pSchema[pColIndex->columnIndex].type; - bytes = pSchema[pColIndex->columnIndex].bytes; + type = pSchema->type; + bytes = pSchema->bytes; } - if (aliasName != NULL) { - tstrncpy(columnName, aliasName, sizeof(columnName)); - } else { - getRevisedName(columnName, cvtFunc.originFuncId, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name); - } - SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false); - tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName)); + tstrncpy(pExpr->aliasName, name, tListLen(pExpr->aliasName)); if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != functionID) { pExpr->colInfo.flag |= TSDB_COL_NULL; @@ -1674,7 +1679,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS // if it is not in the final result, do not add it SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); if (finalResult) { - insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, columnName, pExpr); + insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, pExpr->aliasName, pExpr); } else { tscColumnListInsert(pQueryInfo->colList, &(ids.ids[0])); } @@ -1682,6 +1687,23 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS return TSDB_CODE_SUCCESS; } +void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken) { + if (pItem->aliasName != NULL) { + tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); + } else { + char uname[TSDB_COL_NAME_LEN] = {0}; + int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); + tstrncpy(uname, pToken->z, len); + + int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1; + char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1] = {0}; + + snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname); + + tstrncpy(name, tmp, TSDB_COL_NAME_LEN); + } +} + int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult) { STableMetaInfo* pTableMetaInfo = NULL; int32_t optr = pItem->pNode->nSQLOptr; @@ -1939,8 +1961,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (pParamElem->pNode->nSQLOptr == TK_ALL) { - // select table.* + if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -1950,9 +1971,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); + char name[TSDB_COL_NAME_LEN] = {0}; for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { index.columnIndex = j; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex++, &index, finalResult) != 0) { + SStrToken t = {.z = pSchema[j].name, .n = (uint32_t)strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)}; + setResultColName(name, pItem, cvtFunc.originFuncId, &t); + + if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } } @@ -1963,14 +1988,18 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); // functions can not be applied to tags if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex + i, &index, finalResult) != 0) { + char name[TSDB_COL_NAME_LEN] = {0}; + + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); + setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo); + + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex + i, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2007,7 +2036,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex, &index, finalResult) != 0) { + + char name[TSDB_COL_NAME_LEN] = {0}; + SStrToken t = {.z = pSchema[i].name, .n = (uint32_t)strnlen(pSchema[i].name, TSDB_COL_NAME_LEN)}; + setResultColName(name, pItem, cvtFunc.originFuncId, &t); + + if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2240,10 +2274,6 @@ void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLengt } } -void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName) { - snprintf(resultFieldName, maxLen, "%s(%s)", aAggs[functionId].aName, columnName); -} - static bool isTablenameToken(SStrToken* token) { SStrToken tmpToken = *token; SStrToken tableToken = {0}; @@ -2728,7 +2758,8 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd) { const char* msg1 = "too many columns in group by clause"; const char* msg2 = "invalid column name in group by clause"; -// const char* msg3 = "group by columns must belong to one table"; + const char* msg3 = "columns from one table allowed as group by columns"; + const char* msg4 = "join query does not support group by"; const char* msg7 = "not support group by expression"; const char* msg8 = "not allowed column type for group by"; const char* msg9 = "tags not allowed for table query"; @@ -2749,6 +2780,10 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } + if (pQueryInfo->numOfTables > 1) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + STableMeta* pTableMeta = NULL; SSchema* pSchema = NULL; SSchema s = tscGetTbnameColumnSchema(); @@ -2764,7 +2799,11 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tableIndex = index.tableIndex; + if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { + tableIndex = index.tableIndex; + } else if (tableIndex != index.tableIndex) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); pTableMeta = pTableMetaInfo->pTableMeta; @@ -2803,7 +2842,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* tscColumnListInsert(pTableMetaInfo->tagColList, &index); } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by - if (pSchema->type > TSDB_DATA_TYPE_BINARY) { + if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); } @@ -3349,7 +3388,8 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) { +static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, + int32_t* type, uint64_t* uid) { if (pExpr->nSQLOptr == TK_ID) { if (*type == NON_ARITHMEIC_EXPR) { *type = NORMAL_ARITHMETIC; @@ -3398,13 +3438,22 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer } // Not supported data type in arithmetic expression + uint64_t id = -1; for(int32_t i = 0; i < inc; ++i) { SSqlExpr* p1 = tscSqlExprGet(pQueryInfo, i + outputIndex); int16_t t = p1->resType; if (t == TSDB_DATA_TYPE_BINARY || t == TSDB_DATA_TYPE_NCHAR || t == TSDB_DATA_TYPE_BOOL || t == TSDB_DATA_TYPE_TIMESTAMP) { return TSDB_CODE_TSC_INVALID_SQL; } + + if (i == 0) { + id = p1->uid; + } else if (id != p1->uid){ + return TSDB_CODE_TSC_INVALID_SQL; + } } + + *uid = id; } return TSDB_CODE_SUCCESS; @@ -3416,13 +3465,16 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI } tSQLExpr* pLeft = pExpr->pLeft; + uint64_t uidLeft = 0; + uint64_t uidRight = 0; + if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) { int32_t ret = validateArithmeticSQLExpr(pCmd, pLeft, pQueryInfo, pList, type); if (ret != TSDB_CODE_SUCCESS) { return ret; } } else { - int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type); + int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type, &uidLeft); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -3435,10 +3487,15 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI return ret; } } else { - int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type); + int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type, &uidRight); if (ret != TSDB_CODE_SUCCESS) { return ret; } + + // the expression not from the same table, return error + if (uidLeft != uidRight && uidLeft != 0 && uidRight != 0) { + return TSDB_CODE_TSC_INVALID_SQL; + } } return TSDB_CODE_SUCCESS; @@ -3900,7 +3957,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen); if (ret != TSDB_CODE_SUCCESS) { taosStringBuilderDestroy(&sb1); - taosTFree(segments); + tfree(segments); invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); return ret; @@ -3913,7 +3970,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, pQueryInfo->tagCond.tbnameCond.cond = strdup(str); taosStringBuilderDestroy(&sb1); - taosTFree(segments); + tfree(segments); return TSDB_CODE_SUCCESS; } @@ -4424,8 +4481,8 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema* pSchema) { const char* msg0 = "only support order by primary timestamp"; const char* msg1 = "invalid column name"; - const char* msg2 = "only support order by primary timestamp and queried column"; - const char* msg3 = "only support order by primary timestamp and first tag in groupby clause"; + const char* msg2 = "only support order by primary timestamp or queried column"; + const char* msg3 = "only support order by primary timestamp or first tag in groupby clause"; setDefaultOrderInfo(pQueryInfo); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -5186,7 +5243,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn return TSDB_CODE_SUCCESS; } -static int32_t setKeepOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { +static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { const char* msg = "invalid number of options"; pMsg->daysToKeep = htonl(-1); @@ -5217,7 +5274,7 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* return TSDB_CODE_SUCCESS; } -static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) { +static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) { const char* msg = "invalid time precision"; pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default @@ -5241,7 +5298,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBIn return TSDB_CODE_SUCCESS; } -static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { +static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { pMsg->maxTables = htonl(-1); // max tables can not be set anymore pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); @@ -5255,10 +5312,11 @@ static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { pMsg->replications = pCreateDb->replica; pMsg->quorum = pCreateDb->quorum; pMsg->ignoreExist = pCreateDb->ignoreExists; + pMsg->update = pCreateDb->update; } int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { - SCMCreateDbMsg* pMsg = (SCMCreateDbMsg*)(pCmd->payload); + SCreateDbMsg* pMsg = (SCreateDbMsg *)(pCmd->payload); setCreateDBOption(pMsg, pCreateDbSql); if (setKeepOption(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) { @@ -5281,20 +5339,26 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau if (pParentQueryInfo->groupbyExpr.numOfGroupCols > 0) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, subClauseIndex); + SSqlExpr* pExpr = NULL; + size_t size = taosArrayGetSize(pQueryInfo->exprList); - - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, (int32_t)size - 1); + if (size > 0) { + pExpr = tscSqlExprGet(pQueryInfo, (int32_t)size - 1); + } - if (pExpr->functionId != TSDB_FUNC_TAG) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - int16_t columnInfo = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); - SColumnIndex index = {.tableIndex = 0, .columnIndex = columnInfo}; - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + if (pExpr == NULL || pExpr->functionId != TSDB_FUNC_TAG) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pParentQueryInfo, tableIndex); + + int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); + + SSchema* pTagSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, colId); + int16_t colIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, colId); + SColumnIndex index = {.tableIndex = 0, .columnIndex = colIndex}; + + char* name = pTagSchema->name; + int16_t type = pTagSchema->type; + int16_t bytes = pTagSchema->bytes; - int16_t type = pSchema[index.columnIndex].type; - int16_t bytes = pSchema[index.columnIndex].bytes; - char* name = pSchema[index.columnIndex].name; - pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true); pExpr->colInfo.flag = TSDB_COL_TAG; @@ -5770,7 +5834,7 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ } // can only perform the parameters based on the macro definitation -int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { +int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) { char msg[512] = {0}; if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { @@ -6094,7 +6158,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } // set interval value - if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } else { if ((pQueryInfo->interval.interval > 0) && @@ -6302,7 +6366,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } // set interval value - if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } else { if ((pQueryInfo->interval.interval > 0) && diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index dfd707344c5ab3ec42b49859937f604fa477fe10..441ad8d8382c667c164f7c797fc344cb9ab9118d 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -118,7 +118,7 @@ SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) } // TODO for large number of columns, employ the binary search method -SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId) { +SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) { STableComInfo tinfo = tscGetTableInfo(pTableMeta); for(int32_t i = 0; i < tinfo.numOfColumns + tinfo.numOfTags; ++i) { @@ -140,7 +140,7 @@ struct SSchema tscGetTbnameColumnSchema() { strcpy(s.name, TSQL_TBNAME_L); return s; } -static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo *vgroupInfo) { +static void tscInitCorVgroupInfo(SCorVgroupInfo *corVgroupInfo, SVgroupInfo *vgroupInfo) { corVgroupInfo->version = 0; corVgroupInfo->inUse = 0; corVgroupInfo->numOfEps = vgroupInfo->numOfEps; @@ -166,7 +166,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->id.tid = pTableMetaMsg->tid; pTableMeta->id.uid = pTableMetaMsg->uid; - SCMVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo; + SVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo; pVgroupInfo->numOfEps = pTableMetaMsg->vgroup.numOfEps; pVgroupInfo->vgId = pTableMetaMsg->vgroup.vgId; @@ -177,7 +177,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pVgroupInfo->epAddr[i].port = pEpMsg->port; } - tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo); + tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, pVgroupInfo); pTableMeta->sversion = pTableMetaMsg->sversion; pTableMeta->tversion = pTableMetaMsg->tversion; @@ -197,28 +197,6 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size return pTableMeta; } -/** - * the TableMeta data format in memory is as follows: - * - * +--------------------+ - * |STableMeta Body data| sizeof(STableMeta) - * +--------------------+ - * |Schema data | numOfTotalColumns * sizeof(SSchema) - * +--------------------+ - * |Tags data | tag_col_1.bytes + tag_col_2.bytes + .... - * +--------------------+ - * - * @param pTableMeta - * @return - */ -char* tsGetTagsValue(STableMeta* pTableMeta) { - int32_t offset = 0; -// int32_t numOfTotalCols = pTableMeta->numOfColumns + pTableMeta->numOfTags; -// uint32_t offset = sizeof(STableMeta) + numOfTotalCols * sizeof(SSchema); - - return ((char*)pTableMeta + offset); -} - // todo refactor UNUSED_FUNC static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) { for (int32_t i = 0; i < num; ++i) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index a0841fa2344682facb4a6dc8666716e38a31ea1d..e97e95a31779b5c5a6eab2be1b3a0ade425dc6c5 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -48,7 +48,7 @@ static int32_t getWaitingTimeInterval(int32_t count) { return initial * (2<<(count - 2)); } -static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { +static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) { assert(pSql != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); SRpcEpSet* pEpSet = &pSql->epSet; @@ -100,7 +100,7 @@ void tscUpdateMgmtEpSet(SRpcEpSet *pEpSet) { tscMgmtEpSet.epSet = *pEpSet; taosCorEndWrite(&tscMgmtEpSet.version); } -static void tscDumpEpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) { +static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) { if (pVgroupInfo == NULL) { return;} taosCorBeginRead(&pVgroupInfo->version); int8_t inUse = pVgroupInfo->inUse; @@ -117,14 +117,14 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { SSqlCmd *pCmd = &pObj->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} - SCMCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo; + SCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo; taosCorBeginWrite(&pVgroupInfo->version); tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse); pVgroupInfo->inUse = pEpSet->inUse; pVgroupInfo->numOfEps = pEpSet->numOfEps; for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) { - taosTFree(pVgroupInfo->epAddr[i].fqdn); + tfree(pVgroupInfo->epAddr[i].fqdn); pVgroupInfo->epAddr[i].fqdn = strndup(pEpSet->fqdn[i], tListLen(pEpSet->fqdn[i])); pVgroupInfo->epAddr[i].port = pEpSet->port[i]; } @@ -150,7 +150,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (pObj == NULL) return; if (pObj != pObj->signature) { - tscError("heart beat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); + tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); return; } @@ -158,12 +158,12 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { SSqlRes *pRes = &pSql->res; if (code == 0) { - SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; - SRpcEpSet * epSet = &pRsp->epSet; + SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp; + SRpcEpSet * epSet = &pRsp->epSet; if (epSet->numOfEps > 0) { tscEpSetHtons(epSet); tscUpdateMgmtEpSet(epSet); - } + } pSql->pTscObj->connId = htonl(pRsp->connId); @@ -175,44 +175,44 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId)); } } else { - tscDebug("heartbeat failed, code:%s", tstrerror(code)); + tscDebug("%p heartbeat failed, code:%s", pObj->pHb, tstrerror(code)); } if (pObj->pHb != NULL) { int32_t waitingDuring = tsShellActivityTimer * 500; - tscDebug("%p start heartbeat in %dms", pSql, waitingDuring); + tscDebug("%p send heartbeat in %dms", pSql, waitingDuring); - taosTmrReset(tscProcessActivityTimer, waitingDuring, pObj, tscTmr, &pObj->pTimer); + taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer); } else { tscDebug("%p start to close tscObj:%p, not send heartbeat again", pSql, pObj); } } void tscProcessActivityTimer(void *handle, void *tmrId) { - STscObj *pObj = (STscObj *)handle; - if (pObj == NULL || pObj->signature != pObj) { - return; - } + int64_t rid = (int64_t) handle; + STscObj *pObj = taosAcquireRef(tscRefId, rid); + if (pObj == NULL) return; SSqlObj* pHB = pObj->pHb; - if (pObj->pTimer != tmrId || pHB == NULL) { - return; - } void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE)); if (p == NULL) { tscWarn("%p HB object has been released already", pHB); + taosReleaseRef(tscRefId, pObj->rid); return; } assert(*pHB->self == pHB); + pHB->retry = 0; int32_t code = tscProcessSql(pHB); taosCacheRelease(tscObjCache, (void**) &p, false); if (code != TSDB_CODE_SUCCESS) { tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code)); } + + taosReleaseRef(tscRefId, rid); } int tscSendMsgToServer(SSqlObj *pSql) { @@ -237,7 +237,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { .pCont = pMsg, .contLen = pSql->cmd.payloadLen, .ahandle = pSql, - .handle = &pSql->pRpcCtx, + .handle = NULL, .code = 0 }; @@ -245,7 +245,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { // Otherwise, the pSql object may have been released already during the response function, which is // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // cause crash. - rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); + pSql->rpcRid = rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); return TSDB_CODE_SUCCESS; } @@ -265,7 +265,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { SSqlCmd *pCmd = &pSql->cmd; assert(*pSql->self == pSql); - pSql->pRpcCtx = NULL; + pSql->rpcRid = -1; if (pObj->signature != pObj) { tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature); @@ -481,14 +481,25 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { int32_t vgIndex = pTableMetaInfo->vgroupIndex; - - SVgroupsInfo* pVgroupInfo = pTableMetaInfo->vgroupList; - assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); + if (pTableMetaInfo->pVgroupTables == NULL) { + SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; + assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); + + pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId); + tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d", pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex); + } else { + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(vgIndex >= 0 && vgIndex < numOfVgroups); + + SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); - pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId); + pRetrieveMsg->header.vgId = htonl(pTableIdList->vgInfo.vgId); + tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d", pSql, pTableIdList->vgInfo.vgId, vgIndex); + } } else { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId); + tscDebug("%p build fetch msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgroupInfo.vgId); } pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); @@ -538,11 +549,28 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); - - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); int32_t exprSize = (int32_t)(sizeof(SSqlFuncMsg) * numOfExprs); - - return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + 4096; + + int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; + + int32_t tableSerialize = 0; + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo->pVgroupTables != NULL) { + size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + + int32_t totalTables = 0; + for (int32_t i = 0; i < numOfGroups; ++i) { + SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i); + totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList); + } + + tableSerialize = totalTables * sizeof(STableIdInfo); + } + + return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize + + tableSerialize + 4096; } static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) { @@ -552,7 +580,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { - SCMVgroupInfo* pVgroupInfo = NULL; + SVgroupInfo* pVgroupInfo = NULL; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { int32_t index = pTableMetaInfo->vgroupIndex; assert(index >= 0); @@ -662,12 +690,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->limit = htobe64(pQueryInfo->limit.limit); pQueryMsg->offset = htobe64(pQueryInfo->limit.offset); pQueryMsg->numOfCols = htons((int16_t)taosArrayGetSize(pQueryInfo->colList)); - pQueryMsg->interval.interval = htobe64(pQueryInfo->interval.interval); - pQueryMsg->interval.sliding = htobe64(pQueryInfo->interval.sliding); + pQueryMsg->interval.interval = htobe64(pQueryInfo->interval.interval); + pQueryMsg->interval.sliding = htobe64(pQueryInfo->interval.sliding); pQueryMsg->interval.offset = htobe64(pQueryInfo->interval.offset); pQueryMsg->interval.intervalUnit = pQueryInfo->interval.intervalUnit; - pQueryMsg->interval.slidingUnit = pQueryInfo->interval.slidingUnit; - pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit; + pQueryMsg->interval.slidingUnit = pQueryInfo->interval.slidingUnit; + pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); @@ -846,36 +874,20 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // compressed ts block pQueryMsg->tsOffset = htonl((int32_t)(pMsg - pCmd->payload)); - int32_t tsLen = 0; - int32_t numOfBlocks = 0; if (pQueryInfo->tsBuf != NULL) { - STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, pTableMetaInfo->vgroupIndex); - assert(QUERY_IS_JOIN_QUERY(pQueryInfo->type) && pBlockInfo != NULL); // this query should not be sent - - // todo refactor - if (fseek(pQueryInfo->tsBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { - int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f)); - tscError("%p: fseek failed: %s", pSql, tstrerror(code)); - return code; - } - - size_t s = fread(pMsg, 1, pBlockInfo->compLen, pQueryInfo->tsBuf->f); - if (s != pBlockInfo->compLen) { - int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f)); - tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); + // note: here used the index instead of actual vnode id. + int32_t vnodeIndex = pTableMetaInfo->vgroupIndex; + int32_t code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsLen, &pQueryMsg->tsNumOfBlocks); + if (code != TSDB_CODE_SUCCESS) { return code; } - pMsg += pBlockInfo->compLen; - tsLen = pBlockInfo->compLen; - numOfBlocks = pBlockInfo->numOfBlocks; - } + pMsg += pQueryMsg->tsLen; - pQueryMsg->tsLen = htonl(tsLen); - pQueryMsg->tsNumOfBlocks = htonl(numOfBlocks); - if (pQueryInfo->tsBuf != NULL) { pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); + pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); + pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); } int32_t msgLen = (int32_t)(pMsg - pCmd->payload); @@ -892,10 +904,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMCreateDbMsg); + pCmd->payloadLen = sizeof(SCreateDbMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DB; - SCMCreateDbMsg *pCreateDbMsg = (SCMCreateDbMsg*)pCmd->payload; + SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload; assert(pCmd->numOfClause == 1); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); @@ -906,13 +918,13 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMCreateDnodeMsg); + pCmd->payloadLen = sizeof(SCreateDnodeMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMCreateDnodeMsg *pCreate = (SCMCreateDnodeMsg *)pCmd->payload; + SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload; strncpy(pCreate->ep, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DNODE; @@ -922,13 +934,13 @@ int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMCreateAcctMsg); + pCmd->payloadLen = sizeof(SCreateAcctMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMCreateAcctMsg *pAlterMsg = (SCMCreateAcctMsg *)pCmd->payload; + SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload; SStrToken *pName = &pInfo->pDCLInfo->user.user; SStrToken *pPwd = &pInfo->pDCLInfo->user.passwd; @@ -967,14 +979,14 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMCreateUserMsg); + pCmd->payloadLen = sizeof(SCreateUserMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMCreateUserMsg *pAlterMsg = (SCMCreateUserMsg*)pCmd->payload; + SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload; SUserInfo *pUser = &pInfo->pDCLInfo->user; strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n); @@ -999,21 +1011,21 @@ int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMCfgDnodeMsg); + pCmd->payloadLen = sizeof(SCfgDnodeMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_CONFIG_DNODE; return TSDB_CODE_SUCCESS; } int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMDropDbMsg); + pCmd->payloadLen = sizeof(SDropDbMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMDropDbMsg *pDropDbMsg = (SCMDropDbMsg*)pCmd->payload; + SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); tstrncpy(pDropDbMsg->db, pTableMetaInfo->name, sizeof(pDropDbMsg->db)); @@ -1043,13 +1055,13 @@ int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMDropDnodeMsg); + pCmd->payloadLen = sizeof(SDropDnodeMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload; + SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); tstrncpy(pDrop->ep, pTableMetaInfo->name, sizeof(pDrop->ep)); pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE; @@ -1059,7 +1071,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMDropUserMsg); + pCmd->payloadLen = sizeof(SDropUserMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_USER; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { @@ -1067,7 +1079,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload; + SDropUserMsg * pDropMsg = (SDropUserMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user)); @@ -1076,7 +1088,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMDropUserMsg); + pCmd->payloadLen = sizeof(SDropUserMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_ACCT; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { @@ -1084,7 +1096,7 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload; + SDropUserMsg * pDropMsg = (SDropUserMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user)); @@ -1093,14 +1105,14 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMUseDbMsg); + pCmd->payloadLen = sizeof(SUseDbMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMUseDbMsg *pUseDbMsg = (SCMUseDbMsg*)pCmd->payload; + SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); strcpy(pUseDbMsg->db, pTableMetaInfo->name); pCmd->msgType = TSDB_MSG_TYPE_CM_USE_DB; @@ -1112,14 +1124,14 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STscObj *pObj = pSql->pTscObj; SSqlCmd *pCmd = &pSql->cmd; pCmd->msgType = TSDB_MSG_TYPE_CM_SHOW; - pCmd->payloadLen = sizeof(SCMShowMsg) + 100; + pCmd->payloadLen = sizeof(SShowMsg) + 100; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMShowMsg *pShowMsg = (SCMShowMsg*)pCmd->payload; + SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); size_t nameLen = strlen(pTableMetaInfo->name); @@ -1146,13 +1158,13 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShowMsg->payloadLen = htons(pEpAddr->n); } - pCmd->payloadLen = sizeof(SCMShowMsg) + pShowMsg->payloadLen; + pCmd->payloadLen = sizeof(SShowMsg) + pShowMsg->payloadLen; return TSDB_CODE_SUCCESS; } int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMKillQueryMsg); + pCmd->payloadLen = sizeof(SKillQueryMsg); switch (pCmd->command) { case TSDB_SQL_KILL_QUERY: @@ -1264,8 +1276,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - return minMsgSize() + sizeof(SCMAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + - TSDB_EXTRA_PAYLOAD_SIZE; + return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE; } int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { @@ -1284,7 +1295,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMAlterTableMsg *pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload; + SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload; tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pAlterTableMsg->db); strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name); @@ -1333,10 +1344,10 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMAlterDbMsg); + pCmd->payloadLen = sizeof(SAlterDbMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_ALTER_DB; - SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload; + SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); tstrncpy(pAlterDbMsg->db, pTableMetaInfo->name, sizeof(pAlterDbMsg->db)); @@ -1461,14 +1472,14 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STscObj *pObj = pSql->pTscObj; SSqlCmd *pCmd = &pSql->cmd; pCmd->msgType = TSDB_MSG_TYPE_CM_CONNECT; - pCmd->payloadLen = sizeof(SCMConnectMsg); + pCmd->payloadLen = sizeof(SConnectMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMConnectMsg *pConnect = (SCMConnectMsg*)pCmd->payload; + SConnectMsg *pConnect = (SConnectMsg*)pCmd->payload; // TODO refactor full_name char *db; // ugly code to move the space @@ -1490,11 +1501,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SCMTableInfoMsg* pInfoMsg = (SCMTableInfoMsg *)pCmd->payload; + STableInfoMsg *pInfoMsg = (STableInfoMsg *)pCmd->payload; strcpy(pInfoMsg->tableId, pTableMetaInfo->name); pInfoMsg->createFlag = htons(pSql->cmd.autoCreated ? 1 : 0); - char* pMsg = (char*)pInfoMsg + sizeof(SCMTableInfoMsg); + char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); size_t len = htonl(pCmd->tagData.dataLen); if (pSql->cmd.autoCreated) { @@ -1513,7 +1524,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { /** * multi table meta req pkg format: - * | SMgmtHead | SCMMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... + * | SMgmtHead | SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... * no used 4B **/ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { @@ -1531,16 +1542,16 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SMgmtHead *pMgmt = (SMgmtHead *)(pCmd->payload + tsRpcHeadSize); memset(pMgmt->db, 0, TSDB_TABLE_FNAME_LEN); // server don't need the db - SCMMultiTableInfoMsg *pInfoMsg = (SCMMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead)); + SMultiTableInfoMsg *pInfoMsg = (SMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead)); pInfoMsg->numOfTables = htonl((int32_t)pCmd->count); if (pCmd->payloadLen > 0) { memcpy(pInfoMsg->tableIds, tmpData, pCmd->payloadLen); } - taosTFree(tmpData); + tfree(tmpData); - pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SCMMultiTableInfoMsg); + pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SMultiTableInfoMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_TABLES_META; assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize); @@ -1585,12 +1596,12 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { char* pMsg = pCmd->payload; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - - SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pMsg; + + SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg; pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables); - pMsg += sizeof(SCMSTableVgroupMsg); - - for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + pMsg += sizeof(SSTableVgroupMsg); + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i); size_t size = sizeof(pTableMetaInfo->name); tstrncpy(pMsg, pTableMetaInfo->name, size); @@ -1623,14 +1634,17 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { numOfStreams++; } - int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SCMHeartBeatMsg) + 100; + int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { pthread_mutex_unlock(&pObj->mutex); - tscError("%p failed to malloc for heartbeat msg", pSql); + tscError("%p failed to create heartbeat msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload; + // TODO the expired hb and client can not be identified by server till now. + SHeartBeatMsg *pHeartbeat = (SHeartBeatMsg *)pCmd->payload; + tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer)); + pHeartbeat->numOfQueries = numOfQueries; pHeartbeat->numOfStreams = numOfStreams; @@ -1704,8 +1718,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000); - - // todo handle out of memory case + if (pTableMetaInfo->pTableMeta == NULL) { free(pTableMeta); return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1719,7 +1732,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { /** * multi table meta rsp pkg format: - * | STaosRsp | ieType | SCMMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 + * | STaosRsp | ieType | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 * |...... 1B 1B 4B **/ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { @@ -1736,9 +1749,9 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { rsp++; - SCMMultiTableInfoMsg *pInfo = (SCMMultiTableInfoMsg *)rsp; + SMultiTableInfoMsg *pInfo = (SMultiTableInfoMsg *)rsp; totalNum = htonl(pInfo->numOfTables); - rsp += sizeof(SCMMultiTableInfoMsg); + rsp += sizeof(SMultiTableInfoMsg); for (i = 0; i < totalNum; i++) { SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp; @@ -1830,10 +1843,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { SSqlRes* pRes = &pSql->res; // NOTE: the order of several table must be preserved. - SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pRes->pRsp; + SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp; pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); - char* pMsg = pRes->pRsp + sizeof(SCMSTableVgroupRspMsg); - + char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg); + // master sqlObj locates in param SSqlObj* parent = pSql->param; assert(parent != NULL); @@ -1845,18 +1858,18 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { SVgroupsMsg * pVgroupMsg = (SVgroupsMsg *) pMsg; pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); - size_t size = sizeof(SCMVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg); + size_t size = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg); - size_t vgroupsz = sizeof(SCMVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); + size_t vgroupsz = sizeof(SVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); pInfo->vgroupList = calloc(1, vgroupsz); assert(pInfo->vgroupList != NULL); pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups; for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { //just init, no need to lock - SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; + SVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; - SCMVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; + SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; pVgroups->vgId = htonl(vmsg->vgId); pVgroups->numOfEps = vmsg->numOfEps; @@ -1878,10 +1891,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { * current process do not use the cache at all */ int tscProcessShowRsp(SSqlObj *pSql) { - STableMetaMsg * pMetaMsg; - SCMShowRsp *pShow; - SSchema * pSchema; - char key[20]; + STableMetaMsg *pMetaMsg; + SShowRsp * pShow; + SSchema * pSchema; + char key[20]; SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -1890,7 +1903,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - pShow = (SCMShowRsp *)pRes->pRsp; + pShow = (SShowRsp *)pRes->pRsp; pShow->qhandle = htobe64(pShow->qhandle); pRes->qhandle = pShow->qhandle; @@ -1943,7 +1956,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; tscFieldInfoUpdateOffset(pQueryInfo); - taosTFree(pTableMeta); + tfree(pTableMeta); return 0; } @@ -1968,7 +1981,7 @@ static void createHBObj(STscObj* pObj) { pSql->cmd.command = pQueryInfo->command; if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { - taosTFree(pSql); + tfree(pSql); return; } @@ -1983,11 +1996,12 @@ static void createHBObj(STscObj* pObj) { } int tscProcessConnectRsp(SSqlObj *pSql) { - char temp[TSDB_TABLE_FNAME_LEN * 2]; STscObj *pObj = pSql->pTscObj; SSqlRes *pRes = &pSql->res; - SCMConnectRsp *pConnect = (SCMConnectRsp *)pRes->pRsp; + char temp[TSDB_TABLE_FNAME_LEN * 2] = {0}; + + SConnectRsp *pConnect = (SConnectRsp *)pRes->pRsp; tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId)); // copy acctId from response int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); @@ -2005,7 +2019,9 @@ int tscProcessConnectRsp(SSqlObj *pSql) { pObj->connId = htonl(pConnect->connId); createHBObj(pObj); - taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer); + + //launch a timer to send heartbeat to maintain the connection and send status to mnode + taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pObj->rid, tscTmr, &pObj->pTimer); return 0; } @@ -2271,7 +2287,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); STableMeta *pTableMeta = taosCacheAcquireByData(tscMetaCache, pMInfo->pTableMeta); - tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList); + tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); } if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 8cac9b3398a0bc4569b52acc7858bbc1e50de542..5f8a2eb6b764983a529ff58ffb84dc47447c281b 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -161,6 +161,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa registerSqlObj(pSql); tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg); + pObj->rid = taosAddRef(tscRefId, pObj); return pSql; } @@ -278,9 +279,9 @@ void taos_close(TAOS *taos) { SSqlObj* pHb = pObj->pHb; if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) { - if (pHb->pRpcCtx != NULL) { // wait for rsp from dnode - rpcCancelRequest(pHb->pRpcCtx); - pHb->pRpcCtx = NULL; + if (pHb->rpcRid > 0) { // wait for rsp from dnode + rpcCancelRequest(pHb->rpcRid); + pHb->rpcRid = -1; } tscDebug("%p HB is freed", pHb); @@ -296,7 +297,8 @@ void taos_close(TAOS *taos) { } tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn); - tscCloseTscObj(pObj); + + taosRemoveRef(tscRefId, pObj->rid); } void waitForQueryRsp(void *param, TAOS_RES *tres, int code) { @@ -564,7 +566,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { pRes->rspType = 0; pSql->subState.numOfSub = 0; - taosTFree(pSql->pSubs); + tfree(pSql->pSubs); assert(pSql->fp == NULL); @@ -746,9 +748,9 @@ static void tscKillSTableQuery(SSqlObj *pSql) { assert(pSubObj->self == (SSqlObj**) p); pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - if (pSubObj->pRpcCtx != NULL) { - rpcCancelRequest(pSubObj->pRpcCtx); - pSubObj->pRpcCtx = NULL; + if (pSubObj->rpcRid > 0) { + rpcCancelRequest(pSubObj->rpcRid); + pSubObj->rpcRid = -1; } tscQueueAsyncRes(pSubObj); @@ -773,7 +775,7 @@ void taos_stop_query(TAOS_RES *res) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { - assert(pSql->pRpcCtx == NULL); + assert(pSql->rpcRid <= 0); tscKillSTableQuery(pSql); } else { if (pSql->cmd.command < TSDB_SQL_LOCAL) { @@ -782,9 +784,9 @@ void taos_stop_query(TAOS_RES *res) { * reset and freed in the processMsgFromServer function, and causes the invalid * write problem for rpcCancelRequest. */ - if (pSql->pRpcCtx != NULL) { - rpcCancelRequest(pSql->pRpcCtx); - pSql->pRpcCtx = NULL; + if (pSql->rpcRid > 0) { + rpcCancelRequest(pSql->rpcRid); + pSql->rpcRid = -1; } tscQueueAsyncRes(pSql); @@ -892,7 +894,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { if (sqlLen > tsMaxSQLStringLen) { tscError("%p sql too long", pSql); pRes->code = TSDB_CODE_TSC_INVALID_SQL; - taosTFree(pSql); + tfree(pSql); return pRes->code; } @@ -901,7 +903,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscError("%p failed to malloc sql string buffer", pSql); tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(pSql), pObj); - taosTFree(pSql); + tfree(pSql); return pRes->code; } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 0f67911bbea992503979e5de019e4e10d3bf3c14..68c3bcae165050863cc4bf9c92a1510581531c3a 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -273,7 +273,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); tscFreeSqlResult(pSql); - taosTFree(pSql->pSubs); + tfree(pSql->pSubs); pSql->subState.numOfSub = 0; pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); tscSetNextLaunchTimer(pStream, pSql); @@ -617,6 +617,6 @@ void taos_close_stream(TAOS_STREAM *handle) { pStream->pSql = NULL; taos_free_result(pSql); - taosTFree(pStream); + tfree(pStream); } } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 6b615c3a9b14f97f57013f476b60acc0db53db53..eb32e2490a3aa1a363be632a31e051ad2ca7a6a8 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -23,7 +23,6 @@ #include "tscSubquery.h" #include "tschemautil.h" #include "tsclient.h" -#include "tscSubquery.h" typedef struct SInsertSupporter { SSqlObj* pSql; @@ -33,11 +32,26 @@ typedef struct SInsertSupporter { static void freeJoinSubqueryObj(SSqlObj* pSql); static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql); -static bool tsCompare(int32_t order, int64_t left, int64_t right) { +static int32_t tsCompare(int32_t order, int64_t left, int64_t right) { + if (left == right) { + return 0; + } + if (order == TSDB_ORDER_ASC) { - return left < right; + return left < right? -1:1; } else { - return left > right; + return left > right? -1:1; + } +} + +static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) { + while (tsBufNextPos(pTSBuf)) { + STSElem el1 = tsBufGetElem(pTSBuf); + + int32_t res = tVariantCompare(el1.tag, tag1); + if (res != 0) { // it is a record with new tag + return; + } } } @@ -52,13 +66,15 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ SLimitVal* pLimit = &pQueryInfo->limit; int32_t order = pQueryInfo->order.order; - + SQueryInfo* pSubQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[0]->cmd, 0); SQueryInfo* pSubQueryInfo2 = tscGetQueryInfoDetail(&pSql->pSubs[1]->cmd, 0); - + pSubQueryInfo1->tsBuf = output1; pSubQueryInfo2->tsBuf = output2; + TSKEY st = taosGetTimestampUs(); + // no result generated, return directly if (pSupporter1->pTSBuf == NULL || pSupporter2->pTSBuf == NULL) { tscDebug("%p at least one ts-comp is empty, 0 for secondary query after ts blocks intersecting", pSql); @@ -87,58 +103,74 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ int64_t numOfInput1 = 1; int64_t numOfInput2 = 1; - while (1) { - STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf); - STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); + while(1) { + STSElem elem = tsBufGetElem(pSupporter1->pTSBuf); -#ifdef _DEBUG_VIEW - tscInfo("%" PRId64 ", tags:%"PRId64" \t %" PRId64 ", tags:%"PRId64, elem1.ts, elem1.tag.i64Key, elem2.ts, elem2.tag.i64Key); -#endif + // no data in pSupporter1 anymore, jump out of loop + if (!tsBufIsValidElem(&elem)) { + break; + } - int32_t res = tVariantCompare(&elem1.tag, &elem2.tag); - if (res == -1 || (res == 0 && tsCompare(order, elem1.ts, elem2.ts))) { - if (!tsBufNextPos(pSupporter1->pTSBuf)) { - break; - } + // find the data in supporter2 with the same tag value + STSElem e2 = tsBufFindElemStartPosByTag(pSupporter2->pTSBuf, elem.tag); - numOfInput1++; - } else if ((res > 0) || (res == 0 && tsCompare(order, elem2.ts, elem1.ts))) { - if (!tsBufNextPos(pSupporter2->pTSBuf)) { - break; - } + /** + * there are elements in pSupporter2 with the same tag, continue + */ + tVariant tag1 = {0}; + tVariantAssign(&tag1, elem.tag); - numOfInput2++; - } else { - /* - * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the - * final results which is acquired after the secondry merge of in the client. - */ - if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { - if (win->skey > elem1.ts) { - win->skey = elem1.ts; + if (tsBufIsValidElem(&e2)) { + while (1) { + STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf); + STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); + + // data with current are exhausted + if (!tsBufIsValidElem(&elem1) || tVariantCompare(elem1.tag, &tag1) != 0) { + break; } - - if (win->ekey < elem1.ts) { - win->ekey = elem1.ts; + + if (!tsBufIsValidElem(&elem2) || tVariantCompare(elem2.tag, &tag1) != 0) { // ignore all records with the same tag + skipRemainValue(pSupporter1->pTSBuf, &tag1); + break; } - - tsBufAppend(output1, elem1.vnode, &elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); - tsBufAppend(output2, elem2.vnode, &elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); - } else { - pLimit->offset -= 1; - } - if (!tsBufNextPos(pSupporter1->pTSBuf)) { - break; - } + /* + * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the + * final results which is acquired after the secondary merge of in the client. + */ + int32_t re = tsCompare(order, elem1.ts, elem2.ts); + if (re < 0) { + tsBufNextPos(pSupporter1->pTSBuf); + numOfInput1++; + } else if (re > 0) { + tsBufNextPos(pSupporter2->pTSBuf); + numOfInput2++; + } else { + if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { + if (win->skey > elem1.ts) { + win->skey = elem1.ts; + } + + if (win->ekey < elem1.ts) { + win->ekey = elem1.ts; + } + + tsBufAppend(output1, elem1.id, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); + tsBufAppend(output2, elem2.id, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + } else { + pLimit->offset -= 1;//offset apply to projection? + } - numOfInput1++; + tsBufNextPos(pSupporter1->pTSBuf); + numOfInput1++; - if (!tsBufNextPos(pSupporter2->pTSBuf)) { - break; + tsBufNextPos(pSupporter2->pTSBuf); + numOfInput2++; + } } - - numOfInput2++; + } else { // no data in pSupporter2, ignore current data in pSupporter2 + skipRemainValue(pSupporter1->pTSBuf, &tag1); } } @@ -158,9 +190,11 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ tsBufDestroy(pSupporter1->pTSBuf); tsBufDestroy(pSupporter2->pTSBuf); - tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " for secondary query after ts blocks " - "intersecting, skey:%" PRId64 ", ekey:%" PRId64, pSql, numOfInput1, numOfInput2, output1->numOfTotal, - win->skey, win->ekey); + TSKEY et = taosGetTimestampUs(); + tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks " + "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elapsed time:%" PRId64 " us", + pSql, numOfInput1, numOfInput2, output1->numOfTotal, output1->numOfGroups, win->skey, win->ekey, + tsBufGetNumOfGroup(output1), et - st); return output1->numOfTotal; } @@ -216,7 +250,12 @@ static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) { pSupporter->f = NULL; } - taosTFree(pSupporter->pIdTagList); + if (pSupporter->pVgroupTables != NULL) { + taosArrayDestroy(pSupporter->pVgroupTables); + pSupporter->pVgroupTables = NULL; + } + + tfree(pSupporter->pIdTagList); tscTagCondRelease(&pSupporter->tagCond); free(pSupporter); } @@ -240,6 +279,68 @@ static UNUSED_FUNC bool needSecondaryQuery(SQueryInfo* pQueryInfo) { return false; } +static void filterVgroupTables(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { + int32_t num = 0; + int32_t* list = NULL; + tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list); + + // The virtual node, of which all tables are disqualified after the timestamp intersection, + // is removed to avoid next stage query. + // TODO: If tables from some vnodes are not qualified for next stage query, discard them. + for (int32_t k = 0; k < taosArrayGetSize(pVgroupTables);) { + SVgroupTableInfo* p = taosArrayGet(pVgroupTables, k); + + bool found = false; + for (int32_t f = 0; f < num; ++f) { + if (p->vgInfo.vgId == list[f]) { + found = true; + break; + } + } + + if (!found) { + tscRemoveVgroupTableGroup(pVgroupTables, k); + } else { + k++; + } + } + + assert(taosArrayGetSize(pVgroupTables) > 0); + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); + + tfree(list); +} + +static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { + int32_t num = 0; + int32_t* list = NULL; + tsBufGetGroupIdList(pQueryInfo->tsBuf, &num, &list); + + size_t numOfGroups = taosArrayGetSize(pVgroupTables); + + SArray* pNew = taosArrayInit(num, sizeof(SVgroupTableInfo)); + + SVgroupTableInfo info; + for (int32_t i = 0; i < num; ++i) { + int32_t vnodeId = list[i]; + + for (int32_t j = 0; j < numOfGroups; ++j) { + SVgroupTableInfo* p1 = taosArrayGet(pVgroupTables, j); + if (p1->vgInfo.vgId == vnodeId) { + tscVgroupTableCopy(&info, p1); + break; + } + } + + taosArrayPush(pNew, &info); + } + + tfree(list); + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); + + return pNew; +} + /* * launch secondary stage query to fetch the result that contains timestamp in set */ @@ -305,7 +406,6 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { // set the second stage sub query for join process TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE); - memcpy(&pQueryInfo->interval, &pSupporter->interval, sizeof(pQueryInfo->interval)); tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); @@ -313,24 +413,27 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pQueryInfo->colList = pSupporter->colList; pQueryInfo->exprList = pSupporter->exprList; pQueryInfo->fieldsInfo = pSupporter->fieldsInfo; + pQueryInfo->groupbyExpr = pSupporter->groupInfo; + + assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1); + + tscFieldInfoUpdateOffset(pQueryInfo); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables; pSupporter->exprList = NULL; pSupporter->colList = NULL; + pSupporter->pVgroupTables = NULL; memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo)); - - SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); - - tscFieldInfoUpdateOffset(pNewQueryInfo); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); - + memset(&pSupporter->groupInfo, 0, sizeof(SSqlGroupbyExpr)); + /* * When handling the projection query, the offset value will be modified for table-table join, which is changed * during the timestamp intersection. */ pSupporter->limit = pQueryInfo->limit; - pNewQueryInfo->limit = pSupporter->limit; + pQueryInfo->limit = pSupporter->limit; SColumnIndex index = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; SSchema* s = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, 0); @@ -345,7 +448,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { tscAddSpecialColumnForSelect(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL); tscPrintSelectClause(pNew, 0); - tscFieldInfoUpdateOffset(pNewQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); pExpr = tscSqlExprGet(pQueryInfo, 0); } @@ -356,14 +459,25 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); // set the tag column id for executor to extract correct tag value - pExpr->param[0].i64Key = colId; + pExpr->param[0] = (tVariant) {.i64Key = colId, .nType = TSDB_DATA_TYPE_BIGINT, .nLen = sizeof(int64_t)}; pExpr->numOfParams = 1; } - size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + assert(pTableMetaInfo->pVgroupTables != NULL); + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + SArray* p = buildVgroupTableByResult(pQueryInfo, pTableMetaInfo->pVgroupTables); + tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); + pTableMetaInfo->pVgroupTables = p; + } else { + filterVgroupTables(pQueryInfo, pTableMetaInfo->pVgroupTables); + } + } + + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); tscDebug("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", - pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, taosArrayGetSize(pNewQueryInfo->exprList), - numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name); + pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, taosArrayGetSize(pQueryInfo->exprList), + numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name); } //prepare the subqueries object failed, abort @@ -409,7 +523,7 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) { assert(pSqlObj->subState.numOfRemain > 0); if (atomic_sub_fetch_32(&pSqlObj->subState.numOfRemain, 1) <= 0) { - tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code); + tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code)); freeJoinSubqueryObj(pSqlObj); } } @@ -418,20 +532,40 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) { static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) { assert(pQueryInfo->window.skey <= win->skey && pQueryInfo->window.ekey >= win->ekey); pQueryInfo->window = *win; + + } -int32_t tscCompareTidTags(const void* p1, const void* p2) { - const STidTags* t1 = (const STidTags*) varDataVal(p1); - const STidTags* t2 = (const STidTags*) varDataVal(p2); +int32_t tidTagsCompar(const void* p1, const void* p2) { + const STidTags* t1 = (const STidTags*) (p1); + const STidTags* t2 = (const STidTags*) (p2); if (t1->vgId != t2->vgId) { return (t1->vgId > t2->vgId) ? 1 : -1; } - if (t1->tid != t2->tid) { - return (t1->tid > t2->tid) ? 1 : -1; + tstr* tag1 = (tstr*) t1->tag; + tstr* tag2 = (tstr*) t2->tag; + + if (tag1->len != tag2->len) { + return (tag1->len > tag2->len)? 1: -1; + } + + return strncmp(tag1->data, tag2->data, tag1->len); +} + +int32_t tagValCompar(const void* p1, const void* p2) { + const STidTags* t1 = (const STidTags*) varDataVal(p1); + const STidTags* t2 = (const STidTags*) varDataVal(p2); + + tstr* tag1 = (tstr*) t1->tag; + tstr* tag2 = (tstr*) t2->tag; + + if (tag1->len != tag2->len) { + return (tag1->len > tag2->len)? 1: -1; } - return 0; + + return memcmp(tag1->data, tag2->data, tag1->len); } void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) { @@ -449,7 +583,7 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr SVgroupTableInfo info = {{0}}; for (int32_t m = 0; m < pvg->numOfVgroups; ++m) { if (tt->vgId == pvg->vgroups[m].vgId) { - tscSCMVgroupInfoCopy(&info.vgInfo, &pvg->vgroups[m]); + tscSVgroupInfoCopy(&info.vgInfo, &pvg->vgroups[m]); break; } } @@ -457,27 +591,40 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr vgTables = taosArrayInit(4, sizeof(STableIdInfo)); info.itemList = vgTables; + + if (taosArrayGetSize(result) > 0) { + SVgroupTableInfo* prevGroup = taosArrayGet(result, taosArrayGetSize(result) - 1); + tscDebug("%p vgId:%d, tables:%"PRId64, pSql, prevGroup->vgInfo.vgId, taosArrayGetSize(prevGroup->itemList)); + } + taosArrayPush(result, &info); } - tscDebug("%p tid:%d, uid:%"PRIu64",vgId:%d added for vnode query", pSql, tt->tid, tt->uid, tt->vgId) STableIdInfo item = {.uid = tt->uid, .tid = tt->tid, .key = INT64_MIN}; taosArrayPush(vgTables, &item); + + tscTrace("%p tid:%d, uid:%"PRIu64",vgId:%d added", pSql, tt->tid, tt->uid, tt->vgId); prev = tt; } pTableMetaInfo->pVgroupTables = result; pTableMetaInfo->vgroupIndex = 0; + + if (taosArrayGetSize(result) > 0) { + SVgroupTableInfo* g = taosArrayGet(result, taosArrayGetSize(result) - 1); + tscDebug("%p vgId:%d, tables:%"PRId64, pSql, g->vgInfo.vgId, taosArrayGetSize(g->itemList)); + } } static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) { SSqlCmd* pCmd = &pSql->cmd; tscClearSubqueryInfo(pCmd); tscFreeSqlResult(pSql); - + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + assert(pQueryInfo->numOfTables == 1); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - tscInitQueryInfo(pQueryInfo); TSDB_QUERY_CLEAR_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); @@ -524,13 +671,7 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* tscProcessSql(pSql); } -static bool checkForDuplicateTagVal(SQueryInfo* pQueryInfo, SJoinSupporter* p1, SSqlObj* pPSqlObj) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);// todo: tags mismatch, tags not completed - SColumn *pCol = taosArrayGetP(pTableMetaInfo->tagColList, 0); - SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; - +static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSqlObj* pPSqlObj) { for(int32_t i = 1; i < p1->num; ++i) { STidTags* prev = (STidTags*) varDataVal(p1->pIdTagList + (i - 1) * p1->tagSize); STidTags* p = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); @@ -547,24 +688,26 @@ static bool checkForDuplicateTagVal(SQueryInfo* pQueryInfo, SJoinSupporter* p1, } static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray** s1, SArray** s2) { - tscDebug("%p all subqueries retrieve complete, do tags match", pParentSql); - SJoinSupporter* p1 = pParentSql->pSubs[0]->param; SJoinSupporter* p2 = pParentSql->pSubs[1]->param; - qsort(p1->pIdTagList, p1->num, p1->tagSize, tscCompareTidTags); - qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags); + tscDebug("%p all subquery retrieve complete, do tags match, %d, %d", pParentSql, p1->num, p2->num); + + // sort according to the tag value + qsort(p1->pIdTagList, p1->num, p1->tagSize, tagValCompar); + qsort(p2->pIdTagList, p2->num, p2->tagSize, tagValCompar); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); - SSchema* pColSchema = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); + SSchema* pColSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); // int16_t for padding - *s1 = taosArrayInit(p1->num, p1->tagSize - sizeof(int16_t)); - *s2 = taosArrayInit(p2->num, p2->tagSize - sizeof(int16_t)); + int32_t size = p1->tagSize - sizeof(int16_t); + *s1 = taosArrayInit(p1->num, size); + *s2 = taosArrayInit(p2->num, size); - if (!(checkForDuplicateTagVal(pQueryInfo, p1, pParentSql) && checkForDuplicateTagVal(pQueryInfo, p2, pParentSql))) { + if (!(checkForDuplicateTagVal(pColSchema, p1, pParentSql) && checkForDuplicateTagVal(pColSchema, p2, pParentSql))) { return TSDB_CODE_QRY_DUP_JOIN_KEY; } @@ -590,6 +733,27 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar } } + // reorganize the tid-tag value according to both the vgroup id and tag values + // sort according to the tag value + size_t t1 = taosArrayGetSize(*s1); + size_t t2 = taosArrayGetSize(*s2); + + qsort((*s1)->pData, t1, size, tidTagsCompar); + qsort((*s2)->pData, t2, size, tidTagsCompar); + +#if 0 + for(int32_t k = 0; k < t1; ++k) { + STidTags* p = (*s1)->pData + size * k; + printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data); + } + + for(int32_t k = 0; k < t1; ++k) { + STidTags* p = (*s2)->pData + size * k; + printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data); + } +#endif + + tscDebug("%p tags match complete, result: %"PRId64", %"PRId64, pParentSql, t1, t2); return TSDB_CODE_SUCCESS; } @@ -689,11 +853,15 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow } if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) { // no results,return. + assert(pParentSql->fp != tscJoinQueryCallback); + tscDebug("%p tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", pParentSql); freeJoinSubqueryObj(pParentSql); // set no result command pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + assert(pParentSql->fp != tscJoinQueryCallback); + (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { // proceed to for ts_comp query @@ -708,6 +876,12 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow STableMetaInfo* pTableMetaInfo2 = tscGetMetaInfo(pQueryInfo2, 0); tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo2, s2); + SSqlObj* psub1 = pParentSql->pSubs[0]; + ((SJoinSupporter*)psub1->param)->pVgroupTables = tscVgroupTableInfoClone(pTableMetaInfo1->pVgroupTables); + + SSqlObj* psub2 = pParentSql->pSubs[1]; + ((SJoinSupporter*)psub2->param)->pVgroupTables = tscVgroupTableInfoClone(pTableMetaInfo2->pVgroupTables); + pParentSql->subState.numOfSub = 2; pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub; @@ -766,9 +940,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow pSupporter->pTSBuf = pBuf; } else { assert(pQueryInfo->numOfTables == 1); // for subquery, only one - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - tsBufMerge(pSupporter->pTSBuf, pBuf, pTableMetaInfo->vgroupIndex); + tsBufMerge(pSupporter->pTSBuf, pBuf); tsBufDestroy(pBuf); } @@ -835,6 +1007,8 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow // launch the query the retrieve actual results from vnode along with the filtered timestamp SQueryInfo* pPQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex); updateQueryTimeRange(pPQueryInfo, &win); + + //update the vgroup that involved in real data query tscLaunchRealSubqueries(pParentSql); } @@ -868,20 +1042,28 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR assert(pQueryInfo->numOfTables == 1); // for projection query, need to try next vnode if current vnode is exhausted - if ((++pTableMetaInfo->vgroupIndex) < pTableMetaInfo->vgroupList->numOfVgroups) { - pState->numOfRemain = 1; - pState->numOfSub = 1; + int32_t numOfVgroups = 0; // TODO refactor + if (pTableMetaInfo->pVgroupTables != NULL) { + numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + } else { + numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; + } + if ((++pTableMetaInfo->vgroupIndex) < numOfVgroups) { + tscDebug("%p no result in current vnode anymore, try next vnode, vgIndex:%d", pSql, pTableMetaInfo->vgroupIndex); pSql->cmd.command = TSDB_SQL_SELECT; pSql->fp = tscJoinQueryCallback; - tscProcessSql(pSql); + tscProcessSql(pSql); return; + } else { + tscDebug("%p no result in current subquery anymore", pSql); } } - if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { - tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pParentSql->subState.numOfRemain, pState->numOfSub); + assert(pState->numOfRemain > 0); + if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) { + tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pState->numOfRemain, pState->numOfSub); return; } @@ -893,62 +1075,66 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR } // update the records for each subquery in parent sql object. + bool stableQuery = tscIsTwoStageSTableQuery(pQueryInfo, 0); for (int32_t i = 0; i < pState->numOfSub; ++i) { if (pParentSql->pSubs[i] == NULL) { + tscDebug("%p %p sub:%d not retrieve data", pParentSql, NULL, i); continue; } SSqlRes* pRes1 = &pParentSql->pSubs[i]->res; - pRes1->numOfClauseTotal += pRes1->numOfRows; - } - // data has retrieved to client, build the join results - tscBuildResFromSubqueries(pParentSql); -} - -static SJoinSupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t numOfFetch) { - int32_t notInvolved = 0; - SJoinSupporter* pSupporter = NULL; - SSubqueryState* pState = &pSql->subState; - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - if (pSql->pSubs[i] == NULL) { - notInvolved++; + if (pRes1->row > 0 && pRes1->numOfRows > 0) { + tscDebug("%p sub:%p index:%d numOfRows:%"PRId64" total:%"PRId64 " (not retrieve)", pParentSql, pParentSql->pSubs[i], i, + pRes1->numOfRows, pRes1->numOfTotal); + assert(pRes1->row < pRes1->numOfRows); } else { - pSupporter = (SJoinSupporter*)pSql->pSubs[i]->param; + if (!stableQuery) { + pRes1->numOfClauseTotal += pRes1->numOfRows; + } + + tscDebug("%p sub:%p index:%d numOfRows:%"PRId64" total:%"PRId64, pParentSql, pParentSql->pSubs[i], i, + pRes1->numOfRows, pRes1->numOfTotal); } } - - pState->numOfRemain = numOfFetch; - return pSupporter; + + // data has retrieved to client, build the join results + tscBuildResFromSubqueries(pParentSql); } -void tscFetchDatablockFromSubquery(SSqlObj* pSql) { +void tscFetchDatablockForSubquery(SSqlObj* pSql) { assert(pSql->subState.numOfSub >= 1); int32_t numOfFetch = 0; - bool hasData = true; + bool hasData = true; + bool reachLimit = false; + + // if the subquery is NULL, it does not involved in the final result generation for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - // if the subquery is NULL, it does not involved in the final result generation SSqlObj* pSub = pSql->pSubs[i]; if (pSub == NULL) { continue; } - + SSqlRes *pRes = &pSub->res; + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSub->cmd, 0); if (!tscHasReachLimitation(pQueryInfo, pRes)) { if (pRes->row >= pRes->numOfRows) { + // no data left in current result buffer hasData = false; + // The current query is completed for the active vnode, try next vnode if exists + // If it is completed, no need to fetch anymore. if (!pRes->completed) { numOfFetch++; } } } else { // has reach the limitation, no data anymore if (pRes->row >= pRes->numOfRows) { - hasData = false; + reachLimit = true; + hasData = false; break; } } @@ -958,29 +1144,113 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { if (hasData) { tscBuildResFromSubqueries(pSql); return; - } else if (numOfFetch <= 0) { + } + + // If at least one subquery is completed in current vnode, try the next vnode in case of multi-vnode + // super table projection query. + if (reachLimit) { pSql->res.completed = true; freeJoinSubqueryObj(pSql); - + if (pSql->res.code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, 0); } else { tscQueueAsyncRes(pSql); } - + + return; + } + + if (numOfFetch <= 0) { + bool tryNextVnode = false; + + bool orderedPrjQuery = false; + for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub == NULL) { + continue; + } + + SQueryInfo* p = tscGetQueryInfoDetail(&pSub->cmd, 0); + orderedPrjQuery = tscNonOrderedProjectionQueryOnSTable(p, 0); + if (orderedPrjQuery) { + break; + } + } + + // get the number of subquery that need to retrieve the next vnode. + if (orderedPrjQuery) { + for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub != NULL && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) { + pSql->subState.numOfRemain++; + } + } + } + + for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub == NULL) { + continue; + } + + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSub->cmd, 0); + + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && pSub->res.row >= pSub->res.numOfRows && + pSub->res.completed) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + assert(pQueryInfo->numOfTables == 1); + + // for projection query, need to try next vnode if current vnode is exhausted + int32_t numOfVgroups = 0; // TODO refactor + if (pTableMetaInfo->pVgroupTables != NULL) { + numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + } else { + numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; + } + + if ((++pTableMetaInfo->vgroupIndex) < numOfVgroups) { + tscDebug("%p no result in current vnode anymore, try next vnode, vgIndex:%d", pSub, + pTableMetaInfo->vgroupIndex); + pSub->cmd.command = TSDB_SQL_SELECT; + pSub->fp = tscJoinQueryCallback; + + tscProcessSql(pSub); + tryNextVnode = true; + } else { + tscDebug("%p no result in current subquery anymore", pSub); + } + } + } + + if (tryNextVnode) { + return; + } + + pSql->res.completed = true; + freeJoinSubqueryObj(pSql); + + if (pSql->res.code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, 0); + } else { + tscQueueAsyncRes(pSql); + } + return; } // TODO multi-vnode retrieve for projection query with limitation has bugs, since the global limiation is not handled + // retrieve data from current vnode. tscDebug("%p retrieve data from %d subqueries", pSql, numOfFetch); - SJoinSupporter* pSupporter = tscUpdateSubqueryStatus(pSql, numOfFetch); - + SJoinSupporter* pSupporter = NULL; + pSql->subState.numOfRemain = numOfFetch; + for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSql1 = pSql->pSubs[i]; if (pSql1 == NULL) { continue; } - + SSqlRes* pRes1 = &pSql1->res; SSqlCmd* pCmd1 = &pSql1->cmd; @@ -1013,7 +1283,6 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; - tscDebug("%p all subquery response, retrieve data for subclause:%d", pSql, pCmd->clauseIndex); // the column transfer support struct has been built if (pRes->pColumnIndex != NULL) { @@ -1109,21 +1378,23 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { return; } - // wait for the other subqueries response from vnode - if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { - return; + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + // In case of consequence query from other vnode, do not wait for other query response here. + if (!(pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0))) { + if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { + return; + } } tscSetupOutputColumnIndex(pParentSql); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); /** * if the query is a continue query (vgroupIndex > 0 for projection query) for next vnode, do the retrieval of * data instead of returning to its invoker */ if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub; // reset the record value - pSql->fp = joinRetrieveFinalResCallback; // continue retrieve data pSql->cmd.command = TSDB_SQL_FETCH; tscProcessSql(pSql); @@ -1192,6 +1463,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter return TSDB_CODE_TSC_OUT_OF_MEMORY; } + pSupporter->groupInfo = pNewQueryInfo->groupbyExpr; + memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); + pNew->cmd.numOfCols = 0; pNewQueryInfo->interval.interval = 0; pSupporter->limit = pNewQueryInfo->limit; @@ -1212,17 +1486,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter assert(pTagCond->joinInfo.hasJoin); int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid); - SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); - - // get the tag colId column index - int32_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta); - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - for(int32_t i = 0; i < numOfTags; ++i) { - if (pSchema[i].colId == tagColId) { - colIndex.columnIndex = i; - break; - } - } + SSchema* s = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); + + colIndex.columnIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, tagColId); int16_t bytes = 0; int16_t type = 0; @@ -1303,6 +1569,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { tscDebug("%p start subquery, total:%d", pSql, pQueryInfo->numOfTables); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, i); if (pSupporter == NULL) { // failed to create support struct, abort current query @@ -1357,8 +1624,8 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) { SRetrieveSupport* pSupport = pSub->param; - taosTFree(pSupport->localBuffer); - taosTFree(pSupport); + tfree(pSupport->localBuffer); + tfree(pSupport); taos_free_result(pSub); } @@ -1386,14 +1653,20 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSubqueryState *pState = &pSql->subState; - pState->numOfSub = pTableMetaInfo->vgroupList->numOfVgroups; + pState->numOfSub = 0; + if (pTableMetaInfo->pVgroupTables == NULL) { + pState->numOfSub = pTableMetaInfo->vgroupList->numOfVgroups; + } else { + pState->numOfSub = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + } + assert(pState->numOfSub > 0); int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); if (ret != 0) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscQueueAsyncRes(pSql); - taosTFree(pMemoryBuf); + tfree(pMemoryBuf); return ret; } @@ -1402,7 +1675,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { tscDebug("%p retrieved query data from %d vnode(s)", pSql, pState->numOfSub); if (pSql->pSubs == NULL) { - taosTFree(pSql->pSubs); + tfree(pSql->pSubs); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pState->numOfSub); @@ -1427,7 +1700,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { trs->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage)); if (trs->localBuffer == NULL) { tscError("%p failed to malloc buffer for local buffer, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); - taosTFree(trs); + tfree(trs); break; } @@ -1438,8 +1711,8 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL); if (pNew == NULL) { tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); - taosTFree(trs->localBuffer); - taosTFree(trs); + tfree(trs->localBuffer); + tfree(trs); break; } @@ -1493,8 +1766,8 @@ static void tscFreeRetrieveSup(SSqlObj *pSql) { // SSqlObj *pParentSql = trsupport->pParentSql; // assert(pSql == pParentSql->pSubs[index]); - taosTFree(trsupport->localBuffer); - taosTFree(trsupport); + tfree(trsupport->localBuffer); + tfree(trsupport); } static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); @@ -1518,8 +1791,8 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in SSqlObj *pParentSql = trsupport->pParentSql; int32_t subqueryIndex = trsupport->subqueryIndex; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]); @@ -1721,7 +1994,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - SCMVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; + SVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; @@ -1832,7 +2105,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex]; + SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex]; // stable query killed or other subquery failed, all query stopped if (pParentSql->res.code != TSDB_CODE_SUCCESS) { @@ -1894,7 +2167,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) pParentObj->res.code = pSql->res.code; } - taosTFree(pSupporter); + tfree(pSupporter); if (atomic_sub_fetch_32(&pParentObj->subState.numOfRemain, 1) > 0) { return; @@ -2017,7 +2290,7 @@ static char* getResultBlockPosition(SSqlCmd* pCmd, SSqlRes* pRes, int32_t column assert(pInfo->pSqlExpr != NULL); *bytes = pInfo->pSqlExpr->resBytes; - char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows; + char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + pRes->row * (*bytes); return pData; } @@ -2029,14 +2302,17 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { int32_t numOfRes = INT32_MAX; for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - if (pSql->pSubs[i] == NULL) { + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub == NULL) { continue; } - numOfRes = (int32_t)(MIN(numOfRes, pSql->pSubs[i]->res.numOfRows)); + int32_t remain = (int32_t)(pSub->res.numOfRows - pSub->res.row); + numOfRes = (int32_t)(MIN(numOfRes, remain)); } - if (numOfRes == 0) { + if (numOfRes == 0) { // no result any more, free all subquery objects + freeJoinSubqueryObj(pSql); return; } @@ -2059,14 +2335,23 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); for(int32_t i = 0; i < numOfExprs; ++i) { SColumnIndex* pIndex = &pRes->pColumnIndex[i]; - SSqlRes *pRes1 = &pSql->pSubs[pIndex->tableIndex]->res; - SSqlCmd *pCmd1 = &pSql->pSubs[pIndex->tableIndex]->cmd; + SSqlRes* pRes1 = &pSql->pSubs[pIndex->tableIndex]->res; + SSqlCmd* pCmd1 = &pSql->pSubs[pIndex->tableIndex]->cmd; char* pData = getResultBlockPosition(pCmd1, pRes1, pIndex->columnIndex, &bytes); memcpy(data, pData, bytes * numOfRes); data += bytes * numOfRes; - pRes1->row = numOfRes; + } + + for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub == NULL) { + continue; + } + + pSub->res.row += numOfRes; + assert(pSub->res.row <= pSub->res.numOfRows); } pRes->numOfRows = numOfRes; @@ -2085,6 +2370,8 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + pRes->numOfCols = (int16_t)numOfExprs; + pRes->tsrow = calloc(numOfExprs, POINTER_BYTES); pRes->buffer = calloc(numOfExprs, POINTER_BYTES); pRes->length = calloc(numOfExprs, sizeof(int32_t)); @@ -2155,7 +2442,7 @@ TAOS_ROW doSetResultRowData(SSqlObj *pSql, bool finalResult) { assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker - taosTFree(pRes->tsrow); + tfree(pRes->tsrow); return pRes->tsrow; } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 47c2d35a75cba4787e00c645e2f242ae24788df7..01046a2840aab2e8cdfb27682778f572d3179dfb 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -36,6 +36,7 @@ void * tscTmr; void * tscQhandle; void * tscCheckDiskUsageTmr; int tsInsertHeadSize; +int tscRefId = -1; int tscNumOfThreads; @@ -79,7 +80,7 @@ int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeCon void taos_init_imp(void) { - char temp[128]; + char temp[128] = {0}; errno = TSDB_CODE_SUCCESS; srand(taosGetTimestampSec()); @@ -103,7 +104,6 @@ void taos_init_imp(void) { taosReadGlobalCfg(); taosCheckGlobalCfg(); - taosPrintGlobalCfg(); tscDebug("starting to initialize TAOS client ..."); tscDebug("Local End Point is:%s", tsLocalEp); @@ -146,29 +146,44 @@ void taos_init_imp(void) { tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj"); } + tscRefId = taosOpenRef(200, tscCloseTscObj); + + // in other language APIs, taos_cleanup is not available yet. + // So, to make sure taos_cleanup will be invoked to clean up the allocated + // resource to suppress the valgrind warning. + atexit(taos_cleanup); tscDebug("client is initialized successfully"); } void taos_init() { pthread_once(&tscinit, taos_init_imp); } -void taos_cleanup() { - if (tscMetaCache != NULL) { - taosCacheCleanup(tscMetaCache); - tscMetaCache = NULL; +// this function may be called by user or system, or by both simultaneously. +void taos_cleanup(void) { + tscDebug("start to cleanup client environment"); - taosCacheCleanup(tscObjCache); - tscObjCache = NULL; + void* m = tscMetaCache; + if (m != NULL && atomic_val_compare_exchange_ptr(&tscMetaCache, m, 0) == m) { + taosCacheCleanup(m); } - - if (tscQhandle != NULL) { - taosCleanUpScheduler(tscQhandle); - tscQhandle = NULL; + + m = tscObjCache; + if (m != NULL && atomic_val_compare_exchange_ptr(&tscObjCache, m, 0) == m) { + taosCacheCleanup(m); + } + + m = tscQhandle; + if (m != NULL && atomic_val_compare_exchange_ptr(&tscQhandle, m, 0) == m) { + taosCleanUpScheduler(m); } + taosCloseRef(tscRefId); taosCleanupKeywordsTable(); taosCloseLog(); - - taosTmrCleanUp(tscTmr); + + m = tscTmr; + if (m != NULL && atomic_val_compare_exchange_ptr(&tscTmr, m, 0) == m) { + taosTmrCleanUp(m); + } } static int taos_options_imp(TSDB_OPTION option, const char *pStr) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index b60bf958a84bcefd8fb14970923f4af1074b7a6f..bb9725a74432ac2c06adc86513b62ee32b4b8125 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -245,7 +245,7 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) { } pQueryInfo->fillType = TSDB_FILL_NONE; - taosTFree(pQueryInfo->fillVal); + tfree(pQueryInfo->fillVal); } int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { @@ -259,7 +259,7 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { // not enough memory if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { - taosTFree(pRes->tsrow); + tfree(pRes->tsrow); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; return pRes->code; } @@ -271,24 +271,24 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { static void tscDestroyResPointerInfo(SSqlRes* pRes) { if (pRes->buffer != NULL) { // free all buffers containing the multibyte string for (int i = 0; i < pRes->numOfCols; i++) { - taosTFree(pRes->buffer[i]); + tfree(pRes->buffer[i]); } pRes->numOfCols = 0; } - taosTFree(pRes->pRsp); + tfree(pRes->pRsp); - taosTFree(pRes->tsrow); - taosTFree(pRes->length); - taosTFree(pRes->buffer); + tfree(pRes->tsrow); + tfree(pRes->length); + tfree(pRes->buffer); - taosTFree(pRes->pGroupRec); - taosTFree(pRes->pColumnIndex); + tfree(pRes->pGroupRec); + tfree(pRes->pColumnIndex); if (pRes->pArithSup != NULL) { - taosTFree(pRes->pArithSup->data); - taosTFree(pRes->pArithSup); + tfree(pRes->pArithSup->data); + tfree(pRes->pArithSup); } pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free @@ -305,11 +305,11 @@ static void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) { freeQueryInfoImpl(pQueryInfo); clearAllTableMetaInfo(pQueryInfo, (const char*)addr, removeFromCache); - taosTFree(pQueryInfo); + tfree(pQueryInfo); } pCmd->numOfClause = 0; - taosTFree(pCmd->pQueryInfo); + tfree(pCmd->pQueryInfo); } void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { @@ -338,34 +338,6 @@ void tscFreeSqlResult(SSqlObj* pSql) { memset(&pSql->res, 0, sizeof(SSqlRes)); } -void tscPartiallyFreeSqlObj(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) { - return; - } - - SSqlCmd* pCmd = &pSql->cmd; - int32_t cmd = pCmd->command; - if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || - cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscRemoveFromSqlList(pSql); - } - - // pSql->sqlstr will be used by tscBuildQueryStreamDesc -// if (pObj->signature == pObj) { - //pthread_mutex_lock(&pObj->mutex); - taosTFree(pSql->sqlstr); - //pthread_mutex_unlock(&pObj->mutex); -// } - - tscFreeSqlResult(pSql); - - taosTFree(pSql->pSubs); - pSql->subState.numOfSub = 0; - pSql->self = 0; - - tscResetSqlCmdObj(pCmd, false); -} - static void tscFreeSubobj(SSqlObj* pSql) { if (pSql->subState.numOfSub == 0) { return; @@ -404,7 +376,7 @@ void tscFreeRegisteredSqlObj(void *pSql) { tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref); if (ref == 0) { tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj); - tscCloseTscObj(pTscObj); + taosRemoveRef(tscRefId, pTscObj->rid); } } @@ -415,14 +387,14 @@ void tscFreeTableMetaHelper(void *pTableMeta) { assert(numOfEps >= 0 && numOfEps <= TSDB_MAX_REPLICA); for(int32_t i = 0; i < numOfEps; ++i) { - taosTFree(p->vgroupInfo.epAddr[i].fqdn); + tfree(p->vgroupInfo.epAddr[i].fqdn); } int32_t numOfEps1 = p->corVgroupInfo.numOfEps; assert(numOfEps1 >= 0 && numOfEps1 <= TSDB_MAX_REPLICA); for(int32_t i = 0; i < numOfEps1; ++i) { - taosTFree(p->corVgroupInfo.epAddr[i].fqdn); + tfree(p->corVgroupInfo.epAddr[i].fqdn); } } @@ -434,22 +406,32 @@ void tscFreeSqlObj(SSqlObj* pSql) { tscDebug("%p start to free sqlObj", pSql); pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + tscFreeSubobj(pSql); - tscPartiallyFreeSqlObj(pSql); + SSqlCmd* pCmd = &pSql->cmd; + int32_t cmd = pCmd->command; + if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || + cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) { + tscRemoveFromSqlList(pSql); + } pSql->signature = NULL; pSql->fp = NULL; - - SSqlCmd* pCmd = &pSql->cmd; + tfree(pSql->sqlstr); + + tfree(pSql->pSubs); + pSql->subState.numOfSub = 0; + pSql->self = 0; + + tscFreeSqlResult(pSql); + tscResetSqlCmdObj(pCmd, false); memset(pCmd->payload, 0, (size_t)pCmd->allocSize); - taosTFree(pCmd->payload); + tfree(pCmd->payload); pCmd->allocSize = 0; - taosTFree(pSql->sqlstr); tsem_destroy(&pSql->rspSem); - free(pSql); } @@ -458,15 +440,15 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { return; } - taosTFree(pDataBlock->pData); - taosTFree(pDataBlock->params); + tfree(pDataBlock->pData); + tfree(pDataBlock->params); // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false); } - taosTFree(pDataBlock); + tfree(pDataBlock); } SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, @@ -741,7 +723,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { taosHashCleanup(pVnodeDataBlockHashList); tscDestroyBlockArrayList(pVnodeDataBlockList); - taosTFree(dataBuf->pData); + tfree(dataBuf->pData); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -786,8 +768,8 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { } // TODO: all subqueries should be freed correctly before close this connection. -void tscCloseTscObj(STscObj* pObj) { - assert(pObj != NULL); +void tscCloseTscObj(void *param) { + STscObj *pObj = param; pObj->signature = NULL; taosTmrStopA(&(pObj->pTimer)); @@ -801,7 +783,7 @@ void tscCloseTscObj(STscObj* pObj) { pthread_mutex_destroy(&pObj->mutex); tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p); - taosTFree(pObj); + tfree(pObj); } bool tscIsInsertData(char* sqlstr) { @@ -965,12 +947,12 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { if (pInfo->pArithExprInfo != NULL) { tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL); - taosTFree(pInfo->pArithExprInfo); + tfree(pInfo->pArithExprInfo); } } taosArrayDestroy(pFieldInfo->internalField); - taosTFree(pFieldInfo->final); + tfree(pFieldInfo->final); memset(pFieldInfo, 0, sizeof(SFieldInfo)); } @@ -1086,7 +1068,7 @@ void* sqlExprDestroy(SSqlExpr* pExpr) { tVariantDestroy(&pExpr->param[i]); } - taosTFree(pExpr); + tfree(pExpr); return NULL; } @@ -1121,6 +1103,8 @@ int32_t tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepco } *p1 = *pExpr; + memset(p1->param, 0, sizeof(tVariant) * tListLen(p1->param)); + for (int32_t j = 0; j < pExpr->numOfParams; ++j) { tVariantAssign(&p1->param[j], &pExpr->param[j]); } @@ -1184,11 +1168,11 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { static void destroyFilterInfo(SColumnFilterInfo* pFilterInfo, int32_t numOfFilters) { for(int32_t i = 0; i < numOfFilters; ++i) { if (pFilterInfo[i].filterstr) { - taosTFree(pFilterInfo[i].pz); + tfree(pFilterInfo[i].pz); } } - taosTFree(pFilterInfo); + tfree(pFilterInfo); } SColumn* tscColumnClone(const SColumn* src) { @@ -1476,7 +1460,7 @@ void tscTagCondRelease(STagCond* pTagCond) { size_t s = taosArrayGetSize(pTagCond->pCond); for (int32_t i = 0; i < s; ++i) { SCond* p = taosArrayGet(pTagCond->pCond, i); - taosTFree(p->cond); + tfree(p->cond); } taosArrayDestroy(pTagCond->pCond); @@ -1663,11 +1647,12 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { if (pQueryInfo->groupbyExpr.columnInfo != NULL) { taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo); pQueryInfo->groupbyExpr.columnInfo = NULL; + pQueryInfo->groupbyExpr.numOfGroupCols = 0; } pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf); - taosTFree(pQueryInfo->fillVal); + tfree(pQueryInfo->fillVal); } void tscClearSubqueryInfo(SSqlCmd* pCmd) { @@ -1678,23 +1663,71 @@ void tscClearSubqueryInfo(SSqlCmd* pCmd) { } void tscFreeVgroupTableInfo(SArray* pVgroupTables) { - if (pVgroupTables != NULL) { - size_t num = taosArrayGetSize(pVgroupTables); - for (size_t i = 0; i < num; i++) { - SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); + if (pVgroupTables == NULL) { + return; + } - for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { - taosTFree(pInfo->vgInfo.epAddr[j].fqdn); - } + size_t num = taosArrayGetSize(pVgroupTables); + for (size_t i = 0; i < num; i++) { + SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); - taosArrayDestroy(pInfo->itemList); + for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { + tfree(pInfo->vgInfo.epAddr[j].fqdn); } - taosArrayDestroy(pVgroupTables); + + taosArrayDestroy(pInfo->itemList); + } + + taosArrayDestroy(pVgroupTables); +} + +void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) { + assert(pVgroupTable != NULL && index >= 0); + + size_t size = taosArrayGetSize(pVgroupTable); + assert(size > index); + + SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTable, index); + for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { + tfree(pInfo->vgInfo.epAddr[j].fqdn); } + + taosArrayDestroy(pInfo->itemList); + taosArrayRemove(pVgroupTable, index); +} + +void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) { + memset(info, 0, sizeof(SVgroupTableInfo)); + + info->vgInfo = pInfo->vgInfo; + for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { + info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn); + } + + info->itemList = taosArrayClone(pInfo->itemList); +} + +SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) { + if (pVgroupTables == NULL) { + return NULL; + } + + size_t num = taosArrayGetSize(pVgroupTables); + SArray* pa = taosArrayInit(num, sizeof(SVgroupTableInfo)); + + SVgroupTableInfo info; + for (size_t i = 0; i < num; i++) { + SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); + tscVgroupTableCopy(&info, pInfo); + + taosArrayPush(pa, &info); + } + + return pa; } void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { - tscDebug("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables); + tscDebug("%p unref %d tables in the tableMeta cache", address, pQueryInfo->numOfTables); for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); @@ -1704,11 +1737,11 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool rem free(pTableMetaInfo); } - taosTFree(pQueryInfo->pTableMetaInfo); + tfree(pQueryInfo->pTableMetaInfo); } STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, SArray* pTagCols) { + SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables) { void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1734,6 +1767,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList); } + // TODO handle malloc failure pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); if (pTableMetaInfo->tagColList == NULL) { return NULL; @@ -1742,13 +1776,15 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST if (pTagCols != NULL) { tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1); } + + pTableMetaInfo->pVgroupTables = tscVgroupTableInfoClone(pVgroupTables); pQueryInfo->numOfTables += 1; return pTableMetaInfo; } STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { - return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL); + return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL, NULL); } void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { @@ -1822,7 +1858,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm assert(pSql->cmd.clauseIndex == 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); - tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL); + tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); registerSqlObj(pNew); return pNew; @@ -1987,14 +2023,16 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void STableMeta* pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup assert(pTableMeta != NULL); - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList); + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, + pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); STableMeta* pPrevTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pPrevInfo->pTableMeta); SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList); + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList, + pTableMetaInfo->pVgroupTables); } if (pFinalInfo->pTableMeta == NULL) { @@ -2106,6 +2144,21 @@ int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) { } } +int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId) { + int32_t numOfTags = tscGetNumOfTags(pTableMeta); + + SSchema* pSchema = tscGetTableTagSchema(pTableMeta); + for(int32_t i = 0; i < numOfTags; ++i) { + if (pSchema[i].colId == colId) { + return i; + } + } + + // can not reach here + assert(0); + return INT16_MIN; +} + bool tscIsUpdateQuery(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { terrno = TSDB_CODE_TSC_DISCONNECTED; @@ -2283,7 +2336,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) { pRes->numOfTotal = num; - taosTFree(pSql->pSubs); + tfree(pSql->pSubs); pSql->subState.numOfSub = 0; pSql->fp = fp; @@ -2375,7 +2428,7 @@ SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) { return NULL; } - size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups; + size_t size = sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupList->numOfVgroups; SVgroupsInfo* pNew = calloc(1, size); if (pNew == NULL) { return NULL; @@ -2384,9 +2437,9 @@ SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) { pNew->numOfVgroups = vgroupList->numOfVgroups; for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) { - SCMVgroupInfo* pNewVInfo = &pNew->vgroups[i]; + SVgroupInfo* pNewVInfo = &pNew->vgroups[i]; - SCMVgroupInfo* pvInfo = &vgroupList->vgroups[i]; + SVgroupInfo* pvInfo = &vgroupList->vgroups[i]; pNewVInfo->vgId = pvInfo->vgId; pNewVInfo->numOfEps = pvInfo->numOfEps; @@ -2405,21 +2458,22 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) { } for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) { - SCMVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i]; + SVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i]; for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) { - taosTFree(pVgroupInfo->epAddr[j].fqdn); + tfree(pVgroupInfo->epAddr[j].fqdn); } } - taosTFree(vgroupList); + tfree(vgroupList); return NULL; } -void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src) { +void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src) { dst->vgId = src->vgId; dst->numOfEps = src->numOfEps; for(int32_t i = 0; i < dst->numOfEps; ++i) { + tfree(dst->epAddr[i].fqdn); dst->epAddr[i].port = src->epAddr[i].port; dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn); } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index cc4afeb3f850785ffb18a972961668f91061c6a4..8d4949d9b4364fd1e8c70cbb883aa56468724108 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -80,7 +80,7 @@ typedef struct { #define schemaFLen(s) ((s)->flen) #define schemaVLen(s) ((s)->vlen) #define schemaColAt(s, i) ((s)->columns + i) -#define tdFreeSchema(s) taosTFree((s)) +#define tdFreeSchema(s) tfree((s)) STSchema *tdDupSchema(STSchema *pSchema); int tdEncodeSchema(void **buf, STSchema *pSchema); @@ -119,6 +119,33 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); +// ----------------- Semantic timestamp key definition +typedef uint64_t TKEY; + +#define TKEY_INVALID UINT64_MAX +#define TKEY_NULL TKEY_INVALID +#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63) +#define TKEY_DELETE_FLAG (((TKEY)1) << 62) +#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG)) + +#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) +#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0) +#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG) +#define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) +#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) + +static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { + TSKEY key1 = tdGetKey(*(TKEY *)tkey1); + TSKEY key2 = tdGetKey(*(TKEY *)tkey2); + + if (key1 < key2) { + return -1; + } else if (key1 > key2) { + return 1; + } else { + return 0; + } +} // ----------------- Data row structure /* A data row, the format is like below: @@ -129,6 +156,8 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); * +----------+----------+---------------------------------+---------------------------------+ * | len | sversion | First part | Second part | * +----------+----------+---------------------------------+---------------------------------+ + * + * NOTE: timestamp in this row structure is TKEY instead of TSKEY */ typedef void *SDataRow; @@ -137,11 +166,13 @@ typedef void *SDataRow; #define dataRowLen(r) (*(uint16_t *)(r)) #define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)) #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) -#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r))) +#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) +#define dataRowKey(r) tdGetKey(dataRowTKey(r)) #define dataRowSetLen(r, l) (dataRowLen(r) = (l)) #define dataRowSetVersion(r, v) (dataRowVersion(r) = (v)) #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r)) #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE) +#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r)) SDataRow tdNewDataRowFromSchema(STSchema *pSchema); void tdFreeDataRow(SDataRow row); @@ -154,16 +185,18 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row)); - switch (type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row); - memcpy(ptr, value, varDataTLen(value)); - dataRowLen(row) += varDataTLen(value); - break; - default: + if (IS_VAR_DATA_TYPE(type)) { + *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row); + memcpy(ptr, value, varDataTLen(value)); + dataRowLen(row) += varDataTLen(value); + } else { + if (offset == 0) { + ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP); + TKEY tvalue = tdGetTKEY(*(TSKEY *)value); + memcpy(POINTER_SHIFT(row, toffset), (void *)(&tvalue), TYPE_BYTES[type]); + } else { memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]); - break; + } } return 0; @@ -171,12 +204,10 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i // NOTE: offset here including the header size static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) { - switch (type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); - default: - return POINTER_SHIFT(row, offset); + if (IS_VAR_DATA_TYPE(type)) { + return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); + } else { + return POINTER_SHIFT(row, offset); } } @@ -196,7 +227,6 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints); void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints); -void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows); void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); @@ -204,28 +234,20 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); // Get the data pointer from a column-wised data static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) { - switch (pCol->type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); - break; - - default: - return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); - break; + if (IS_VAR_DATA_TYPE(pCol->type)) { + return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); + } else { + return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); } } static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) { ASSERT(rows > 0); - switch (pDataCol->type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1)); - break; - default: - return TYPE_BYTES[pDataCol->type] * rows; + if (IS_VAR_DATA_TYPE(pDataCol->type)) { + return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1)); + } else { + return TYPE_BYTES[pDataCol->type] * rows; } } @@ -243,9 +265,14 @@ typedef struct { } SDataCols; #define keyCol(pCols) (&((pCols)->cols[0])) // Key column -#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)] -#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0) -#define dataColsKeyLast(pCols) ((pCols->numOfRows == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfRows - 1)) +#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] +#define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) +#define dataColsTKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0)) +#define dataColsKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0)) +#define dataColsTKeyLast(pCols) \ + (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1)) +#define dataColsKeyLast(pCols) \ + (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1)) SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); @@ -253,10 +280,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols); -void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!! int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); -void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows); // ----------------- K-V data row structure /* @@ -284,7 +308,7 @@ typedef struct { #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) -#define kvRowFree(r) taosTFree(r) +#define kvRowFree(r) tfree(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) SKVRow tdKVRowDup(SKVRow row); diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 515115c323294a67318b5eb1dd17660e651d09f8..4087f638a95b063fcf6081db2556758643ef1c9a 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -44,13 +44,18 @@ extern int32_t tsMaxShellConns; extern int32_t tsShellActivityTimer; extern uint32_t tsMaxTmrCtrl; extern float tsNumOfThreadsPerCore; -extern float tsRatioOfQueryThreads; +extern int32_t tsNumOfCommitThreads; +extern float tsRatioOfQueryThreads; // todo remove it extern int8_t tsDaylight; extern char tsTimezone[]; extern char tsLocale[]; -extern char tsCharset[]; // default encode string +extern char tsCharset[]; // default encode string extern int32_t tsEnableCoreFile; extern int32_t tsCompressMsgSize; +extern char tsTempDir[]; + +//query buffer management +extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing // client extern int32_t tsTableMetaKeepTimer; @@ -84,6 +89,7 @@ extern int16_t tsWAL; extern int32_t tsFsyncPeriod; extern int32_t tsReplications; extern int32_t tsQuorum; +extern int32_t tsUpdate; // balance extern int32_t tsEnableBalance; @@ -180,13 +186,13 @@ extern int32_t debugFlag; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) -void taosInitGlobalCfg(); -bool taosCheckGlobalCfg(); -void taosSetAllDebugFlag(); -bool taosCfgDynamicOptions(char *msg); -int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); -bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); - +void taosInitGlobalCfg(); +int32_t taosCheckGlobalCfg(); +void taosSetAllDebugFlag(); +bool taosCfgDynamicOptions(char *msg); +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); +bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); + #ifdef __cplusplus } #endif diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 28289b051e4ef32e1b1e22847df584238857002b..f21205479396ba606a1212f30350df2e0b3f59b5 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -18,6 +18,9 @@ #include "tcoding.h" #include "wchar.h" +static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows); + /** * Duplicate the schema and return a new object */ @@ -94,7 +97,7 @@ int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) { if (pBuilder) { - taosTFree(pBuilder->columns); + tfree(pBuilder->columns); } } @@ -202,7 +205,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE; pDataCol->len = 0; - if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { + if (IS_VAR_DATA_TYPE(pDataCol->type)) { pDataCol->dataOff = (VarDataOffsetT *)(*pBuf); pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints); pDataCol->spaceSize = pDataCol->bytes * maxPoints; @@ -215,60 +218,29 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) } } +// value from timestamp should be TKEY here instead of TSKEY void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) { ASSERT(pCol != NULL && value != NULL); - switch (pCol->type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - // set offset - pCol->dataOff[numOfRows] = pCol->len; - // Copy data - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); - // Update the length - pCol->len += varDataTLen(value); - break; - default: - ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); - pCol->len += pCol->bytes; - break; - } -} - -void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows) { - int pointsLeft = numOfRows - pointsToPop; - - ASSERT(pointsLeft > 0); - - if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) { - ASSERT(pCol->len > 0); - VarDataOffsetT toffset = pCol->dataOff[pointsToPop]; - pCol->len = pCol->len - toffset; - ASSERT(pCol->len > 0); - memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len); - dataColSetOffset(pCol, pointsLeft); + if (IS_VAR_DATA_TYPE(pCol->type)) { + // set offset + pCol->dataOff[numOfRows] = pCol->len; + // Copy data + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); + // Update the length + pCol->len += varDataTLen(value); } else { ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - pCol->len = TYPE_BYTES[pCol->type] * pointsLeft; - memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); + pCol->len += pCol->bytes; } } bool isNEleNull(SDataCol *pCol, int nEle) { - switch (pCol->type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - for (int i = 0; i < nEle; i++) { - if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false; - } - return true; - default: - for (int i = 0; i < nEle; i++) { - if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false; - } - return true; + for (int i = 0; i < nEle; i++) { + if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false; } + return true; } void dataColSetNullAt(SDataCol *pCol, int index) { @@ -367,8 +339,8 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { void tdFreeDataCols(SDataCols *pCols) { if (pCols) { - taosTFree(pCols->buf); - taosTFree(pCols->cols); + tfree(pCols->buf); + tfree(pCols->cols); free(pCols); } } @@ -390,7 +362,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize; pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf))); - if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) { + if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { ASSERT(pDataCols->cols[i].dataOff != NULL); pRet->cols[i].dataOff = (int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf))); @@ -400,7 +372,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { pRet->cols[i].len = pDataCols->cols[i].len; if (pDataCols->cols[i].len > 0) { memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); - if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) { + if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints); } } @@ -420,58 +392,54 @@ void tdResetDataCols(SDataCols *pCols) { } void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) { - ASSERT(dataColsKeyLast(pCols) < dataRowKey(row)); + ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row)); int rcol = 0; int dcol = 0; - while (dcol < pCols->numOfCols) { - SDataCol *pDataCol = &(pCols->cols[dcol]); - if (rcol >= schemaNCols(pSchema)) { - dataColSetNullAt(pDataCol, pCols->numOfRows); - dcol++; - continue; + if (dataRowDeleted(row)) { + for (; dcol < pCols->numOfCols; dcol++) { + SDataCol *pDataCol = &(pCols->cols[dcol]); + if (dcol == 0) { + dataColAppendVal(pDataCol, dataRowTuple(row), pCols->numOfRows, pCols->maxPoints); + } else { + dataColSetNullAt(pDataCol, pCols->numOfRows); + } } + } else { + while (dcol < pCols->numOfCols) { + SDataCol *pDataCol = &(pCols->cols[dcol]); + if (rcol >= schemaNCols(pSchema)) { + dataColSetNullAt(pDataCol, pCols->numOfRows); + dcol++; + continue; + } - STColumn *pRowCol = schemaColAt(pSchema, rcol); - if (pRowCol->colId == pDataCol->colId) { - void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset+TD_DATA_ROW_HEAD_SIZE); - dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); - dcol++; - rcol++; - } else if (pRowCol->colId < pDataCol->colId) { - rcol++; - } else { - dataColSetNullAt(pDataCol, pCols->numOfRows); - dcol++; + STColumn *pRowCol = schemaColAt(pSchema, rcol); + if (pRowCol->colId == pDataCol->colId) { + void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE); + dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); + dcol++; + rcol++; + } else if (pRowCol->colId < pDataCol->colId) { + rcol++; + } else { + dataColSetNullAt(pDataCol, pCols->numOfRows); + dcol++; + } } } pCols->numOfRows++; } -// Pop pointsToPop points from the SDataCols -void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) { - int pointsLeft = pCols->numOfRows - pointsToPop; - if (pointsLeft <= 0) { - tdResetDataCols(pCols); - return; - } - - for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { - SDataCol *pCol = pCols->cols + iCol; - dataColPopPoints(pCol, pointsToPop, pCols->numOfRows); - } - pCols->numOfRows = pointsLeft; -} - int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); - ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); ASSERT(target->numOfCols == source->numOfCols); SDataCols *pTarget = NULL; if (dataColsKeyLast(target) < dataColsKeyFirst(source)) { // No overlap + ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); for (int i = 0; i < rowsToMerge; i++) { for (int j = 0; j < source->numOfCols; j++) { if (source->cols[j].len > 0) { @@ -499,17 +467,23 @@ _err: return -1; } -void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) { +// src2 data has more priority than src1 +static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows) { tdResetDataCols(target); ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows); while (target->numOfRows < tRows) { if (*iter1 >= limit1 && *iter2 >= limit2) break; - TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1]; - TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2]; + TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1); + TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1); + TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2); + TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2); - if (key1 <= key2) { + ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1))); + + if (key1 < key2) { for (int i = 0; i < src1->numOfCols; i++) { ASSERT(target->cols[i].type == src1->cols[i].type); if (src1->cols[i].len > 0) { @@ -520,19 +494,23 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi target->numOfRows++; (*iter1)++; - if (key1 == key2) (*iter2)++; - } else { - for (int i = 0; i < src2->numOfCols; i++) { - ASSERT(target->cols[i].type == src2->cols[i].type); - if (src2->cols[i].len > 0) { - dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, - target->maxPoints); + } else if (key1 >= key2) { + if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) { + for (int i = 0; i < src2->numOfCols; i++) { + ASSERT(target->cols[i].type == src2->cols[i].type); + if (src2->cols[i].len > 0) { + dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, + target->maxPoints); + } } + target->numOfRows++; } - target->numOfRows++; (*iter2)++; + if (key1 == key2) (*iter1)++; } + + ASSERT(target->numOfRows <= target->maxPoints); } } @@ -691,8 +669,8 @@ int tdInitKVRowBuilder(SKVRowBuilder *pBuilder) { } void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder) { - taosTFree(pBuilder->pColIdx); - taosTFree(pBuilder->buf); + tfree(pBuilder->pColIdx); + tfree(pBuilder->buf); } void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) { diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index c24ba490ba7f4cb25ba032b0404790d68540c826..4495c3d9288a5df0b8944aa444625c143b5c7670 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -45,19 +45,21 @@ int32_t tsEnableTelemetryReporting = 1; char tsEmail[TSDB_FQDN_LEN] = {0}; // common -int32_t tsRpcTimer = 1000; -int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsMaxShellConns = 5000; +int32_t tsRpcTimer = 1000; +int32_t tsRpcMaxTime = 600; // seconds; +int32_t tsMaxShellConns = 5000; int32_t tsMaxConnections = 5000; -int32_t tsShellActivityTimer = 3; // second -float tsNumOfThreadsPerCore = 1.0; -float tsRatioOfQueryThreads = 0.5; -int8_t tsDaylight = 0; +int32_t tsShellActivityTimer = 3; // second +float tsNumOfThreadsPerCore = 1.0f; +int32_t tsNumOfCommitThreads = 1; +float tsRatioOfQueryThreads = 0.5f; +int8_t tsDaylight = 0; char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string int32_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; +char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -99,6 +101,12 @@ float tsStreamComputDelayRatio = 0.1f; int32_t tsProjectExecInterval = 10000; // every 10sec, the projection will be executed once int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance +// the maximum allowed query buffer size during query processing for each data node. +// -1 no limit (default) +// 0 no query allowed, queries are disabled +// positive value (in MB) +int32_t tsQueryBufferSize = -1; + // db parameters int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS; @@ -113,6 +121,7 @@ int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; +int32_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; int32_t tsMaxVgroupsPerDb = 0; int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; @@ -210,6 +219,8 @@ int32_t (*monitorStartSystemFp)() = NULL; void (*monitorStopSystemFp)() = NULL; void (*monitorExecuteSQLFp)(char *sql) = NULL; +char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"}; + static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; void taosSetAllDebugFlag() { @@ -416,6 +427,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "numOfCommitThreads"; + cfg.ptr = &tsNumOfCommitThreads; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; + cfg.minValue = 1; + cfg.maxValue = 100; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + cfg.option = "ratioOfQueryThreads"; cfg.ptr = &tsRatioOfQueryThreads; cfg.valType = TAOS_CFG_VTYPE_FLOAT; @@ -676,7 +697,7 @@ static void doInitGlobalConfig(void) { cfg.minValue = TSDB_MIN_CACHE_BLOCK_SIZE; cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE; cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_Mb; + cfg.unitType = TAOS_CFG_UTYPE_MB; taosInitConfigOption(cfg); cfg.option = "blocks"; @@ -779,6 +800,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "update"; + cfg.ptr = &tsUpdate; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_DB_UPDATE; + cfg.maxValue = TSDB_MAX_DB_UPDATE; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + cfg.option = "mqttHostName"; cfg.ptr = tsMqttHostName; cfg.valType = TAOS_CFG_VTYPE_STRING; @@ -839,6 +870,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "queryBufferSize"; + cfg.ptr = &tsQueryBufferSize; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = -1; + cfg.maxValue = 500000000000.0f; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_BYTE; + taosInitConfigOption(cfg); + // locale & charset cfg.option = "timezone"; cfg.ptr = tsTimezone; @@ -1294,13 +1335,23 @@ static void doInitGlobalConfig(void) { cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + + cfg.option = "tempDir"; + cfg.ptr = tsTempDir; + cfg.valType = TAOS_CFG_VTYPE_STRING; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; + cfg.minValue = 0; + cfg.maxValue = 0; + cfg.ptrLength = tListLen(tsTempDir); + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); } void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } -bool taosCheckGlobalCfg() { +int32_t taosCheckGlobalCfg() { char fqdn[TSDB_FQDN_LEN]; uint16_t port; @@ -1359,7 +1410,9 @@ bool taosCheckGlobalCfg() { tsSyncPort = tsServerPort + TSDB_PORT_SYNC; tsHttpPort = tsServerPort + TSDB_PORT_HTTP; - return true; + taosPrintGlobalCfg(); + + return 0; } int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 005def6dc597361436b03c15535840af2bd3461e..6dd065382265cb09e26f8d1b6f86c4b5b3f5110b 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -108,7 +108,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 break; } case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method - pVar->pz = calloc(len, sizeof(char)); + pVar->pz = calloc(len + 1, sizeof(char)); memcpy(pVar->pz, pz, len); pVar->nLen = (int32_t)len; break; @@ -125,7 +125,7 @@ void tVariantDestroy(tVariant *pVar) { if (pVar == NULL) return; if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) { - taosTFree(pVar->pz); + tfree(pVar->pz); pVar->nLen = 0; } @@ -144,21 +144,24 @@ void tVariantDestroy(tVariant *pVar) { void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { if (pSrc == NULL || pDst == NULL) return; - *pDst = *pSrc; - + pDst->nType = pSrc->nType; if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = pSrc->nLen + 1; - if (pSrc->nType == TSDB_DATA_TYPE_NCHAR) { - len = len * TSDB_NCHAR_SIZE; - } - - pDst->pz = calloc(1, len); - memcpy(pDst->pz, pSrc->pz, len); + int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; + char* p = realloc(pDst->pz, len); + assert(p); + + memset(p, 0, len); + pDst->pz = p; + + memcpy(pDst->pz, pSrc->pz, pSrc->nLen); + pDst->nLen = pSrc->nLen; return; + } - // this is only for string array - if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) { + if (pSrc->nType >= TSDB_DATA_TYPE_BOOL && pSrc->nType <= TSDB_DATA_TYPE_DOUBLE) { + pDst->i64Key = pSrc->i64Key; + } else if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) { // this is only for string array size_t num = taosArrayGetSize(pSrc->arr); pDst->arr = taosArrayInit(num, sizeof(char*)); for(size_t i = 0; i < num; i++) { @@ -166,8 +169,6 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { char* n = strdup(p); taosArrayPush(pDst->arr, &n); } - - return; } pDst->nLen = tDataTypeDesc[pDst->nType].nSize; diff --git a/src/connector/C#/TDengineDriver.cs b/src/connector/C#/TDengineDriver.cs new file mode 100644 index 0000000000000000000000000000000000000000..b6f143e1813d60c1ac4ae8356efdca4929c51345 --- /dev/null +++ b/src/connector/C#/TDengineDriver.cs @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace TDengineDriver +{ + enum TDengineDataType { + TSDB_DATA_TYPE_NULL = 0, // 1 bytes + TSDB_DATA_TYPE_BOOL = 1, // 1 bytes + TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes + TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes + TSDB_DATA_TYPE_INT = 4, // 4 bytes + TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes + TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes + TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes + TSDB_DATA_TYPE_BINARY = 8, // string + TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes + TSDB_DATA_TYPE_NCHAR = 10 // unicode string + } + + enum TDengineInitOption + { + TSDB_OPTION_LOCALE = 0, + TSDB_OPTION_CHARSET = 1, + TSDB_OPTION_TIMEZONE = 2, + TDDB_OPTION_CONFIGDIR = 3, + TDDB_OPTION_SHELL_ACTIVITY_TIMER = 4 + } + + class TDengineMeta + { + public string name; + public short size; + public byte type; + public string TypeName() + { + switch ((TDengineDataType)type) + { + case TDengineDataType.TSDB_DATA_TYPE_BOOL: + return "BOOLEAN"; + case TDengineDataType.TSDB_DATA_TYPE_TINYINT: + return "BYTE"; + case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: + return "SHORT"; + case TDengineDataType.TSDB_DATA_TYPE_INT: + return "INT"; + case TDengineDataType.TSDB_DATA_TYPE_BIGINT: + return "LONG"; + case TDengineDataType.TSDB_DATA_TYPE_FLOAT: + return "FLOAT"; + case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: + return "DOUBLE"; + case TDengineDataType.TSDB_DATA_TYPE_BINARY: + return "STRING"; + case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: + return "TIMESTAMP"; + case TDengineDataType.TSDB_DATA_TYPE_NCHAR: + return "NCHAR"; + default: + return "undefine"; + } + } + } + + class TDengine + { + public const int TSDB_CODE_SUCCESS = 0; + + [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)] + static extern public void Init(); + + [DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)] + static extern public void Cleanup(); + + [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)] + static extern public void Options(int option, string value); + + [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Connect(string ip, string user, string password, string db, short port); + + [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_errstr(IntPtr res); + static public string Error(IntPtr res) + { + IntPtr errPtr = taos_errstr(res); + return Marshal.PtrToStringAnsi(errPtr); + } + + [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)] + static extern public int ErrorNo(IntPtr res); + + [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Query(IntPtr conn, string sqlstr); + + [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)] + static extern public int AffectRows(IntPtr res); + + [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)] + static extern public int FieldCount(IntPtr res); + + [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_fetch_fields(IntPtr res); + static public List FetchFields(IntPtr res) + { + const int fieldSize = 68; + + List metas = new List(); + if (res == IntPtr.Zero) + { + return metas; + } + + int fieldCount = FieldCount(res); + IntPtr fieldsPtr = taos_fetch_fields(res); + + for (int i = 0; i < fieldCount; ++i) + { + int offset = i * fieldSize; + + TDengineMeta meta = new TDengineMeta(); + meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset); + meta.type = Marshal.ReadByte(fieldsPtr + offset + 65); + meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66); + metas.Add(meta); + } + + return metas; + } + + [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FetchRows(IntPtr res); + + [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FreeResult(IntPtr res); + + [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] + static extern public int Close(IntPtr taos); + } +} \ No newline at end of file diff --git a/src/connector/go b/src/connector/go index 8d7bf743852897110cbdcc7c4322cd7a74d4167b..8c58c512b6acda8bcdfa48fdc7140227b5221766 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 8d7bf743852897110cbdcc7c4322cd7a74d4167b +Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766 diff --git a/src/connector/hivemq-tdengine-extension b/src/connector/hivemq-tdengine-extension new file mode 160000 index 0000000000000000000000000000000000000000..b62a26ecc164a310104df57691691b237e091c89 --- /dev/null +++ b/src/connector/hivemq-tdengine-extension @@ -0,0 +1 @@ +Subproject commit b62a26ecc164a310104df57691691b237e091c89 diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 3f6ebeff03f538e4963540e015c7fd4b310acc42..893290868ddc4d87e7052e2fe27a13461c02508d 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.6 + 2.0.9 jar JDBCDriver diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index ac0e4eb84aa0fbcc8162e68d94de00cb2f4e79f5..294544ed4742205f1aaae61a52f1a42c02bec030 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -52,7 +52,6 @@ public class TSDBConnection implements Connection { public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException { this.dbMetaData = meta; - connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST), Integer.parseInt(info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "0")), info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), info.getProperty(TSDBDriver.PROPERTY_KEY_USER), @@ -197,12 +196,13 @@ public class TSDBConnection implements Connection { } public SQLWarning getWarnings() throws SQLException { + //todo: implement getWarnings according to the warning messages returned from TDengine throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } public void clearWarnings() throws SQLException { // left blank to support HikariCP connection - //todo: implement getWarnings according to the warning messages returned from TDengine + //todo: implement clearWarnings according to the warning messages returned from TDengine } public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java index c1d9d2af8e5a5c24dcfed6039e3ce06530b95276..f4dee67adf03a3474f582dbe662a8c5e988e3fab 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java @@ -96,7 +96,7 @@ public class TSDBDatabaseMetaData implements java.sql.DatabaseMetaData { } public int getDriverMajorVersion() { - return 0; + return 2; } public int getDriverMinorVersion() { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index 97d93fb0a18fa887465583bce1492c8305faaec5..63c42ca399b6098a5fa8c34d6fafeb51bdc8f588 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -14,13 +14,9 @@ *****************************************************************************/ package com.taosdata.jdbc; - import java.io.*; - import java.sql.*; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; +import java.util.*; import java.util.logging.Logger; /** @@ -44,76 +40,53 @@ import java.util.logging.Logger; */ public class TSDBDriver implements java.sql.Driver { - @Deprecated private static final String URL_PREFIX1 = "jdbc:TSDB://"; private static final String URL_PREFIX = "jdbc:TAOS://"; - /** - * Key used to retrieve the database value from the properties instance passed - * to the driver. - */ - public static final String PROPERTY_KEY_DBNAME = "dbname"; - /** * Key used to retrieve the host value from the properties instance passed to * the driver. */ public static final String PROPERTY_KEY_HOST = "host"; - /** - * Key used to retrieve the password value from the properties instance passed - * to the driver. - */ - public static final String PROPERTY_KEY_PASSWORD = "password"; - /** * Key used to retrieve the port number value from the properties instance * passed to the driver. */ public static final String PROPERTY_KEY_PORT = "port"; - + /** + * Key used to retrieve the database value from the properties instance passed + * to the driver. + */ + public static final String PROPERTY_KEY_DBNAME = "dbname"; /** * Key used to retrieve the user value from the properties instance passed to * the driver. */ public static final String PROPERTY_KEY_USER = "user"; - - + /** + * Key used to retrieve the password value from the properties instance passed + * to the driver. + */ + public static final String PROPERTY_KEY_PASSWORD = "password"; /** * Key for the configuration file directory of TSDB client in properties instance */ public static final String PROPERTY_KEY_CONFIG_DIR = "cfgdir"; - /** * Key for the timezone used by the TSDB client in properties instance */ public static final String PROPERTY_KEY_TIME_ZONE = "timezone"; - /** * Key for the locale used by the TSDB client in properties instance */ public static final String PROPERTY_KEY_LOCALE = "locale"; - - /** * Key for the char encoding used by the TSDB client in properties instance */ public static final String PROPERTY_KEY_CHARSET = "charset"; - public static final String PROPERTY_KEY_PROTOCOL = "protocol"; - - - /** - * Index for port coming out of parseHostPortPair(). - */ - public final static int PORT_NUMBER_INDEX = 1; - - /** - * Index for host coming out of parseHostPortPair(). - */ - public final static int HOST_NAME_INDEX = 0; - private TSDBDatabaseMetaData dbMetaData = null; static { @@ -169,9 +142,11 @@ public class TSDBDriver implements java.sql.Driver { } public Connection connect(String url, Properties info) throws SQLException { - if (url == null) { + if (url == null) throw new SQLException(TSDBConstants.WrapErrMsg("url is not set!")); - } + + if (!acceptsURL(url)) + return null; Properties props = null; if ((props = parseURL(url, info)) == null) { @@ -179,7 +154,10 @@ public class TSDBDriver implements java.sql.Driver { } //load taos.cfg start - if (info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null && info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null) { + if ((info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null || + info.getProperty(TSDBDriver.PROPERTY_KEY_HOST).isEmpty()) && ( + info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null || + info.getProperty(TSDBDriver.PROPERTY_KEY_PORT).isEmpty())) { File cfgDir = loadConfigDir(info.getProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR)); File cfgFile = cfgDir.listFiles((dir, name) -> "taos.cfg".equalsIgnoreCase(name))[0]; List endpoints = loadConfigEndpoints(cfgFile); @@ -190,7 +168,9 @@ public class TSDBDriver implements java.sql.Driver { } try { - TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE), (String) props.get(PROPERTY_KEY_CHARSET), + TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), + (String) props.get(PROPERTY_KEY_LOCALE), + (String) props.get(PROPERTY_KEY_CHARSET), (String) props.get(PROPERTY_KEY_TIME_ZONE)); Connection newConn = new TSDBConnection(props, this.dbMetaData); return newConn; @@ -208,43 +188,15 @@ public class TSDBDriver implements java.sql.Driver { } /** - * Parses hostPortPair in the form of [host][:port] into an array, with the - * element of index HOST_NAME_INDEX being the host (or null if not specified), - * and the element of index PORT_NUMBER_INDEX being the port (or null if not - * specified). - * - * @param hostPortPair host and port in form of of [host][:port] - * @return array containing host and port as Strings - * @throws SQLException if a parse error occurs + * @param url the URL of the database + * @return true if this driver understands the given URL; + * false otherwise + * @throws SQLException if a database access error occurs or the url is {@code null} */ - protected static String[] parseHostPortPair(String hostPortPair) throws SQLException { - String[] splitValues = new String[2]; - - int portIndex = hostPortPair.indexOf(":"); - - String hostname = null; - - if (portIndex != -1) { - if ((portIndex + 1) < hostPortPair.length()) { - String portAsString = hostPortPair.substring(portIndex + 1); - hostname = hostPortPair.substring(0, portIndex); - - splitValues[HOST_NAME_INDEX] = hostname; - - splitValues[PORT_NUMBER_INDEX] = portAsString; - } else { - throw new SQLException(TSDBConstants.WrapErrMsg("port is not proper!")); - } - } else { - splitValues[HOST_NAME_INDEX] = hostPortPair; - splitValues[PORT_NUMBER_INDEX] = null; - } - - return splitValues; - } - public boolean acceptsURL(String url) throws SQLException { - return (url != null && url.length() > 0 && url.trim().length() > 0) && url.startsWith(URL_PREFIX); + if (url == null) + throw new SQLException(TSDBConstants.WrapErrMsg("url is null")); + return (url != null && url.length() > 0 && url.trim().length() > 0) && (url.startsWith(URL_PREFIX) || url.startsWith(URL_PREFIX1)); } public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException { @@ -252,15 +204,17 @@ public class TSDBDriver implements java.sql.Driver { info = new Properties(); } - if ((url != null) && (url.startsWith(URL_PREFIX) || url.startsWith(URL_PREFIX1))) { + if (acceptsURL(url)) { info = parseURL(url, info); } DriverPropertyInfo hostProp = new DriverPropertyInfo(PROPERTY_KEY_HOST, info.getProperty(PROPERTY_KEY_HOST)); - hostProp.required = true; + hostProp.required = false; + hostProp.description = "Hostname"; DriverPropertyInfo portProp = new DriverPropertyInfo(PROPERTY_KEY_PORT, info.getProperty(PROPERTY_KEY_PORT, TSDBConstants.DEFAULT_PORT)); portProp.required = false; + portProp.description = "Port"; DriverPropertyInfo dbProp = new DriverPropertyInfo(PROPERTY_KEY_DBNAME, info.getProperty(PROPERTY_KEY_DBNAME)); dbProp.required = false; @@ -268,9 +222,11 @@ public class TSDBDriver implements java.sql.Driver { DriverPropertyInfo userProp = new DriverPropertyInfo(PROPERTY_KEY_USER, info.getProperty(PROPERTY_KEY_USER)); userProp.required = true; + userProp.description = "User"; DriverPropertyInfo passwordProp = new DriverPropertyInfo(PROPERTY_KEY_PASSWORD, info.getProperty(PROPERTY_KEY_PASSWORD)); passwordProp.required = true; + passwordProp.description = "Password"; DriverPropertyInfo[] propertyInfo = new DriverPropertyInfo[5]; propertyInfo[0] = hostProp; @@ -283,20 +239,68 @@ public class TSDBDriver implements java.sql.Driver { } /** - * example: jdbc:TSDB://127.0.0.1:0/db?user=root&password=your_password + * example: jdbc:TAOS://127.0.0.1:0/db?user=root&password=your_password */ - public Properties parseURL(String url, Properties defaults) throws java.sql.SQLException { + public Properties parseURL(String url, Properties defaults) { Properties urlProps = (defaults != null) ? defaults : new Properties(); - if (url == null) { + if (url == null || url.length() <= 0 || url.trim().length() <= 0) return null; - } - - if (!url.startsWith(URL_PREFIX) && !url.startsWith(URL_PREFIX1)) { + if (!url.startsWith(URL_PREFIX) && !url.startsWith(URL_PREFIX1)) return null; - } + // parse properties String urlForMeta = url; + int beginningOfSlashes = url.indexOf("//"); + int index = url.indexOf("?"); + if (index != -1) { + String paramString = url.substring(index + 1, url.length()); + url = url.substring(0, index); + StringTokenizer queryParams = new StringTokenizer(paramString, "&"); + while (queryParams.hasMoreElements()) { + String parameterValuePair = queryParams.nextToken(); + int indexOfEqual = parameterValuePair.indexOf("="); + String parameter = null; + String value = null; + if (indexOfEqual != -1) { + parameter = parameterValuePair.substring(0, indexOfEqual); + if (indexOfEqual + 1 < parameterValuePair.length()) { + value = parameterValuePair.substring(indexOfEqual + 1); + } + } + if ((value != null && value.length() > 0) && (parameter != null && parameter.length() > 0)) { + urlProps.setProperty(parameter, value); + } + } + } + // parse Product Name + String dbProductName = url.substring(0, beginningOfSlashes); + dbProductName = dbProductName.substring(dbProductName.indexOf(":") + 1); + dbProductName = dbProductName.substring(0, dbProductName.indexOf(":")); + // parse dbname + url = url.substring(beginningOfSlashes + 2); + int indexOfSlash = url.indexOf("/"); + if (indexOfSlash != -1) { + if (indexOfSlash + 1 < url.length()) { + urlProps.setProperty(TSDBDriver.PROPERTY_KEY_DBNAME, url.substring(indexOfSlash + 1)); + } + url = url.substring(0, indexOfSlash); + } + // parse port + int indexOfColon = url.indexOf(":"); + if (indexOfColon != -1) { + if (indexOfColon + 1 < url.length()) { + urlProps.setProperty(TSDBDriver.PROPERTY_KEY_PORT, url.substring(indexOfColon + 1)); + } + url = url.substring(0, indexOfColon); + } + if (url != null && url.length() > 0 && url.trim().length() > 0) { + urlProps.setProperty(TSDBDriver.PROPERTY_KEY_HOST, url); + } + this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, urlProps.getProperty(TSDBDriver.PROPERTY_KEY_USER)); + + /* + String urlForMeta = url; String dbProductName = url.substring(url.indexOf(":") + 1); dbProductName = dbProductName.substring(0, dbProductName.indexOf(":")); int beginningOfSlashes = url.indexOf("//"); @@ -345,11 +349,11 @@ public class TSDBDriver implements java.sql.Driver { user = urlProps.getProperty(PROPERTY_KEY_USER).toString(); this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, user); - +*/ return urlProps; } - public void setPropertyValue(Properties property, String[] keyValuePair) { + private void setPropertyValue(Properties property, String[] keyValuePair) { switch (keyValuePair[0].toLowerCase()) { case PROPERTY_KEY_USER: property.setProperty(PROPERTY_KEY_USER, keyValuePair[1]); @@ -372,13 +376,12 @@ public class TSDBDriver implements java.sql.Driver { } } - public int getMajorVersion() { - return 1; + return 2; } public int getMinorVersion() { - return 1; + return 0; } public boolean jdbcCompliant() { @@ -389,33 +392,4 @@ public class TSDBDriver implements java.sql.Driver { return null; } - /** - * Returns the host property - * - * @param props the java.util.Properties instance to retrieve the hostname from. - * @return the host - */ - public String host(Properties props) { - return props.getProperty(PROPERTY_KEY_HOST, "localhost"); - } - - /** - * Returns the port number property - * - * @param props the properties to get the port number from - * @return the port number - */ - public int port(Properties props) { - return Integer.parseInt(props.getProperty(PROPERTY_KEY_PORT, TSDBConstants.DEFAULT_PORT)); - } - - /** - * Returns the database property from props - * - * @param props the Properties to look for the database property. - * @return the database name. - */ - public String database(Properties props) { - return props.getProperty(PROPERTY_KEY_DBNAME); - } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 0cd185de50d858aec78b60acc4055c1ae99a4cb5..edc160e323d2dc9c885b5404545a6d0689857a31 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -111,8 +111,8 @@ public class TSDBJNIConnector { * @throws SQLException */ public long executeQuery(String sql) throws SQLException { - // close previous result set if the user forgets to invoke the - // free method to close previous result set. + // close previous result set if the user forgets to invoke the + // free method to close previous result set. if (!this.isResultsetClosed) { freeResultSet(taosResultSetPointer); } @@ -122,23 +122,23 @@ public class TSDBJNIConnector { pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos); } catch (Exception e) { e.printStackTrace(); - this.freeResultSet(pSql); + this.freeResultSetImp(this.taos, pSql); throw new SQLException(TSDBConstants.WrapErrMsg("Unsupported encoding")); } - + int code = this.getErrCode(pSql); if (code != 0) { affectedRows = -1; String msg = this.getErrMsg(pSql); - - this.freeResultSet(pSql); + + this.freeResultSetImp(this.taos, pSql); throw new SQLException(TSDBConstants.WrapErrMsg(msg), "", code); } // Try retrieving result set for the executed SQL using the current connection pointer. taosResultSetPointer = this.getResultSetImp(this.taos, pSql); isResultsetClosed = (taosResultSetPointer == TSDBConstants.JNI_NULL_POINTER); - + return pSql; } @@ -171,11 +171,11 @@ public class TSDBJNIConnector { } private native long getResultSetImp(long connection, long pSql); - + public boolean isUpdateQuery(long pSql) { - return isUpdateQueryImp(this.taos, pSql) == 1? true:false; + return isUpdateQueryImp(this.taos, pSql) == 1 ? true : false; } - + private native long isUpdateQueryImp(long connection, long pSql); /** @@ -191,7 +191,7 @@ public class TSDBJNIConnector { res = this.freeResultSetImp(this.taos, result); taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; } - + isResultsetClosed = true; return res; } @@ -274,7 +274,7 @@ public class TSDBJNIConnector { * Consume a subscription */ long consume(long subscription) { - return this.consumeImp(subscription); + return this.consumeImp(subscription); } private native long consumeImp(long subscription); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index a0981063a5052f04b849b2187a78352a2c2560be..8adcdefb2974d5817793297091eee1c2f40a52b9 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -1,47 +1,210 @@ package com.taosdata.jdbc; +import org.junit.BeforeClass; import org.junit.Test; -import java.sql.SQLException; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.sql.*; import java.util.Properties; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; public class TSDBDriverTest { + private static final String[] validURLs = { + "jdbc:TAOS://localhost:0", + "jdbc:TAOS://localhost", + "jdbc:TAOS://localhost:6030/test", + "jdbc:TAOS://localhost:6030", + "jdbc:TAOS://localhost:6030/", + "jdbc:TSDB://localhost:6030", + "jdbc:TSDB://localhost:6030/", + "jdbc:TAOS://127.0.0.1:0/db?user=root&password=taosdata", + "jdbc:TAOS://:", + "jdbc:TAOS://:/", + "jdbc:TAOS://:/test", + "jdbc:TAOS://localhost:0/?user=root&password=taosdata" + }; + private static boolean islibLoaded = false; + private static boolean isTaosdActived; + + private Connection conn; + + @BeforeClass + public static void before() { + String osName = System.getProperty("os.name").toLowerCase(); + if (!osName.equals("linux")) + return; + // try to load taos lib + try { + System.loadLibrary("taos"); + islibLoaded = true; + } catch (UnsatisfiedLinkError error) { + System.out.println("load tdengine lib failed."); + error.printStackTrace(); + } + // check taosd is activated + try { + String[] cmd = {"/bin/bash", "-c", "ps -ef | grep taosd | grep -v \"grep\""}; + Process exec = Runtime.getRuntime().exec(cmd); + BufferedReader reader = new BufferedReader(new InputStreamReader(exec.getInputStream())); + int lineCnt = 0; + while (reader.readLine() != null) { + lineCnt++; + } + if (lineCnt > 0) + isTaosdActived = true; + } catch (IOException e) { + e.printStackTrace(); + } + + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + + @Test + public void testConnectWithJdbcURL() { + final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; + try { + if (islibLoaded && isTaosdActived) { + conn = DriverManager.getConnection(url); + assertNotNull("failure - connection should not be null", conn); + } + } catch (SQLException e) { + e.printStackTrace(); + fail("failure - should not throw Exception"); + } + } + + @Test + public void testConnectWithProperties() { + final String jdbcUrl = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + try { + if (islibLoaded && isTaosdActived) { + conn = DriverManager.getConnection(jdbcUrl, connProps); + assertNotNull("failure - connection should not be null", conn); + } + } catch (SQLException e) { + e.printStackTrace(); + fail("failure - should not throw Exception"); + } + } + @Test - public void urlParserTest() throws SQLException { + public void testConnectWithConfigFile() { + String jdbcUrl = "jdbc:TAOS://:/log?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + try { + if (islibLoaded && isTaosdActived) { + conn = DriverManager.getConnection(jdbcUrl, connProps); + assertNotNull("failure - connection should not be null", conn); + } + } catch (SQLException e) { + e.printStackTrace(); + fail("failure - should not throw Exception"); + } + } + + @Test(expected = SQLException.class) + public void testAcceptsURL() throws SQLException { + Driver driver = new TSDBDriver(); + for (String url : validURLs) { + assertTrue("failure - acceptsURL(\" " + url + " \") should be true", driver.acceptsURL(url)); + } + driver.acceptsURL(null); + fail("acceptsURL throws exception when parameter is null"); + } + + @Test + public void testParseURL() { TSDBDriver driver = new TSDBDriver(); - String url = "jdbc:TSDB://127.0.0.1:0/db"; - - Properties properties = new Properties(); - driver.parseURL(url, properties); - assertEquals(properties.get("host"), "127.0.0.1"); - assertEquals(properties.get("port"), "0"); - assertEquals(properties.get("dbname"), "db"); - assertEquals(properties.get("user"), "root"); - assertEquals(properties.get("password"), "your_password"); - - url = "jdbc:TSDB://127.0.0.1:0/log?charset=UTF-8"; - properties = new Properties(); - driver.parseURL(url, properties); - assertEquals(properties.get("host"), "127.0.0.1"); - assertEquals(properties.get("port"), "0"); - assertEquals(properties.get("dbname"), "log"); - assertEquals(properties.get("charset"), "UTF-8"); - - url = "jdbc:TSDB://127.0.0.1:0/"; - properties = new Properties(); - driver.parseURL(url, properties); - assertEquals(properties.get("host"), "127.0.0.1"); - assertEquals(properties.get("port"), "0"); - assertEquals(properties.get("dbname"), null); - - url = "jdbc:TSDB://127.0.0.1:0/db"; - properties = new Properties(); - driver.parseURL(url, properties); - assertEquals(properties.get("host"), "127.0.0.1"); - assertEquals(properties.get("port"), "0"); - assertEquals(properties.get("dbname"), "db"); + + String url = "jdbc:TAOS://127.0.0.1:0/db?user=root&password=taosdata&charset=UTF-8"; + Properties config = new Properties(); + Properties actual = driver.parseURL(url, config); + assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.get("host")); + assertEquals("failure - port should be 0", "0", actual.get("port")); + assertEquals("failure - dbname should be db", "db", actual.get("dbname")); + assertEquals("failure - user should be root", "root", actual.get("user")); + assertEquals("failure - password should be taosdata", "taosdata", actual.get("password")); + assertEquals("failure - charset should be UTF-8", "UTF-8", actual.get("charset")); + + url = "jdbc:TAOS://127.0.0.1:0"; + config = new Properties(); + actual = driver.parseURL(url, config); + assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.getProperty("host")); + assertEquals("failure - port should be 0", "0", actual.get("port")); + assertNull("failure - dbname should be null", actual.get("dbname")); + + url = "jdbc:TAOS://127.0.0.1:0/db"; + config = new Properties(); + actual = driver.parseURL(url, config); + assertEquals("failure - host should be 127.0.0.1", "127.0.0.1", actual.getProperty("host")); + assertEquals("failure - port should be 0", "0", actual.get("port")); + assertEquals("failure - dbname should be db", "db", actual.get("dbname")); + + url = "jdbc:TAOS://:/?"; + config = new Properties(); + config.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root"); + config.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata"); + actual = driver.parseURL(url, config); + assertEquals("failure - user should be root", "root", actual.getProperty("user")); + assertEquals("failure - password should be taosdata", "taosdata", actual.getProperty("password")); + assertNull("failure - host should be null", actual.getProperty("host")); + assertNull("failure - port should be null", actual.getProperty("port")); + assertNull("failure - dbname should be null", actual.getProperty("dbname")); + } + + @Test + public void testGetPropertyInfo() throws SQLException { + Driver driver = new TSDBDriver(); + final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; + Properties connProps = new Properties(); + DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, connProps); + for (DriverPropertyInfo info : propertyInfo) { + if (info.name.equals(TSDBDriver.PROPERTY_KEY_HOST)) + assertEquals("failure - host should be localhost", "localhost", info.value); + if (info.name.equals(TSDBDriver.PROPERTY_KEY_PORT)) + assertEquals("failure - port should be 6030", "6030", info.value); + if (info.name.equals(TSDBDriver.PROPERTY_KEY_DBNAME)) + assertEquals("failure - dbname should be test", "log", info.value); + if (info.name.equals(TSDBDriver.PROPERTY_KEY_USER)) + assertEquals("failure - user should be root", "root", info.value); + if (info.name.equals(TSDBDriver.PROPERTY_KEY_PASSWORD)) + assertEquals("failure - password should be root", "taosdata", info.value); + } + } + + @Test + public void testGetMajorVersion() { + assertEquals("failure - getMajorVersion should be 2", 2, new TSDBDriver().getMajorVersion()); + } + + @Test + public void testGetMinorVersion() { + assertEquals("failure - getMinorVersion should be 0", 0, new TSDBDriver().getMinorVersion()); + } + + @Test + public void testJdbcCompliant() { + assertFalse("failure - jdbcCompliant should be false", new TSDBDriver().jdbcCompliant()); } + + @Test + public void testGetParentLogger() throws SQLFeatureNotSupportedException { + assertNull("failure - getParentLogger should be be null", new TSDBDriver().getParentLogger()); + } + } \ No newline at end of file diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 1a99a84b8ebfe8ed503213299646da39b5b1d27a..1be7552a892e25cf44317a99fe52ff57689ad338 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -39,16 +39,16 @@ #define cTrace(...) { if (cqDebugFlag & DEBUG_TRACE) { taosPrintLog("CQ ", cqDebugFlag, __VA_ARGS__); }} typedef struct { - int vgId; + int32_t vgId; char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; char db[TSDB_DB_NAME_LEN]; FCqWrite cqWrite; void *ahandle; - int num; // number of continuous streams + int32_t num; // number of continuous streams struct SCqObj *pHead; void *dbConn; - int master; + int32_t master; void *tmrCtrl; pthread_mutex_t mutex; } SCqContext; @@ -57,7 +57,7 @@ typedef struct SCqObj { tmr_h tmrId; uint64_t uid; int32_t tid; // table ID - int rowSize; // bytes of a row + int32_t rowSize; // bytes of a row char * sqlStr; // SQL string STSchema * pSchema; // pointer to schema array void * pStream; @@ -115,7 +115,7 @@ void cqClose(void *handle) { SCqObj *pTemp = pObj; pObj = pObj->next; tdFreeSchema(pTemp->pSchema); - taosTFree(pTemp->sqlStr); + tfree(pTemp->sqlStr); free(pTemp); } @@ -175,7 +175,7 @@ void cqStop(void *handle) { pthread_mutex_unlock(&pContext->mutex); } -void *cqCreate(void *handle, uint64_t uid, int tid, char *sqlStr, STSchema *pSchema) { +void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) { SCqContext *pContext = handle; SCqObj *pObj = calloc(sizeof(SCqObj), 1); @@ -237,7 +237,7 @@ void cqDrop(void *handle) { pthread_mutex_unlock(&pContext->mutex); } -static void doCreateStream(void *param, TAOS_RES *result, int code) { +static void doCreateStream(void *param, TAOS_RES *result, int32_t code) { SCqObj* pObj = (SCqObj*)param; SCqContext* pContext = pObj->pContext; SSqlObj* pSql = (SSqlObj*)result; @@ -288,7 +288,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr); - int size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize; + int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize; char *buffer = calloc(size, 1); SWalHead *pHead = (SWalHead *)buffer; @@ -334,7 +334,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { pHead->version = 0; // write into vnode write queue - pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ); + pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ, NULL); free(buffer); } diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c index 1daee644a7effc5fa50fbc340ad7df8e2b5d70b0..e1114fc024054f7a4af32ca1497a63f7da942147 100644 --- a/src/cq/test/cqtest.c +++ b/src/cq/test/cqtest.c @@ -24,7 +24,7 @@ int64_t ver = 0; void *pCq = NULL; -int writeToQueue(void *pVnode, void *data, int type) { +int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { return 0; } diff --git a/src/dnode/inc/dnodeCfg.h b/src/dnode/inc/dnodeCfg.h new file mode 100644 index 0000000000000000000000000000000000000000..35d889646021a079abe18c01a26fa5891718aaa0 --- /dev/null +++ b/src/dnode/inc/dnodeCfg.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_DNODE_CFG_H +#define TDENGINE_DNODE_CFG_H + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t dnodeInitCfg(); +void dnodeCleanupCfg(); +void dnodeUpdateCfg(SDnodeCfg *cfg); +int32_t dnodeGetDnodeId(); +void dnodeGetCfg(int32_t *dnodeId, char *clusterId); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/dnode/inc/dnodeEps.h b/src/dnode/inc/dnodeEps.h new file mode 100644 index 0000000000000000000000000000000000000000..2a203498c1f270c7e456694a4e1e195cbd9022cd --- /dev/null +++ b/src/dnode/inc/dnodeEps.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_DNODE_EP_H +#define TDENGINE_DNODE_EP_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taosmsg.h" + +int32_t dnodeInitEps(); +void dnodeCleanupEps(); +void dnodeUpdateEps(SDnodeEps *eps); +void dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port); +bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/dnode/inc/dnodeMInfos.h b/src/dnode/inc/dnodeMInfos.h new file mode 100644 index 0000000000000000000000000000000000000000..9c3c85c47e2dbcc11c5b5a80fbf091bd93855149 --- /dev/null +++ b/src/dnode/inc/dnodeMInfos.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_DNODE_MINFOS_H +#define TDENGINE_DNODE_MINFOS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taosmsg.h" + +int32_t dnodeInitMInfos(); +void dnodeCleanupMInfos(); +void dnodeUpdateMInfos(SMnodeInfos *minfos); +void dnodeUpdateEpSetForPeer(SRpcEpSet *epSet); +void dnodeGetMInfos(SMnodeInfos *minfos); +bool dnodeIsMasterEp(char *ep); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/dnode/inc/dnodeMPeer.h b/src/dnode/inc/dnodeMPeer.h index 9a48703110c389f10ac623ccaeaa85420a32817f..00221baa221a411d614c1fa1bb3dc4525de2ed71 100644 --- a/src/dnode/inc/dnodeMPeer.h +++ b/src/dnode/inc/dnodeMPeer.h @@ -20,11 +20,11 @@ extern "C" { #endif -int32_t dnodeInitMnodePeer(); -void dnodeCleanupMnodePeer(); -int32_t dnodeAllocateMnodePqueue(); -void dnodeFreeMnodePqueue(); -void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg); +int32_t dnodeInitMPeer(); +void dnodeCleanupMPeer(); +int32_t dnodeAllocateMPeerQueue(); +void dnodeFreeMPeerQueue(); +void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/src/dnode/inc/dnodeMRead.h b/src/dnode/inc/dnodeMRead.h index 4e93838b7998850e1bec79f017fdfbd28f286a11..8a8e71227ddf3cd7b7c77ef664ac3d4c935889db 100644 --- a/src/dnode/inc/dnodeMRead.h +++ b/src/dnode/inc/dnodeMRead.h @@ -20,11 +20,11 @@ extern "C" { #endif -int32_t dnodeInitMnodeRead(); -void dnodeCleanupMnodeRead(); -int32_t dnodeAllocateMnodeRqueue(); -void dnodeFreeMnodeRqueue(); -void dnodeDispatchToMnodeReadQueue(SRpcMsg *rpcMsg); +int32_t dnodeInitMRead(); +void dnodeCleanupMRead(); +int32_t dnodeAllocMReadQueue(); +void dnodeFreeMReadQueue(); +void dnodeDispatchToMReadQueue(SRpcMsg *rpcMsg); #ifdef __cplusplus } diff --git a/src/dnode/inc/dnodeMWrite.h b/src/dnode/inc/dnodeMWrite.h index 498fea81c59329b4d30874d36d10182b6e3ae54f..6a3d41bc81a03465a2e097bc711c83a8d8af362e 100644 --- a/src/dnode/inc/dnodeMWrite.h +++ b/src/dnode/inc/dnodeMWrite.h @@ -20,11 +20,11 @@ extern "C" { #endif -int32_t dnodeInitMnodeWrite(); -void dnodeCleanupMnodeWrite(); -int32_t dnodeAllocateMnodeWqueue(); -void dnodeFreeMnodeWqueue(); -void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg); +int32_t dnodeInitMWrite(); +void dnodeCleanupMWrite(); +int32_t dnodeAllocMWritequeue(); +void dnodeFreeMWritequeue(); +void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/src/dnode/inc/dnodeMgmt.h b/src/dnode/inc/dnodeMgmt.h index e8f4a0823f3cc9846b85804d323e73a8cfc0476e..2038ef5286b32522b11409ba5a253b33228b984d 100644 --- a/src/dnode/inc/dnodeMgmt.h +++ b/src/dnode/inc/dnodeMgmt.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include "trpc.h" + int32_t dnodeInitMgmt(); void dnodeCleanupMgmt(); int32_t dnodeInitMgmtTimer(); @@ -35,8 +37,8 @@ void* dnodeGetVnodeTsdb(void *pVnode); void dnodeReleaseVnode(void *pVnode); void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell); -void dnodeGetMnodeEpSetForPeer(void *epSet); -void dnodeGetMnodeEpSetForShell(void *epSet); +void dnodeGetEpSetForPeer(SRpcEpSet *epSet); +void dnodeGetEpSetForShell(SRpcEpSet *epSet); #ifdef __cplusplus } diff --git a/src/dnode/inc/dnodeVRead.h b/src/dnode/inc/dnodeVRead.h index a1035200475259bd91757f934d35a0dd5a69b1fe..b3c3df80b2e201ff9f4b40ebfcceafba57de1366 100644 --- a/src/dnode/inc/dnodeVRead.h +++ b/src/dnode/inc/dnodeVRead.h @@ -20,9 +20,11 @@ extern "C" { #endif -int32_t dnodeInitVnodeRead(); -void dnodeCleanupVnodeRead(); -void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg); +int32_t dnodeInitVRead(); +void dnodeCleanupVRead(); +void dnodeDispatchToVReadQueue(SRpcMsg *pMsg); +void * dnodeAllocVReadQueue(void *pVnode); +void dnodeFreeVReadQueue(void *rqueue); #ifdef __cplusplus } diff --git a/src/dnode/inc/dnodeVWrite.h b/src/dnode/inc/dnodeVWrite.h index 7da701a8e270239ee2f48e090138f048cc81f880..323405143fd10b7b4f73bac5103ad63ed870044f 100644 --- a/src/dnode/inc/dnodeVWrite.h +++ b/src/dnode/inc/dnodeVWrite.h @@ -20,9 +20,12 @@ extern "C" { #endif -int32_t dnodeInitVnodeWrite(); -void dnodeCleanupVnodeWrite(); -void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg); +int32_t dnodeInitVWrite(); +void dnodeCleanupVWrite(); +void dnodeDispatchToVWriteQueue(SRpcMsg *pMsg); +void * dnodeAllocVWriteQueue(void *pVnode); +void dnodeFreeVWriteQueue(void *wqueue); +void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code); #ifdef __cplusplus } diff --git a/src/dnode/src/dnodeCfg.c b/src/dnode/src/dnodeCfg.c new file mode 100644 index 0000000000000000000000000000000000000000..16d109a13a7600c5414b1f1edd175a8f15aa4cf0 --- /dev/null +++ b/src/dnode/src/dnodeCfg.c @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "cJSON.h" +#include "tglobal.h" +#include "dnode.h" +#include "dnodeInt.h" +#include "dnodeCfg.h" + +static SDnodeCfg tsCfg = {0}; +static pthread_mutex_t tsCfgMutex; + +static int32_t dnodeReadCfg(); +static int32_t dnodeWriteCfg(); +static void dnodeResetCfg(SDnodeCfg *cfg); +static void dnodePrintCfg(SDnodeCfg *cfg); + +int32_t dnodeInitCfg() { + pthread_mutex_init(&tsCfgMutex, NULL); + dnodeResetCfg(NULL); + int32_t ret = dnodeReadCfg(); + if (ret == 0) { + dInfo("dnode cfg is initialized"); + } + return ret; +} + +void dnodeCleanupCfg() { pthread_mutex_destroy(&tsCfgMutex); } + +void dnodeUpdateCfg(SDnodeCfg *cfg) { + if (tsCfg.dnodeId != 0) return; + dnodeResetCfg(cfg); +} + +int32_t dnodeGetDnodeId() { + int32_t dnodeId = 0; + pthread_mutex_lock(&tsCfgMutex); + dnodeId = tsCfg.dnodeId; + pthread_mutex_unlock(&tsCfgMutex); + return dnodeId; +} + +void dnodeGetCfg(int32_t *dnodeId, char *clusterId) { + pthread_mutex_lock(&tsCfgMutex); + *dnodeId = tsCfg.dnodeId; + tstrncpy(clusterId, tsCfg.clusterId, TSDB_CLUSTER_ID_LEN); + pthread_mutex_unlock(&tsCfgMutex); +} + +static void dnodeResetCfg(SDnodeCfg *cfg) { + if (cfg == NULL) return; + if (cfg->dnodeId == 0) return; + + pthread_mutex_lock(&tsCfgMutex); + tsCfg.dnodeId = cfg->dnodeId; + tstrncpy(tsCfg.clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN); + dnodePrintCfg(cfg); + dnodeWriteCfg(); + pthread_mutex_unlock(&tsCfgMutex); +} + +static void dnodePrintCfg(SDnodeCfg *cfg) { + dInfo("dnodeId is set to %d, clusterId is set to %s", cfg->dnodeId, cfg->clusterId); +} + +static int32_t dnodeReadCfg() { + int32_t len = 0; + int32_t maxLen = 200; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + SDnodeCfg cfg = {0}; + + char file[TSDB_FILENAME_LEN + 20] = {0}; + sprintf(file, "%s/dnodeCfg.json", tsDnodeDir); + + fp = fopen(file, "r"); + if (!fp) { + dDebug("failed to read %s, file not exist", file); + goto PARSE_CFG_OVER; + } + + len = fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s, content is null", file); + goto PARSE_CFG_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s, invalid json format", file); + goto PARSE_CFG_OVER; + } + + cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); + if (!dnodeId || dnodeId->type != cJSON_Number) { + dError("failed to read %s, dnodeId not found", file); + goto PARSE_CFG_OVER; + } + cfg.dnodeId = dnodeId->valueint; + + cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); + if (!clusterId || clusterId->type != cJSON_String) { + dError("failed to read %s, clusterId not found", file); + goto PARSE_CFG_OVER; + } + tstrncpy(cfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN); + + dInfo("read file %s successed", file); + +PARSE_CFG_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + terrno = 0; + + dnodeResetCfg(&cfg); + return 0; +} + +static int32_t dnodeWriteCfg() { + char file[TSDB_FILENAME_LEN + 20] = {0}; + sprintf(file, "%s/dnodeCfg.json", tsDnodeDir); + + FILE *fp = fopen(file, "w"); + if (!fp) { + dError("failed to write %s, reason:%s", file, strerror(errno)); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 200; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", tsCfg.dnodeId); + len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%s\"\n", tsCfg.clusterId); + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + fflush(fp); + fclose(fp); + free(content); + terrno = 0; + + dInfo("successed to write %s", file); + return 0; +} diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c index 9b68fc1f6c7a4db266eeb36cc20459bc71870739..a9ee4ac649c2f4b2734ce7f0dd59004b08e2fb67 100644 --- a/src/dnode/src/dnodeCheck.c +++ b/src/dnode/src/dnodeCheck.c @@ -15,9 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosdef.h" #include "tglobal.h" -#include "mnode.h" #include "dnodeInt.h" #include "dnodeCheck.h" @@ -30,8 +28,8 @@ typedef struct { void (*stopFp)(); } SCheckItem; -static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; -int64_t tsMinFreeMemSizeForStart = 0; +static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; +int64_t tsMinFreeMemSizeForStart = 0; static int bindTcpPort(int port) { int serverSocket; @@ -264,8 +262,6 @@ int32_t dnodeInitCheck() { } } + dInfo("dnode check is initialized"); return 0; } - - - diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c new file mode 100644 index 0000000000000000000000000000000000000000..9c90c391813f57024ecf090136dfa86f5e5f91e6 --- /dev/null +++ b/src/dnode/src/dnodeEps.c @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "cJSON.h" +#include "tglobal.h" +#include "hash.h" +#include "dnode.h" +#include "dnodeInt.h" +#include "dnodeEps.h" + +static SDnodeEps *tsEps = NULL; +static SHashObj * tsEpsHash = NULL; +static pthread_mutex_t tsEpsMutex; + +static int32_t dnodeReadEps(); +static int32_t dnodeWriteEps(); +static void dnodeResetEps(SDnodeEps *eps); +static void dnodePrintEps(SDnodeEps *eps); + +int32_t dnodeInitEps() { + pthread_mutex_init(&tsEpsMutex, NULL); + tsEpsHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); + dnodeResetEps(NULL); + int32_t ret = dnodeReadEps(); + if (ret == 0) { + dInfo("dnode eps is initialized"); + } + return ret; +} + +void dnodeCleanupEps() { + pthread_mutex_lock(&tsEpsMutex); + if (tsEps) { + free(tsEps); + tsEps = NULL; + } + if (tsEpsHash) { + taosHashCleanup(tsEpsHash); + tsEpsHash = NULL; + } + pthread_mutex_unlock(&tsEpsMutex); + pthread_mutex_destroy(&tsEpsMutex); +} + +void dnodeUpdateEps(SDnodeEps *eps) { + if (eps == NULL) return; + + eps->dnodeNum = htonl(eps->dnodeNum); + for (int32_t i = 0; i < eps->dnodeNum; ++i) { + eps->dnodeEps[i].dnodeId = htonl(eps->dnodeEps[i].dnodeId); + eps->dnodeEps[i].dnodePort = htons(eps->dnodeEps[i].dnodePort); + } + + pthread_mutex_lock(&tsEpsMutex); + if (eps->dnodeNum != tsEps->dnodeNum) { + dnodeResetEps(eps); + dnodeWriteEps(); + } else { + int32_t size = sizeof(SDnodeEps) + eps->dnodeNum * sizeof(SDnodeEp); + if (memcmp(eps, tsEps, size) != 0) { + dnodeResetEps(eps); + dnodeWriteEps(); + } + } + pthread_mutex_unlock(&tsEpsMutex); +} + +bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr) { + bool changed = false; + pthread_mutex_lock(&tsEpsMutex); + SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t)); + if (ep != NULL) { + char epSaved[TSDB_EP_LEN + 1]; + snprintf(epSaved, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort); + changed = strcmp(epstr, epSaved) != 0; + tstrncpy(epstr, epSaved, TSDB_EP_LEN); + } + pthread_mutex_unlock(&tsEpsMutex); + return changed; +} + +void dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) { + pthread_mutex_lock(&tsEpsMutex); + SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t)); + if (ep != NULL) { + if (port) *port = ep->dnodePort; + if (fqdn) tstrncpy(fqdn, ep->dnodeFqdn, TSDB_FQDN_LEN); + if (epstr) snprintf(epstr, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort); + } + pthread_mutex_unlock(&tsEpsMutex); +} + +static void dnodeResetEps(SDnodeEps *eps) { + if (eps == NULL) { + int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp); + if (tsEps == NULL) { + tsEps = calloc(1, size); + } else { + tsEps->dnodeNum = 0; + } + } else { + assert(tsEps); + + int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp) * eps->dnodeNum; + if (eps->dnodeNum > tsEps->dnodeNum) { + tsEps = realloc(tsEps, size); + } + memcpy(tsEps, eps, size); + dnodePrintEps(eps); + } + + for (int32_t i = 0; i < tsEps->dnodeNum; ++i) { + SDnodeEp *ep = &tsEps->dnodeEps[i]; + taosHashPut(tsEpsHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp)); + } +} + +static void dnodePrintEps(SDnodeEps *eps) { + dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum); + for (int32_t i = 0; i < eps->dnodeNum; i++) { + SDnodeEp *ep = &eps->dnodeEps[i]; + dDebug("dnodeId:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort); + } +} + +static int32_t dnodeReadEps() { + int32_t ret = -1; + int32_t len = 0; + int32_t maxLen = 30000; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + SDnodeEps *eps = NULL; + + char file[TSDB_FILENAME_LEN + 20] = {0}; + sprintf(file, "%s/dnodeEps.json", tsDnodeDir); + + fp = fopen(file, "r"); + if (!fp) { + dDebug("failed to read %s, file not exist", file); + goto PRASE_EPS_OVER; + } + + len = fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s, content is null", file); + goto PRASE_EPS_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s, invalid json format", file); + goto PRASE_EPS_OVER; + } + + cJSON *dnodeNum = cJSON_GetObjectItem(root, "dnodeNum"); + if (!dnodeNum || dnodeNum->type != cJSON_Number) { + dError("failed to read %s, dnodeNum not found", file); + goto PRASE_EPS_OVER; + } + + cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos"); + if (!dnodeInfos || dnodeInfos->type != cJSON_Array) { + dError("failed to read %s, dnodeInfos not found", file); + goto PRASE_EPS_OVER; + } + + int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos); + if (dnodeInfosSize != dnodeNum->valueint) { + dError("failed to read %s, dnodeInfos size:%d not matched dnodeNum:%d", file, dnodeInfosSize, + (int32_t)dnodeNum->valueint); + goto PRASE_EPS_OVER; + } + + int32_t epsSize = sizeof(SDnodeEps) + dnodeInfosSize * sizeof(SDnodeEp); + eps = calloc(1, epsSize); + eps->dnodeNum = dnodeInfosSize; + + for (int32_t i = 0; i < dnodeInfosSize; ++i) { + cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i); + if (dnodeInfo == NULL) break; + + SDnodeEp *ep = &eps->dnodeEps[i]; + + cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId"); + if (!dnodeId || dnodeId->type != cJSON_Number) { + dError("failed to read %s, dnodeId not found", file); + goto PRASE_EPS_OVER; + } + ep->dnodeId = dnodeId->valueint; + + cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn"); + if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { + dError("failed to read %s, dnodeFqdn not found", file); + goto PRASE_EPS_OVER; + } + strncpy(ep->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); + + cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort"); + if (!dnodePort || dnodePort->type != cJSON_Number) { + dError("failed to read %s, dnodePort not found", file); + goto PRASE_EPS_OVER; + } + ep->dnodePort = (uint16_t)dnodePort->valueint; + } + + ret = 0; + + dInfo("read file %s successed", file); + dnodePrintEps(eps); + +PRASE_EPS_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + if (ret != 0) { + if (eps) free(eps); + eps = NULL; + } + + dnodeResetEps(eps); + if (eps) free(eps); + + dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort); + + terrno = 0; + return 0; +} + +static int32_t dnodeWriteEps() { + char file[TSDB_FILENAME_LEN + 20] = {0}; + sprintf(file, "%s/dnodeEps.json", tsDnodeDir); + + FILE *fp = fopen(file, "w"); + if (!fp) { + dError("failed to write %s, reason:%s", file, strerror(errno)); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 30000; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"dnodeNum\": %d,\n", tsEps->dnodeNum); + len += snprintf(content + len, maxLen - len, " \"dnodeInfos\": [{\n"); + for (int32_t i = 0; i < tsEps->dnodeNum; ++i) { + SDnodeEp *ep = &tsEps->dnodeEps[i]; + len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", ep->dnodeId); + len += snprintf(content + len, maxLen - len, " \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn); + len += snprintf(content + len, maxLen - len, " \"dnodePort\": %u\n", ep->dnodePort); + if (i < tsEps->dnodeNum - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + fflush(fp); + fclose(fp); + free(content); + terrno = 0; + + dInfo("successed to write %s", file); + return 0; +} diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c new file mode 100644 index 0000000000000000000000000000000000000000..cefe44aebe7f87803141ce3d75c45dca18463849 --- /dev/null +++ b/src/dnode/src/dnodeMInfos.c @@ -0,0 +1,288 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "cJSON.h" +#include "tglobal.h" +#include "mnode.h" +#include "dnode.h" +#include "dnodeInt.h" +#include "dnodeMInfos.h" + +static SMnodeInfos tsMInfos; +static SRpcEpSet tsMEpSet; +static pthread_mutex_t tsMInfosMutex; + +static void dnodeResetMInfos(SMnodeInfos *minfos); +static void dnodePrintMInfos(SMnodeInfos *minfos); +static int32_t dnodeReadMInfos(); +static int32_t dnodeWriteMInfos(); + +int32_t dnodeInitMInfos() { + pthread_mutex_init(&tsMInfosMutex, NULL); + dnodeResetMInfos(NULL); + int32_t ret = dnodeReadMInfos(); + if (ret == 0) { + dInfo("dnode minfos is initialized"); + } + + return ret; +} + +void dnodeCleanupMInfos() { pthread_mutex_destroy(&tsMInfosMutex); } + +void dnodeUpdateMInfos(SMnodeInfos *minfos) { + if (minfos->mnodeNum <= 0 || minfos->mnodeNum > 3) { + dError("invalid mnode infos, mnodeNum:%d", minfos->mnodeNum); + return; + } + + for (int32_t i = 0; i < minfos->mnodeNum; ++i) { + SMnodeInfo *minfo = &minfos->mnodeInfos[i]; + minfo->mnodeId = htonl(minfo->mnodeId); + if (minfo->mnodeId <= 0 || strlen(minfo->mnodeEp) <= 5) { + dError("invalid mnode info:%d, mnodeId:%d mnodeEp:%s", i, minfo->mnodeId, minfo->mnodeEp); + return; + } + } + + pthread_mutex_lock(&tsMInfosMutex); + if (minfos->mnodeNum != tsMInfos.mnodeNum) { + dnodeResetMInfos(minfos); + dnodeWriteMInfos(); + sdbUpdateAsync(); + } else { + int32_t size = sizeof(SMnodeInfos); + if (memcmp(minfos, &tsMInfos, size) != 0) { + dnodeResetMInfos(minfos); + dnodeWriteMInfos(); + sdbUpdateAsync(); + } + } + pthread_mutex_unlock(&tsMInfosMutex); +} + +void dnodeUpdateEpSetForPeer(SRpcEpSet *ep) { + if (ep->numOfEps <= 0) { + dError("minfos is changed, but content is invalid, discard it"); + return; + } + + pthread_mutex_lock(&tsMInfosMutex); + dInfo("minfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse); + for (int i = 0; i < ep->numOfEps; ++i) { + ep->port[i] -= TSDB_PORT_DNODEDNODE; + dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]); + } + tsMEpSet = *ep; + pthread_mutex_unlock(&tsMInfosMutex); +} + +bool dnodeIsMasterEp(char *ep) { + pthread_mutex_lock(&tsMInfosMutex); + bool isMaster = strcmp(ep, tsMInfos.mnodeInfos[tsMEpSet.inUse].mnodeEp) == 0; + pthread_mutex_unlock(&tsMInfosMutex); + + return isMaster; +} + +void dnodeGetMInfos(SMnodeInfos *minfos) { + pthread_mutex_lock(&tsMInfosMutex); + memcpy(minfos, &tsMInfos, sizeof(SMnodeInfos)); + for (int32_t i = 0; i < tsMInfos.mnodeNum; ++i) { + minfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId); + } + pthread_mutex_unlock(&tsMInfosMutex); +} + +void dnodeGetEpSetForPeer(SRpcEpSet *epSet) { + pthread_mutex_lock(&tsMInfosMutex); + *epSet = tsMEpSet; + for (int i = 0; i < epSet->numOfEps; ++i) { + epSet->port[i] += TSDB_PORT_DNODEDNODE; + } + pthread_mutex_unlock(&tsMInfosMutex); +} + +void dnodeGetEpSetForShell(SRpcEpSet *epSet) { + pthread_mutex_lock(&tsMInfosMutex); + *epSet = tsMEpSet; + pthread_mutex_unlock(&tsMInfosMutex); +} + +static void dnodePrintMInfos(SMnodeInfos *minfos) { + dInfo("print mnode infos, mnodeNum:%d inUse:%d", minfos->mnodeNum, minfos->inUse); + for (int32_t i = 0; i < minfos->mnodeNum; i++) { + dInfo("mnode index:%d, %s", minfos->mnodeInfos[i].mnodeId, minfos->mnodeInfos[i].mnodeEp); + } +} + +static void dnodeResetMInfos(SMnodeInfos *minfos) { + if (minfos == NULL) { + tsMEpSet.numOfEps = 1; + taosGetFqdnPortFromEp(tsFirst, tsMEpSet.fqdn[0], &tsMEpSet.port[0]); + + if (strcmp(tsSecond, tsFirst) != 0) { + tsMEpSet.numOfEps = 2; + taosGetFqdnPortFromEp(tsSecond, tsMEpSet.fqdn[1], &tsMEpSet.port[1]); + } + return; + } + + if (minfos->mnodeNum == 0) return; + + int32_t size = sizeof(SMnodeInfos); + memcpy(&tsMInfos, minfos, size); + + tsMEpSet.inUse = tsMInfos.inUse; + tsMEpSet.numOfEps = tsMInfos.mnodeNum; + for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) { + taosGetFqdnPortFromEp(tsMInfos.mnodeInfos[i].mnodeEp, tsMEpSet.fqdn[i], &tsMEpSet.port[i]); + } + + dnodePrintMInfos(minfos); +} + +static int32_t dnodeReadMInfos() { + int32_t len = 0; + int32_t maxLen = 2000; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + SMnodeInfos minfos = {0}; + + char file[TSDB_FILENAME_LEN + 20] = {0}; + sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir); + + fp = fopen(file, "r"); + if (!fp) { + dDebug("failed to read %s, file not exist", file); + goto PARSE_MINFOS_OVER; + } + + len = fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s, content is null", file); + goto PARSE_MINFOS_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s, invalid json format", file); + goto PARSE_MINFOS_OVER; + } + + cJSON *inUse = cJSON_GetObjectItem(root, "inUse"); + if (!inUse || inUse->type != cJSON_Number) { + dError("failed to read mnodeEpSet.json, inUse not found"); + goto PARSE_MINFOS_OVER; + } + tsMInfos.inUse = inUse->valueint; + + cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum"); + if (!nodeNum || nodeNum->type != cJSON_Number) { + dError("failed to read mnodeEpSet.json, nodeNum not found"); + goto PARSE_MINFOS_OVER; + } + minfos.mnodeNum = nodeNum->valueint; + + cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); + if (!nodeInfos || nodeInfos->type != cJSON_Array) { + dError("failed to read mnodeEpSet.json, nodeInfos not found"); + goto PARSE_MINFOS_OVER; + } + + int size = cJSON_GetArraySize(nodeInfos); + if (size != minfos.mnodeNum) { + dError("failed to read mnodeEpSet.json, nodeInfos size not matched"); + goto PARSE_MINFOS_OVER; + } + + for (int i = 0; i < size; ++i) { + cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); + if (nodeInfo == NULL) continue; + + cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); + if (!nodeId || nodeId->type != cJSON_Number) { + dError("failed to read mnodeEpSet.json, nodeId not found"); + goto PARSE_MINFOS_OVER; + } + minfos.mnodeInfos[i].mnodeId = nodeId->valueint; + + cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); + if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { + dError("failed to read mnodeEpSet.json, nodeName not found"); + goto PARSE_MINFOS_OVER; + } + strncpy(minfos.mnodeInfos[i].mnodeEp, nodeEp->valuestring, TSDB_EP_LEN); + } + + dInfo("read file %s successed", file); + dnodePrintMInfos(&minfos); + +PARSE_MINFOS_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + terrno = 0; + + for (int32_t i = 0; i < minfos.mnodeNum; ++i) { + SMnodeInfo *mInfo = &minfos.mnodeInfos[i]; + dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL); + } + dnodeResetMInfos(&minfos); + return 0; +} + +static int32_t dnodeWriteMInfos() { + char file[TSDB_FILENAME_LEN + 20] = {0}; + sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir); + + FILE *fp = fopen(file, "w"); + if (!fp) { + dError("failed to write %s, reason:%s", file, strerror(errno)); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 2000; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsMInfos.inUse); + len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsMInfos.mnodeNum); + len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); + for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) { + len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMInfos.mnodeInfos[i].mnodeId); + len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", tsMInfos.mnodeInfos[i].mnodeEp); + if (i < tsMInfos.mnodeNum - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + fflush(fp); + fclose(fp); + free(content); + terrno = 0; + + dInfo("successed to write %s", file); + return 0; +} diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index 8414d79a9815287dfdfae3af5cc123304745c56d..05b37bd3388c22046d2a0dd1827655c97c2e40ad 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -35,44 +35,44 @@ typedef struct { typedef struct { int32_t curNum; int32_t maxNum; - SMPeerWorker *peerWorker; + SMPeerWorker *worker; } SMPeerWorkerPool; -static SMPeerWorkerPool tsMPeerPool; +static SMPeerWorkerPool tsMPeerWP; static taos_qset tsMPeerQset; static taos_queue tsMPeerQueue; -static void *dnodeProcessMnodePeerQueue(void *param); +static void *dnodeProcessMPeerQueue(void *param); -int32_t dnodeInitMnodePeer() { +int32_t dnodeInitMPeer() { tsMPeerQset = taosOpenQset(); - tsMPeerPool.maxNum = 1; - tsMPeerPool.curNum = 0; - tsMPeerPool.peerWorker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerPool.maxNum); + tsMPeerWP.maxNum = 1; + tsMPeerWP.curNum = 0; + tsMPeerWP.worker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerWP.maxNum); - if (tsMPeerPool.peerWorker == NULL) return -1; - for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) { - SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i; + if (tsMPeerWP.worker == NULL) return -1; + for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { + SMPeerWorker *pWorker = tsMPeerWP.worker + i; pWorker->workerId = i; dDebug("dnode mpeer worker:%d is created", i); } - dDebug("dnode mpeer is opened, workers:%d qset:%p", tsMPeerPool.maxNum, tsMPeerQset); + dDebug("dnode mpeer is initialized, workers:%d qset:%p", tsMPeerWP.maxNum, tsMPeerQset); return 0; } -void dnodeCleanupMnodePeer() { - for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) { - SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i; +void dnodeCleanupMPeer() { + for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { + SMPeerWorker *pWorker = tsMPeerWP.worker + i; if (pWorker->thread) { taosQsetThreadResume(tsMPeerQset); } dDebug("dnode mpeer worker:%d is closed", i); } - for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) { - SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i; + for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { + SMPeerWorker *pWorker = tsMPeerWP.worker + i; dDebug("dnode mpeer worker:%d start to join", i); if (pWorker->thread) { pthread_join(pWorker->thread, NULL); @@ -84,61 +84,60 @@ void dnodeCleanupMnodePeer() { taosCloseQset(tsMPeerQset); tsMPeerQset = NULL; - taosTFree(tsMPeerPool.peerWorker); + tfree(tsMPeerWP.worker); } -int32_t dnodeAllocateMnodePqueue() { +int32_t dnodeAllocateMPeerQueue() { tsMPeerQueue = taosOpenQueue(); if (tsMPeerQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; taosAddIntoQset(tsMPeerQset, tsMPeerQueue, NULL); - for (int32_t i = tsMPeerPool.curNum; i < tsMPeerPool.maxNum; ++i) { - SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i; + for (int32_t i = tsMPeerWP.curNum; i < tsMPeerWP.maxNum; ++i) { + SMPeerWorker *pWorker = tsMPeerWP.worker + i; pWorker->workerId = i; pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodePeerQueue, pWorker) != 0) { + if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMPeerQueue, pWorker) != 0) { dError("failed to create thread to process mpeer queue, reason:%s", strerror(errno)); } pthread_attr_destroy(&thAttr); - tsMPeerPool.curNum = i + 1; - dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerPool.maxNum); + tsMPeerWP.curNum = i + 1; + dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerWP.maxNum); } dDebug("dnode mpeer queue:%p is allocated", tsMPeerQueue); return TSDB_CODE_SUCCESS; } -void dnodeFreeMnodePqueue() { +void dnodeFreeMPeerQueue() { dDebug("dnode mpeer queue:%p is freed", tsMPeerQueue); taosCloseQueue(tsMPeerQueue); tsMPeerQueue = NULL; } -void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg) { +void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg) { if (!mnodeIsRunning() || tsMPeerQueue == NULL) { dnodeSendRedirectMsg(pMsg, false); - rpcFreeCont(pMsg->pCont); - return; + } else { + SMnodeMsg *pPeer = mnodeCreateMsg(pMsg); + taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer); } - SMnodeMsg *pPeer = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg)); - mnodeCreateMsg(pPeer, pMsg); - taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer); + rpcFreeCont(pMsg->pCont); } -static void dnodeFreeMnodePeerMsg(SMnodeMsg *pPeer) { +static void dnodeFreeMPeerMsg(SMnodeMsg *pPeer) { mnodeCleanupMsg(pPeer); taosFreeQitem(pPeer); } -static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) { +static void dnodeSendRpcMPeerRsp(SMnodeMsg *pPeer, int32_t code) { if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return; SRpcMsg rpcRsp = { @@ -149,10 +148,10 @@ static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) { }; rpcSendResponse(&rpcRsp); - dnodeFreeMnodePeerMsg(pPeer); + dnodeFreeMPeerMsg(pPeer); } -static void *dnodeProcessMnodePeerQueue(void *param) { +static void *dnodeProcessMPeerQueue(void *param) { SMnodeMsg *pPeerMsg; int32_t type; void * unUsed; @@ -165,7 +164,7 @@ static void *dnodeProcessMnodePeerQueue(void *param) { dDebug("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]); int32_t code = mnodeProcessPeerReq(pPeerMsg); - dnodeSendRpcMnodePeerRsp(pPeerMsg, code); + dnodeSendRpcMPeerRsp(pPeerMsg, code); } return NULL; diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index fdcbb5889f766ddebdb3f1e56ccffa0b5b129552..c14c7a81586d306ff1423e8dba91e0844293c407 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -35,46 +35,46 @@ typedef struct { typedef struct { int32_t curNum; int32_t maxNum; - SMReadWorker *readWorker; + SMReadWorker *worker; } SMReadWorkerPool; -static SMReadWorkerPool tsMReadPool; +static SMReadWorkerPool tsMReadWP; static taos_qset tsMReadQset; static taos_queue tsMReadQueue; -static void *dnodeProcessMnodeReadQueue(void *param); +static void *dnodeProcessMReadQueue(void *param); -int32_t dnodeInitMnodeRead() { +int32_t dnodeInitMRead() { tsMReadQset = taosOpenQset(); - tsMReadPool.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2; - tsMReadPool.maxNum = MAX(2, tsMReadPool.maxNum); - tsMReadPool.maxNum = MIN(4, tsMReadPool.maxNum); - tsMReadPool.curNum = 0; - tsMReadPool.readWorker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadPool.maxNum); + tsMReadWP.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2; + tsMReadWP.maxNum = MAX(2, tsMReadWP.maxNum); + tsMReadWP.maxNum = MIN(4, tsMReadWP.maxNum); + tsMReadWP.curNum = 0; + tsMReadWP.worker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadWP.maxNum); - if (tsMReadPool.readWorker == NULL) return -1; - for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) { - SMReadWorker *pWorker = tsMReadPool.readWorker + i; + if (tsMReadWP.worker == NULL) return -1; + for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { + SMReadWorker *pWorker = tsMReadWP.worker + i; pWorker->workerId = i; dDebug("dnode mread worker:%d is created", i); } - dDebug("dnode mread is opened, workers:%d qset:%p", tsMReadPool.maxNum, tsMReadQset); + dDebug("dnode mread is initialized, workers:%d qset:%p", tsMReadWP.maxNum, tsMReadQset); return 0; } -void dnodeCleanupMnodeRead() { - for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) { - SMReadWorker *pWorker = tsMReadPool.readWorker + i; +void dnodeCleanupMRead() { + for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { + SMReadWorker *pWorker = tsMReadWP.worker + i; if (pWorker->thread) { taosQsetThreadResume(tsMReadQset); } dDebug("dnode mread worker:%d is closed", i); } - for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) { - SMReadWorker *pWorker = tsMReadPool.readWorker + i; + for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { + SMReadWorker *pWorker = tsMReadWP.worker + i; dDebug("dnode mread worker:%d start to join", i); if (pWorker->thread) { pthread_join(pWorker->thread, NULL); @@ -86,64 +86,63 @@ void dnodeCleanupMnodeRead() { taosCloseQset(tsMReadQset); tsMReadQset = NULL; - free(tsMReadPool.readWorker); + free(tsMReadWP.worker); } -int32_t dnodeAllocateMnodeRqueue() { +int32_t dnodeAllocMReadQueue() { tsMReadQueue = taosOpenQueue(); if (tsMReadQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; taosAddIntoQset(tsMReadQset, tsMReadQueue, NULL); - for (int32_t i = tsMReadPool.curNum; i < tsMReadPool.maxNum; ++i) { - SMReadWorker *pWorker = tsMReadPool.readWorker + i; + for (int32_t i = tsMReadWP.curNum; i < tsMReadWP.maxNum; ++i) { + SMReadWorker *pWorker = tsMReadWP.worker + i; pWorker->workerId = i; pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeReadQueue, pWorker) != 0) { + if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMReadQueue, pWorker) != 0) { dError("failed to create thread to process mread queue, reason:%s", strerror(errno)); } pthread_attr_destroy(&thAttr); - tsMReadPool.curNum = i + 1; - dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadPool.maxNum); + tsMReadWP.curNum = i + 1; + dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadWP.maxNum); } dDebug("dnode mread queue:%p is allocated", tsMReadQueue); return TSDB_CODE_SUCCESS; } -void dnodeFreeMnodeRqueue() { +void dnodeFreeMReadQueue() { dDebug("dnode mread queue:%p is freed", tsMReadQueue); taosCloseQueue(tsMReadQueue); tsMReadQueue = NULL; } -void dnodeDispatchToMnodeReadQueue(SRpcMsg *pMsg) { +void dnodeDispatchToMReadQueue(SRpcMsg *pMsg) { if (!mnodeIsRunning() || tsMReadQueue == NULL) { dnodeSendRedirectMsg(pMsg, true); - rpcFreeCont(pMsg->pCont); - return; + } else { + SMnodeMsg *pRead = mnodeCreateMsg(pMsg); + taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead); } - SMnodeMsg *pRead = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg)); - mnodeCreateMsg(pRead, pMsg); - taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead); + rpcFreeCont(pMsg->pCont); } -static void dnodeFreeMnodeReadMsg(SMnodeMsg *pRead) { +static void dnodeFreeMReadMsg(SMnodeMsg *pRead) { mnodeCleanupMsg(pRead); taosFreeQitem(pRead); } -static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) { +static void dnodeSendRpcMReadRsp(SMnodeMsg *pRead, int32_t code) { if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return; if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) { // may be a auto create req, should put into write queue - dnodeReprocessMnodeWriteMsg(pRead); + dnodeReprocessMWriteMsg(pRead); return; } @@ -155,23 +154,23 @@ static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) { }; rpcSendResponse(&rpcRsp); - dnodeFreeMnodeReadMsg(pRead); + dnodeFreeMReadMsg(pRead); } -static void *dnodeProcessMnodeReadQueue(void *param) { - SMnodeMsg *pReadMsg; +static void *dnodeProcessMReadQueue(void *param) { + SMnodeMsg *pRead; int32_t type; void * unUsed; - + while (1) { - if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pReadMsg, &unUsed) == 0) { + if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pRead, &unUsed) == 0) { dDebug("qset:%p, mnode read got no message from qset, exiting", tsMReadQset); break; } - dDebug("%p, msg:%s will be processed in mread queue", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]); - int32_t code = mnodeProcessRead(pReadMsg); - dnodeSendRpcMnodeReadRsp(pReadMsg, code); + dDebug("%p, msg:%s will be processed in mread queue", pRead->rpcMsg.ahandle, taosMsg[pRead->rpcMsg.msgType]); + int32_t code = mnodeProcessRead(pRead); + dnodeSendRpcMReadRsp(pRead, code); } return NULL; diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 384a0fae75088197d7fb01dd67c6e1b9d38739cd..bde4b95bc6d0019178312cfa76a34bb381608097 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -36,45 +36,45 @@ typedef struct { typedef struct { int32_t curNum; int32_t maxNum; - SMWriteWorker *writeWorker; + SMWriteWorker *worker; } SMWriteWorkerPool; -static SMWriteWorkerPool tsMWritePool; +static SMWriteWorkerPool tsMWriteWP; static taos_qset tsMWriteQset; static taos_queue tsMWriteQueue; extern void * tsDnodeTmr; -static void *dnodeProcessMnodeWriteQueue(void *param); +static void *dnodeProcessMWriteQueue(void *param); -int32_t dnodeInitMnodeWrite() { +int32_t dnodeInitMWrite() { tsMWriteQset = taosOpenQset(); - tsMWritePool.maxNum = 1; - tsMWritePool.curNum = 0; - tsMWritePool.writeWorker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWritePool.maxNum); + tsMWriteWP.maxNum = 1; + tsMWriteWP.curNum = 0; + tsMWriteWP.worker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWriteWP.maxNum); - if (tsMWritePool.writeWorker == NULL) return -1; - for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) { - SMWriteWorker *pWorker = tsMWritePool.writeWorker + i; + if (tsMWriteWP.worker == NULL) return -1; + for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { + SMWriteWorker *pWorker = tsMWriteWP.worker + i; pWorker->workerId = i; dDebug("dnode mwrite worker:%d is created", i); } - dDebug("dnode mwrite is opened, workers:%d qset:%p", tsMWritePool.maxNum, tsMWriteQset); + dDebug("dnode mwrite is initialized, workers:%d qset:%p", tsMWriteWP.maxNum, tsMWriteQset); return 0; } -void dnodeCleanupMnodeWrite() { - for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) { - SMWriteWorker *pWorker = tsMWritePool.writeWorker + i; +void dnodeCleanupMWrite() { + for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { + SMWriteWorker *pWorker = tsMWriteWP.worker + i; if (pWorker->thread) { taosQsetThreadResume(tsMWriteQset); } dDebug("dnode mwrite worker:%d is closed", i); } - for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) { - SMWriteWorker *pWorker = tsMWritePool.writeWorker + i; + for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { + SMWriteWorker *pWorker = tsMWriteWP.worker + i; dDebug("dnode mwrite worker:%d start to join", i); if (pWorker->thread) { pthread_join(pWorker->thread, NULL); @@ -86,58 +86,56 @@ void dnodeCleanupMnodeWrite() { taosCloseQset(tsMWriteQset); tsMWriteQset = NULL; - taosTFree(tsMWritePool.writeWorker); + tfree(tsMWriteWP.worker); } -int32_t dnodeAllocateMnodeWqueue() { +int32_t dnodeAllocMWritequeue() { tsMWriteQueue = taosOpenQueue(); if (tsMWriteQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; taosAddIntoQset(tsMWriteQset, tsMWriteQueue, NULL); - for (int32_t i = tsMWritePool.curNum; i < tsMWritePool.maxNum; ++i) { - SMWriteWorker *pWorker = tsMWritePool.writeWorker + i; + for (int32_t i = tsMWriteWP.curNum; i < tsMWriteWP.maxNum; ++i) { + SMWriteWorker *pWorker = tsMWriteWP.worker + i; pWorker->workerId = i; pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeWriteQueue, pWorker) != 0) { + if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMWriteQueue, pWorker) != 0) { dError("failed to create thread to process mwrite queue, reason:%s", strerror(errno)); } pthread_attr_destroy(&thAttr); - tsMWritePool.curNum = i + 1; - dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWritePool.maxNum); + tsMWriteWP.curNum = i + 1; + dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWriteWP.maxNum); } dDebug("dnode mwrite queue:%p is allocated", tsMWriteQueue); return TSDB_CODE_SUCCESS; } -void dnodeFreeMnodeWqueue() { +void dnodeFreeMWritequeue() { dDebug("dnode mwrite queue:%p is freed", tsMWriteQueue); taosCloseQueue(tsMWriteQueue); tsMWriteQueue = NULL; } -void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) { +void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) { if (!mnodeIsRunning() || tsMWriteQueue == NULL) { dnodeSendRedirectMsg(pMsg, true); - rpcFreeCont(pMsg->pCont); - return; + } else { + SMnodeMsg *pWrite = mnodeCreateMsg(pMsg); + dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite, + taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); + taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); } - SMnodeMsg *pWrite = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg)); - mnodeCreateMsg(pWrite, pMsg); - - dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite, - taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); - taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); + rpcFreeCont(pMsg->pCont); } -static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) { +static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) { dDebug("app:%p:%p, msg:%s is freed from mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite, taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); @@ -145,12 +143,12 @@ static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) { taosFreeQitem(pWrite); } -void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) { +void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code) { SMnodeMsg *pWrite = pMsg; if (pWrite == NULL) return; if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return; if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) { - dnodeReprocessMnodeWriteMsg(pWrite); + dnodeReprocessMWriteMsg(pWrite); return; } @@ -162,10 +160,10 @@ void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) { }; rpcSendResponse(&rpcRsp); - dnodeFreeMnodeWriteMsg(pWrite); + dnodeFreeMWriteMsg(pWrite); } -static void *dnodeProcessMnodeWriteQueue(void *param) { +static void *dnodeProcessMWriteQueue(void *param) { SMnodeMsg *pWrite; int32_t type; void * unUsed; @@ -180,13 +178,13 @@ static void *dnodeProcessMnodeWriteQueue(void *param) { taosMsg[pWrite->rpcMsg.msgType]); int32_t code = mnodeProcessWrite(pWrite); - dnodeSendRpcMnodeWriteRsp(pWrite, code); + dnodeSendRpcMWriteRsp(pWrite, code); } return NULL; } -void dnodeReprocessMnodeWriteMsg(void *pMsg) { +void dnodeReprocessMWriteMsg(void *pMsg) { SMnodeMsg *pWrite = pMsg; if (!mnodeIsRunning() || tsMWriteQueue == NULL) { @@ -194,7 +192,7 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) { taosMsg[pWrite->rpcMsg.msgType], pWrite->retry); dnodeSendRedirectMsg(pMsg, true); - dnodeFreeMnodeWriteMsg(pWrite); + dnodeFreeMWriteMsg(pWrite); } else { dDebug("app:%p:%p, msg:%s is reput into mwrite queue:%p, retry times:%d", pWrite->rpcMsg.ahandle, pWrite, taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue, pWrite->retry); @@ -203,12 +201,12 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) { } } -static void dnodeDoDelayReprocessMnodeWriteMsg(void *param, void *tmrId) { - dnodeReprocessMnodeWriteMsg(param); +static void dnodeDoDelayReprocessMWriteMsg(void *param, void *tmrId) { + dnodeReprocessMWriteMsg(param); } -void dnodeDelayReprocessMnodeWriteMsg(void *pMsg) { +void dnodeDelayReprocessMWriteMsg(void *pMsg) { SMnodeMsg *mnodeMsg = pMsg; void *unUsed = NULL; - taosTmrReset(dnodeDoDelayReprocessMnodeWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed); + taosTmrReset(dnodeDoDelayReprocessMWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed); } diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 97e6f2ce6debe7ffc273ca103e3f55576da0eb31..f4c0ee565e4246e0d8a5c4fe4d67a590721e8de2 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -19,11 +19,15 @@ #include "tutil.h" #include "tconfig.h" #include "tglobal.h" +#include "twal.h" #include "dnode.h" #include "dnodeInt.h" #include "dnodeMgmt.h" #include "dnodePeer.h" #include "dnodeModule.h" +#include "dnodeEps.h" +#include "dnodeMInfos.h" +#include "dnodeCfg.h" #include "dnodeCheck.h" #include "dnodeVRead.h" #include "dnodeVWrite.h" @@ -33,29 +37,35 @@ #include "dnodeShell.h" #include "dnodeTelemetry.h" +static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED; + static int32_t dnodeInitStorage(); -static void dnodeCleanupStorage(); -static void dnodeSetRunStatus(SDnodeRunStatus status); -static void dnodeCheckDataDirOpenned(char *dir); -static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED; +static void dnodeCleanupStorage(); +static void dnodeSetRunStatus(SRunStatus status); +static void dnodeCheckDataDirOpenned(char *dir); static int32_t dnodeInitComponents(); -static void dnodeCleanupComponents(int32_t stepId); -static int dnodeCreateDir(const char *dir); +static void dnodeCleanupComponents(int32_t stepId); +static int dnodeCreateDir(const char *dir); typedef struct { const char *const name; - int (*init)(); - void (*cleanup)(); + int32_t (*init)(); + void (*cleanup)(); } SDnodeComponent; static const SDnodeComponent tsDnodeComponents[] = { {"storage", dnodeInitStorage, dnodeCleanupStorage}, + {"dnodecfg", dnodeInitCfg, dnodeCleanupCfg}, + {"dnodeeps", dnodeInitEps, dnodeCleanupEps}, + {"globalcfg" ,taosCheckGlobalCfg, NULL}, + {"mnodeinfos",dnodeInitMInfos, dnodeCleanupMInfos}, + {"wal", walInit, walCleanUp}, {"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!! - {"vread", dnodeInitVnodeRead, dnodeCleanupVnodeRead}, - {"vwrite", dnodeInitVnodeWrite, dnodeCleanupVnodeWrite}, - {"mread", dnodeInitMnodeRead, dnodeCleanupMnodeRead}, - {"mwrite", dnodeInitMnodeWrite, dnodeCleanupMnodeWrite}, - {"mpeer", dnodeInitMnodePeer, dnodeCleanupMnodePeer}, + {"vread", dnodeInitVRead, dnodeCleanupVRead}, + {"vwrite", dnodeInitVWrite, dnodeCleanupVWrite}, + {"mread", dnodeInitMRead, dnodeCleanupMRead}, + {"mwrite", dnodeInitMWrite, dnodeCleanupMWrite}, + {"mpeer", dnodeInitMPeer, dnodeCleanupMPeer}, {"client", dnodeInitClient, dnodeCleanupClient}, {"server", dnodeInitServer, dnodeCleanupServer}, {"mgmt", dnodeInitMgmt, dnodeCleanupMgmt}, @@ -75,7 +85,9 @@ static int dnodeCreateDir(const char *dir) { static void dnodeCleanupComponents(int32_t stepId) { for (int32_t i = stepId; i >= 0; i--) { - tsDnodeComponents[i].cleanup(); + if (tsDnodeComponents[i].cleanup) { + (*tsDnodeComponents[i].cleanup)(); + } } } @@ -92,7 +104,7 @@ static int32_t dnodeInitComponents() { } int32_t dnodeInitSystem() { - dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_INITIALIZE); + dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE); tscEmbedded = 1; taosBlockSIGPIPE(); taosResolveCRC(); @@ -112,21 +124,20 @@ int32_t dnodeInitSystem() { printf("failed to init log file\n"); } - if (!taosReadGlobalCfg() || !taosCheckGlobalCfg()) { + if (!taosReadGlobalCfg()) { taosPrintGlobalCfg(); dError("TDengine read global config failed"); return -1; } - taosPrintGlobalCfg(); - dInfo("start to initialize TDengine on %s", tsLocalEp); + dInfo("start to initialize TDengine"); if (dnodeInitComponents() != 0) { return -1; } dnodeStartModules(); - dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_RUNING); + dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING); dInfo("TDengine is initialized successfully"); @@ -134,20 +145,20 @@ int32_t dnodeInitSystem() { } void dnodeCleanUpSystem() { - if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) { - dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED); + if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) { + dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED); dnodeCleanupComponents(sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]) - 1); taos_cleanup(); taosCloseLog(); } } -SDnodeRunStatus dnodeGetRunStatus() { - return tsDnodeRunStatus; +SRunStatus dnodeGetRunStatus() { + return tsRunStatus; } -static void dnodeSetRunStatus(SDnodeRunStatus status) { - tsDnodeRunStatus = status; +static void dnodeSetRunStatus(SRunStatus status) { + tsRunStatus = status; } static void dnodeCheckDataDirOpenned(char *dir) { @@ -198,7 +209,7 @@ static int32_t dnodeInitStorage() { dnodeCheckDataDirOpenned(tsDnodeDir); - dInfo("storage directory is initialized"); + dInfo("dnode storage is initialized at %s", tsDnodeDir); return 0; } diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 968a8d9759e5618753996476b40efc3be77f7925..dcb48f7833abf921d1085e1cbb14b9cfbf2aa8b7 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -31,12 +31,13 @@ #include "mnode.h" #include "dnodeInt.h" #include "dnodeMgmt.h" +#include "dnodeEps.h" +#include "dnodeCfg.h" +#include "dnodeMInfos.h" #include "dnodeVRead.h" #include "dnodeVWrite.h" #include "dnodeModule.h" -#define MPEER_CONTENT_LEN 2000 - typedef struct { pthread_t thread; int32_t threadIndex; @@ -46,23 +47,18 @@ typedef struct { int32_t * vnodeList; } SOpenVnodeThread; -void * tsDnodeTmr = NULL; -static void * tsStatusTimer = NULL; -static uint32_t tsRebootTime; - -static SRpcEpSet tsDMnodeEpSet = {0}; -static SDMMnodeInfos tsDMnodeInfos = {0}; -static SDMDnodeCfg tsDnodeCfg = {0}; -static taos_qset tsMgmtQset = NULL; -static taos_queue tsMgmtQueue = NULL; -static pthread_t tsQthread; - -static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes); -static bool dnodeReadMnodeInfos(); -static void dnodeSaveMnodeInfos(); -static void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg); -static bool dnodeReadDnodeCfg(); -static void dnodeSaveDnodeCfg(); +typedef struct { + SRpcMsg rpcMsg; + char pCont[]; +} SMgmtMsg; + +void * tsDnodeTmr = NULL; +static void * tsStatusTimer = NULL; +static uint32_t tsRebootTime; +static taos_qset tsMgmtQset = NULL; +static taos_queue tsMgmtQueue = NULL; +static pthread_t tsQthread; + static void dnodeProcessStatusRsp(SRpcMsg *pMsg); static void dnodeSendStatusMsg(void *handle, void *tmrId); static void *dnodeProcessMgmtQueue(void *param); @@ -74,7 +70,7 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg); static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg); +static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg); static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg); int32_t dnodeInitMgmt() { @@ -86,28 +82,8 @@ int32_t dnodeInitMgmt() { dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg; dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP, dnodeProcessStatusRsp); - dnodeReadDnodeCfg(); tsRebootTime = taosGetTimestampSec(); - if (!dnodeReadMnodeInfos()) { - memset(&tsDMnodeEpSet, 0, sizeof(SRpcEpSet)); - memset(&tsDMnodeInfos, 0, sizeof(SDMMnodeInfos)); - - tsDMnodeEpSet.numOfEps = 1; - taosGetFqdnPortFromEp(tsFirst, tsDMnodeEpSet.fqdn[0], &tsDMnodeEpSet.port[0]); - - if (strcmp(tsSecond, tsFirst) != 0) { - tsDMnodeEpSet.numOfEps = 2; - taosGetFqdnPortFromEp(tsSecond, tsDMnodeEpSet.fqdn[1], &tsDMnodeEpSet.port[1]); - } - } else { - tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse; - tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum; - for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]); - } - } - int32_t code = vnodeInitResources(); if (code != TSDB_CODE_SUCCESS) { dnodeCleanupMgmt(); @@ -201,38 +177,46 @@ void dnodeCleanupMgmt() { vnodeCleanupResources(); } -void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) { - void *item; +static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) { + int32_t size = sizeof(SMgmtMsg) + pMsg->contLen; + SMgmtMsg *pMgmt = taosAllocateQitem(size); + if (pMgmt == NULL) { + return TSDB_CODE_DND_OUT_OF_MEMORY; + } - item = taosAllocateQitem(sizeof(SRpcMsg)); - if (item) { - memcpy(item, pMsg, sizeof(SRpcMsg)); - taosWriteQitem(tsMgmtQueue, 1, item); - } else { - SRpcMsg rsp = { - .handle = pMsg->handle, - .pCont = NULL, - .code = TSDB_CODE_DND_OUT_OF_MEMORY - }; - + pMgmt->rpcMsg = *pMsg; + pMgmt->rpcMsg.pCont = pMgmt->pCont; + memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen); + taosWriteQitem(tsMgmtQueue, TAOS_QTYPE_RPC, pMgmt); + + return TSDB_CODE_SUCCESS; +} + +void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) { + int32_t code = dnodeWriteToMgmtQueue(pMsg); + if (code != TSDB_CODE_SUCCESS) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = code}; rpcSendResponse(&rsp); - rpcFreeCont(pMsg->pCont); } + + rpcFreeCont(pMsg->pCont); } static void *dnodeProcessMgmtQueue(void *param) { - SRpcMsg *pMsg; - SRpcMsg rsp = {0}; - int type; - void * handle; + SMgmtMsg *pMgmt; + SRpcMsg * pMsg; + SRpcMsg rsp = {0}; + int32_t qtype; + void * handle; while (1) { - if (taosReadQitemFromQset(tsMgmtQset, &type, (void **) &pMsg, &handle) == 0) { + if (taosReadQitemFromQset(tsMgmtQset, &qtype, (void **)&pMgmt, &handle) == 0) { dDebug("qset:%p, dnode mgmt got no message from qset, exit", tsMgmtQset); break; } - dDebug("%p, msg:%s will be processed", pMsg->ahandle, taosMsg[pMsg->msgType]); + pMsg = &pMgmt->rpcMsg; + dDebug("%p, msg:%p:%s will be processed", pMsg->ahandle, pMgmt, taosMsg[pMsg->msgType]); if (dnodeProcessMgmtMsgFp[pMsg->msgType]) { rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg); } else { @@ -240,10 +224,9 @@ static void *dnodeProcessMgmtQueue(void *param) { } rsp.handle = pMsg->handle; - rsp.pCont = NULL; + rsp.pCont = NULL; rpcSendResponse(&rsp); - rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -381,7 +364,7 @@ static void dnodeCloseVnodes() { } static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) { - SMDCreateVnodeMsg *pCreate = rpcMsg->pCont; + SCreateVnodeMsg *pCreate = rpcMsg->pCont; pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion); pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables); @@ -404,7 +387,7 @@ static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) { } static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { - SMDCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg); + SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg); void *pVnode = vnodeAcquire(pCreate->cfg.vgId); if (pVnode != NULL) { @@ -418,7 +401,7 @@ static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { } static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) { - SMDAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg); + SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg); void *pVnode = vnodeAcquire(pAlter->cfg.vgId); if (pVnode != NULL) { @@ -433,14 +416,14 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) { } static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) { - SMDDropVnodeMsg *pDrop = rpcMsg->pCont; + SDropVnodeMsg *pDrop = rpcMsg->pCont; pDrop->vgId = htonl(pDrop->vgId); return vnodeDrop(pDrop->vgId); } static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) { -// SMDAlterStreamMsg *pStream = pCont; +// SAlterStreamMsg *pStream = pCont; // pStream->uid = htobe64(pStream->uid); // pStream->stime = htobe64(pStream->stime); // pStream->vnode = htonl(pStream->vnode); @@ -453,12 +436,12 @@ static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) { } static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) { - SMDCfgDnodeMsg *pCfg = pMsg->pCont; + SCfgDnodeMsg *pCfg = pMsg->pCont; return taosCfgDynamicOptions(pCfg->config); } static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) { - SMDCreateMnodeMsg *pCfg = pMsg->pCont; + SCreateMnodeMsg *pCfg = pMsg->pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); if (pCfg->dnodeId != dnodeGetDnodeId()) { dError("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId()); @@ -470,10 +453,10 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) { return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED; } - dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.nodeNum); - for (int i = 0; i < pCfg->mnodes.nodeNum; ++i) { - pCfg->mnodes.nodeInfos[i].nodeId = htonl(pCfg->mnodes.nodeInfos[i].nodeId); - dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.nodeInfos[i].nodeId, pCfg->mnodes.nodeInfos[i].nodeEp); + dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum); + for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) { + pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId); + dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp); } dnodeStartMnode(&pCfg->mnodes); @@ -481,34 +464,6 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } -void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet) { - if (pEpSet->numOfEps <= 0) { - dError("mnode EP list for peer is changed, but content is invalid, discard it"); - return; - } - - dInfo("mnode EP list for peer is changed, numOfEps:%d inUse:%d", pEpSet->numOfEps, pEpSet->inUse); - for (int i = 0; i < pEpSet->numOfEps; ++i) { - pEpSet->port[i] -= TSDB_PORT_DNODEDNODE; - dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]); - } - - tsDMnodeEpSet = *pEpSet; -} - -void dnodeGetMnodeEpSetForPeer(void *epSetRaw) { - SRpcEpSet *epSet = epSetRaw; - *epSet = tsDMnodeEpSet; - - for (int i=0; inumOfEps; ++i) - epSet->port[i] += TSDB_PORT_DNODEDNODE; -} - -void dnodeGetMnodeEpSetForShell(void *epSetRaw) { - SRpcEpSet *epSet = epSetRaw; - *epSet = tsDMnodeEpSet; -} - static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { if (pMsg->code != TSDB_CODE_SUCCESS) { dError("status rsp is received, error:%s", tstrerror(pMsg->code)); @@ -516,202 +471,24 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { return; } - SDMStatusRsp *pStatusRsp = pMsg->pCont; - SDMMnodeInfos *pMnodes = &pStatusRsp->mnodes; - if (pMnodes->nodeNum <= 0) { - dError("status msg is invalid, num of ips is %d", pMnodes->nodeNum); - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); - return; - } + SStatusRsp *pStatusRsp = pMsg->pCont; + SMnodeInfos *minfos = &pStatusRsp->mnodes; + dnodeUpdateMInfos(minfos); - SDMDnodeCfg *pCfg = &pStatusRsp->dnodeCfg; - pCfg->numOfVnodes = htonl(pCfg->numOfVnodes); + SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg; + pCfg->numOfVnodes = htonl(pCfg->numOfVnodes); pCfg->moduleStatus = htonl(pCfg->moduleStatus); pCfg->dnodeId = htonl(pCfg->dnodeId); - - for (int32_t i = 0; i < pMnodes->nodeNum; ++i) { - SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i]; - pMnodeInfo->nodeId = htonl(pMnodeInfo->nodeId); - } + dnodeUpdateCfg(pCfg); vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes); - // will not set mnode in status msg - // dnodeProcessModuleStatus(pCfg->moduleStatus); - dnodeUpdateDnodeCfg(pCfg); + SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess)); + dnodeUpdateEps(pEps); - dnodeUpdateMnodeInfos(pMnodes); taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); } -static bool dnodeCheckMnodeInfos(SDMMnodeInfos *pMnodes) { - if (pMnodes->nodeNum <= 0 || pMnodes->nodeNum > 3) { - dError("invalid mnode infos, num:%d", pMnodes->nodeNum); - return false; - } - - for (int32_t i = 0; i < pMnodes->nodeNum; ++i) { - SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i]; - if (pMnodeInfo->nodeId <= 0 || strlen(pMnodeInfo->nodeEp) <= 5) { - dError("invalid mnode info:%d, nodeId:%d nodeEp:%s", i, pMnodeInfo->nodeId, pMnodeInfo->nodeEp); - return false; - } - } - - return true; -} - -static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) { - bool mnodesChanged = (memcmp(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos)) != 0); - bool mnodesNotInit = (tsDMnodeInfos.nodeNum == 0); - if (!(mnodesChanged || mnodesNotInit)) return; - - if (!dnodeCheckMnodeInfos(pMnodes)) return; - - memcpy(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos)); - dInfo("mnode infos is changed, nodeNum:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse); - for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - dInfo("mnode index:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp); - } - - tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse; - tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum; - for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]); - } - - dnodeSaveMnodeInfos(); - sdbUpdateAsync(); -} - -static bool dnodeReadMnodeInfos() { - char ipFile[TSDB_FILENAME_LEN*2] = {0}; - - sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir); - FILE *fp = fopen(ipFile, "r"); - if (!fp) { - dDebug("failed to read mnodeEpSet.json, file not exist"); - return false; - } - - bool ret = false; - int maxLen = 2000; - char *content = calloc(1, maxLen + 1); - int len = fread(content, 1, maxLen, fp); - if (len <= 0) { - free(content); - fclose(fp); - dError("failed to read mnodeEpSet.json, content is null"); - return false; - } - - content[len] = 0; - cJSON* root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read mnodeEpSet.json, invalid json format"); - goto PARSE_OVER; - } - - cJSON* inUse = cJSON_GetObjectItem(root, "inUse"); - if (!inUse || inUse->type != cJSON_Number) { - dError("failed to read mnodeEpSet.json, inUse not found"); - goto PARSE_OVER; - } - tsDMnodeInfos.inUse = inUse->valueint; - - cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum"); - if (!nodeNum || nodeNum->type != cJSON_Number) { - dError("failed to read mnodeEpSet.json, nodeNum not found"); - goto PARSE_OVER; - } - tsDMnodeInfos.nodeNum = nodeNum->valueint; - - cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); - if (!nodeInfos || nodeInfos->type != cJSON_Array) { - dError("failed to read mnodeEpSet.json, nodeInfos not found"); - goto PARSE_OVER; - } - - int size = cJSON_GetArraySize(nodeInfos); - if (size != tsDMnodeInfos.nodeNum) { - dError("failed to read mnodeEpSet.json, nodeInfos size not matched"); - goto PARSE_OVER; - } - - for (int i = 0; i < size; ++i) { - cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); - if (nodeInfo == NULL) continue; - - cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); - if (!nodeId || nodeId->type != cJSON_Number) { - dError("failed to read mnodeEpSet.json, nodeId not found"); - goto PARSE_OVER; - } - tsDMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint; - - cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); - if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { - dError("failed to read mnodeEpSet.json, nodeName not found"); - goto PARSE_OVER; - } - strncpy(tsDMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN); - } - - ret = true; - - dInfo("read mnode epSet successed, numOfEps:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse); - for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - dInfo("mnode:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp); - } - -PARSE_OVER: - free(content); - cJSON_Delete(root); - fclose(fp); - return ret; -} - -static void dnodeSaveMnodeInfos() { - char ipFile[TSDB_FILENAME_LEN] = {0}; - sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir); - FILE *fp = fopen(ipFile, "w"); - if (!fp) return; - - int32_t len = 0; - int32_t maxLen = 2000; - char * content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsDMnodeInfos.inUse); - len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsDMnodeInfos.nodeNum); - len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); - for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsDMnodeInfos.nodeInfos[i].nodeId); - len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", tsDMnodeInfos.nodeInfos[i].nodeEp); - if (i < tsDMnodeInfos.nodeNum -1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } - } - len += snprintf(content + len, maxLen - len, "}\n"); - - fwrite(content, 1, len, fp); - fflush(fp); - fclose(fp); - free(content); - - dInfo("save mnode epSet successed"); -} - -char *dnodeGetMnodeMasterEp() { - return tsDMnodeInfos.nodeInfos[tsDMnodeEpSet.inUse].nodeEp; -} - -void* dnodeGetMnodeInfos() { - return &tsDMnodeInfos; -} - static void dnodeSendStatusMsg(void *handle, void *tmrId) { if (tsDnodeTmr == NULL) { dError("dnode timer is already released"); @@ -724,22 +501,21 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { return; } - int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); - SDMStatusMsg *pStatus = rpcMallocCont(contLen); + int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); + SStatusMsg *pStatus = rpcMallocCont(contLen); if (pStatus == NULL) { taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); dError("failed to malloc status message"); return; } - //strcpy(pStatus->dnodeName, tsDnodeName); + dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId); + pStatus->dnodeId = htonl(dnodeGetDnodeId()); pStatus->version = htonl(tsVersion); - pStatus->dnodeId = htonl(tsDnodeCfg.dnodeId); pStatus->lastReboot = htonl(tsRebootTime); pStatus->numOfCores = htons((uint16_t) tsNumOfCores); pStatus->diskAvailable = tsAvailDataDirGB; pStatus->alternativeRole = (uint8_t) tsAlternativeRole; - tstrncpy(pStatus->clusterId, tsDnodeCfg.clusterId, TSDB_CLUSTER_ID_LEN); tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN); // fill cluster cfg parameters @@ -759,7 +535,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); vnodeBuildStatusMsg(pStatus); - contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); + contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); pStatus->openVnodes = htons(pStatus->openVnodes); SRpcMsg rpcMsg = { @@ -769,110 +545,19 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { }; SRpcEpSet epSet; - dnodeGetMnodeEpSetForPeer(&epSet); + dnodeGetEpSetForPeer(&epSet); dnodeSendMsgToDnode(&epSet, &rpcMsg); } -static bool dnodeReadDnodeCfg() { - char dnodeCfgFile[TSDB_FILENAME_LEN*2] = {0}; - - sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir); - - FILE *fp = fopen(dnodeCfgFile, "r"); - if (!fp) { - dDebug("failed to read dnodeCfg.json, file not exist"); - return false; - } - - bool ret = false; - int maxLen = 100; - char *content = calloc(1, maxLen + 1); - int len = fread(content, 1, maxLen, fp); - if (len <= 0) { - free(content); - fclose(fp); - dError("failed to read dnodeCfg.json, content is null"); - return false; - } - - content[len] = 0; - cJSON* root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read dnodeCfg.json, invalid json format"); - goto PARSE_CFG_OVER; - } - - cJSON* dnodeId = cJSON_GetObjectItem(root, "dnodeId"); - if (!dnodeId || dnodeId->type != cJSON_Number) { - dError("failed to read dnodeCfg.json, dnodeId not found"); - goto PARSE_CFG_OVER; - } - tsDnodeCfg.dnodeId = dnodeId->valueint; - - cJSON* clusterId = cJSON_GetObjectItem(root, "clusterId"); - if (!clusterId || clusterId->type != cJSON_String) { - dError("failed to read dnodeCfg.json, clusterId not found"); - goto PARSE_CFG_OVER; - } - tstrncpy(tsDnodeCfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN); - - ret = true; - - dInfo("read numOfVnodes successed, dnodeId:%d", tsDnodeCfg.dnodeId); - -PARSE_CFG_OVER: - free(content); - cJSON_Delete(root); - fclose(fp); - return ret; -} - -static void dnodeSaveDnodeCfg() { - char dnodeCfgFile[TSDB_FILENAME_LEN] = {0}; - sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir); - - FILE *fp = fopen(dnodeCfgFile, "w"); - if (!fp) return; - - int32_t len = 0; - int32_t maxLen = 200; - char * content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", tsDnodeCfg.dnodeId); - len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%s\"\n", tsDnodeCfg.clusterId); - len += snprintf(content + len, maxLen - len, "}\n"); - - fwrite(content, 1, len, fp); - fflush(fp); - fclose(fp); - free(content); - - dInfo("save dnodeId successed"); -} - -void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) { - if (tsDnodeCfg.dnodeId == 0) { - dInfo("dnodeId is set to %d, clusterId is set to %s", pCfg->dnodeId, pCfg->clusterId); - tsDnodeCfg.dnodeId = pCfg->dnodeId; - tstrncpy(tsDnodeCfg.clusterId, pCfg->clusterId, TSDB_CLUSTER_ID_LEN); - dnodeSaveDnodeCfg(); - } -} - -int32_t dnodeGetDnodeId() { - return tsDnodeCfg.dnodeId; -} - void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { SRpcConnInfo connInfo = {0}; rpcGetConnInfo(rpcMsg->handle, &connInfo); SRpcEpSet epSet = {0}; if (forShell) { - dnodeGetMnodeEpSetForShell(&epSet); + dnodeGetEpSetForShell(&epSet); } else { - dnodeGetMnodeEpSetForPeer(&epSet); + dnodeGetEpSetForPeer(&epSet); } dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType], diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index 46376159c6782efde7adbd19c75af83aa4cde397..bd9500ba51226d138fd3fe52f027d144d289681b 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -114,6 +114,7 @@ int32_t dnodeInitModules() { } } + dInfo("dnode modules is initialized"); return 0; } @@ -146,8 +147,8 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) { } } -bool dnodeStartMnode(void *pMnodes) { - SDMMnodeInfos *mnodes = pMnodes; +bool dnodeStartMnode(SMnodeInfos *minfos) { + SMnodeInfos *mnodes = minfos; if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) { dDebug("mnode module is already started, module status:%d", tsModuleStatus); diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 3bc2f7b48b319f3c9e6215a05463ebedc74035fa..afa712a965e8233baaf9d4f5f14abeee8e88cc38 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -19,6 +19,7 @@ * to dnode. All theses messages are handled from here */ +#define _DEFAULT_SOURCE #include "os.h" #include "taosmsg.h" #include "tglobal.h" @@ -28,20 +29,20 @@ #include "dnodeMgmt.h" #include "dnodeVWrite.h" #include "dnodeMPeer.h" +#include "dnodeMInfos.h" -extern void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet); static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *); static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg); static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet); -static void *tsDnodeServerRpc = NULL; -static void *tsDnodeClientRpc = NULL; +static void *tsServerRpc = NULL; +static void *tsClientRpc = NULL; int32_t dnodeInitServer() { - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVnodeWriteQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = dnodeDispatchToVnodeWriteQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = dnodeDispatchToVnodeWriteQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVnodeWriteQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVWriteQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = dnodeDispatchToVWriteQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = dnodeDispatchToVWriteQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVWriteQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeDispatchToMgmtQueue; @@ -50,11 +51,11 @@ int32_t dnodeInitServer() { dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMnodePeerQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMnodePeerQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = dnodeDispatchToMnodePeerQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = dnodeDispatchToMnodePeerQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = dnodeDispatchToMnodePeerQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = dnodeDispatchToMPeerQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = dnodeDispatchToMPeerQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = dnodeDispatchToMPeerQueue; SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); @@ -66,20 +67,20 @@ int32_t dnodeInitServer() { rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; - tsDnodeServerRpc = rpcOpen(&rpcInit); - if (tsDnodeServerRpc == NULL) { + tsServerRpc = rpcOpen(&rpcInit); + if (tsServerRpc == NULL) { dError("failed to init inter-dnodes RPC server"); return -1; } - dInfo("inter-dnodes RPC server is opened"); + dInfo("dnode inter-dnodes RPC server is initialized"); return 0; } void dnodeCleanupServer() { - if (tsDnodeServerRpc) { - rpcClose(tsDnodeServerRpc); - tsDnodeServerRpc = NULL; + if (tsServerRpc) { + rpcClose(tsServerRpc); + tsServerRpc = NULL; dInfo("inter-dnodes RPC server is closed"); } } @@ -93,7 +94,7 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { if (pMsg->pCont == NULL) return; - if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) { + if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) { rspMsg.code = TSDB_CODE_APP_NOT_READY; rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); @@ -131,27 +132,27 @@ int32_t dnodeInitClient() { rpcInit.ckey = "key"; rpcInit.secret = secret; - tsDnodeClientRpc = rpcOpen(&rpcInit); - if (tsDnodeClientRpc == NULL) { + tsClientRpc = rpcOpen(&rpcInit); + if (tsClientRpc == NULL) { dError("failed to init mnode rpc client"); return -1; } - dInfo("inter-dnodes rpc client is opened"); + dInfo("dnode inter-dnodes rpc client is initialized"); return 0; } void dnodeCleanupClient() { - if (tsDnodeClientRpc) { - rpcClose(tsDnodeClientRpc); - tsDnodeClientRpc = NULL; - dInfo("inter-dnodes rpc client is closed"); + if (tsClientRpc) { + rpcClose(tsClientRpc); + tsClientRpc = NULL; + dInfo("dnode inter-dnodes rpc client is closed"); } } static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) { - dnodeUpdateMnodeEpSetForPeer(pEpSet); + dnodeUpdateEpSetForPeer(pEpSet); } if (dnodeProcessRspMsgFp[pMsg->msgType]) { @@ -168,15 +169,15 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) { } void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) { - rpcSendRequest(tsDnodeClientRpc, epSet, rpcMsg); + rpcSendRequest(tsClientRpc, epSet, rpcMsg); } void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { SRpcEpSet epSet = {0}; - dnodeGetMnodeEpSetForPeer(&epSet); - rpcSendRecv(tsDnodeClientRpc, &epSet, rpcMsg, rpcRsp); + dnodeGetEpSetForPeer(&epSet); + rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp); } void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) { - rpcSendRecv(tsDnodeClientRpc, epSet, rpcMsg, rpcRsp); + rpcSendRecv(tsClientRpc, epSet, rpcMsg, rpcRsp); } \ No newline at end of file diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 4c6c2100e048e9aaf23f4155d3c54604104a8f9b..89f657f78986b8e57c0b5e1dedb841c451db00ba 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -33,46 +33,46 @@ static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *); static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey); -static void * tsDnodeShellRpc = NULL; -static int32_t tsDnodeQueryReqNum = 0; -static int32_t tsDnodeSubmitReqNum = 0; +static void * tsShellRpc = NULL; +static int32_t tsQueryReqNum = 0; +static int32_t tsSubmitReqNum = 0; int32_t dnodeInitShell() { - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVnodeWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue; // the following message shall be treated as mnode write - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMnodeWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMWriteQueue; // the following message shall be treated as mnode query - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMnodeReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue; int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore; numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0); @@ -91,24 +91,24 @@ int32_t dnodeInitShell() { rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = dnodeRetrieveUserAuthInfo; - tsDnodeShellRpc = rpcOpen(&rpcInit); - if (tsDnodeShellRpc == NULL) { + tsShellRpc = rpcOpen(&rpcInit); + if (tsShellRpc == NULL) { dError("failed to init shell rpc server"); return -1; } - dInfo("shell rpc server is opened"); + dInfo("dnode shell rpc server is initialized"); return 0; } void dnodeCleanupShell() { - if (tsDnodeShellRpc) { - rpcClose(tsDnodeShellRpc); - tsDnodeShellRpc = NULL; + if (tsShellRpc) { + rpcClose(tsShellRpc); + tsShellRpc = NULL; } } -void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { +static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { SRpcMsg rpcMsg = { .handle = pMsg->handle, .pCont = NULL, @@ -117,7 +117,7 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { if (pMsg->pCont == NULL) return; - if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) { + if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) { dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); rpcMsg.code = TSDB_CODE_APP_NOT_READY; rpcSendResponse(&rpcMsg); @@ -126,9 +126,9 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { } if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) { - atomic_fetch_add_32(&tsDnodeQueryReqNum, 1); + atomic_fetch_add_32(&tsQueryReqNum, 1); } else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { - atomic_fetch_add_32(&tsDnodeSubmitReqNum, 1); + atomic_fetch_add_32(&tsSubmitReqNum, 1); } else {} if ( dnodeProcessShellMsgFp[pMsg->msgType] ) { @@ -146,12 +146,12 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey); if (code != TSDB_CODE_APP_NOT_READY) return code; - SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg)); + SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg)); tstrncpy(pMsg->user, user, sizeof(pMsg->user)); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pMsg; - rpcMsg.contLen = sizeof(SDMAuthMsg); + rpcMsg.contLen = sizeof(SAuthMsg); rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH; dDebug("user:%s, send auth msg to mnodes", user); @@ -161,7 +161,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char if (rpcRsp.code != 0) { dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code)); } else { - SDMAuthRsp *pRsp = rpcRsp.pCont; + SAuthRsp *pRsp = rpcRsp.pCont; dDebug("user:%s, auth msg received from mnodes", user); memcpy(secret, pRsp->secret, TSDB_KEY_LEN); memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN); @@ -176,8 +176,8 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) { dDebug("vgId:%d, tid:%d send config table msg to mnode", vgId, tid); - int32_t contLen = sizeof(SDMConfigTableMsg); - SDMConfigTableMsg *pMsg = rpcMallocCont(contLen); + int32_t contLen = sizeof(SConfigTableMsg); + SConfigTableMsg *pMsg = rpcMallocCont(contLen); pMsg->dnodeId = htonl(dnodeGetDnodeId()); pMsg->vgId = htonl(vgId); @@ -211,12 +211,12 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) { } } -SDnodeStatisInfo dnodeGetStatisInfo() { - SDnodeStatisInfo info = {0}; - if (dnodeGetRunStatus() == TSDB_DNODE_RUN_STATUS_RUNING) { +SStatisInfo dnodeGetStatisInfo() { + SStatisInfo info = {0}; + if (dnodeGetRunStatus() == TSDB_RUN_STATUS_RUNING) { info.httpReqNum = httpGetReqCount(); - info.queryReqNum = atomic_exchange_32(&tsDnodeQueryReqNum, 0); - info.submitReqNum = atomic_exchange_32(&tsDnodeSubmitReqNum, 0); + info.queryReqNum = atomic_exchange_32(&tsQueryReqNum, 0); + info.submitReqNum = atomic_exchange_32(&tsSubmitReqNum, 0); } return info; diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index 4fdc0b8a73b58efa801f33ea99ffb7d123a72dcf..e973f9901f19b7aa6f4d4958d88e01ba258a82bb 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -268,7 +268,7 @@ static void dnodeGetEmail(char* filepath) { return; } - if (taosTRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) { + if (taosRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) { dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); } @@ -299,6 +299,7 @@ int32_t dnodeInitTelemetry() { dTrace("failed to create telemetry thread, reason:%s", strerror(errno)); } + dInfo("dnode telemetry is initialized"); return 0; } diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index e61158ef30dddd6037af36c61be1fd522f8af10b..34df11adcce72a6a21718b81c1895c30c66bea1c 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -17,225 +17,184 @@ #include "os.h" #include "taoserror.h" #include "taosmsg.h" -#include "tutil.h" -#include "tqueue.h" -#include "twal.h" #include "tglobal.h" -#include "dnodeInt.h" -#include "dnodeMgmt.h" -#include "dnodeVRead.h" +#include "tqueue.h" #include "vnode.h" +#include "dnodeInt.h" typedef struct { - pthread_t thread; // thread - int32_t workerId; // worker ID -} SReadWorker; + pthread_t thread; // thread + int32_t workerId; // worker ID +} SVReadWorker; typedef struct { - int32_t max; // max number of workers - int32_t min; // min number of workers - int32_t num; // current number of workers - SReadWorker *readWorker; + int32_t max; // max number of workers + int32_t min; // min number of workers + int32_t num; // current number of workers + SVReadWorker * worker; pthread_mutex_t mutex; -} SReadWorkerPool; +} SVReadWorkerPool; static void *dnodeProcessReadQueue(void *param); -static void dnodeHandleIdleReadWorker(SReadWorker *); // module global variable -static SReadWorkerPool readPool; -static taos_qset readQset; +static SVReadWorkerPool tsVReadWP; +static taos_qset tsVReadQset; -int32_t dnodeInitVnodeRead() { - readQset = taosOpenQset(); +int32_t dnodeInitVRead() { + tsVReadQset = taosOpenQset(); - readPool.min = tsNumOfCores; - readPool.max = tsNumOfCores * tsNumOfThreadsPerCore; - if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min; - readPool.readWorker = (SReadWorker *)calloc(sizeof(SReadWorker), readPool.max); - pthread_mutex_init(&readPool.mutex, NULL); + tsVReadWP.min = tsNumOfCores; + tsVReadWP.max = tsNumOfCores * tsNumOfThreadsPerCore; + if (tsVReadWP.max <= tsVReadWP.min * 2) tsVReadWP.max = 2 * tsVReadWP.min; + tsVReadWP.worker = (SVReadWorker *)calloc(sizeof(SVReadWorker), tsVReadWP.max); + pthread_mutex_init(&tsVReadWP.mutex, NULL); - if (readPool.readWorker == NULL) return -1; - for (int i = 0; i < readPool.max; ++i) { - SReadWorker *pWorker = readPool.readWorker + i; + if (tsVReadWP.worker == NULL) return -1; + for (int i = 0; i < tsVReadWP.max; ++i) { + SVReadWorker *pWorker = tsVReadWP.worker + i; pWorker->workerId = i; } - dInfo("dnode read is opened, min worker:%d max worker:%d", readPool.min, readPool.max); + dInfo("dnode vread is initialized, min worker:%d max worker:%d", tsVReadWP.min, tsVReadWP.max); return 0; } -void dnodeCleanupVnodeRead() { - for (int i = 0; i < readPool.max; ++i) { - SReadWorker *pWorker = readPool.readWorker + i; +void dnodeCleanupVRead() { + for (int i = 0; i < tsVReadWP.max; ++i) { + SVReadWorker *pWorker = tsVReadWP.worker + i; if (pWorker->thread) { - taosQsetThreadResume(readQset); + taosQsetThreadResume(tsVReadQset); } } - for (int i = 0; i < readPool.max; ++i) { - SReadWorker *pWorker = readPool.readWorker + i; + for (int i = 0; i < tsVReadWP.max; ++i) { + SVReadWorker *pWorker = tsVReadWP.worker + i; if (pWorker->thread) { pthread_join(pWorker->thread, NULL); } } - free(readPool.readWorker); - taosCloseQset(readQset); - pthread_mutex_destroy(&readPool.mutex); + free(tsVReadWP.worker); + taosCloseQset(tsVReadQset); + pthread_mutex_destroy(&tsVReadWP.mutex); - dInfo("dnode read is closed"); + dInfo("dnode vread is closed"); } -void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) { - int32_t queuedMsgNum = 0; - int32_t leftLen = pMsg->contLen; - char *pCont = (char *) pMsg->pCont; +void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { + int32_t queuedMsgNum = 0; + int32_t leftLen = pMsg->contLen; + char * pCont = (char *)pMsg->pCont; while (leftLen > 0) { - SMsgHead *pHead = (SMsgHead *) pCont; - pHead->vgId = htonl(pHead->vgId); + SMsgHead *pHead = (SMsgHead *)pCont; + pHead->vgId = htonl(pHead->vgId); pHead->contLen = htonl(pHead->contLen); - taos_queue queue = vnodeAcquireRqueue(pHead->vgId); - - if (queue == NULL) { - leftLen -= pHead->contLen; - pCont -= pHead->contLen; - continue; + void *pVnode = vnodeAcquire(pHead->vgId); + if (pVnode != NULL) { + int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); + if (code == TSDB_CODE_SUCCESS) queuedMsgNum++; + vnodeRelease(pVnode); } - // put message into queue - SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); - pRead->rpcMsg = *pMsg; - pRead->pCont = pCont; - pRead->contLen = pHead->contLen; - - // next vnode leftLen -= pHead->contLen; pCont -= pHead->contLen; - queuedMsgNum++; - - taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead); } if (queuedMsgNum == 0) { - SRpcMsg rpcRsp = { - .handle = pMsg->handle, - .pCont = NULL, - .contLen = 0, - .code = TSDB_CODE_VND_INVALID_VGROUP_ID, - .msgType = 0 - }; + SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; rpcSendResponse(&rpcRsp); - rpcFreeCont(pMsg->pCont); } + + rpcFreeCont(pMsg->pCont); } -void *dnodeAllocateVnodeRqueue(void *pVnode) { - pthread_mutex_lock(&readPool.mutex); +void *dnodeAllocVReadQueue(void *pVnode) { + pthread_mutex_lock(&tsVReadWP.mutex); taos_queue queue = taosOpenQueue(); if (queue == NULL) { - pthread_mutex_unlock(&readPool.mutex); + pthread_mutex_unlock(&tsVReadWP.mutex); return NULL; } - taosAddIntoQset(readQset, queue, pVnode); + taosAddIntoQset(tsVReadQset, queue, pVnode); // spawn a thread to process queue - if (readPool.num < readPool.max) { + if (tsVReadWP.num < tsVReadWP.max) { do { - SReadWorker *pWorker = readPool.readWorker + readPool.num; + SVReadWorker *pWorker = tsVReadWP.worker + tsVReadWP.num; pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessReadQueue, pWorker) != 0) { - dError("failed to create thread to process read queue, reason:%s", strerror(errno)); + dError("failed to create thread to process vread vqueue since %s", strerror(errno)); } pthread_attr_destroy(&thAttr); - readPool.num++; - dDebug("read worker:%d is launched, total:%d", pWorker->workerId, readPool.num); - } while (readPool.num < readPool.min); + tsVReadWP.num++; + dDebug("dnode vread worker:%d is launched, total:%d", pWorker->workerId, tsVReadWP.num); + } while (tsVReadWP.num < tsVReadWP.min); } - pthread_mutex_unlock(&readPool.mutex); - dDebug("pVnode:%p, read queue:%p is allocated", pVnode, queue); + pthread_mutex_unlock(&tsVReadWP.mutex); + dDebug("pVnode:%p, dnode vread queue:%p is allocated", pVnode, queue); return queue; } -void dnodeFreeVnodeRqueue(void *rqueue) { +void dnodeFreeVReadQueue(void *rqueue) { taosCloseQueue(rqueue); - - // dynamically adjust the number of threads } -void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) { +void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) { SRpcMsg rpcRsp = { - .handle = pRead->rpcMsg.handle, + .handle = pRead->rpcHandle, .pCont = pRead->rspRet.rsp, .contLen = pRead->rspRet.len, .code = code, }; rpcSendResponse(&rpcRsp); - rpcFreeCont(pRead->rpcMsg.pCont); vnodeRelease(pVnode); } -void dnodeDispatchNonRspMsg(void *pVnode, SReadMsg *pRead, int32_t code) { - rpcFreeCont(pRead->rpcMsg.pCont); +void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) { vnodeRelease(pVnode); } static void *dnodeProcessReadQueue(void *param) { - SReadMsg *pReadMsg; - int type; - void *pVnode; + SVReadMsg *pRead; + int32_t qtype; + void * pVnode; while (1) { - if (taosReadQitemFromQset(readQset, &type, (void **)&pReadMsg, &pVnode) == 0) { - dDebug("qset:%p dnode read got no message from qset, exiting", readQset); + if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pRead, &pVnode) == 0) { + dDebug("qset:%p dnode vread got no message from qset, exiting", tsVReadQset); break; } - dDebug("%p, msg:%s will be processed in vread queue, qtype:%d, msg:%p", pReadMsg->rpcMsg.ahandle, - taosMsg[pReadMsg->rpcMsg.msgType], type, pReadMsg); + dDebug("%p, msg:%p:%s will be processed in vread queue, qtype:%d", pRead->rpcAhandle, pRead, + taosMsg[pRead->msgType], qtype); - int32_t code = vnodeProcessRead(pVnode, pReadMsg); + int32_t code = vnodeProcessRead(pVnode, pRead); - if (type == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { - dnodeSendRpcReadRsp(pVnode, pReadMsg, code); + if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { + dnodeSendRpcVReadRsp(pVnode, pRead, code); } else { if (code == TSDB_CODE_QRY_HAS_RSP) { - dnodeSendRpcReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code); - } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client - assert(pReadMsg->rpcMsg.handle == NULL || (pReadMsg->rpcMsg.handle != NULL && pReadMsg->rpcMsg.msgType == 5)); - dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); + dnodeSendRpcVReadRsp(pVnode, pRead, pRead->code); + } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client + assert(pRead->rpcHandle == NULL || (pRead->rpcHandle != NULL && pRead->msgType == 5)); + dnodeDispatchNonRspMsg(pVnode, pRead, code); } } - taosFreeQitem(pReadMsg); + taosFreeQitem(pRead); } return NULL; } - - -UNUSED_FUNC -static void dnodeHandleIdleReadWorker(SReadWorker *pWorker) { - int32_t num = taosGetQueueNumber(readQset); - - if (num == 0 || (num <= readPool.min && readPool.num > readPool.min)) { - readPool.num--; - dDebug("read worker:%d is released, total:%d", pWorker->workerId, readPool.num); - pthread_exit(NULL); - } else { - usleep(30000); - sched_yield(); - } -} - diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index f2740bf6b810197283926a602db4fc423067e231..e0345eb1f6f72c28a99b99546f2d0c2a96b8ed37 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -15,74 +15,55 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosmsg.h" #include "taoserror.h" -#include "tutil.h" +#include "taosmsg.h" +#include "tglobal.h" #include "tqueue.h" -#include "trpc.h" -#include "tsdb.h" #include "twal.h" -#include "tdataformat.h" -#include "tglobal.h" -#include "tsync.h" #include "vnode.h" #include "dnodeInt.h" -#include "syncInt.h" -#include "dnodeVWrite.h" -#include "dnodeMgmt.h" - -typedef struct { - taos_qall qall; - taos_qset qset; // queue set - pthread_t thread; // thread - int32_t workerId; // worker ID -} SWriteWorker; typedef struct { - SRspRet rspRet; - int32_t processedCount; - int32_t code; - void *pCont; - int32_t contLen; - SRpcMsg rpcMsg; -} SWriteMsg; + taos_qall qall; + taos_qset qset; // queue set + int32_t workerId; // worker ID + pthread_t thread; // thread +} SVWriteWorker; typedef struct { - int32_t max; // max number of workers - int32_t nextId; // from 0 to max-1, cyclic - SWriteWorker *writeWorker; + int32_t max; // max number of workers + int32_t nextId; // from 0 to max-1, cyclic + SVWriteWorker * worker; pthread_mutex_t mutex; -} SWriteWorkerPool; +} SVWriteWorkerPool; -static void *dnodeProcessWriteQueue(void *param); -static void dnodeHandleIdleWorker(SWriteWorker *pWorker); +static SVWriteWorkerPool tsVWriteWP; +static void *dnodeProcessVWriteQueue(void *param); -SWriteWorkerPool wWorkerPool; +int32_t dnodeInitVWrite() { + tsVWriteWP.max = tsNumOfCores; + tsVWriteWP.worker = (SVWriteWorker *)tcalloc(sizeof(SVWriteWorker), tsVWriteWP.max); + if (tsVWriteWP.worker == NULL) return -1; + pthread_mutex_init(&tsVWriteWP.mutex, NULL); -int32_t dnodeInitVnodeWrite() { - wWorkerPool.max = tsNumOfCores; - wWorkerPool.writeWorker = (SWriteWorker *)calloc(sizeof(SWriteWorker), wWorkerPool.max); - if (wWorkerPool.writeWorker == NULL) return -1; - pthread_mutex_init(&wWorkerPool.mutex, NULL); - - for (int32_t i = 0; i < wWorkerPool.max; ++i) { - wWorkerPool.writeWorker[i].workerId = i; + for (int32_t i = 0; i < tsVWriteWP.max; ++i) { + tsVWriteWP.worker[i].workerId = i; } - dInfo("dnode write is opened, max worker %d", wWorkerPool.max); + dInfo("dnode vwrite is initialized, max worker %d", tsVWriteWP.max); return 0; } -void dnodeCleanupVnodeWrite() { - for (int32_t i = 0; i < wWorkerPool.max; ++i) { - SWriteWorker *pWorker = wWorkerPool.writeWorker + i; +void dnodeCleanupVWrite() { + for (int32_t i = 0; i < tsVWriteWP.max; ++i) { + SVWriteWorker *pWorker = tsVWriteWP.worker + i; if (pWorker->thread) { taosQsetThreadResume(pWorker->qset); } } - for (int32_t i = 0; i < wWorkerPool.max; ++i) { - SWriteWorker *pWorker = wWorkerPool.writeWorker + i; + for (int32_t i = 0; i < tsVWriteWP.max; ++i) { + SVWriteWorker *pWorker = tsVWriteWP.worker + i; if (pWorker->thread) { pthread_join(pWorker->thread, NULL); taosFreeQall(pWorker->qall); @@ -90,52 +71,51 @@ void dnodeCleanupVnodeWrite() { } } - pthread_mutex_destroy(&wWorkerPool.mutex); - free(wWorkerPool.writeWorker); - dInfo("dnode write is closed"); + pthread_mutex_destroy(&tsVWriteWP.mutex); + tfree(tsVWriteWP.worker); + dInfo("dnode vwrite is closed"); } -void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) { - char *pCont = (char *)pMsg->pCont; +void dnodeDispatchToVWriteQueue(SRpcMsg *pRpcMsg) { + int32_t code; + char *pCont = pRpcMsg->pCont; - if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { + if (pRpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { SMsgDesc *pDesc = (SMsgDesc *)pCont; pDesc->numOfVnodes = htonl(pDesc->numOfVnodes); pCont += sizeof(SMsgDesc); } - SMsgHead *pHead = (SMsgHead *) pCont; - pHead->vgId = htonl(pHead->vgId); - pHead->contLen = htonl(pHead->contLen); - - taos_queue queue = vnodeAcquireWqueue(pHead->vgId); - if (queue) { - // put message into queue - SWriteMsg *pWrite = (SWriteMsg *)taosAllocateQitem(sizeof(SWriteMsg)); - pWrite->rpcMsg = *pMsg; - pWrite->pCont = pCont; - pWrite->contLen = pHead->contLen; + SMsgHead *pMsg = (SMsgHead *)pCont; + pMsg->vgId = htonl(pMsg->vgId); + pMsg->contLen = htonl(pMsg->contLen); - taosWriteQitem(queue, TAOS_QTYPE_RPC, pWrite); + void *pVnode = vnodeAcquire(pMsg->vgId); + if (pVnode == NULL) { + code = TSDB_CODE_VND_INVALID_VGROUP_ID; } else { - SRpcMsg rpcRsp = { - .handle = pMsg->handle, - .pCont = NULL, - .contLen = 0, - .code = TSDB_CODE_VND_INVALID_VGROUP_ID, - .msgType = 0 - }; + SWalHead *pHead = (SWalHead *)(pCont - sizeof(SWalHead)); + pHead->msgType = pRpcMsg->msgType; + pHead->version = 0; + pHead->len = pMsg->contLen; + code = vnodeWriteToWQueue(pVnode, pHead, TAOS_QTYPE_RPC, pRpcMsg); + } + + if (code != TSDB_CODE_SUCCESS) { + SRpcMsg rpcRsp = {.handle = pRpcMsg->handle, .code = code}; rpcSendResponse(&rpcRsp); - rpcFreeCont(pMsg->pCont); } + + vnodeRelease(pVnode); + rpcFreeCont(pRpcMsg->pCont); } -void *dnodeAllocateVnodeWqueue(void *pVnode) { - pthread_mutex_lock(&wWorkerPool.mutex); - SWriteWorker *pWorker = wWorkerPool.writeWorker + wWorkerPool.nextId; - void *queue = taosOpenQueue(); +void *dnodeAllocVWriteQueue(void *pVnode) { + pthread_mutex_lock(&tsVWriteWP.mutex); + SVWriteWorker *pWorker = tsVWriteWP.worker + tsVWriteWP.nextId; + taos_queue *queue = taosOpenQueue(); if (queue == NULL) { - pthread_mutex_unlock(&wWorkerPool.mutex); + pthread_mutex_unlock(&tsVWriteWP.mutex); return NULL; } @@ -143,7 +123,7 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) { pWorker->qset = taosOpenQset(); if (pWorker->qset == NULL) { taosCloseQueue(queue); - pthread_mutex_unlock(&wWorkerPool.mutex); + pthread_mutex_unlock(&tsVWriteWP.mutex); return NULL; } @@ -152,45 +132,43 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) { if (pWorker->qall == NULL) { taosCloseQset(pWorker->qset); taosCloseQueue(queue); - pthread_mutex_unlock(&wWorkerPool.mutex); + pthread_mutex_unlock(&tsVWriteWP.mutex); return NULL; } pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessWriteQueue, pWorker) != 0) { - dError("failed to create thread to process read queue, reason:%s", strerror(errno)); + if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessVWriteQueue, pWorker) != 0) { + dError("failed to create thread to process vwrite queue since %s", strerror(errno)); taosFreeQall(pWorker->qall); taosCloseQset(pWorker->qset); taosCloseQueue(queue); queue = NULL; } else { - dDebug("write worker:%d is launched", pWorker->workerId); - wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max; + dDebug("dnode vwrite worker:%d is launched", pWorker->workerId); + tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max; } pthread_attr_destroy(&thAttr); } else { taosAddIntoQset(pWorker->qset, queue, pVnode); - wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max; + tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max; } - pthread_mutex_unlock(&wWorkerPool.mutex); - dDebug("pVnode:%p, write queue:%p is allocated", pVnode, queue); + pthread_mutex_unlock(&tsVWriteWP.mutex); + dDebug("pVnode:%p, dnode vwrite queue:%p is allocated", pVnode, queue); return queue; } -void dnodeFreeVnodeWqueue(void *wqueue) { +void dnodeFreeVWriteQueue(void *wqueue) { taosCloseQueue(wqueue); - - // dynamically adjust the number of threads } -void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) { - SWriteMsg *pWrite = (SWriteMsg *)param; - if (pWrite == NULL) return; +void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code) { + if (param == NULL) return; + SVWriteMsg *pWrite = param; if (code < 0) pWrite->code = code; int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1); @@ -198,86 +176,61 @@ void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) { if (count <= 1) return; SRpcMsg rpcRsp = { - .handle = pWrite->rpcMsg.handle, + .handle = pWrite->rpcHandle, .pCont = pWrite->rspRet.rsp, .contLen = pWrite->rspRet.len, .code = pWrite->code, }; rpcSendResponse(&rpcRsp); - rpcFreeCont(pWrite->rpcMsg.pCont); taosFreeQitem(pWrite); vnodeRelease(pVnode); } -static void *dnodeProcessWriteQueue(void *param) { - SWriteWorker *pWorker = (SWriteWorker *)param; - SWriteMsg * pWrite; - SWalHead * pHead; - int32_t numOfMsgs; - int type; - void * pVnode, *item; - SRspRet * pRspRet; +static void *dnodeProcessVWriteQueue(void *param) { + SVWriteWorker *pWorker = param; + SVWriteMsg * pWrite; + void * pVnode; + int32_t numOfMsgs; + int32_t qtype; - dDebug("write worker:%d is running", pWorker->workerId); + dDebug("dnode vwrite worker:%d is running", pWorker->workerId); while (1) { numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode); if (numOfMsgs == 0) { - dDebug("qset:%p, dnode write got no message from qset, exiting", pWorker->qset); + dDebug("qset:%p, dnode vwrite got no message from qset, exiting", pWorker->qset); break; } + bool forceFsync = false; for (int32_t i = 0; i < numOfMsgs; ++i) { - pWrite = NULL; - pRspRet = NULL; - taosGetQitem(pWorker->qall, &type, &item); - if (type == TAOS_QTYPE_RPC) { - pWrite = (SWriteMsg *)item; - pRspRet = &pWrite->rspRet; - pHead = (SWalHead *)(pWrite->pCont - sizeof(SWalHead)); - pHead->msgType = pWrite->rpcMsg.msgType; - pHead->version = 0; - pHead->len = pWrite->contLen; - dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle, - taosMsg[pWrite->rpcMsg.msgType]); - } else if (type == TAOS_QTYPE_CQ) { - pHead = (SWalHead *)((char*)item + sizeof(SSyncHead)); - dTrace("%p, CQ wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType], - pHead->version); - } else { - pHead = (SWalHead *)item; - dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType], - pHead->version); - } + taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite); + dTrace("%p, msg:%p:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite->rpcAhandle, pWrite, + taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version); - int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet); - dTrace("%p, msg:%s is processed in vwrite queue, version:%" PRIu64 ", result:%s", pHead, taosMsg[pHead->msgType], - pHead->version, tstrerror(code)); + pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet); + if (pWrite->code <= 0) pWrite->processedCount = 1; + if (pWrite->pHead->msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true; - if (pWrite) { - pWrite->rpcMsg.code = code; - if (code <= 0) pWrite->processedCount = 1; - } + dTrace("msg:%p is processed in vwrite queue, result:%s", pWrite, tstrerror(pWrite->code)); } - walFsync(vnodeGetWal(pVnode)); + walFsync(vnodeGetWal(pVnode), forceFsync); // browse all items, and process them one by one taosResetQitems(pWorker->qall); for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(pWorker->qall, &type, &item); - if (type == TAOS_QTYPE_RPC) { - pWrite = (SWriteMsg *)item; - dnodeSendRpcVnodeWriteRsp(pVnode, item, pWrite->rpcMsg.code); - } else if (type == TAOS_QTYPE_FWD) { - pHead = (SWalHead *)item; - vnodeConfirmForward(pVnode, pHead->version, 0); - taosFreeQitem(item); + taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite); + if (qtype == TAOS_QTYPE_RPC) { + dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code); + } else if (qtype == TAOS_QTYPE_FWD) { + vnodeConfirmForward(pVnode, pWrite->pHead->version, 0); + taosFreeQitem(pWrite); vnodeRelease(pVnode); } else { - taosFreeQitem(item); + taosFreeQitem(pWrite); vnodeRelease(pVnode); } } @@ -285,19 +238,3 @@ static void *dnodeProcessWriteQueue(void *param) { return NULL; } - -UNUSED_FUNC -static void dnodeHandleIdleWorker(SWriteWorker *pWorker) { - int32_t num = taosGetQueueNumber(pWorker->qset); - - if (num > 0) { - usleep(30000); - sched_yield(); - } else { - taosFreeQall(pWorker->qall); - taosCloseQset(pWorker->qset); - pWorker->qset = NULL; - dDebug("write worker:%d is released", pWorker->workerId); - pthread_exit(NULL); - } -} diff --git a/src/inc/dnode.h b/src/inc/dnode.h index e84545be1753f3fedb4ee78acf397c91d824ad8b..9454b97e388e6d51b52e7fd6c1fe586b1e5bdbbd 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -21,29 +21,31 @@ extern "C" { #endif #include "trpc.h" +#include "taosmsg.h" typedef struct { int32_t queryReqNum; int32_t submitReqNum; int32_t httpReqNum; -} SDnodeStatisInfo; +} SStatisInfo; typedef enum { - TSDB_DNODE_RUN_STATUS_INITIALIZE, - TSDB_DNODE_RUN_STATUS_RUNING, - TSDB_DNODE_RUN_STATUS_STOPPED -} SDnodeRunStatus; + TSDB_RUN_STATUS_INITIALIZE, + TSDB_RUN_STATUS_RUNING, + TSDB_RUN_STATUS_STOPPED +} SRunStatus; -SDnodeRunStatus dnodeGetRunStatus(); -SDnodeStatisInfo dnodeGetStatisInfo(); +SRunStatus dnodeGetRunStatus(); +SStatisInfo dnodeGetStatisInfo(); bool dnodeIsFirstDeploy(); -char * dnodeGetMnodeMasterEp(); -void dnodeGetMnodeEpSetForPeer(void *epSet); -void dnodeGetMnodeEpSetForShell(void *epSet); -void * dnodeGetMnodeInfos(); +bool dnodeIsMasterEp(char *ep); +void dnodeGetEpSetForPeer(SRpcEpSet *epSet); +void dnodeGetEpSetForShell(SRpcEpSet *epSet); int32_t dnodeGetDnodeId(); -bool dnodeStartMnode(void *pModes); +void dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); +bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr); +bool dnodeStartMnode(SMnodeInfos *minfos); void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); @@ -51,21 +53,21 @@ void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp); void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet); void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid); -void *dnodeAllocateVnodeWqueue(void *pVnode); -void dnodeFreeVnodeWqueue(void *queue); -void *dnodeAllocateVnodeRqueue(void *pVnode); -void dnodeFreeVnodeRqueue(void *rqueue); -void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code); +void *dnodeAllocVWriteQueue(void *pVnode); +void dnodeFreeVWriteQueue(void *wqueue); +void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code); +void *dnodeAllocVReadQueue(void *pVnode); +void dnodeFreeVReadQueue(void *rqueue); -int32_t dnodeAllocateMnodePqueue(); -void dnodeFreeMnodePqueue(); -int32_t dnodeAllocateMnodeRqueue(); -void dnodeFreeMnodeRqueue(); -int32_t dnodeAllocateMnodeWqueue(); -void dnodeFreeMnodeWqueue(); -void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code); -void dnodeReprocessMnodeWriteMsg(void *pMsg); -void dnodeDelayReprocessMnodeWriteMsg(void *pMsg); +int32_t dnodeAllocateMPeerQueue(); +void dnodeFreeMPeerQueue(); +int32_t dnodeAllocMReadQueue(); +void dnodeFreeMReadQueue(); +int32_t dnodeAllocMWritequeue(); +void dnodeFreeMWritequeue(); +void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code); +void dnodeReprocessMWriteMsg(void *pMsg); +void dnodeDelayReprocessMWriteMsg(void *pMsg); void dnodeSendStatusMsgToMnode(); diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 5bef7402e30a0cc9f1964cc2430e0b1f5141590c..bdc30b0c46ced0961715bd48623fdb9e52fb440e 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -35,24 +35,26 @@ typedef struct { } SMnodeRsp; typedef struct SMnodeMsg { - SRpcMsg rpcMsg; + struct SAcctObj * pAcct; + struct SDnodeObj *pDnode; + struct SUserObj * pUser; + struct SDbObj * pDb; + struct SVgObj * pVgroup; + struct STableObj *pTable; + struct SSTableObj*pSTable; SMnodeRsp rpcRsp; int8_t received; int8_t successed; int8_t expected; int8_t retry; + int32_t incomingTs; int32_t code; void * pObj; - struct SAcctObj * pAcct; - struct SDnodeObj *pDnode; - struct SUserObj * pUser; - struct SDbObj * pDb; - struct SVgObj * pVgroup; - struct STableObj *pTable; - struct SSuperTableObj *pSTable; + SRpcMsg rpcMsg; + char pCont[]; } SMnodeMsg; -void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg); +void * mnodeCreateMsg(SRpcMsg *pRpcMsg); int32_t mnodeInitMsg(SMnodeMsg *pMsg); void mnodeCleanupMsg(SMnodeMsg *pMsg); diff --git a/src/inc/query.h b/src/inc/query.h index 0c18f85dc31bae5e77bae7228d5390a8d32df07a..5e1de77889cc469566cc94b729c55622e5462bd6 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -78,7 +78,6 @@ int32_t qKillQuery(qinfo_t qinfo); int32_t qQueryCompleted(qinfo_t qinfo); - /** * destroy query info structure * @param qHandle diff --git a/src/inc/taos.h b/src/inc/taos.h index 315313734753de73bf477b1f67783a45c38c87c9..2c6454ced1911de603f120c79af5237c2f064c48 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -64,7 +64,7 @@ typedef struct taosField { #endif DLL_EXPORT void taos_init(); -DLL_EXPORT void taos_cleanup(); +DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT void taos_close(TAOS *taos); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index aee60da20147c674979540aa73da5ac101fae7f0..875092b88da1aa03b7eb017e14fff3148f3644a8 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -75,6 +75,7 @@ extern const int32_t TYPE_BYTES[11]; #define TSDB_DATA_SMALLINT_NULL 0x8000 #define TSDB_DATA_INT_NULL 0x80000000L #define TSDB_DATA_BIGINT_NULL 0x8000000000000000L +#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL #define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN #define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN @@ -363,6 +364,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_MAX_WAL_LEVEL 2 #define TSDB_DEFAULT_WAL_LEVEL 1 +#define TSDB_MIN_DB_UPDATE 0 +#define TSDB_MAX_DB_UPDATE 1 +#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 + #define TSDB_MIN_FSYNC_PERIOD 0 #define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond #define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second @@ -424,42 +429,47 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_PORT_DNODEDNODE 5 #define TSDB_PORT_SYNC 10 #define TSDB_PORT_HTTP 11 -#define TSDB_PORT_ARBITRATOR 12 +#define TSDB_PORT_ARBITRATOR 12 + +#define TSDB_MAX_WAL_SIZE (1024*1024) -#define TAOS_QTYPE_RPC 0 -#define TAOS_QTYPE_FWD 1 -#define TAOS_QTYPE_WAL 2 -#define TAOS_QTYPE_CQ 3 -#define TAOS_QTYPE_QUERY 4 +typedef enum { + TAOS_QTYPE_RPC = 0, + TAOS_QTYPE_FWD = 1, + TAOS_QTYPE_WAL = 2, + TAOS_QTYPE_CQ = 3, + TAOS_QTYPE_QUERY = 4 +} EQType; typedef enum { - TSDB_SUPER_TABLE = 0, // super table - TSDB_CHILD_TABLE = 1, // table created from super table - TSDB_NORMAL_TABLE = 2, // ordinary table - TSDB_STREAM_TABLE = 3, // table created from stream computing - TSDB_TABLE_MAX = 4 + TSDB_SUPER_TABLE = 0, // super table + TSDB_CHILD_TABLE = 1, // table created from super table + TSDB_NORMAL_TABLE = 2, // ordinary table + TSDB_STREAM_TABLE = 3, // table created from stream computing + TSDB_TABLE_MAX = 4 } ETableType; typedef enum { - TSDB_MOD_MNODE, - TSDB_MOD_HTTP, - TSDB_MOD_MONITOR, - TSDB_MOD_MQTT, - TSDB_MOD_MAX + TSDB_MOD_MNODE = 0, + TSDB_MOD_HTTP = 1, + TSDB_MOD_MONITOR = 2, + TSDB_MOD_MQTT = 3, + TSDB_MOD_MAX = 4 } EModuleType; - typedef enum { - TSDB_CHECK_ITEM_NETWORK, - TSDB_CHECK_ITEM_MEM, - TSDB_CHECK_ITEM_CPU, - TSDB_CHECK_ITEM_DISK, - TSDB_CHECK_ITEM_OS, - TSDB_CHECK_ITEM_ACCESS, - TSDB_CHECK_ITEM_VERSION, - TSDB_CHECK_ITEM_DATAFILE, - TSDB_CHECK_ITEM_MAX - } ECheckItemType; - +typedef enum { + TSDB_CHECK_ITEM_NETWORK, + TSDB_CHECK_ITEM_MEM, + TSDB_CHECK_ITEM_CPU, + TSDB_CHECK_ITEM_DISK, + TSDB_CHECK_ITEM_OS, + TSDB_CHECK_ITEM_ACCESS, + TSDB_CHECK_ITEM_VERSION, + TSDB_CHECK_ITEM_DATAFILE, + TSDB_CHECK_ITEM_MAX +} ECheckItemType; + +extern char *qtypeStr[]; #ifdef __cplusplus } diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index bb111d2da0da75e6a3e3812ac21364f9a18fb6f3..1919747a0b019239e5d6b717921c6a957728d455 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -28,7 +28,7 @@ extern "C" { #else #define TAOS_DEFINE_ERROR(name, mod, code, msg) static const int32_t name = (0x80000000 | ((mod)<<16) | (code)); #endif - + #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) #define TAOS_SUCCEEDED(err) ((err) >= 0) #define TAOS_FAILED(err) ((err) < 0) @@ -37,7 +37,7 @@ const char* tstrerror(int32_t err); int32_t* taosGetErrno(); #define terrno (*taosGetErrno()) - + #define TSDB_CODE_SUCCESS 0 #ifdef TAOS_ERROR_C @@ -74,6 +74,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_COM_MEMORY_CORRUPTED, 0, 0x0101, "Memory cor TAOS_DEFINE_ERROR(TSDB_CODE_COM_OUT_OF_MEMORY, 0, 0x0102, "Out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_COM_INVALID_CFG_MSG, 0, 0x0103, "Invalid config message") TAOS_DEFINE_ERROR(TSDB_CODE_COM_FILE_CORRUPTED, 0, 0x0104, "Data file corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, 0, 0x0105, "Ref out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, 0, 0x0106, "too many Ref Objs") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, 0, 0x0107, "Ref ID is removed") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID, 0, 0x0108, "Invalid Ref ID") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_ALREADY_EXIST, 0, 0x0109, "Ref is already there") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, 0, 0x010A, "Ref is not there") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_SQL, 0, 0x0200, "Invalid SQL statement") @@ -175,6 +181,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, 0, 0x0383, "Invalid da TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN, 0, 0x0384, "Cannot delete monitor database") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, 0, 0x0385, "Too many databases for account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, 0, 0x0386, "Database not available") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY, 0, 0x0387, "Database unsynced") // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, 0, 0x0400, "Message not processed") @@ -182,7 +189,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, 0, 0x0401, "Dnode out TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, 0, 0x0402, "No permission for disk files in dnode") TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, 0, 0x0403, "Invalid message length") -// vnode +// vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, 0, 0x0500, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_VND_MSG_NOT_PROCESSED, 0, 0x0501, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_NEED_REPROCESSED, 0, 0x0502, "Action need to be reprocessed") @@ -230,6 +237,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, 0, 0x070B, "Query buffer limit has reached") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") @@ -253,6 +261,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, 0, 0x0902, "Invalid Sy // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, 0, 0x1001, "WAL file is corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, 0, 0x1002, "WAL size exceeds limit") // http TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE, 0, 0x1100, "http server is not onlin") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 600347c44fdb2a89e03bf24413f22240bb2815f2..743616e08ed7601650bd144e6f4a6f1d9fc72230 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -106,6 +106,10 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" ) + +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "network-test" ) + + #ifndef TAOS_MESSAGE_C TSDB_MSG_TYPE_MAX // 105 #endif @@ -291,7 +295,7 @@ typedef struct { SSchema schema[]; // tagVal is padded after schema // char tagVal[]; -} SCMAlterTableMsg; +} SAlterTableMsg; typedef struct { SMsgHead head; @@ -308,12 +312,12 @@ typedef struct { } SUpdateTableTagValMsg; typedef struct { - char clientVersion[TSDB_VERSION_LEN]; - char msgVersion[TSDB_VERSION_LEN]; - char db[TSDB_TABLE_FNAME_LEN]; - char appName[TSDB_APPNAME_LEN]; + char clientVersion[TSDB_VERSION_LEN]; + char msgVersion[TSDB_VERSION_LEN]; + char db[TSDB_TABLE_FNAME_LEN]; + char appName[TSDB_APPNAME_LEN]; int32_t pid; -} SCMConnectMsg; +} SConnectMsg; typedef struct { char acctId[TSDB_ACCT_LEN]; @@ -324,7 +328,7 @@ typedef struct { int8_t reserved2; int32_t connId; SRpcEpSet epSet; -} SCMConnectRsp; +} SConnectRsp; typedef struct { int32_t maxUsers; @@ -344,18 +348,18 @@ typedef struct { char user[TSDB_USER_LEN]; char pass[TSDB_KEY_LEN]; SAcctCfg cfg; -} SCMCreateAcctMsg, SCMAlterAcctMsg; +} SCreateAcctMsg, SAlterAcctMsg; typedef struct { char user[TSDB_USER_LEN]; -} SCMDropUserMsg, SCMDropAcctMsg; +} SDropUserMsg, SDropAcctMsg; typedef struct { char user[TSDB_USER_LEN]; char pass[TSDB_KEY_LEN]; int8_t privilege; int8_t flag; -} SCMCreateUserMsg, SCMAlterUserMsg; +} SCreateUserMsg, SAlterUserMsg; typedef struct { int32_t contLen; @@ -370,11 +374,11 @@ typedef struct { int32_t vgId; uint64_t uid; char tableId[TSDB_TABLE_FNAME_LEN]; -} SMDDropSTableMsg; +} SDropSTableMsg; typedef struct { int32_t vgId; -} SMDDropVnodeMsg; +} SDropVnodeMsg; typedef struct SColIndex { int16_t colId; // column id @@ -540,12 +544,14 @@ typedef struct { int8_t replications; int8_t quorum; int8_t ignoreExist; -} SCMCreateDbMsg, SCMAlterDbMsg; + int8_t update; + int8_t reserve[9]; +} SCreateDbMsg, SAlterDbMsg; typedef struct { char db[TSDB_TABLE_FNAME_LEN]; uint8_t ignoreNotExists; -} SCMDropDbMsg, SCMUseDbMsg; +} SDropDbMsg, SUseDbMsg; // IMPORTANT: sizeof(SVnodeStatisticInfo) should not exceed // TSDB_FILE_HEADER_LEN/4 - TSDB_FILE_HEADER_VERSION_SIZE @@ -560,7 +566,7 @@ typedef struct { typedef struct { int32_t vgId; int8_t accessState; -} SDMVgroupAccess; +} SVgroupAccess; typedef struct { int32_t dnodeId; @@ -568,18 +574,29 @@ typedef struct { uint32_t numOfVnodes; char clusterId[TSDB_CLUSTER_ID_LEN]; char reserved[16]; -} SDMDnodeCfg; +} SDnodeCfg; typedef struct { - int32_t nodeId; - char nodeEp[TSDB_EP_LEN]; -} SDMMnodeInfo; + int32_t dnodeId; + uint16_t dnodePort; + char dnodeFqdn[TSDB_FQDN_LEN]; +} SDnodeEp; typedef struct { - int8_t inUse; - int8_t nodeNum; - SDMMnodeInfo nodeInfos[TSDB_MAX_REPLICA]; -} SDMMnodeInfos; + int32_t dnodeNum; + SDnodeEp dnodeEps[]; +} SDnodeEps; + +typedef struct { + int32_t mnodeId; + char mnodeEp[TSDB_EP_LEN]; +} SMnodeInfo; + +typedef struct { + int8_t inUse; + int8_t mnodeNum; + SMnodeInfo mnodeInfos[TSDB_MAX_REPLICA]; +} SMnodeInfos; typedef struct { int32_t numOfMnodes; // tsNumOfMnodes @@ -611,13 +628,13 @@ typedef struct { uint8_t reserve2[15]; SClusterCfg clusterCfg; SVnodeLoad load[]; -} SDMStatusMsg; +} SStatusMsg; typedef struct { - SDMMnodeInfos mnodes; - SDMDnodeCfg dnodeCfg; - SDMVgroupAccess vgAccess[]; -} SDMStatusRsp; + SMnodeInfos mnodes; + SDnodeCfg dnodeCfg; + SVgroupAccess vgAccess[]; +} SStatusRsp; typedef struct { uint32_t vgId; @@ -639,55 +656,56 @@ typedef struct { int8_t replications; int8_t wals; int8_t quorum; - int8_t reserved[16]; -} SMDVnodeCfg; + int8_t update; + int8_t reserved[15]; +} SVnodeCfg; typedef struct { int32_t nodeId; char nodeEp[TSDB_EP_LEN]; -} SMDVnodeDesc; +} SVnodeDesc; typedef struct { - char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; - SMDVnodeCfg cfg; - SMDVnodeDesc nodes[TSDB_MAX_REPLICA]; -} SMDCreateVnodeMsg, SMDAlterVnodeMsg; + char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; + SVnodeCfg cfg; + SVnodeDesc nodes[TSDB_MAX_REPLICA]; +} SCreateVnodeMsg, SAlterVnodeMsg; typedef struct { char tableId[TSDB_TABLE_FNAME_LEN]; int16_t createFlag; char tags[]; -} SCMTableInfoMsg; +} STableInfoMsg; typedef struct { int32_t numOfTables; char tableIds[]; -} SCMMultiTableInfoMsg; +} SMultiTableInfoMsg; -typedef struct SCMSTableVgroupMsg { +typedef struct SSTableVgroupMsg { int32_t numOfTables; -} SCMSTableVgroupMsg, SCMSTableVgroupRspMsg; +} SSTableVgroupMsg, SSTableVgroupRspMsg; typedef struct { int32_t vgId; int8_t numOfEps; SEpAddr1 epAddr[TSDB_MAX_REPLICA]; -} SCMVgroupInfo; +} SVgroupInfo; typedef struct { int32_t vgId; int8_t numOfEps; SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; -} SCMVgroupMsg; +} SVgroupMsg; typedef struct { int32_t numOfVgroups; - SCMVgroupInfo vgroups[]; + SVgroupInfo vgroups[]; } SVgroupsInfo; typedef struct { int32_t numOfVgroups; - SCMVgroupMsg vgroups[]; + SVgroupMsg vgroups[]; } SVgroupsMsg; typedef struct STableMetaMsg { @@ -702,7 +720,7 @@ typedef struct STableMetaMsg { int16_t tversion; int32_t tid; uint64_t uid; - SCMVgroupMsg vgroup; + SVgroupMsg vgroup; SSchema schema[]; } STableMetaMsg; @@ -728,38 +746,38 @@ typedef struct { char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; uint16_t payloadLen; char payload[]; -} SCMShowMsg; +} SShowMsg; -typedef struct SCMShowRsp { +typedef struct SShowRsp { uint64_t qhandle; STableMetaMsg tableMeta; -} SCMShowRsp; +} SShowRsp; typedef struct { - char ep[TSDB_EP_LEN]; // end point, hostname:port -} SCMCreateDnodeMsg, SCMDropDnodeMsg; + char ep[TSDB_EP_LEN]; // end point, hostname:port +} SCreateDnodeMsg, SDropDnodeMsg; typedef struct { int32_t dnodeId; char dnodeEp[TSDB_EP_LEN]; // end point, hostname:port - SDMMnodeInfos mnodes; -} SMDCreateMnodeMsg; + SMnodeInfos mnodes; +} SCreateMnodeMsg; typedef struct { int32_t dnodeId; int32_t vgId; int32_t tid; -} SDMConfigTableMsg; +} SConfigTableMsg; typedef struct { uint32_t dnodeId; int32_t vgId; -} SDMConfigVnodeMsg; +} SConfigVnodeMsg; typedef struct { char ep[TSDB_EP_LEN]; // end point, hostname:port char config[64]; -} SMDCfgDnodeMsg, SCMCfgDnodeMsg; +} SCfgDnodeMsg; typedef struct { char sql[TSDB_SHOW_SQL_LEN]; @@ -781,13 +799,14 @@ typedef struct { } SStreamDesc; typedef struct { + char clientVer[TSDB_VERSION_LEN]; uint32_t connId; int32_t pid; int32_t numOfQueries; int32_t numOfStreams; char appName[TSDB_APPNAME_LEN]; char pData[]; -} SCMHeartBeatMsg; +} SHeartBeatMsg; typedef struct { uint32_t queryId; @@ -797,11 +816,11 @@ typedef struct { uint32_t connId; int8_t killConnection; SRpcEpSet epSet; -} SCMHeartBeatRsp; +} SHeartBeatRsp; typedef struct { char queryId[TSDB_KILL_MSG_LEN + 1]; -} SCMKillQueryMsg, SCMKillStreamMsg, SCMKillConnMsg; +} SKillQueryMsg, SKillStreamMsg, SKillConnMsg; typedef struct { int32_t vnode; @@ -810,7 +829,7 @@ typedef struct { uint64_t stime; // stream starting time int32_t status; char tableId[TSDB_TABLE_FNAME_LEN]; -} SMDAlterStreamMsg; +} SAlterStreamMsg; typedef struct { char user[TSDB_USER_LEN]; @@ -818,7 +837,7 @@ typedef struct { char encrypt; char secret[TSDB_KEY_LEN]; char ckey[TSDB_KEY_LEN]; -} SDMAuthMsg, SDMAuthRsp; +} SAuthMsg, SAuthRsp; #pragma pack(pop) diff --git a/src/inc/tcq.h b/src/inc/tcq.h index 32b75674c3278b3273fd4b98dd645f4168543155..7a0727f1b8dd7cdea1a815c174b9f85004eedf87 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -21,10 +21,10 @@ extern "C" { #include "tdataformat.h" -typedef int (*FCqWrite)(void *ahandle, void *pHead, int type); +typedef int32_t (*FCqWrite)(void *ahandle, void *pHead, int32_t qtype, void *pMsg); typedef struct { - int vgId; + int32_t vgId; char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; char db[TSDB_DB_NAME_LEN]; @@ -42,12 +42,12 @@ void cqStart(void *handle); void cqStop(void *handle); // cqCreate is called by TSDB to start an instance of CQ -void *cqCreate(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema); +void *cqCreate(void *handle, uint64_t uid, int32_t sid, char *sqlStr, STSchema *pSchema); // cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate void cqDrop(void *handle); -extern int cqDebugFlag; +extern int32_t cqDebugFlag; #ifdef __cplusplus diff --git a/src/inc/trpc.h b/src/inc/trpc.h index bdee917b5e8203743a79ca27d8fbc987569c35ab..e430a43807bc6fb0dd5c48d8a912223c1ba015af 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -83,13 +83,13 @@ void rpcClose(void *); void *rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); void *rpcReallocCont(void *ptr, int contLen); -void rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg); +int64_t rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg); void rpcSendResponse(const SRpcMsg *pMsg); void rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet); int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); int rpcReportProgress(void *pConn, char *pCont, int contLen); -void rpcCancelRequest(void *pContext); +void rpcCancelRequest(int64_t rid); #ifdef __cplusplus } diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 85f9b3bdc7f1cf806309f597ffad6955151b4c36..d7515a14956d7029d909241b2985023d33e58622 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -65,6 +65,7 @@ typedef struct { int32_t maxRowsPerFileBlock; // maximum rows per file block int8_t precision; int8_t compression; + int8_t update; } STsdbCfg; // --------- TSDB REPOSITORY USAGE STATISTICS @@ -163,6 +164,12 @@ typedef struct STsdbQueryCond { SColumnInfo *colList; } STsdbQueryCond; +typedef struct SMemRef { + int32_t ref; + void *mem; + void *imem; +} SMemRef; + typedef struct SDataBlockInfo { STimeWindow window; int32_t rows; @@ -192,7 +199,7 @@ typedef struct { * @param qinfo query info handle from query processor * @return */ -TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo); +TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo, SMemRef* pRef); /** * Get the last row of the given query time window for all the tables in STableGroupInfo object. @@ -204,7 +211,7 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab * @param tableInfo table list. * @return */ -TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo); +TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo, SMemRef* pRef); /** * get the queried table object list @@ -222,7 +229,7 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle); * @return */ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, - void *qinfo); + void *qinfo, SMemRef* pRef); /** * move to next block if exists @@ -314,6 +321,10 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle); */ void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage); +int tsdbInitCommitQueue(int nthreads); +void tsdbDestroyCommitQueue(); +int tsdbSyncCommit(TSDB_REPO_T *repo); + #ifdef __cplusplus } #endif diff --git a/src/inc/tsync.h b/src/inc/tsync.h index ca0f70d104d603d176d89dd5b92979433f390466..d57433eba9bacb622081032f4ed6a2a72406248d 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -51,9 +51,9 @@ typedef struct { } SSyncCfg; typedef struct { - int selfIndex; - uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; - int role[TAOS_SYNC_MAX_REPLICA]; + int32_t selfIndex; + uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; + int32_t role[TAOS_SYNC_MAX_REPLICA]; } SNodesRole; /* @@ -68,10 +68,10 @@ typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uin // get the wal file from index or after // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file -typedef int (*FGetWalInfo)(void *ahandle, char *name, uint32_t *index); +typedef int32_t (*FGetWalInfo)(void *ahandle, char *fileName, int64_t *fileId); -// when a forward pkt is received, call this to handle data -typedef int (*FWriteToCache)(void *ahandle, void *pHead, int type); +// when a forward pkt is received, call this to handle data +typedef int32_t (*FWriteToCache)(void *ahandle, void *pHead, int32_t qtype, void *pMsg); // when forward is confirmed by peer, master call this API to notify app typedef void (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code); @@ -83,48 +83,47 @@ typedef void (*FNotifyRole)(void *ahandle, int8_t role); typedef void (*FNotifyFlowCtrl)(void *ahandle, int32_t mseconds); // when data file is synced successfully, notity app -typedef int (*FNotifyFileSynced)(void *ahandle, uint64_t fversion); +typedef int32_t (*FNotifyFileSynced)(void *ahandle, uint64_t fversion); typedef struct { - int32_t vgId; // vgroup ID - uint64_t version; // initial version - SSyncCfg syncCfg; // configuration from mgmt - char path[128]; // path to the file - - void *ahandle; // handle provided by APP - FGetFileInfo getFileInfo; - FGetWalInfo getWalInfo; - FWriteToCache writeToCache; - FConfirmForward confirmForward; - FNotifyRole notifyRole; - FNotifyFlowCtrl notifyFlowCtrl; + int32_t vgId; // vgroup ID + uint64_t version; // initial version + SSyncCfg syncCfg; // configuration from mgmt + char path[128]; // path to the file + void * ahandle; // handle provided by APP + FGetFileInfo getFileInfo; + FGetWalInfo getWalInfo; + FWriteToCache writeToCache; + FConfirmForward confirmForward; + FNotifyRole notifyRole; + FNotifyFlowCtrl notifyFlowCtrl; FNotifyFileSynced notifyFileSynced; } SSyncInfo; -typedef void* tsync_h; +typedef void *tsync_h; int32_t syncInit(); void syncCleanUp(); -tsync_h syncStart(const SSyncInfo *); -void syncStop(tsync_h shandle); -int32_t syncReconfig(tsync_h shandle, const SSyncCfg *); -int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype); -void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code); -void syncRecover(tsync_h shandle); // recover from other nodes: -int syncGetNodesRole(tsync_h shandle, SNodesRole *); +int64_t syncStart(const SSyncInfo *); +void syncStop(int64_t rid); +int32_t syncReconfig(int64_t rid, const SSyncCfg *); +int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype); +void syncConfirmForward(int64_t rid, uint64_t version, int32_t code); +void syncRecover(int64_t rid); // recover from other nodes: +int32_t syncGetNodesRole(int64_t rid, SNodesRole *); -extern char *syncRole[]; +extern char *syncRole[]; //global configurable parameters -extern int tsMaxSyncNum; -extern int tsSyncTcpThreads; -extern int tsMaxWatchFiles; -extern int tsSyncTimer; -extern int tsMaxFwdInfo; -extern int sDebugFlag; -extern char tsArbitrator[]; -extern uint16_t tsSyncPort; +extern int32_t tsMaxSyncNum; +extern int32_t tsSyncTcpThreads; +extern int32_t tsMaxWatchFiles; +extern int32_t tsSyncTimer; +extern int32_t tsMaxFwdInfo; +extern int32_t sDebugFlag; +extern char tsArbitrator[]; +extern uint16_t tsSyncPort; #ifdef __cplusplus } diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index a94cdaad15a5c2abe35b7a02502ff8ae95102cd9..32f2a2d32ea9000b098b0feadaf5b4cf9e6e8fc9 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -114,114 +114,115 @@ #define TK_FSYNC 95 #define TK_COMP 96 #define TK_PRECISION 97 -#define TK_LP 98 -#define TK_RP 99 -#define TK_TAGS 100 -#define TK_USING 101 -#define TK_AS 102 -#define TK_COMMA 103 -#define TK_NULL 104 -#define TK_SELECT 105 -#define TK_UNION 106 -#define TK_ALL 107 -#define TK_FROM 108 -#define TK_VARIABLE 109 -#define TK_INTERVAL 110 -#define TK_FILL 111 -#define TK_SLIDING 112 -#define TK_ORDER 113 -#define TK_BY 114 -#define TK_ASC 115 -#define TK_DESC 116 -#define TK_GROUP 117 -#define TK_HAVING 118 -#define TK_LIMIT 119 -#define TK_OFFSET 120 -#define TK_SLIMIT 121 -#define TK_SOFFSET 122 -#define TK_WHERE 123 -#define TK_NOW 124 -#define TK_RESET 125 -#define TK_QUERY 126 -#define TK_ADD 127 -#define TK_COLUMN 128 -#define TK_TAG 129 -#define TK_CHANGE 130 -#define TK_SET 131 -#define TK_KILL 132 -#define TK_CONNECTION 133 -#define TK_STREAM 134 -#define TK_COLON 135 -#define TK_ABORT 136 -#define TK_AFTER 137 -#define TK_ATTACH 138 -#define TK_BEFORE 139 -#define TK_BEGIN 140 -#define TK_CASCADE 141 -#define TK_CLUSTER 142 -#define TK_CONFLICT 143 -#define TK_COPY 144 -#define TK_DEFERRED 145 -#define TK_DELIMITERS 146 -#define TK_DETACH 147 -#define TK_EACH 148 -#define TK_END 149 -#define TK_EXPLAIN 150 -#define TK_FAIL 151 -#define TK_FOR 152 -#define TK_IGNORE 153 -#define TK_IMMEDIATE 154 -#define TK_INITIALLY 155 -#define TK_INSTEAD 156 -#define TK_MATCH 157 -#define TK_KEY 158 -#define TK_OF 159 -#define TK_RAISE 160 -#define TK_REPLACE 161 -#define TK_RESTRICT 162 -#define TK_ROW 163 -#define TK_STATEMENT 164 -#define TK_TRIGGER 165 -#define TK_VIEW 166 -#define TK_COUNT 167 -#define TK_SUM 168 -#define TK_AVG 169 -#define TK_MIN 170 -#define TK_MAX 171 -#define TK_FIRST 172 -#define TK_LAST 173 -#define TK_TOP 174 -#define TK_BOTTOM 175 -#define TK_STDDEV 176 -#define TK_PERCENTILE 177 -#define TK_APERCENTILE 178 -#define TK_LEASTSQUARES 179 -#define TK_HISTOGRAM 180 -#define TK_DIFF 181 -#define TK_SPREAD 182 -#define TK_TWA 183 -#define TK_INTERP 184 -#define TK_LAST_ROW 185 -#define TK_RATE 186 -#define TK_IRATE 187 -#define TK_SUM_RATE 188 -#define TK_SUM_IRATE 189 -#define TK_AVG_RATE 190 -#define TK_AVG_IRATE 191 -#define TK_TBID 192 -#define TK_SEMI 193 -#define TK_NONE 194 -#define TK_PREV 195 -#define TK_LINEAR 196 -#define TK_IMPORT 197 -#define TK_METRIC 198 -#define TK_TBNAME 199 -#define TK_JOIN 200 -#define TK_METRICS 201 -#define TK_STABLE 202 -#define TK_INSERT 203 -#define TK_INTO 204 -#define TK_VALUES 205 +#define TK_UPDATE 98 +#define TK_LP 99 +#define TK_RP 100 +#define TK_TAGS 101 +#define TK_USING 102 +#define TK_AS 103 +#define TK_COMMA 104 +#define TK_NULL 105 +#define TK_SELECT 106 +#define TK_UNION 107 +#define TK_ALL 108 +#define TK_FROM 109 +#define TK_VARIABLE 110 +#define TK_INTERVAL 111 +#define TK_FILL 112 +#define TK_SLIDING 113 +#define TK_ORDER 114 +#define TK_BY 115 +#define TK_ASC 116 +#define TK_DESC 117 +#define TK_GROUP 118 +#define TK_HAVING 119 +#define TK_LIMIT 120 +#define TK_OFFSET 121 +#define TK_SLIMIT 122 +#define TK_SOFFSET 123 +#define TK_WHERE 124 +#define TK_NOW 125 +#define TK_RESET 126 +#define TK_QUERY 127 +#define TK_ADD 128 +#define TK_COLUMN 129 +#define TK_TAG 130 +#define TK_CHANGE 131 +#define TK_SET 132 +#define TK_KILL 133 +#define TK_CONNECTION 134 +#define TK_STREAM 135 +#define TK_COLON 136 +#define TK_ABORT 137 +#define TK_AFTER 138 +#define TK_ATTACH 139 +#define TK_BEFORE 140 +#define TK_BEGIN 141 +#define TK_CASCADE 142 +#define TK_CLUSTER 143 +#define TK_CONFLICT 144 +#define TK_COPY 145 +#define TK_DEFERRED 146 +#define TK_DELIMITERS 147 +#define TK_DETACH 148 +#define TK_EACH 149 +#define TK_END 150 +#define TK_EXPLAIN 151 +#define TK_FAIL 152 +#define TK_FOR 153 +#define TK_IGNORE 154 +#define TK_IMMEDIATE 155 +#define TK_INITIALLY 156 +#define TK_INSTEAD 157 +#define TK_MATCH 158 +#define TK_KEY 159 +#define TK_OF 160 +#define TK_RAISE 161 +#define TK_REPLACE 162 +#define TK_RESTRICT 163 +#define TK_ROW 164 +#define TK_STATEMENT 165 +#define TK_TRIGGER 166 +#define TK_VIEW 167 +#define TK_COUNT 168 +#define TK_SUM 169 +#define TK_AVG 170 +#define TK_MIN 171 +#define TK_MAX 172 +#define TK_FIRST 173 +#define TK_LAST 174 +#define TK_TOP 175 +#define TK_BOTTOM 176 +#define TK_STDDEV 177 +#define TK_PERCENTILE 178 +#define TK_APERCENTILE 179 +#define TK_LEASTSQUARES 180 +#define TK_HISTOGRAM 181 +#define TK_DIFF 182 +#define TK_SPREAD 183 +#define TK_TWA 184 +#define TK_INTERP 185 +#define TK_LAST_ROW 186 +#define TK_RATE 187 +#define TK_IRATE 188 +#define TK_SUM_RATE 189 +#define TK_SUM_IRATE 190 +#define TK_AVG_RATE 191 +#define TK_AVG_IRATE 192 +#define TK_TBID 193 +#define TK_SEMI 194 +#define TK_NONE 195 +#define TK_PREV 196 +#define TK_LINEAR 197 +#define TK_IMPORT 198 +#define TK_METRIC 199 +#define TK_TBNAME 200 +#define TK_JOIN 201 +#define TK_METRICS 202 +#define TK_STABLE 203 +#define TK_INSERT 204 +#define TK_INTO 205 +#define TK_VALUES 206 #define TK_SPACE 300 #define TK_COMMENT 301 diff --git a/src/inc/twal.h b/src/inc/twal.h index 92204abd7d34a9ee2eebf1b74c2e8d58b9599f17..8dd3a8a91209e840abeb9560f94a52ce362492a9 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -19,42 +19,53 @@ extern "C" { #endif -#define TAOS_WAL_NOLOG 0 -#define TAOS_WAL_WRITE 1 -#define TAOS_WAL_FSYNC 2 - +typedef enum { + TAOS_WAL_NOLOG = 0, + TAOS_WAL_WRITE = 1, + TAOS_WAL_FSYNC = 2 +} EWalType; + +typedef enum { + TAOS_WAL_NOT_KEEP = 0, + TAOS_WAL_KEEP = 1 +} EWalKeep; + typedef struct { - int8_t msgType; - int8_t reserved[3]; - int32_t len; - uint64_t version; - uint32_t signature; - uint32_t cksum; - char cont[]; + int8_t msgType; + int8_t sver; + int8_t reserved[2]; + int32_t len; + uint64_t version; + uint32_t signature; + uint32_t cksum; + char cont[]; } SWalHead; typedef struct { - int8_t walLevel; // wal level - int32_t fsyncPeriod; // millisecond - int8_t wals; // number of WAL files; - int8_t keep; // keep the wal file when closed + int32_t vgId; + int32_t fsyncPeriod; // millisecond + EWalType walLevel; // wal level + EWalKeep keep; // keep the wal file when closed } SWalCfg; -typedef void* twalh; // WAL HANDLE -typedef int (*FWalWrite)(void *ahandle, void *pHead, int type); - -twalh walOpen(const char *path, const SWalCfg *pCfg); -int walAlter(twalh pWal, const SWalCfg *pCfg); -void walClose(twalh); -int walRenew(twalh); -int walWrite(twalh, SWalHead *); -void walFsync(twalh); -int walRestore(twalh, void *pVnode, FWalWrite writeFp); -int walGetWalFile(twalh, char *name, uint32_t *index); -int64_t walGetVersion(twalh); +typedef void * twalh; // WAL HANDLE +typedef int32_t FWalWrite(void *ahandle, void *pHead, int32_t qtype, void *pMsg); -extern int wDebugFlag; +int32_t walInit(); +void walCleanUp(); +twalh walOpen(char *path, SWalCfg *pCfg); +int32_t walAlter(twalh pWal, SWalCfg *pCfg); +void walStop(twalh); +void walClose(twalh); +int32_t walRenew(twalh); +void walRemoveOneOldFile(twalh); +void walRemoveAllOldFiles(twalh); +int32_t walWrite(twalh, SWalHead *); +void walFsync(twalh, bool forceFsync); +int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp); +int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId); +uint64_t walGetVersion(twalh); #ifdef __cplusplus } diff --git a/src/inc/vnode.h b/src/inc/vnode.h index fdce4d62794075bd2e7027b125780fbd7a2deaed..018e96e1938336809fc8829be42ed3e946c6ac98 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include "twal.h" + typedef enum _VN_STATUS { TAOS_VN_STATUS_INIT, TAOS_VN_STATUS_READY, @@ -29,44 +31,57 @@ typedef enum _VN_STATUS { } EVnStatus; typedef struct { - int len; - void *rsp; - void *qhandle; //used by query and retrieve msg + int32_t len; + void * rsp; + void * qhandle; // used by query and retrieve msg } SRspRet; typedef struct { + int32_t code; + int32_t contLen; + void * rpcHandle; + void * rpcAhandle; + void * qhandle; + int8_t qtype; + int8_t msgType; + SRspRet rspRet; + char pCont[]; +} SVReadMsg; + +typedef struct { + int32_t code; + int32_t processedCount; + void * rpcHandle; + void * rpcAhandle; SRspRet rspRet; - void *pCont; - int32_t contLen; - SRpcMsg rpcMsg; -} SReadMsg; + char reserveForSync[16]; + SWalHead pHead[]; +} SVWriteMsg; extern char *vnodeStatus[]; -int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg); +int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg); int32_t vnodeDrop(int32_t vgId); int32_t vnodeOpen(int32_t vgId, char *rootDir); -int32_t vnodeAlter(void *pVnode, SMDCreateVnodeMsg *pVnodeCfg); +int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); int32_t vnodeClose(int32_t vgId); void* vnodeAcquire(int32_t vgId); // add refcount -void* vnodeAcquireRqueue(int32_t vgId); // add refCount, get read queue -void* vnodeAcquireWqueue(int32_t vgId); // add recCount, get write queue void vnodeRelease(void *pVnode); // dec refCount void* vnodeGetWal(void *pVnode); -int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item); -int32_t vnodeCheckWrite(void *pVnode); +int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam); +int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam); int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); void vnodeBuildStatusMsg(void *param); void vnodeConfirmForward(void *param, uint64_t version, int32_t code); -void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes); +void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes); int32_t vnodeInitResources(); void vnodeCleanupResources(); -int32_t vnodeProcessRead(void *pVnode, SReadMsg *pReadMsg); -int32_t vnodeCheckRead(void *pVnode); +int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam); +int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead); #ifdef __cplusplus } diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index f508d186083c84482080f2f8fe251173733f1366..d65c943e28d7d5a63aba4fc1839a4ba9cf744746 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -60,7 +60,7 @@ typedef struct SShellArguments { extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments); extern TAOS* shellInit(SShellArguments* args); extern void* shellLoopQuery(void* arg); -extern void taos_error(TAOS_RES* tres); +extern void taos_error(TAOS_RES* tres, int64_t st); extern int regex_match(const char* s, const char* reg, int cflags); void shellReadCommand(TAOS* con, char command[]); int32_t shellRunCommand(TAOS* con, char* command); diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index ffe537dd91834247460f04c47efff5ebe8687d7e..995b56f341b36b84e413aec154ae1abcc832741d 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -229,8 +229,8 @@ void shellReadCommand(TAOS *con, char *command) { printf("\n"); if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - taosTFree(cmd.buffer); - taosTFree(cmd.command); + tfree(cmd.buffer); + tfree(cmd.command); return; } else { updateBuffer(&cmd); diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 748b7e792982352de611227d82a844448d251ee2..22f01ac142c72603b9fa0595ce8c1fc89c626d7e 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -193,7 +193,7 @@ int32_t shellRunCommand(TAOS* con, char* command) { history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE] == NULL || strcmp(command, history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE]) != 0) { if (history.hist[history.hend] != NULL) { - taosTFree(history.hist[history.hend]); + tfree(history.hist[history.hend]); } history.hist[history.hend] = strdup(command); @@ -244,7 +244,7 @@ int32_t shellRunCommand(TAOS* con, char* command) { } *p++ = c; - if (c == ';') { + if (c == ';' && quote == 0) { c = *p; *p = 0; if (shellRunSingleCommand(con, cmd) < 0) { @@ -296,7 +296,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { TAOS_RES* pSql = taos_query_h(con, command, &result); if (taos_errno(pSql)) { - taos_error(pSql); + taos_error(pSql, st); return; } @@ -770,7 +770,7 @@ void read_history() { return; } - while ((read_size = taosGetline(&line, &line_size, f)) != -1) { + while ((read_size = tgetline(&line, &line_size, f)) != -1) { line[read_size - 1] = '\0'; history.hist[history.hend] = strdup(line); @@ -800,16 +800,17 @@ void write_history() { for (int i = history.hstart; i != history.hend;) { if (history.hist[i] != NULL) { fprintf(f, "%s\n", history.hist[i]); - taosTFree(history.hist[i]); + tfree(history.hist[i]); } i = (i + 1) % MAX_HISTORY_SIZE; } fclose(f); } -void taos_error(TAOS_RES *tres) { +void taos_error(TAOS_RES *tres, int64_t st) { + int64_t et = taosGetTimestampUs(); atomic_store_ptr(&result, 0); - fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres)); + fprintf(stderr, "\nDB error: %s (%.6fs)\n", taos_errstr(tres), (et - st) / 1E6); taos_free_result(tres); } @@ -853,7 +854,7 @@ void source_file(TAOS *con, char *fptr) { return; } - while ((read_len = taosGetline(&line, &line_len, f)) != -1) { + while ((read_len = tgetline(&line, &line_len, f)) != -1) { if (read_len >= tsMaxSQLStringLen) continue; line[--read_len] = '\0'; diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 6f5ea33d79f09219f49ee4094de2c377648b2290..57f0f65be4315dda479f751cee20c833dc4b4c75 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -232,8 +232,8 @@ void shellReadCommand(TAOS *con, char *command) { printf("\n"); if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - taosTFree(cmd.buffer); - taosTFree(cmd.command); + tfree(cmd.buffer); + tfree(cmd.command); return; } else { updateBuffer(&cmd); @@ -351,7 +351,7 @@ void *shellLoopQuery(void *arg) { reset_terminal_mode(); } while (shellRunCommand(con, command) == 0); - taosTFree(command); + tfree(command); exitShell(); pthread_cleanup_pop(1); diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 6cb7c669cc7a08434b2558588067d007b51b3595..2083ad3e9b7d5a101f35af4fe20dcd38957d5be2 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -80,7 +80,10 @@ int main(int argc, char* argv[]) { shellParseArgument(argc, argv, &args); if (args.netTestRole && args.netTestRole[0] != 0) { - taosNetTest(args.host, (uint16_t)args.port, (uint16_t)args.endPort, args.pktLen, args.netTestRole); + taos_init(); + CmdArguments cmdArgs; + memcpy(&cmdArgs, &args, sizeof(SShellArguments)); + taosNetTest(&cmdArgs); exit(0); } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 699e96428e201e998c1a4bb9cb8cd3c835e61364..60785ec004776aa39ab719de5ca0975922b70d10 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -475,6 +475,7 @@ typedef struct { tsem_t mutex_sem; int notFinished; tsem_t lock_sem; + int counter; } info; typedef struct { @@ -766,6 +767,7 @@ int main(int argc, char *argv[]) { t_info->data_of_rate = rate; t_info->end_table_id = i < b ? last + a : last + a - 1; last = t_info->end_table_id + 1; + t_info->counter = 0; tsem_init(&(t_info->mutex_sem), 0, 1); t_info->notFinished = t_info->end_table_id - t_info->start_table_id + 1; @@ -788,14 +790,14 @@ int main(int argc, char *argv[]) { printf("ASYNC Insert with %d connections:\n", threads); } - fprintf(fp, "|%10.d | %10.2f | %10.2f | %10.4f |\n\n", - ntables * nrecords_per_table, ntables * nrecords_per_table / t, - (ntables * nrecords_per_table) / (t * nrecords_per_request), + fprintf(fp, "|%"PRIu64" | %10.2f | %10.2f | %10.4f |\n\n", + (int64_t)ntables * nrecords_per_table, ntables * nrecords_per_table / t, + ((int64_t)ntables * nrecords_per_table) / (t * nrecords_per_request), t * 1000); - printf("Spent %.4f seconds to insert %d records with %d record(s) per request: %.2f records/second\n", - t, ntables * nrecords_per_table, nrecords_per_request, - ntables * nrecords_per_table / t); + printf("Spent %.4f seconds to insert %"PRIu64" records with %d record(s) per request: %.2f records/second\n", + t, (int64_t)ntables * nrecords_per_table, nrecords_per_request, + (int64_t)ntables * nrecords_per_table / t); for (int i = 0; i < threads; i++) { info *t_info = infos + i; @@ -955,7 +957,7 @@ void querySqlFile(TAOS* taos, char* sqlFile) double t = getCurrentTime(); - while ((read_len = taosGetline(&line, &line_len, fp)) != -1) { + while ((read_len = tgetline(&line, &line_len, fp)) != -1) { if (read_len >= MAX_SQL_SIZE) continue; line[--read_len] = '\0'; @@ -1283,68 +1285,39 @@ void *syncWrite(void *sarg) { void *asyncWrite(void *sarg) { info *winfo = (info *)sarg; - - sTable *tb_infos = (sTable *)malloc(sizeof(sTable) * (winfo->end_table_id - winfo->start_table_id + 1)); - - for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { - sTable *tb_info = tb_infos + tID - winfo->start_table_id; - tb_info->data_type = winfo->datatype; - tb_info->ncols_per_record = winfo->ncols_per_record; - tb_info->taos = winfo->taos; - sprintf(tb_info->tb_name, "%s.%s%d", winfo->db_name, winfo->tb_prefix, tID); - tb_info->timestamp = winfo->start_time; - tb_info->counter = 0; - tb_info->target = winfo->nrecords_per_table; - tb_info->len_of_binary = winfo->len_of_binary; - tb_info->nrecords_per_request = winfo->nrecords_per_request; - tb_info->mutex_sem = &(winfo->mutex_sem); - tb_info->notFinished = &(winfo->notFinished); - tb_info->lock_sem = &(winfo->lock_sem); - tb_info->data_of_order = winfo->data_of_order; - tb_info->data_of_rate = winfo->data_of_rate; - - /* char buff[BUFFER_SIZE] = "\0"; */ - /* sprintf(buff, "insert into %s values (0, 0)", tb_info->tb_name); */ - /* queryDB(tb_info->taos,buff); */ - - taos_query_a(winfo->taos, "show databases", callBack, tb_info); - } + taos_query_a(winfo->taos, "show databases", callBack, winfo); tsem_wait(&(winfo->lock_sem)); - free(tb_infos); return NULL; } void callBack(void *param, TAOS_RES *res, int code) { - sTable *tb_info = (sTable *)param; - char **datatype = tb_info->data_type; - int ncols_per_record = tb_info->ncols_per_record; - int len_of_binary = tb_info->len_of_binary; - int64_t tmp_time = tb_info->timestamp; - - if (code < 0) { - fprintf(stderr, "failed to insert data %d:reason; %s\n", code, taos_errstr(res)); - exit(EXIT_FAILURE); - } + info* winfo = (info*)param; + char **datatype = winfo->datatype; + int ncols_per_record = winfo->ncols_per_record; + int len_of_binary = winfo->len_of_binary; - // If finished; - if (tb_info->counter >= tb_info->target) { - tsem_wait(tb_info->mutex_sem); - (*(tb_info->notFinished))--; - if (*(tb_info->notFinished) == 0) tsem_post(tb_info->lock_sem); - tsem_post(tb_info->mutex_sem); + int64_t tmp_time = winfo->start_time; + char *buffer = calloc(1, BUFFER_SIZE); + char *data = calloc(1, MAX_DATA_SIZE); + char *pstr = buffer; + pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, winfo->start_table_id); + if (winfo->counter >= winfo->nrecords_per_table) { + winfo->start_table_id++; + winfo->counter = 0; + } + if (winfo->start_table_id > winfo->end_table_id) { + tsem_post(&winfo->lock_sem); + free(buffer); + free(data); + taos_free_result(res); return; } - - char buffer[BUFFER_SIZE] = "\0"; - char data[MAX_DATA_SIZE]; - char *pstr = buffer; - pstr += sprintf(pstr, "insert into %s values", tb_info->tb_name); - - for (int i = 0; i < tb_info->nrecords_per_request; i++) { + + for (int i = 0; i < winfo->nrecords_per_request; i++) { int rand_num = rand() % 100; - if (tb_info->data_of_order ==1 && rand_num < tb_info->data_of_rate) + if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate) { int64_t d = tmp_time - rand() % 1000000 + rand_num; generateData(data, datatype, ncols_per_record, d, len_of_binary); @@ -1353,15 +1326,15 @@ void callBack(void *param, TAOS_RES *res, int code) { generateData(data, datatype, ncols_per_record, tmp_time += 1000, len_of_binary); } pstr += sprintf(pstr, "%s", data); - tb_info->counter++; + winfo->counter++; - if (tb_info->counter >= tb_info->target) { + if (winfo->counter >= winfo->nrecords_per_table) { break; } } - tb_info->timestamp = tmp_time; - - taos_query_a(tb_info->taos, buffer, callBack, tb_info); + taos_query_a(winfo->taos, buffer, callBack, winfo); + free(buffer); + free(data); taos_free_result(res); } diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 4bc840f026d41dc4fe4e8d61c84d53140501bff6..6d3061c4269f54f0075a07ee75115f43e856479b 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -89,7 +89,7 @@ typedef struct STableObj { int8_t type; } STableObj; -typedef struct SSuperTableObj { +typedef struct SSTableObj { STableObj info; int8_t reserved0[9]; // for fill struct STableObj to 4byte align int16_t nextColId; @@ -104,7 +104,7 @@ typedef struct SSuperTableObj { int32_t numOfTables; SSchema * schema; void * vgHash; -} SSuperTableObj; +} SSTableObj; typedef struct { STableObj info; @@ -122,8 +122,8 @@ typedef struct { int32_t refCount; char* sql; //used by normal table SSchema* schema; //used by normal table - SSuperTableObj *superTable; -} SChildTableObj; + SSTableObj*superTable; +} SCTableObj; typedef struct { int32_t dnodeId; @@ -172,7 +172,8 @@ typedef struct { int8_t walLevel; int8_t replications; int8_t quorum; - int8_t reserved[12]; + int8_t update; + int8_t reserved[11]; } SDbCfg; typedef struct SDbObj { diff --git a/src/mnode/inc/mnodeProfile.h b/src/mnode/inc/mnodeProfile.h index 1e5b1c0f9c7c1c220a88339f00d46538c78cf938..ee57c5da1f66b620e8624a74ba774643c589e818 100644 --- a/src/mnode/inc/mnodeProfile.h +++ b/src/mnode/inc/mnodeProfile.h @@ -45,7 +45,7 @@ void mnodeCleanupProfile(); SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port, int32_t pid, const char* app); SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t port); void mnodeReleaseConn(SConnObj *pConn); -int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg); +int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeTable.h b/src/mnode/inc/mnodeTable.h index ed0dbe4ecfc4b344052c70487c3b740d81eaa0f3..7c0077aa609843aa99e9f5114573aea8f7446fc1 100644 --- a/src/mnode/inc/mnodeTable.h +++ b/src/mnode/inc/mnodeTable.h @@ -29,8 +29,8 @@ int64_t mnodeGetChildTableNum(); void * mnodeGetTable(char *tableId); void mnodeIncTableRef(void *pTable); void mnodeDecTableRef(void *pTable); -void * mnodeGetNextChildTable(void *pIter, SChildTableObj **pTable); -void * mnodeGetNextSuperTable(void *pIter, SSuperTableObj **pTable); +void * mnodeGetNextChildTable(void *pIter, SCTableObj **pTable); +void * mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable); void mnodeDropAllChildTables(SDbObj *pDropDb); void mnodeDropAllSuperTables(SDbObj *pDropDb); void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup); diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 7aa662b81cc08c90d6453ff5c73191ad81416073..0e6d9dfde463dc50810654921b2006fda212e496 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -43,8 +43,8 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle); void mnodeAlterVgroup(SVgObj *pVgroup, void *ahandle); int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_t *sid); -void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable); -void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable); +void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable); +void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable); void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle); void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); void mnodeSendAlterVgroupMsg(SVgObj *pVgroup); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index e161940a2b1a07572d9beff5222b8eb8d3829075..365cf656de283a53899ccd866682e8d0a06caaf8 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -34,7 +34,7 @@ static int32_t mnodeCreateRootAcct(); static int32_t mnodeAcctActionDestroy(SSdbOper *pOper) { SAcctObj *pAcct = pOper->pObj; pthread_mutex_destroy(&pAcct->mutex); - taosTFree(pOper->pObj); + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 35b6a67ab24d5ee50a4f4da7bb902ab0d7232b7e..98587cf53d673b847a321ded9c369e2085195081 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -33,7 +33,7 @@ static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void * static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeClusterActionDestroy(SSdbOper *pOper) { - taosTFree(pOper->pObj); + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 8d7c267ab7cb328bb187518a88be3d344b0d53e2..3b3a21aeb18f08d9af20a17ee78dcba124bd00d7 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -41,7 +41,7 @@ static void * tsDbSdb = NULL; static int32_t tsDbUpdateSize; -static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg); +static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *pMsg); static int32_t mnodeDropDb(SMnodeMsg *newMsg); static int32_t mnodeSetDbDropping(SDbObj *pDb); static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); @@ -52,8 +52,8 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg); static void mnodeDestroyDb(SDbObj *pDb) { pthread_mutex_destroy(&pDb->mutex); - taosTFree(pDb->vgList); - taosTFree(pDb); + tfree(pDb->vgList); + tfree(pDb); } static int32_t mnodeDbActionDestroy(SSdbOper *pOper) { @@ -319,6 +319,11 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) { } #endif + if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) { + mError("invalid db option update:%d valid range: [%d, %d]", pCfg->update, TSDB_MIN_DB_UPDATE, TSDB_MAX_DB_UPDATE); + return TSDB_CODE_MND_INVALID_DB_OPTION; + } + return TSDB_CODE_SUCCESS; } @@ -339,6 +344,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL; if (pCfg->replications < 0) pCfg->replications = tsReplications; if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; + if (pCfg->update < 0) pCfg->update = tsUpdate; } static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) { @@ -352,7 +358,7 @@ static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) { return code; } -static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg) { +static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *pMsg) { int32_t code = acctCheck(pAcct, ACCT_GRANT_DB); if (code != 0) return code; @@ -391,14 +397,15 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg .compression = pCreate->compression, .walLevel = pCreate->walLevel, .replications = pCreate->replications, - .quorum = pCreate->quorum + .quorum = pCreate->quorum, + .update = pCreate->update }; mnodeSetDefaultDbCfg(&pDb->cfg); code = mnodeCheckDbCfg(&pDb->cfg); if (code != TSDB_CODE_SUCCESS) { - taosTFree(pDb); + tfree(pDb); return code; } @@ -610,6 +617,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "update"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); @@ -749,6 +762,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2); cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.update; + cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; if (pDb->status == TSDB_DB_STATUS_READY) { const char *src = "ready"; @@ -805,7 +822,7 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb) { } static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) { - SCMCreateDbMsg *pCreate = pMsg->rpcMsg.pCont; + SCreateDbMsg *pCreate = pMsg->rpcMsg.pCont; pCreate->maxTables = htonl(pCreate->maxTables); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); pCreate->totalBlocks = htonl(pCreate->totalBlocks); @@ -830,7 +847,7 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) { return code; } -static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { +static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) { SDbCfg newCfg = pDb->cfg; int32_t maxTables = htonl(pAlter->maxTables); int32_t cacheBlockSize = htonl(pAlter->cacheBlockSize); @@ -848,6 +865,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { int8_t replications = pAlter->replications; int8_t quorum = pAlter->quorum; int8_t precision = pAlter->precision; + int8_t update = pAlter->update; terrno = TSDB_CODE_SUCCESS; @@ -950,6 +968,16 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { newCfg.quorum = quorum; } + if (update >= 0 && update != pDb->cfg.update) { +#if 0 + mDebug("db:%s, update:%d change to %d", pDb->name, pDb->cfg.update, update); + newCfg.update = update; +#else + mError("db:%s, can't alter update option", pDb->name); + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; +#endif + } + return newCfg; } @@ -977,7 +1005,7 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) { +static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) { SDbCfg newCfg = mnodeGetAlterDbOption(pDb, pAlter); if (terrno != TSDB_CODE_SUCCESS) { return terrno; @@ -1009,7 +1037,7 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) { } static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) { - SCMAlterDbMsg *pAlter = pMsg->rpcMsg.pCont; + SAlterDbMsg *pAlter = pMsg->rpcMsg.pCont; mDebug("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->rpcMsg.handle); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pAlter->db); @@ -1060,7 +1088,7 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) { } static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) { - SCMDropDbMsg *pDrop = pMsg->rpcMsg.pCont; + SDropDbMsg *pDrop = pMsg->rpcMsg.pCont; mDebug("db:%s, drop db msg is received from thandle:%p", pDrop->db, pMsg->rpcMsg.handle); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pDrop->db); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 1cd861e22303d5f6f0dcb75c7ef83cf66696578f..7e34e0837389e8a288ba70a2b18d4627f91219ae 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -39,11 +39,15 @@ #include "mnodeCluster.h" int32_t tsAccessSquence = 0; -static void *tsDnodeSdb = NULL; +static void * tsDnodeSdb = NULL; static int32_t tsDnodeUpdateSize = 0; extern void * tsMnodeSdb; extern void * tsVgroupSdb; +static SDnodeEps*tsDnodeEps; +static int32_t tsDnodeEpsSize; +static pthread_mutex_t tsDnodeEpsMutex; + static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg); static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg); @@ -59,6 +63,7 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole); +static void mnodeUpdateDnodeEps(); static char* offlineReason[] = { "", @@ -83,7 +88,7 @@ static char* offlineReason[] = { }; static int32_t mnodeDnodeActionDestroy(SSdbOper *pOper) { - taosTFree(pOper->pObj); + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -95,6 +100,9 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) { pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED; } + dnodeUpdateEp(pDnode->dnodeId, pDnode->dnodeEp, pDnode->dnodeFqdn, &pDnode->dnodePort); + mnodeUpdateDnodeEps(); + mInfo("dnode:%d, fqdn:%s ep:%s port:%d, do insert action", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort); return TSDB_CODE_SUCCESS; } @@ -107,6 +115,7 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) { #endif mnodeDropMnodeLocal(pDnode->dnodeId); balanceAsyncNotify(); + mnodeUpdateDnodeEps(); mDebug("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId); return TSDB_CODE_SUCCESS; @@ -121,6 +130,7 @@ static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) { } mnodeDecDnodeRef(pDnode); + mnodeUpdateDnodeEps(); return TSDB_CODE_SUCCESS; } @@ -152,12 +162,14 @@ static int32_t mnodeDnodeActionRestored() { } } + mnodeUpdateDnodeEps(); return TSDB_CODE_SUCCESS; } int32_t mnodeInitDnodes() { SDnodeObj tObj; tsDnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + pthread_mutex_init(&tsDnodeEpsMutex, NULL); SSdbTableDesc tableDesc = { .tableId = SDB_TABLE_DNODE, @@ -201,6 +213,9 @@ int32_t mnodeInitDnodes() { void mnodeCleanupDnodes() { sdbCloseTable(tsDnodeSdb); + pthread_mutex_destroy(&tsDnodeEpsMutex); + free(tsDnodeEps); + tsDnodeEps = NULL; tsDnodeSdb = NULL; } @@ -299,7 +314,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_NO_RIGHTS; } - SCMCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont; + SCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont; if (pCmCfgDnode->ep[0] == 0) { tstrncpy(pCmCfgDnode->ep, tsLocalEp, TSDB_EP_LEN); } @@ -334,7 +349,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { mnodeDecDnodeRef(pDnode); return code; } else { - SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg)); + SCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SCfgDnodeMsg)); strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep); strcpy(pMdCfgDnode->config, pCmCfgDnode->config); @@ -343,7 +358,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { .code = 0, .msgType = TSDB_MSG_TYPE_MD_CONFIG_DNODE, .pCont = pMdCfgDnode, - .contLen = sizeof(SMDCfgDnodeMsg) + .contLen = sizeof(SCfgDnodeMsg) }; mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user); @@ -418,9 +433,54 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { return 0; } +static int32_t mnodeGetDnodeEpsSize() { + pthread_mutex_lock(&tsDnodeEpsMutex); + int32_t size = tsDnodeEpsSize; + pthread_mutex_unlock(&tsDnodeEpsMutex); + return size; +} + +static void mnodeGetDnodeEpsData(SDnodeEps *pEps, int32_t epsSize) { + pthread_mutex_lock(&tsDnodeEpsMutex); + if (epsSize == tsDnodeEpsSize) { + memcpy(pEps, tsDnodeEps, tsDnodeEpsSize); + } + pthread_mutex_unlock(&tsDnodeEpsMutex); +} + +static void mnodeUpdateDnodeEps() { + pthread_mutex_lock(&tsDnodeEpsMutex); + + int32_t totalDnodes = mnodeGetDnodesNum(); + tsDnodeEpsSize = sizeof(SDnodeEps) + totalDnodes * sizeof(SDnodeEp); + free(tsDnodeEps); + tsDnodeEps = calloc(1, tsDnodeEpsSize); + tsDnodeEps->dnodeNum = htonl(totalDnodes); + + SDnodeObj *pDnode = NULL; + void * pIter = NULL; + int32_t dnodesNum = 0; + + while (1) { + pIter = mnodeGetNextDnode(pIter, &pDnode); + if (pDnode == NULL) break; + if (dnodesNum >= totalDnodes) break; + + SDnodeEp *pEp = &tsDnodeEps->dnodeEps[dnodesNum]; + dnodesNum++; + pEp->dnodeId = htonl(pDnode->dnodeId); + pEp->dnodePort = htons(pDnode->dnodePort); + tstrncpy(pEp->dnodeFqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); + mnodeDecDnodeRef(pDnode); + } + + sdbFreeIter(pIter); + pthread_mutex_unlock(&tsDnodeEpsMutex); +} + static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { SDnodeObj *pDnode = NULL; - SDMStatusMsg *pStatus = pMsg->rpcMsg.pCont; + SStatusMsg *pStatus = pMsg->rpcMsg.pCont; pStatus->dnodeId = htonl(pStatus->dnodeId); pStatus->moduleStatus = htonl(pStatus->moduleStatus); pStatus->lastReboot = htonl(pStatus->lastReboot); @@ -477,8 +537,11 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { } int32_t openVnodes = htons(pStatus->openVnodes); - int32_t contLen = sizeof(SDMStatusRsp) + openVnodes * sizeof(SDMVgroupAccess); - SDMStatusRsp *pRsp = rpcMallocCont(contLen); + int32_t epsSize = mnodeGetDnodeEpsSize(); + int32_t vgAccessSize = openVnodes * sizeof(SVgroupAccess); + int32_t contLen = sizeof(SStatusRsp) + vgAccessSize + epsSize; + + SStatusRsp *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { mnodeDecDnodeRef(pDnode); return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -488,8 +551,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt); pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes); tstrncpy(pRsp->dnodeCfg.clusterId, mnodeGetClusterId(), TSDB_CLUSTER_ID_LEN); - SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp)); - + SVgroupAccess *pAccess = (SVgroupAccess *)((char *)pRsp + sizeof(SStatusRsp)); + for (int32_t j = 0; j < openVnodes; ++j) { SVnodeLoad *pVload = &pStatus->load[j]; pVload->vgId = htonl(pVload->vgId); @@ -521,7 +584,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT; } - mDebug("dnode:%d, from offline to online", pDnode->dnodeId); + mInfo("dnode:%d, from offline to online", pDnode->dnodeId); pDnode->status = TAOS_DN_STATUS_READY; pDnode->offlineReason = TAOS_DN_OFF_ONLINE; balanceSyncNotify(); @@ -539,6 +602,9 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { mnodeDecDnodeRef(pDnode); + SDnodeEps *pEps = (SDnodeEps *)((char *)pRsp + sizeof(SStatusRsp) + vgAccessSize); + mnodeGetDnodeEpsData(pEps, epsSize); + pMsg->rpcRsp.len = contLen; pMsg->rpcRsp.rsp = pRsp; @@ -589,7 +655,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { int32_t code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { int dnodeId = pDnode->dnodeId; - taosTFree(pDnode); + tfree(pDnode); mError("failed to create dnode:%d, reason:%s", dnodeId, tstrerror(code)); } else { mLInfo("dnode:%d is created", pDnode->dnodeId); @@ -645,7 +711,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) { } static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) { - SCMCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont; + SCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont; if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) { return TSDB_CODE_MND_NO_RIGHTS; @@ -655,7 +721,7 @@ static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) { } static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg) { - SCMDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont; + SDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont; if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) { return TSDB_CODE_MND_NO_RIGHTS; diff --git a/src/mnode/src/mnodeInt.c b/src/mnode/src/mnodeInt.c index fb1b8741a9f8895c579ad0e549dc8f3c15f7618c..8431baf4f455a2c577321ec18ff3bfcd7fbe5d46 100644 --- a/src/mnode/src/mnodeInt.c +++ b/src/mnode/src/mnodeInt.c @@ -18,7 +18,7 @@ #include "taosmsg.h" #include "taoserror.h" #include "trpc.h" -#include "tcache.h" +#include "tqueue.h" #include "mnode.h" #include "dnode.h" #include "mnodeDef.h" @@ -34,8 +34,16 @@ #include "mnodeUser.h" #include "mnodeVgroup.h" -void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg) { - pMsg->rpcMsg = *rpcMsg; +void *mnodeCreateMsg(SRpcMsg *pRpcMsg) { + int32_t size = sizeof(SMnodeMsg) + pRpcMsg->contLen; + SMnodeMsg *pMsg = taosAllocateQitem(size); + + pMsg->rpcMsg = *pRpcMsg; + pMsg->rpcMsg.pCont = pMsg->pCont; + pMsg->incomingTs = taosGetTimestampSec(); + memcpy(pMsg->pCont, pRpcMsg->pCont, pRpcMsg->contLen); + + return pMsg; } int32_t mnodeInitMsg(SMnodeMsg *pMsg) { @@ -54,7 +62,9 @@ int32_t mnodeInitMsg(SMnodeMsg *pMsg) { void mnodeCleanupMsg(SMnodeMsg *pMsg) { if (pMsg != NULL) { - if (pMsg->rpcMsg.pCont) rpcFreeCont(pMsg->rpcMsg.pCont); + if (pMsg->rpcMsg.pCont != pMsg->pCont) { + tfree(pMsg->rpcMsg.pCont); + } if (pMsg->pUser) mnodeDecUserRef(pMsg->pUser); if (pMsg->pDb) mnodeDecDbRef(pMsg->pDb); if (pMsg->pVgroup) mnodeDecVgroupRef(pMsg->pVgroup); diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 2bb8a810566a676a4ae991bc79e5eb78234747c7..1f5ad42bdead75064c60f8626ce4916e7b08fd9f 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -96,9 +96,9 @@ int32_t mnodeStartSystem() { return -1; } - dnodeAllocateMnodeWqueue(); - dnodeAllocateMnodeRqueue(); - dnodeAllocateMnodePqueue(); + dnodeAllocMWritequeue(); + dnodeAllocMReadQueue(); + dnodeAllocateMPeerQueue(); if (mnodeInitComponents() != 0) { return -1; @@ -127,9 +127,9 @@ void mnodeCleanupSystem() { mInfo("starting to clean up mnode"); tsMgmtIsRunning = false; - dnodeFreeMnodeWqueue(); - dnodeFreeMnodeRqueue(); - dnodeFreeMnodePqueue(); + dnodeFreeMWritequeue(); + dnodeFreeMReadQueue(); + dnodeFreeMPeerQueue(); mnodeCleanupTimer(); mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 89b2f50b731c90c1261957da93623ba16cebf2ab..092f246c139533f806ce1d311162e0516088d879 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -38,7 +38,7 @@ static void * tsMnodeSdb = NULL; static int32_t tsMnodeUpdateSize = 0; static SRpcEpSet tsMnodeEpSetForShell; static SRpcEpSet tsMnodeEpSetForPeer; -static SDMMnodeInfos tsMnodeInfos; +static SMnodeInfos tsMnodeInfos; static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); @@ -59,7 +59,7 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo #endif static int32_t mnodeMnodeActionDestroy(SSdbOper *pOper) { - taosTFree(pOper->pObj); + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -70,8 +70,9 @@ static int32_t mnodeMnodeActionInsert(SSdbOper *pOper) { pDnode->isMgmt = true; mnodeDecDnodeRef(pDnode); - - mInfo("mnode:%d, fqdn:%s ep:%s port:%d, do insert action", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort); + + mInfo("mnode:%d, fqdn:%s ep:%s port:%u, do insert action", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, + pDnode->dnodePort); return TSDB_CODE_SUCCESS; } @@ -202,7 +203,7 @@ void mnodeUpdateMnodeEpSet() { memset(&tsMnodeEpSetForShell, 0, sizeof(SRpcEpSet)); memset(&tsMnodeEpSetForPeer, 0, sizeof(SRpcEpSet)); - memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos)); + memset(&tsMnodeInfos, 0, sizeof(SMnodeInfos)); int32_t index = 0; void * pIter = NULL; @@ -221,8 +222,8 @@ void mnodeUpdateMnodeEpSet() { tsMnodeEpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE); mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForPeer.fqdn[index], htons(tsMnodeEpSetForPeer.port[index])); - tsMnodeInfos.nodeInfos[index].nodeId = htonl(pMnode->mnodeId); - strcpy(tsMnodeInfos.nodeInfos[index].nodeEp, pDnode->dnodeEp); + tsMnodeInfos.mnodeInfos[index].mnodeId = htonl(pMnode->mnodeId); + strcpy(tsMnodeInfos.mnodeInfos[index].mnodeEp, pDnode->dnodeEp); if (pMnode->role == TAOS_SYNC_ROLE_MASTER) { tsMnodeEpSetForShell.inUse = index; @@ -238,7 +239,7 @@ void mnodeUpdateMnodeEpSet() { mnodeDecMnodeRef(pMnode); } - tsMnodeInfos.nodeNum = index; + tsMnodeInfos.mnodeNum = index; tsMnodeEpSetForShell.numOfEps = index; tsMnodeEpSetForPeer.numOfEps = index; @@ -260,19 +261,19 @@ void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) { } char* mnodeGetMnodeMasterEp() { - return tsMnodeInfos.nodeInfos[tsMnodeInfos.inUse].nodeEp; + return tsMnodeInfos.mnodeInfos[tsMnodeInfos.inUse].mnodeEp; } void mnodeGetMnodeInfos(void *mnodeInfos) { mnodeMnodeRdLock(); - *(SDMMnodeInfos *)mnodeInfos = tsMnodeInfos; + *(SMnodeInfos *)mnodeInfos = tsMnodeInfos; mnodeMnodeUnLock(); } static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) { mDebug("dnode:%d, send create mnode msg to dnode %s", dnodeId, dnodeEp); - SMDCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SMDCreateMnodeMsg)); + SCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SCreateMnodeMsg)); if (pCreate == NULL) { return TSDB_CODE_MND_OUT_OF_MEMORY; } else { @@ -280,21 +281,21 @@ static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) { tstrncpy(pCreate->dnodeEp, dnodeEp, sizeof(pCreate->dnodeEp)); pCreate->mnodes = tsMnodeInfos; bool found = false; - for (int i = 0; i < pCreate->mnodes.nodeNum; ++i) { - if (pCreate->mnodes.nodeInfos[i].nodeId == htonl(dnodeId)) { + for (int i = 0; i < pCreate->mnodes.mnodeNum; ++i) { + if (pCreate->mnodes.mnodeInfos[i].mnodeId == htonl(dnodeId)) { found = true; } } if (!found) { - pCreate->mnodes.nodeInfos[pCreate->mnodes.nodeNum].nodeId = htonl(dnodeId); - tstrncpy(pCreate->mnodes.nodeInfos[pCreate->mnodes.nodeNum].nodeEp, dnodeEp, sizeof(pCreate->dnodeEp)); - pCreate->mnodes.nodeNum++; + pCreate->mnodes.mnodeInfos[pCreate->mnodes.mnodeNum].mnodeId = htonl(dnodeId); + tstrncpy(pCreate->mnodes.mnodeInfos[pCreate->mnodes.mnodeNum].mnodeEp, dnodeEp, sizeof(pCreate->dnodeEp)); + pCreate->mnodes.mnodeNum++; } } SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pCreate; - rpcMsg.contLen = sizeof(SMDCreateMnodeMsg); + rpcMsg.contLen = sizeof(SCreateMnodeMsg); rpcMsg.msgType = TSDB_MSG_TYPE_MD_CREATE_MNODE; SRpcMsg rpcRsp = {0}; @@ -341,14 +342,14 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { } if (code != TSDB_CODE_SUCCESS) { - taosTFree(pMnode); + tfree(pMnode); return; } code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("dnode:%d, failed to create mnode, ep:%s reason:%s", dnodeId, dnodeEp, tstrerror(code)); - taosTFree(pMnode); + tfree(pMnode); } } diff --git a/src/mnode/src/mnodePeer.c b/src/mnode/src/mnodePeer.c index 2a04f541c51483ccba93369a65061507f83ead23..e43f8c1b786e6126253cdb40c0184bc34dc4dab2 100644 --- a/src/mnode/src/mnodePeer.c +++ b/src/mnode/src/mnodePeer.c @@ -63,9 +63,9 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) { for (int32_t i = 0; i < epSet->numOfEps; ++i) { if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) { epSet->inUse = (i + 1) % epSet->numOfEps; - mDebug("mnode index:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); + mDebug("mpeer:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); } else { - mDebug("mnode index:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); + mDebug("mpeer:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); } } diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index f8f99e22c6f437b1eece0d704c8c4551bc434110..12ac64854c6834ca0cd1509f9bfedd7b65a0900c 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -131,8 +131,8 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po static void mnodeFreeConn(void *data) { SConnObj *pConn = data; - taosTFree(pConn->pQueries); - taosTFree(pConn->pStreams); + tfree(pConn->pQueries); + tfree(pConn->pStreams); mDebug("connId:%d, is destroyed", pConn->connId); } @@ -182,7 +182,7 @@ static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC // app name pShow->bytes[cols] = TSDB_APPNAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "app_name"); + strcpy(pSchema[cols].name, "program"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -280,7 +280,7 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi } // not thread safe, need optimized -int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg) { +int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg) { pConn->numOfQueries = htonl(pHBMsg->numOfQueries); if (pConn->numOfQueries > 0) { if (pConn->pQueries == NULL) { @@ -561,7 +561,7 @@ static int32_t mnodeProcessKillQueryMsg(SMnodeMsg *pMsg) { SUserObj *pUser = pMsg->pUser; if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS; - SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont; + SKillQueryMsg *pKill = pMsg->rpcMsg.pCont; mInfo("kill query msg is received, queryId:%s", pKill->queryId); const char delim = ':'; @@ -592,7 +592,7 @@ static int32_t mnodeProcessKillStreamMsg(SMnodeMsg *pMsg) { SUserObj *pUser = pMsg->pUser; if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS; - SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont; + SKillQueryMsg *pKill = pMsg->rpcMsg.pCont; mInfo("kill stream msg is received, streamId:%s", pKill->queryId); const char delim = ':'; @@ -623,7 +623,7 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg) { SUserObj *pUser = pMsg->pUser; if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS; - SCMKillConnMsg *pKill = pMsg->rpcMsg.pCont; + SKillConnMsg *pKill = pMsg->rpcMsg.pCont; int32_t connId = atoi(pKill->queryId); SConnObj *pConn = taosCacheAcquireByKey(tsMnodeConnCache, &connId, sizeof(int32_t)); if (pConn == NULL) { diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 14558485aa743a42bee016226dfc2e16a4f81d0c..12d977f32b224f9ce85db93fb4afc3d6f0e550f8 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -72,7 +72,7 @@ typedef struct { ESyncRole role; ESdbStatus status; int64_t version; - void * sync; + int64_t sync; void * wal; SSyncCfg cfg; int32_t numOfTables; @@ -98,8 +98,8 @@ static taos_qall tsSdbWriteQall; static taos_queue tsSdbWriteQueue; static SSdbWriteWorkerPool tsSdbPool; -static int sdbWrite(void *param, void *data, int type); -static int sdbWriteToQueue(void *param, void *data, int type); +static int32_t sdbWrite(void *param, void *data, int32_t type, void *pMsg); +static int32_t sdbWriteToQueue(void *param, void *data, int32_t type, void *pMsg); static void * sdbWorkerFp(void *param); static int32_t sdbInitWriteWorker(); static void sdbCleanupWriteWorker(); @@ -175,7 +175,7 @@ static void *sdbGetTableFromId(int32_t tableId) { } static int32_t sdbInitWal() { - SWalCfg walCfg = {.walLevel = 2, .wals = 2, .keep = 1, .fsyncPeriod = 0}; + SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0}; char temp[TSDB_FILENAME_LEN]; sprintf(temp, "%s/wal", tsMnodeDir); tsSdbObj.wal = walOpen(temp, &walCfg); @@ -212,7 +212,7 @@ static void sdbRestoreTables() { } void sdbUpdateMnodeRoles() { - if (tsSdbObj.sync == NULL) return; + if (tsSdbObj.sync <= 0) return; SNodesRole roles = {0}; syncGetNodesRole(tsSdbObj.sync, &roles); @@ -237,8 +237,8 @@ static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint3 return 0; } -static int sdbGetWalInfo(void *ahandle, char *name, uint32_t *index) { - return walGetWalFile(tsSdbObj.wal, name, index); +static int32_t sdbGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) { + return walGetWalFile(tsSdbObj.wal, fileName, fileId); } static void sdbNotifyRole(void *ahandle, int8_t role) { @@ -281,14 +281,21 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { ((SSdbTable *)pOper->table)->tableName, pOper->pObj, sdbGetKeyStr(pOper->table, pHead->cont), pHead->version, action, tstrerror(pOper->retCode)); if (action == SDB_ACTION_INSERT) { - sdbDeleteHash(pOper->table, pOper); + // It's better to create a table in two stages, create it first and then set it success + //sdbDeleteHash(pOper->table, pOper); + SSdbOper oper = { + .type = SDB_OPER_GLOBAL, + .table = pOper->table, + .pObj = pOper->pObj + }; + sdbDeleteRow(&oper); } } if (pOper->writeCb != NULL) { pOper->retCode = (*pOper->writeCb)(pMsg, pOper->retCode); } - dnodeSendRpcMnodeWriteRsp(pMsg, pOper->retCode); + dnodeSendRpcMWriteRsp(pMsg, pOper->retCode); // if ahandle, means this func is called by sdb write if (ahandle == NULL) { @@ -305,7 +312,7 @@ void sdbUpdateAsync() { } void sdbUpdateSync(void *pMnodes) { - SDMMnodeInfos *mnodes = pMnodes; + SMnodeInfos *mnodes = pMnodes; if (!mnodeIsRunning()) { mDebug("mnode not start yet, update sync config later"); return; @@ -339,10 +346,10 @@ void sdbUpdateSync(void *pMnodes) { syncCfg.replica = index; mDebug("mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica); } else { - for (index = 0; index < mnodes->nodeNum; ++index) { - SDMMnodeInfo *node = &mnodes->nodeInfos[index]; - syncCfg.nodeInfo[index].nodeId = node->nodeId; - taosGetFqdnPortFromEp(node->nodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort); + for (index = 0; index < mnodes->mnodeNum; ++index) { + SMnodeInfo *node = &mnodes->mnodeInfos[index]; + syncCfg.nodeInfo[index].nodeId = node->mnodeId; + taosGetFqdnPortFromEp(node->mnodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort); syncCfg.nodeInfo[index].nodePort += TSDB_PORT_SYNC; } syncCfg.replica = index; @@ -426,7 +433,7 @@ void sdbCleanUp() { if (tsSdbObj.sync) { syncStop(tsSdbObj.sync); - tsSdbObj.sync = NULL; + tsSdbObj.sync = -1; } if (tsSdbObj.wal) { @@ -568,7 +575,7 @@ static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int sdbWrite(void *param, void *data, int type) { +static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) { SSdbOper *pOper = param; SWalHead *pHead = data; int32_t tableId = pHead->msgType / 10; @@ -977,7 +984,7 @@ void sdbCleanupWriteWorker() { } sdbFreeWritequeue(); - taosTFree(tsSdbPool.writeWorker); + tfree(tsSdbPool.writeWorker); mInfo("sdb write is closed"); } @@ -1033,13 +1040,13 @@ void sdbFreeWritequeue() { tsSdbWriteQueue = NULL; } -int sdbWriteToQueue(void *param, void *data, int type) { +int32_t sdbWriteToQueue(void *param, void *data, int32_t qtype, void *pMsg) { SWalHead *pHead = data; - int size = sizeof(SWalHead) + pHead->len; - SWalHead *pWal = (SWalHead *)taosAllocateQitem(size); + int32_t size = sizeof(SWalHead) + pHead->len; + SWalHead *pWal = taosAllocateQitem(size); memcpy(pWal, pHead, size); - taosWriteQitem(tsSdbWriteQueue, type, pWal); + taosWriteQitem(tsSdbWriteQueue, qtype, pWal); return 0; } @@ -1074,7 +1081,7 @@ static void *sdbWorkerFp(void *param) { pOper = NULL; } - int32_t code = sdbWrite(pOper, pHead, type); + int32_t code = sdbWrite(pOper, pHead, type, NULL); if (code > 0) code = 0; if (pOper) { pOper->retCode = code; @@ -1083,7 +1090,7 @@ static void *sdbWorkerFp(void *param) { } } - walFsync(tsSdbObj.wal); + walFsync(tsSdbObj.wal, true); // browse all items, and process them one by one taosResetQitems(tsSdbWriteQall); diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 80909e99aec6d752d35042ca2d761a6e8b923441..0d53fa9617b500d61457889e2228e5e756a244ae 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -110,7 +110,7 @@ static char *mnodeGetShowType(int32_t showType) { } static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) { - SCMShowMsg *pShowMsg = pMsg->rpcMsg.pCont; + SShowMsg *pShowMsg = pMsg->rpcMsg.pCont; if (pShowMsg->type >= TSDB_MGMT_TABLE_MAX) { return TSDB_CODE_MND_INVALID_MSG_TYPE; } @@ -132,8 +132,8 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_OUT_OF_MEMORY; } - int32_t size = sizeof(SCMShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE; - SCMShowRsp *pShowRsp = rpcMallocCont(size); + int32_t size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE; + SShowRsp *pShowRsp = rpcMallocCont(size); if (pShowRsp == NULL) { mnodeReleaseShowObj(pShow, true); return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -146,7 +146,7 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) { if (code == TSDB_CODE_SUCCESS) { pMsg->rpcRsp.rsp = pShowRsp; - pMsg->rpcRsp.len = sizeof(SCMShowRsp) + sizeof(SSchema) * pShow->numOfColumns; + pMsg->rpcRsp.len = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns; mnodeReleaseShowObj(pShow, false); return TSDB_CODE_SUCCESS; } else { @@ -232,12 +232,17 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) { } static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { - SCMHeartBeatRsp *pHBRsp = (SCMHeartBeatRsp *) rpcMallocCont(sizeof(SCMHeartBeatRsp)); - if (pHBRsp == NULL) { + SHeartBeatRsp *pRsp = (SHeartBeatRsp *)rpcMallocCont(sizeof(SHeartBeatRsp)); + if (pRsp == NULL) { return TSDB_CODE_MND_OUT_OF_MEMORY; } - SCMHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont; + SHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont; + if (taosCheckVersion(pHBMsg->clientVer, version, 3) != TSDB_CODE_SUCCESS) { + rpcFreeCont(pRsp); + return TSDB_CODE_TSC_INVALID_VERSION; // todo change the error code + } + SRpcConnInfo connInfo = {0}; rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo); @@ -251,40 +256,40 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { if (pConn == NULL) { // do not close existing links, otherwise // mError("failed to create connId, close connect"); - // pHBRsp->killConnection = 1; + // pRsp->killConnection = 1; } else { - pHBRsp->connId = htonl(pConn->connId); + pRsp->connId = htonl(pConn->connId); mnodeSaveQueryStreamList(pConn, pHBMsg); if (pConn->killed != 0) { - pHBRsp->killConnection = 1; + pRsp->killConnection = 1; } if (pConn->streamId != 0) { - pHBRsp->streamId = htonl(pConn->streamId); + pRsp->streamId = htonl(pConn->streamId); pConn->streamId = 0; } if (pConn->queryId != 0) { - pHBRsp->queryId = htonl(pConn->queryId); + pRsp->queryId = htonl(pConn->queryId); pConn->queryId = 0; } } - pHBRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum()); - pHBRsp->totalDnodes = htonl(mnodeGetDnodesNum()); - mnodeGetMnodeEpSetForShell(&pHBRsp->epSet); + pRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum()); + pRsp->totalDnodes = htonl(mnodeGetDnodesNum()); + mnodeGetMnodeEpSetForShell(&pRsp->epSet); + + pMsg->rpcRsp.rsp = pRsp; + pMsg->rpcRsp.len = sizeof(SHeartBeatRsp); - pMsg->rpcRsp.rsp = pHBRsp; - pMsg->rpcRsp.len = sizeof(SCMHeartBeatRsp); - mnodeReleaseConn(pConn); return TSDB_CODE_SUCCESS; } static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { - SCMConnectMsg *pConnectMsg = pMsg->rpcMsg.pCont; - SCMConnectRsp *pConnectRsp = NULL; + SConnectMsg *pConnectMsg = pMsg->rpcMsg.pCont; + SConnectRsp *pConnectRsp = NULL; int32_t code = TSDB_CODE_SUCCESS; SRpcConnInfo connInfo = {0}; @@ -320,7 +325,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { mnodeDecDbRef(pDb); } - pConnectRsp = rpcMallocCont(sizeof(SCMConnectRsp)); + pConnectRsp = rpcMallocCont(sizeof(SConnectRsp)); if (pConnectRsp == NULL) { code = TSDB_CODE_MND_OUT_OF_MEMORY; goto connect_over; @@ -349,14 +354,14 @@ connect_over: } else { mLInfo("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); pMsg->rpcRsp.rsp = pConnectRsp; - pMsg->rpcRsp.len = sizeof(SCMConnectRsp); + pMsg->rpcRsp.len = sizeof(SConnectRsp); } return code; } static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) { - SCMUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont; + SUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont; int32_t code = TSDB_CODE_SUCCESS; if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pUseDbMsg->db); @@ -410,7 +415,7 @@ static void mnodeFreeShowObj(void *data) { sdbFreeIter(pShow->pIter); mDebug("%p, show is destroyed, data:%p index:%d", pShow, data, pShow->index); - taosTFree(pShow); + tfree(pShow); } static void mnodeReleaseShowObj(SShowObj *pShow, bool forceRemove) { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 82a062169a591d26d665437273183a4c5e1f27b4..2b5e6455c0ec731d0697f2b8f6b638a1ce139788 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -45,6 +45,10 @@ #include "mnodeRead.h" #include "mnodePeer.h" +#define ALTER_CTABLE_RETRY_TIMES 3 +#define CREATE_CTABLE_RETRY_TIMES 10 +#define CREATE_CTABLE_RETRY_SEC 14 + static void * tsChildTableSdb; static void * tsSuperTableSdb; static int32_t tsChildTableUpdateSize; @@ -52,9 +56,9 @@ static int32_t tsSuperTableUpdateSize; static void * mnodeGetChildTable(char *tableId); static void * mnodeGetSuperTable(char *tableId); static void * mnodeGetSuperTableByUid(uint64_t uid); -static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable); -static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable); -static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable); +static void mnodeDropAllChildTablesInStable(SSTableObj *pStable); +static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable); +static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable); static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); @@ -86,13 +90,13 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg); static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg); static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg); -static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName); +static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName); -static void mnodeDestroyChildTable(SChildTableObj *pTable) { - taosTFree(pTable->info.tableId); - taosTFree(pTable->schema); - taosTFree(pTable->sql); - taosTFree(pTable); +static void mnodeDestroyChildTable(SCTableObj *pTable) { + tfree(pTable->info.tableId); + tfree(pTable->schema); + tfree(pTable->sql); + tfree(pTable); } static int32_t mnodeChildTableActionDestroy(SSdbOper *pOper) { @@ -101,7 +105,7 @@ static int32_t mnodeChildTableActionDestroy(SSdbOper *pOper) { } static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) { - SChildTableObj *pTable = pOper->pObj; + SCTableObj *pTable = pOper->pObj; SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId); if (pVgroup == NULL) { @@ -150,7 +154,7 @@ static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) { } static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) { - SChildTableObj *pTable = pOper->pObj; + SCTableObj *pTable = pOper->pObj; if (pTable->vgId == 0) { return TSDB_CODE_MND_VGROUP_NOT_EXIST; } @@ -186,8 +190,8 @@ static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) { } static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) { - SChildTableObj *pNew = pOper->pObj; - SChildTableObj *pTable = mnodeGetChildTable(pNew->info.tableId); + SCTableObj *pNew = pOper->pObj; + SCTableObj *pTable = mnodeGetChildTable(pNew->info.tableId); if (pTable != pNew) { void *oldTableId = pTable->info.tableId; void *oldSql = pTable->sql; @@ -195,7 +199,7 @@ static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) { void *oldSTable = pTable->superTable; int32_t oldRefCount = pTable->refCount; - memcpy(pTable, pNew, sizeof(SChildTableObj)); + memcpy(pTable, pNew, sizeof(SCTableObj)); pTable->refCount = oldRefCount; pTable->sql = pNew->sql; @@ -213,7 +217,7 @@ static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) { } static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) { - SChildTableObj *pTable = pOper->pObj; + SCTableObj *pTable = pOper->pObj; assert(pTable != NULL && pOper->rowData != NULL); int32_t len = strlen(pTable->info.tableId); @@ -244,7 +248,7 @@ static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) { static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) { assert(pOper->rowData != NULL); - SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); + SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; int32_t len = strlen(pOper->rowData); @@ -284,7 +288,7 @@ static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) { static int32_t mnodeChildTableActionRestored() { void *pIter = NULL; - SChildTableObj *pTable = NULL; + SCTableObj *pTable = NULL; while (1) { pIter = mnodeGetNextChildTable(pIter, &pTable); @@ -323,7 +327,7 @@ static int32_t mnodeChildTableActionRestored() { } if (pTable->info.type == TSDB_CHILD_TABLE) { - SSuperTableObj *pSuperTable = mnodeGetSuperTableByUid(pTable->suid); + SSTableObj *pSuperTable = mnodeGetSuperTableByUid(pTable->suid); if (pSuperTable == NULL) { mError("ctable:%s, stable:%" PRIu64 " not exist", pTable->info.tableId, pTable->suid); pTable->vgId = 0; @@ -344,14 +348,14 @@ static int32_t mnodeChildTableActionRestored() { } static int32_t mnodeInitChildTables() { - SChildTableObj tObj; + SCTableObj tObj; tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; SSdbTableDesc tableDesc = { .tableId = SDB_TABLE_CTABLE, .tableName = "ctables", .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE, - .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE, + .maxRowSize = sizeof(SCTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_VAR_STRING, .insertFp = mnodeChildTableActionInsert, @@ -386,7 +390,7 @@ int64_t mnodeGetChildTableNum() { return sdbGetNumOfRows(tsChildTableSdb); } -static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) { +static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable) { atomic_add_fetch_32(&pStable->numOfTables, 1); if (pStable->vgHash == NULL) { @@ -402,7 +406,7 @@ static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCt } } -static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable) { +static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable) { atomic_sub_fetch_32(&pStable->numOfTables, 1); if (pStable->vgHash == NULL) return; @@ -416,14 +420,14 @@ static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj * mnodeDecVgroupRef(pVgroup); } -static void mnodeDestroySuperTable(SSuperTableObj *pStable) { +static void mnodeDestroySuperTable(SSTableObj *pStable) { if (pStable->vgHash != NULL) { taosHashCleanup(pStable->vgHash); pStable->vgHash = NULL; } - taosTFree(pStable->info.tableId); - taosTFree(pStable->schema); - taosTFree(pStable); + tfree(pStable->info.tableId); + tfree(pStable->schema); + tfree(pStable); } static int32_t mnodeSuperTableActionDestroy(SSdbOper *pOper) { @@ -432,7 +436,7 @@ static int32_t mnodeSuperTableActionDestroy(SSdbOper *pOper) { } static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) { - SSuperTableObj *pStable = pOper->pObj; + SSTableObj *pStable = pOper->pObj; SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); if (pDb != NULL && pDb->status == TSDB_DB_STATUS_READY) { mnodeAddSuperTableIntoDb(pDb); @@ -443,11 +447,11 @@ static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) { } static int32_t mnodeSuperTableActionDelete(SSdbOper *pOper) { - SSuperTableObj *pStable = pOper->pObj; + SSTableObj *pStable = pOper->pObj; SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); if (pDb != NULL) { mnodeRemoveSuperTableFromDb(pDb); - mnodeDropAllChildTablesInStable((SSuperTableObj *)pStable); + mnodeDropAllChildTablesInStable((SSTableObj *)pStable); } mnodeDecDbRef(pDb); @@ -455,8 +459,8 @@ static int32_t mnodeSuperTableActionDelete(SSdbOper *pOper) { } static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) { - SSuperTableObj *pNew = pOper->pObj; - SSuperTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId); + SSTableObj *pNew = pOper->pObj; + SSTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId); if (pTable != NULL && pTable != pNew) { void *oldTableId = pTable->info.tableId; void *oldSchema = pTable->schema; @@ -464,7 +468,7 @@ static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) { int32_t oldRefCount = pTable->refCount; int32_t oldNumOfTables = pTable->numOfTables; - memcpy(pTable, pNew, sizeof(SSuperTableObj)); + memcpy(pTable, pNew, sizeof(SSTableObj)); pTable->vgHash = oldVgHash; pTable->refCount = oldRefCount; @@ -480,7 +484,7 @@ static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) { } static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) { - SSuperTableObj *pStable = pOper->pObj; + SSTableObj *pStable = pOper->pObj; assert(pOper->pObj != NULL && pOper->rowData != NULL); int32_t len = strlen(pStable->info.tableId); @@ -504,7 +508,7 @@ static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) { static int32_t mnodeSuperTableActionDecode(SSdbOper *pOper) { assert(pOper->rowData != NULL); - SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj)); + SSTableObj *pStable = (SSTableObj *) calloc(1, sizeof(SSTableObj)); if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; int32_t len = strlen(pOper->rowData); @@ -537,14 +541,14 @@ static int32_t mnodeSuperTableActionRestored() { } static int32_t mnodeInitSuperTables() { - SSuperTableObj tObj; + SSTableObj tObj; tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; SSdbTableDesc tableDesc = { .tableId = SDB_TABLE_STABLE, .tableName = "stables", .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE, - .maxRowSize = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN, + .maxRowSize = sizeof(SSTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_VAR_STRING, .insertFp = mnodeSuperTableActionInsert, @@ -615,7 +619,7 @@ static void *mnodeGetSuperTable(char *tableId) { } static void *mnodeGetSuperTableByUid(uint64_t uid) { - SSuperTableObj *pStable = NULL; + SSTableObj *pStable = NULL; void *pIter = NULL; while (1) { @@ -647,11 +651,11 @@ void *mnodeGetTable(char *tableId) { return NULL; } -void *mnodeGetNextChildTable(void *pIter, SChildTableObj **pTable) { +void *mnodeGetNextChildTable(void *pIter, SCTableObj **pTable) { return sdbFetchRow(tsChildTableSdb, pIter, (void **)pTable); } -void *mnodeGetNextSuperTable(void *pIter, SSuperTableObj **pTable) { +void *mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable) { return sdbFetchRow(tsSuperTableSdb, pIter, (void **)pTable); } @@ -765,12 +769,12 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { } if (pMsg->pTable->type == TSDB_SUPER_TABLE) { - SSuperTableObj *pSTable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pSTable = (SSTableObj *)pMsg->pTable; mInfo("app:%p:%p, table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pSTable->uid, pSTable->numOfTables, (int32_t)taosHashGetSize(pSTable->vgHash)); return mnodeProcessDropSuperTableMsg(pMsg); } else { - SChildTableObj *pCTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pCTable = (SCTableObj *)pMsg->pTable; mInfo("app:%p:%p, table:%s, start to drop ctable, vgId:%d tid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pCTable->vgId, pCTable->tid, pCTable->uid); return mnodeProcessDropChildTableMsg(pMsg); @@ -778,7 +782,7 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { } static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) { - SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont; + STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; pInfo->createFlag = htons(pInfo->createFlag); mDebug("app:%p:%p, table:%s, table meta msg is received from thandle:%p, createFlag:%d", pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag); @@ -816,7 +820,7 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) { } static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pTable = (SSTableObj *)pMsg->pTable; assert(pTable); if (code == TSDB_CODE_SUCCESS) { @@ -835,7 +839,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; - SSuperTableObj * pStable = calloc(1, sizeof(SSuperTableObj)); + SSTableObj * pStable = calloc(1, sizeof(SSTableObj)); if (pStable == NULL) { mError("app:%p:%p, table:%s, failed to create, no enough memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -878,7 +882,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, - .rowSize = sizeof(SSuperTableObj) + schemaSize, + .rowSize = sizeof(SSTableObj) + schemaSize, .pMsg = pMsg, .writeCb = mnodeCreateSuperTableCb }; @@ -894,7 +898,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { } static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pTable = (SSTableObj *)pMsg->pTable; if (code != TSDB_CODE_SUCCESS) { mError("app:%p:%p, stable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); } else { @@ -907,7 +911,7 @@ static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) { SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash); while (taosHashIterNext(pIter)) { @@ -915,8 +919,8 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { SVgObj *pVgroup = mnodeGetVgroup(*pVgId); if (pVgroup == NULL) break; - SMDDropSTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropSTableMsg)); - pDrop->contLen = htonl(sizeof(SMDDropSTableMsg)); + SDropSTableMsg *pDrop = rpcMallocCont(sizeof(SDropSTableMsg)); + pDrop->contLen = htonl(sizeof(SDropSTableMsg)); pDrop->vgId = htonl(pVgroup->vgId); pDrop->uid = htobe64(pStable->uid); mnodeExtractTableName(pStable->info.tableId, pDrop->tableId); @@ -924,7 +928,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { mInfo("app:%p:%p, stable:%s, send drop stable msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, pVgroup->vgId); SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pVgroup); - SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; + SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; dnodeSendMsgToDnode(&epSet, &rpcMsg); mnodeDecVgroupRef(pVgroup); } @@ -950,7 +954,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { return code; } -static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) { +static int32_t mnodeFindSuperTableTagIndex(SSTableObj *pStable, const char *tagName) { SSchema *schema = (SSchema *) pStable->schema; for (int32_t tag = 0; tag < pStable->numOfTags; tag++) { if (strcasecmp(schema[pStable->numOfColumns + tag].name, tagName) == 0) { @@ -962,7 +966,7 @@ static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char * } static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; mLInfo("app:%p:%p, stable %s, add tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); @@ -970,7 +974,7 @@ static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { } static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t ntags) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) { mError("app:%p:%p, stable:%s, add tag, too many tags", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); return TSDB_CODE_MND_TOO_MANY_TAGS; @@ -1018,14 +1022,14 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t } static int32_t mnodeDropSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; mLInfo("app:%p:%p, stable %s, drop tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); return code; } static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableTagIndex(pStable, tagName); if (col < 0) { mError("app:%p:%p, stable:%s, drop tag, tag:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, @@ -1052,14 +1056,14 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) { } static int32_t mnodeModifySuperTableTagNameCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; mLInfo("app:%p:%p, stable %s, modify tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); return code; } static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, char *newTagName) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableTagIndex(pStable, oldTagName); if (col < 0) { mError("app:%p:%p, stable:%s, failed to modify table tag, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg, @@ -1095,7 +1099,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c return sdbUpdateRow(&oper); } -static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName) { +static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName) { SSchema *schema = (SSchema *) pStable->schema; for (int32_t col = 0; col < pStable->numOfColumns; col++) { if (strcasecmp(schema[col].name, colName) == 0) { @@ -1107,7 +1111,7 @@ static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *col } static int32_t mnodeAddSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; mLInfo("app:%p:%p, stable %s, add column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); return code; @@ -1115,7 +1119,7 @@ static int32_t mnodeAddSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32_t ncols) { SDbObj *pDb = pMsg->pDb; - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; if (ncols <= 0) { mError("app:%p:%p, stable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, ncols); return TSDB_CODE_MND_APP_ERROR; @@ -1170,7 +1174,7 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32 } static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; mLInfo("app:%p:%p, stable %s, delete column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); return code; @@ -1178,7 +1182,7 @@ static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { SDbObj *pDb = pMsg->pDb; - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableColumnIndex(pStable, colName); if (col <= 0) { mError("app:%p:%p, stable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, @@ -1215,14 +1219,14 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { } static int32_t mnodeChangeSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; mLInfo("app:%p:%p, stable %s, change column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); return code; } static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableColumnIndex(pStable, oldName); if (col < 0) { mError("app:%p:%p, stable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg, @@ -1321,7 +1325,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, int32_t numOfRows = 0; char * pWrite; int32_t cols = 0; - SSuperTableObj *pTable = NULL; + SSTableObj *pTable = NULL; char prefix[64] = {0}; int32_t prefixLen; @@ -1399,7 +1403,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void mnodeDropAllSuperTables(SDbObj *pDropDb) { void * pIter= NULL; int32_t numOfTables = 0; - SSuperTableObj *pTable = NULL; + SSTableObj *pTable = NULL; char prefix[64] = {0}; tstrncpy(prefix, pDropDb->name, 64); @@ -1430,7 +1434,7 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { mInfo("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } -static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) { +static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSTableObj *pTable) { int32_t numOfCols = pTable->numOfColumns + pTable->numOfTags; assert(numOfCols <= TSDB_MAX_COLUMNS); @@ -1446,7 +1450,7 @@ static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pT } static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { - SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pTable = (SSTableObj *)pMsg->pTable; STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)); if (pMeta == NULL) { return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -1472,32 +1476,32 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { } static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { - SCMSTableVgroupMsg *pInfo = pMsg->rpcMsg.pCont; + SSTableVgroupMsg *pInfo = pMsg->rpcMsg.pCont; int32_t numOfTable = htonl(pInfo->numOfTables); // reserve space - int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupMsg) + sizeof(SVgroupsMsg); + int32_t contLen = sizeof(SSTableVgroupRspMsg) + 32 * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg); for (int32_t i = 0; i < numOfTable; ++i) { - char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN) * i; - SSuperTableObj *pTable = mnodeGetSuperTable(stableName); + char *stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; + SSTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable != NULL && pTable->vgHash != NULL) { - contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupMsg) + sizeof(SVgroupsMsg)); - } - + contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg)); + } + mnodeDecTableRef(pTable); } - SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen); + SSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { return TSDB_CODE_MND_OUT_OF_MEMORY; } pRsp->numOfTables = 0; - char *msg = (char *)pRsp + sizeof(SCMSTableVgroupRspMsg); + char *msg = (char *)pRsp + sizeof(SSTableVgroupRspMsg); for (int32_t i = 0; i < numOfTable; ++i) { - char * stableName = (char *)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; - SSuperTableObj *pTable = mnodeGetSuperTable(stableName); + char * stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; + SSTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable == NULL) { mError("app:%p:%p, stable:%s, not exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, stableName); mnodeDecTableRef(pTable); @@ -1548,7 +1552,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { pVgroupMsg->numOfVgroups = htonl(vgSize); // one table is done, try the next table - msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SCMVgroupMsg); + msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SVgroupMsg); pRsp->numOfTables++; } } @@ -1569,7 +1573,7 @@ static void mnodeProcessDropSuperTableRsp(SRpcMsg *rpcMsg) { mInfo("drop stable rsp received, result:%s", tstrerror(rpcMsg->code)); } -static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) { +static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SCTableObj *pTable) { STagData * pTagData = NULL; int32_t tagDataLen = 0; int32_t totalCols = 0; @@ -1643,7 +1647,7 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableO } static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; assert(pTable); mDebug("app:%p:%p, table:%s, created in mnode, vgId:%d sid:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, @@ -1669,7 +1673,7 @@ static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) { } static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; assert(pTable); @@ -1679,12 +1683,12 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { pTable->info.tableId, pMsg->rpcMsg.handle); pMsg->retry = 0; - dnodeReprocessMnodeWriteMsg(pMsg); + dnodeReprocessMWriteMsg(pMsg); } else { mDebug("app:%p:%p, table:%s, created in dnode, thandle:%p", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pMsg->rpcMsg.handle); - dnodeSendRpcMnodeWriteRsp(pMsg, TSDB_CODE_SUCCESS); + dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS); } return TSDB_CODE_MND_ACTION_IN_PROGRESS; } else { @@ -1699,7 +1703,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { SVgObj *pVgroup = pMsg->pVgroup; SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; - SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); + SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) { mError("app:%p:%p, table:%s, failed to alloc memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -1842,7 +1846,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { } static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; mLInfo("app:%p:%p, ctable:%s, is dropped from sdb", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg)); @@ -1880,7 +1884,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) { if (code != TSDB_CODE_SUCCESS) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; mError("app:%p:%p, ctable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); return code; } @@ -1889,7 +1893,7 @@ static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) { } static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); if (pMsg->pVgroup == NULL) { mError("app:%p:%p, table:%s, failed to drop ctable, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg, @@ -1914,7 +1918,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { return code; } -static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) { +static int32_t mnodeFindNormalTableColumnIndex(SCTableObj *pTable, char *colName) { SSchema *schema = (SSchema *) pTable->schema; for (int32_t col = 0; col < pTable->numOfColumns; col++) { if (strcasecmp(schema[col].name, colName) == 0) { @@ -1926,7 +1930,7 @@ static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *col } static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; if (code != TSDB_CODE_SUCCESS) { mError("app:%p:%p, ctable %s, failed to alter column, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, tstrerror(code)); @@ -1965,7 +1969,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { } static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32_t ncols) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; SDbObj *pDb = pMsg->pDb; if (ncols <= 0) { mError("app:%p:%p, ctable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, ncols); @@ -2014,7 +2018,7 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3 static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { SDbObj *pDb = pMsg->pDb; - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; int32_t col = mnodeFindNormalTableColumnIndex(pTable, colName); if (col <= 0) { mError("app:%p:%p, ctable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, @@ -2046,7 +2050,7 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { } static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; int32_t col = mnodeFindNormalTableColumnIndex(pTable, oldName); if (col < 0) { mError("app:%p:%p, ctable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg, @@ -2082,7 +2086,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char return sdbUpdateRow(&oper); } -static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pTable) { +static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SCTableObj *pTable) { int32_t numOfCols = pTable->numOfColumns; for (int32_t i = 0; i < numOfCols; ++i) { strcpy(pSchema->name, pTable->schema[i].name); @@ -2097,7 +2101,7 @@ static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *p static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { SDbObj *pDb = pMsg->pDb; - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; pMeta->uid = htobe64(pTable->uid); pMeta->tid = htonl(pTable->tid); @@ -2146,7 +2150,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { } static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { - SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont; + STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; STagData *pTags = (STagData *)pInfo->tags; int32_t tagLen = htonl(pTags->dataLen); if (pTags->name[0] == 0) { @@ -2156,7 +2160,7 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { } int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + tagLen; - SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen); + SCMCreateTableMsg *pCreateMsg = calloc(1, contLen); if (pCreateMsg == NULL) { mError("app:%p:%p, table:%s, failed to create table while get meta info, no enough memory", pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId); @@ -2174,11 +2178,13 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { mDebug("app:%p:%p, table:%s, start to create on demand, tagLen:%d stable:%s", pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId, tagLen, pTags->name); - rpcFreeCont(pMsg->rpcMsg.pCont); + if (pMsg->rpcMsg.pCont != pMsg->pCont) { + tfree(pMsg->rpcMsg.pCont); + } pMsg->rpcMsg.msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE; pMsg->rpcMsg.pCont = pCreateMsg; pMsg->rpcMsg.contLen = contLen; - + return TSDB_CODE_MND_ACTION_NEED_REPROCESSED; } @@ -2203,7 +2209,7 @@ static int32_t mnodeGetChildTableMeta(SMnodeMsg *pMsg) { void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) { void * pIter = NULL; int32_t numOfTables = 0; - SChildTableObj *pTable = NULL; + SCTableObj *pTable = NULL; mInfo("vgId:%d, all child tables will be dropped from sdb", pVgroup->vgId); @@ -2231,7 +2237,7 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) { void mnodeDropAllChildTables(SDbObj *pDropDb) { void * pIter = NULL; int32_t numOfTables = 0; - SChildTableObj *pTable = NULL; + SCTableObj *pTable = NULL; char prefix[64] = {0}; tstrncpy(prefix, pDropDb->name, 64); @@ -2261,10 +2267,10 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) { mInfo("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); } -static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) { +static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) { void * pIter = NULL; int32_t numOfTables = 0; - SChildTableObj *pTable = NULL; + SCTableObj *pTable = NULL; mInfo("stable:%s uid:%" PRIu64 ", all child tables:%d will be dropped from sdb", pStable->info.tableId, pStable->uid, pStable->numOfTables); @@ -2292,11 +2298,11 @@ static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) { } #if 0 -static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) { +static SCTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) { SVgObj *pVgroup = mnodeGetVgroup(vnode); if (pVgroup == NULL) return NULL; - SChildTableObj *pTable = pVgroup->tableList[tid - 1]; + SCTableObj *pTable = pVgroup->tableList[tid - 1]; mnodeIncTableRef((STableObj *)pTable); mnodeDecVgroupRef(pVgroup); @@ -2307,14 +2313,14 @@ static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) { static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { return TSDB_CODE_COM_OPS_NOT_SUPPORT; #if 0 - SDMConfigTableMsg *pCfg = pMsg->rpcMsg.pCont; + SConfigTableMsg *pCfg = pMsg->rpcMsg.pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->vgId = htonl(pCfg->vgId); pCfg->sid = htonl(pCfg->sid); mDebug("app:%p:%p, dnode:%d, vgId:%d sid:%d, receive table config msg", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId, pCfg->vgId, pCfg->sid); - SChildTableObj *pTable = mnodeGetTableByPos(pCfg->vgId, pCfg->sid); + SCTableObj *pTable = mnodeGetTableByPos(pCfg->vgId, pCfg->sid); if (pTable == NULL) { mError("app:%p:%p, dnode:%d, vgId:%d sid:%d, table not found", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId, pCfg->vgId, pCfg->sid); @@ -2322,7 +2328,7 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { } SMDCreateTableMsg *pCreate = NULL; - pCreate = mnodeBuildCreateChildTableMsg(NULL, (SChildTableObj *)pTable); + pCreate = mnodeBuildCreateChildTableMsg(NULL, (SCTableObj *)pTable); mnodeDecTableRef(pTable); if (pCreate == NULL) return terrno; @@ -2340,7 +2346,7 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; - SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; + SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable; assert(pTable); mInfo("app:%p:%p, table:%s, drop table rsp received, vgId:%d sid:%d uid:%" PRIu64 ", thandle:%p result:%s", @@ -2351,14 +2357,14 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { mError("app:%p:%p, table:%s, failed to drop in dnode, vgId:%d sid:%d uid:%" PRIu64 ", reason:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, tstrerror(rpcMsg->code)); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); + dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code); return; } if (mnodeMsg->pVgroup == NULL) mnodeMsg->pVgroup = mnodeGetVgroup(pTable->vgId); if (mnodeMsg->pVgroup == NULL) { mError("app:%p:%p, table:%s, failed to get vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST); + dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST); return; } @@ -2368,7 +2374,7 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { mnodeDropVgroup(mnodeMsg->pVgroup, NULL); } - dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); + dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); } /* @@ -2381,7 +2387,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; - SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; + SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable; assert(pTable); // If the table is deleted by another thread during creation, stop creating and send drop msg to vnode @@ -2399,7 +2405,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { mnodeSendDropChildTableMsg(mnodeMsg, false); rpcMsg->code = TSDB_CODE_SUCCESS; - dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); + dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code); return; } @@ -2416,25 +2422,32 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeMsg->pTable = NULL; mnodeDestroyChildTable(pTable); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, code); + dnodeSendRpcMWriteRsp(mnodeMsg, code); } } else { - if (mnodeMsg->retry++ < 10) { + mnodeMsg->retry++; + int32_t sec = taosGetTimestampSec(); + if (mnodeMsg->retry < CREATE_CTABLE_RETRY_TIMES && ABS(sec - mnodeMsg->incomingTs) < CREATE_CTABLE_RETRY_SEC) { mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64 " result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->tid, pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); - dnodeDelayReprocessMnodeWriteMsg(mnodeMsg); + dnodeDelayReprocessMWriteMsg(mnodeMsg); } else { - mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 ", result:%s thandle:%p", + mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 + ", result:%s thandle:%p incomingTs:%d curTs:%d retryTimes:%d", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, - tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); + tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle, mnodeMsg->incomingTs, sec, mnodeMsg->retry); SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable}; sdbDeleteRow(&oper); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); + if (rpcMsg->code == TSDB_CODE_APP_NOT_READY) { + //Avoid retry again in client + rpcMsg->code = TSDB_CODE_MND_VGROUP_NOT_READY; + } + dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code); } } } @@ -2445,31 +2458,31 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) { SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; - SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; + SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable; assert(pTable); if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { mDebug("app:%p:%p, ctable:%s, altered in dnode, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); + dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); } else { - if (mnodeMsg->retry++ < 3) { + if (mnodeMsg->retry++ < ALTER_CTABLE_RETRY_TIMES) { mDebug("app:%p:%p, table:%s, alter table rsp received, need retry, times:%d result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); - dnodeDelayReprocessMnodeWriteMsg(mnodeMsg); + dnodeDelayReprocessMWriteMsg(mnodeMsg); } else { mError("app:%p:%p, table:%s, failed to alter in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); + dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code); } } } static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { - SCMMultiTableInfoMsg *pInfo = pMsg->rpcMsg.pCont; + SMultiTableInfoMsg *pInfo = pMsg->rpcMsg.pCont; pInfo->numOfTables = htonl(pInfo->numOfTables); int32_t totalMallocLen = 4 * 1024 * 1024; // first malloc 4 MB, subsequent reallocation as twice @@ -2483,7 +2496,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { for (int32_t t = 0; t < pInfo->numOfTables; ++t) { char * tableId = (char *)(pInfo->tableIds + t * TSDB_TABLE_FNAME_LEN); - SChildTableObj *pTable = mnodeGetChildTable(tableId); + SCTableObj *pTable = mnodeGetChildTable(tableId); if (pTable == NULL) continue; if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(tableId); @@ -2607,7 +2620,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows int32_t cols = 0; int32_t numOfRows = 0; - SChildTableObj *pTable = NULL; + SCTableObj *pTable = NULL; SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char prefix[64] = {0}; @@ -2705,7 +2718,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows } static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { - SCMAlterTableMsg *pAlter = pMsg->rpcMsg.pCont; + SAlterTableMsg *pAlter = pMsg->rpcMsg.pCont; mDebug("app:%p:%p, table:%s, alter table msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId, pMsg->rpcMsg.handle); @@ -2843,7 +2856,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro } int32_t numOfRows = 0; - SChildTableObj *pTable = NULL; + SCTableObj *pTable = NULL; SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char prefix[64] = {0}; diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index 779e25254897ff37f1ca884e83469233d7197b8c..cd02d5a935c3ef3cf822d518a9cc3bbd795db46b 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -43,7 +43,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg); static int32_t mnodeUserActionDestroy(SSdbOper *pOper) { - taosTFree(pOper->pObj); + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -270,7 +270,7 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to create by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code)); - taosTFree(pUser); + tfree(pUser); } else { mLInfo("user:%s, is created by %s", pUser->user, mnodeGetUserFromMsg(pMsg)); } @@ -414,7 +414,7 @@ static int32_t mnodeProcessCreateUserMsg(SMnodeMsg *pMsg) { SUserObj *pOperUser = pMsg->pUser; if (pOperUser->superAuth) { - SCMCreateUserMsg *pCreate = pMsg->rpcMsg.pCont; + SCreateUserMsg *pCreate = pMsg->rpcMsg.pCont; return mnodeCreateUser(pOperUser->pAcct, pCreate->user, pCreate->pass, pMsg); } else { mError("user:%s, no rights to create user", pOperUser->user); @@ -426,7 +426,7 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) { int32_t code; SUserObj *pOperUser = pMsg->pUser; - SCMAlterUserMsg *pAlter = pMsg->rpcMsg.pCont; + SAlterUserMsg *pAlter = pMsg->rpcMsg.pCont; SUserObj *pUser = mnodeGetUser(pAlter->user); if (pUser == NULL) { return TSDB_CODE_MND_INVALID_USER; @@ -514,7 +514,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) { int32_t code; SUserObj *pOperUser = pMsg->pUser; - SCMDropUserMsg *pDrop = pMsg->rpcMsg.pCont; + SDropUserMsg *pDrop = pMsg->rpcMsg.pCont; SUserObj *pUser = mnodeGetUser(pDrop->user); if (pUser == NULL) { return TSDB_CODE_MND_INVALID_USER; @@ -604,11 +604,11 @@ int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, cha } static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg) { - SDMAuthMsg *pAuthMsg = pMsg->rpcMsg.pCont; - SDMAuthRsp *pAuthRsp = rpcMallocCont(sizeof(SDMAuthRsp)); + SAuthMsg *pAuthMsg = pMsg->rpcMsg.pCont; + SAuthRsp *pAuthRsp = rpcMallocCont(sizeof(SAuthRsp)); pMsg->rpcRsp.rsp = pAuthRsp; - pMsg->rpcRsp.len = sizeof(SDMAuthRsp); + pMsg->rpcRsp.len = sizeof(SAuthRsp); return mnodeRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey); } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 5084f1276ae883289939bd74c50a0502f6b3c71d..9e398e94f136de7324f18ee6bef32fdd4a700b33 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -69,7 +69,7 @@ static void mnodeDestroyVgroup(SVgObj *pVgroup) { pVgroup->idPool = NULL; } - taosTFree(pVgroup); + tfree(pVgroup); } static int32_t mnodeVgroupActionDestroy(SSdbOper *pOper) { @@ -529,7 +529,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; (void)sdbUpdateRow(&desc); - dnodeReprocessMnodeWriteMsg(pMsg); + dnodeReprocessMWriteMsg(pMsg); return TSDB_CODE_MND_ACTION_IN_PROGRESS; // if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_READY) { // mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, @@ -537,7 +537,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { // pVgroup->status = TAOS_VG_STATUS_READY; // SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; // (void)sdbUpdateRow(&desc); - // dnodeReprocessMnodeWriteMsg(pMsg); + // dnodeReprocessMWriteMsg(pMsg); // return TSDB_CODE_MND_ACTION_IN_PROGRESS; // } else { // mError("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d, but vgroup is dropping", pMsg->rpcMsg.ahandle, @@ -663,13 +663,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p for (int32_t i = 0; i < pShow->maxReplica; ++i) { pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "dnode%d", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dstatus", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } @@ -694,7 +694,7 @@ static bool mnodeFilterVgroups(SVgObj *pVgroup, STableObj *pTable) { return true; } - SChildTableObj *pCTable = (SChildTableObj *)pTable; + SCTableObj *pCTable = (SCTableObj *)pTable; if (pVgroup->vgId == pCTable->vgId) { return true; } else { @@ -791,7 +791,7 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v return numOfRows; } -void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { +void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable) { int32_t idPoolSize = taosIdPoolMaxSize(pVgroup->idPool); if (pTable->tid > idPoolSize) { mnodeAllocVgroupIdPool(pVgroup); @@ -807,7 +807,7 @@ void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { } } -void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { +void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable) { if (pTable->tid >= 1) { taosFreeId(pVgroup->idPool, pTable->tid); pVgroup->numOfTables--; @@ -818,11 +818,11 @@ void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { } } -static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { +static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { SDbObj *pDb = pVgroup->pDb; if (pDb == NULL) return NULL; - SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg)); + SCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SCreateVnodeMsg)); if (pVnode == NULL) return NULL; strcpy(pVnode->db, pVgroup->dbName); @@ -830,7 +830,7 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { //TODO: dynamic alloc tables in tsdb maxTables = MAX(10000, tsMaxTablePerVnode); - SMDVnodeCfg *pCfg = &pVnode->cfg; + SVnodeCfg *pCfg = &pVnode->cfg; pCfg->vgId = htonl(pVgroup->vgId); pCfg->cfgVersion = htonl(pDb->cfgVersion); pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); @@ -850,8 +850,9 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { pCfg->replications = (int8_t) pVgroup->numOfVnodes; pCfg->wals = 3; pCfg->quorum = pDb->cfg.quorum; + pCfg->update = pDb->cfg.update; - SMDVnodeDesc *pNodes = pVnode->nodes; + SVnodeDesc *pNodes = pVnode->nodes; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { SDnodeObj *pDnode = pVgroup->vnodeGid[j].pDnode; if (pDnode != NULL) { @@ -886,11 +887,11 @@ SRpcEpSet mnodeGetEpSetFromIp(char *ep) { } static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet) { - SMDAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup); + SAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup); SRpcMsg rpcMsg = { .ahandle = NULL, .pCont = pAlter, - .contLen = pAlter ? sizeof(SMDAlterVnodeMsg) : 0, + .contLen = pAlter ? sizeof(SAlterVnodeMsg) : 0, .code = 0, .msgType = TSDB_MSG_TYPE_MD_ALTER_VNODE }; @@ -909,11 +910,11 @@ void mnodeSendAlterVgroupMsg(SVgObj *pVgroup) { } static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet, void *ahandle) { - SMDCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup); + SCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup); SRpcMsg rpcMsg = { .ahandle = ahandle, .pCont = pCreate, - .contLen = pCreate ? sizeof(SMDCreateVnodeMsg) : 0, + .contLen = pCreate ? sizeof(SCreateVnodeMsg) : 0, .code = 0, .msgType = TSDB_MSG_TYPE_MD_CREATE_VNODE }; @@ -969,7 +970,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeMsg->pVgroup = NULL; mnodeDestroyVgroup(pVgroup); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, code); + dnodeSendRpcMWriteRsp(mnodeMsg, code); } } else { SSdbOper oper = { @@ -978,12 +979,12 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { .pObj = pVgroup }; sdbDeleteRow(&oper); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, mnodeMsg->code); + dnodeSendRpcMWriteRsp(mnodeMsg, mnodeMsg->code); } } -static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) { - SMDDropVnodeMsg *pDrop = rpcMallocCont(sizeof(SMDDropVnodeMsg)); +static SDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) { + SDropVnodeMsg *pDrop = rpcMallocCont(sizeof(SDropVnodeMsg)); if (pDrop == NULL) return NULL; pDrop->vgId = htonl(vgId); @@ -991,11 +992,11 @@ static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) { } void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle) { - SMDDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId); + SDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId); SRpcMsg rpcMsg = { .ahandle = ahandle, .pCont = pDrop, - .contLen = pDrop ? sizeof(SMDDropVnodeMsg) : 0, + .contLen = pDrop ? sizeof(SDropVnodeMsg) : 0, .code = 0, .msgType = TSDB_MSG_TYPE_MD_DROP_VNODE }; @@ -1040,11 +1041,11 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) { code = TSDB_CODE_MND_SDB_ERROR; } - dnodeReprocessMnodeWriteMsg(mnodeMsg); + dnodeReprocessMWriteMsg(mnodeMsg); } static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) { - SDMConfigVnodeMsg *pCfg = pMsg->rpcMsg.pCont; + SConfigVnodeMsg *pCfg = pMsg->rpcMsg.pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->vgId = htonl(pCfg->vgId); diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 86e16db8b1446308060945d3a7db2531287c62ec..9383ae48dc5c6151aea0d9e8a2641603b63da144 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -52,9 +52,9 @@ extern "C" { #include "osWindows.h" #endif +#include "osDef.h" #include "osAtomic.h" #include "osCommon.h" -#include "osDef.h" #include "osDir.h" #include "osFile.h" #include "osLz4.h" diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index c1a950fbe6375b14e3cf25277eb933b96e518f10..7bb844831e03dd1e21abd389a61976e14927ebbd 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -72,8 +72,6 @@ extern "C" { #include #define TAOS_OS_FUNC_FILE_SENDIFLE - #define taosFSendFile(outfile, infile, offset, count) taosFSendFileImp(outfile, infile, offset, size) - #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size) #define TAOS_OS_FUNC_SEMPHONE #define tsem_t dispatch_semaphore_t diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index dc19c8177c62307ca110486c130f6a6b56047ef4..62e44d8eb0b70fb1526693895847637daa72247a 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -20,46 +20,52 @@ extern "C" { #endif -ssize_t taosTReadImp(int fd, void *buf, size_t count); -ssize_t taosTWriteImp(int fd, void *buf, size_t count); +#define tread(fd, buf, count) read(fd, buf, count) +#define twrite(fd, buf, count) write(fd, buf, count) +#define tlseek(fd, offset, whence) lseek(fd, offset, whence) +#define tclose(fd) \ + { \ + if (FD_VALID(fd)) { \ + close(fd); \ + fd = FD_INITIALIZER; \ + } \ + } -ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size); -int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count); +int64_t taosReadImp(int32_t fd, void *buf, int64_t count); +int64_t taosWriteImp(int32_t fd, void *buf, int64_t count); +int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence); +int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath); -#ifndef TAOS_OS_FUNC_FILE_SENDIFLE - #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size) - #define taosFSendFile(outfile, infile, offset, count) taosTSendFileImp(fileno(outfile), fileno(infile), offset, size) -#endif +#define taosRead(fd, buf, count) taosReadImp(fd, buf, count) +#define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count) +#define taosLSeek(fd, offset, whence) taosLSeekImp(fd, offset, whence) +#define taosClose(x) tclose(x) -#define taosTRead(fd, buf, count) taosTReadImp(fd, buf, count) -#define taosTWrite(fd, buf, count) taosTWriteImp(fd, buf, count) -#define taosLSeek(fd, offset, whence) lseek(fd, offset, whence) +// TAOS_OS_FUNC_FILE_SENDIFLE +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size); +int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); #ifdef TAOS_RANDOM_FILE_FAIL - void taosSetRandomFileFailFactor(int factor); + void taosSetRandomFileFailFactor(int32_t factor); void taosSetRandomFileFailOutput(const char *path); #ifdef TAOS_RANDOM_FILE_FAIL_TEST - ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); - ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); - off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line); - #undef taosTRead - #undef taosTWrite + int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line); + int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line); + int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line); + #undef taosRead + #undef taosWrite #undef taosLSeek - #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__) - #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__) + #define taosRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__) + #define taosWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__) #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__) #endif #endif -int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath); - // TAOS_OS_FUNC_FILE_GETTMPFILEPATH void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath); -#ifndef TAOS_OS_FUNC_FILE_FTRUNCATE - #define taosFtruncate ftruncate -#endif - +// TAOS_OS_FUNC_FILE_FTRUNCATE +int32_t taosFtruncate(int32_t fd, int64_t length); #ifdef __cplusplus } #endif diff --git a/src/os/inc/osMemory.h b/src/os/inc/osMemory.h index 37d9dc9828ec1f51f36542e6c297ec3df8709f7e..0616006650eb2d53cdf7bd70b68c9f60748deac5 100644 --- a/src/os/inc/osMemory.h +++ b/src/os/inc/osMemory.h @@ -31,6 +31,7 @@ typedef enum { void taosSetAllocMode(int mode, const char *path, bool autoDump); void taosDumpMemoryLeak(); +// used in tsdb module void * taosTMalloc(size_t size); void * taosTCalloc(size_t nmemb, size_t size); void * taosTRealloc(void *ptr, size_t size); @@ -38,7 +39,14 @@ void taosTZfree(void *ptr); size_t taosTSizeof(void *ptr); void taosTMemset(void *ptr, int c); -#define taosTFree(x) \ +// used in other module +#define tmalloc(size) malloc(size) +#define tcalloc(num, size) calloc(num, size) +#define trealloc(ptr, size) realloc(ptr, size) +#define tstrdup(str) taosStrdupImp(str) +#define tstrndup(str, size) taosStrndupImp(str, size) +#define tgetline(lineptr, n, stream) taosGetlineImp(lineptr, n, stream) +#define tfree(x) \ do { \ if (x) { \ free((void *)(x)); \ @@ -46,37 +54,30 @@ void taosTMemset(void *ptr, int c); } \ } while (0); -#define taosMalloc(size) malloc(size) -#define taosCalloc(num, size) calloc(num, size) -#define taosRealloc(ptr, size) realloc(ptr, size) -#define taosFree(ptr) free(ptr) -#define taosStrdup(str) taosStrdupImp(str) -#define taosStrndup(str, size) taosStrndupImp(str, size) -#define taosGetline(lineptr, n, stream) taosGetlineImp(lineptr, n, stream) - #ifdef TAOS_MEM_CHECK #ifdef TAOS_MEM_CHECK_TEST - void * taos_malloc(size_t size, const char *file, uint32_t line); - void * taos_calloc(size_t num, size_t size, const char *file, uint32_t line); - void * taos_realloc(void *ptr, size_t size, const char *file, uint32_t line); - void taos_free(void *ptr, const char *file, uint32_t line); - char * taos_strdup(const char *str, const char *file, uint32_t line); - char * taos_strndup(const char *str, size_t size, const char *file, uint32_t line); - ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line); - #undef taosMalloc - #undef taosCalloc - #undef taosRealloc - #undef taosFree - #undef taosStrdup - #undef taosStrndup - #undef taosGetline - #define taosMalloc(size) taos_malloc(size, __FILE__, __LINE__) - #define taosCalloc(num, size) taos_calloc(num, size, __FILE__, __LINE__) - #define taosRealloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__) - #define taosFree(ptr) taos_free(ptr, __FILE__, __LINE__) - //#define taosStrdup(str) taos_strdup(str, __FILE__, __LINE__) - //#define taosStrndup(str, size) taos_strndup(str, size, __FILE__, __LINE__) - //#define taosGetline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__) + void * taosMallocMem(size_t size, const char *file, uint32_t line); + void * taosCallocMem(size_t num, size_t size, const char *file, uint32_t line); + void * taosReallocMem(void *ptr, size_t size, const char *file, uint32_t line); + void taosFreeMem(void *ptr, const char *file, uint32_t line); + char * taosStrdupMem(const char *str, const char *file, uint32_t line); + char * taosStrndupMem(const char *str, size_t size, const char *file, uint32_t line); + ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line); + #undef tmalloc + #undef tcalloc + #undef trealloc + #undef tfree + #define tmalloc(size) taosMallocMem(size, __FILE__, __LINE__) + #define tcalloc(num, size) taosCallocMem(num, size, __FILE__, __LINE__) + #define trealloc(ptr, size) taosReallocMem(ptr, size, __FILE__, __LINE__) + #define tfree(ptr) taosFreeMem(ptr, __FILE__, __LINE__) + + // #undef tstrdup + // #undef tstrndup + // #undef tgetline + // #define taosStrdup(str) taos_strdup(str, __FILE__, __LINE__) + // #define taosStrndup(str, size) taos_strndup(str, size, __FILE__, __LINE__) + // #define tgetline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__) #endif #endif diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index 0ab3ff0fcafa43a404533f145a0bb1595bb29061..cbfdedef485f73f0005b2da1e1d9cc3adf9dd377 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -33,21 +33,19 @@ extern "C" { x = FD_INITIALIZER; \ } \ } - typedef int SOCKET; + typedef int32_t SOCKET; #endif #ifndef TAOS_OS_DEF_EPOLL #define TAOS_EPOLL_WAIT_TIME -1 #endif -#define taosClose(x) taosCloseSocket(x) - #ifdef TAOS_RANDOM_NETWORK_FAIL #ifdef TAOS_RANDOM_NETWORK_FAIL_TEST - ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags); - ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen); - ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count); - ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count); + int64_t taosSendRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags); + int64_t taosSendToRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags, const struct sockaddr *dest_addr, socklen_t addrlen); + int64_t taosReadSocketRandomFail(int32_t fd, void *buf, size_t count); + int64_t taosWriteSocketRandomFail(int32_t fd, const void *buf, size_t count); #undef taosSend #undef taosSendto #undef taosReadSocket @@ -60,14 +58,14 @@ extern "C" { #endif // TAOS_OS_FUNC_SOCKET -int taosSetNonblocking(SOCKET sock, int on); -void taosBlockSIGPIPE(); +int32_t taosSetNonblocking(SOCKET sock, int32_t on); +void taosBlockSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT -int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen); +int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); // TAOS_OS_FUNC_SOCKET_INET -uint32_t taosInetAddr(char *ipAddr); +uint32_t taosInetAddr(char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt); #ifdef __cplusplus diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index dc1da35037dc06c258856a1a143146492eeed0e5..36e30528bf9d84739fd391442145d7e65e1f7df2 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -62,11 +62,8 @@ extern "C" { #define TAOS_OS_FUNC_FILE_ISDIR #define TAOS_OS_FUNC_FILE_ISLNK #define TAOS_OS_FUNC_FILE_SENDIFLE - #define taosFSendFile(outfile, infile, offset, count) taosFSendFileImp(outfile, infile, offset, size) - #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size) #define TAOS_OS_FUNC_FILE_GETTMPFILEPATH -#define TAOS_OS_FUNC_FILE_FTRUNCATE - extern int taosFtruncate(int fd, int64_t length); +#define TAOS_OS_FUNC_FILE_FTRUNCATE #define TAOS_OS_FUNC_MATH #define SWAP(a, b, c) \ @@ -139,7 +136,6 @@ typedef int (*__compar_fn_t)(const void *, const void *); #define in_addr_t unsigned long #define socklen_t int #define htobe64 htonll -#define twrite write #define getpid _getpid struct tm *localtime_r(const time_t *timep, struct tm *result); diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c index 66bdb5b939d77ab03b30f024466fe3e3ff4e9388..dacf4db74137b8cc58db2b390ba971cf9dfbc012 100644 --- a/src/os/src/darwin/darwinFile.c +++ b/src/os/src/darwin/darwinFile.c @@ -19,21 +19,19 @@ #define _SEND_FILE_STEP_ 1000 -int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) { +int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { fseek(in_file, (int32_t)(*offset), 0); - int writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; + int writeLen = 0; + uint8_t buffer[_SEND_FILE_STEP_] = {0}; for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); if (rlen <= 0) { return writeLen; - } - else if (rlen < _SEND_FILE_STEP_) { + } else if (rlen < _SEND_FILE_STEP_) { fwrite(buffer, 1, rlen, out_file); return (int)(writeLen + rlen); - } - else { + } else { fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); writeLen += _SEND_FILE_STEP_; } @@ -44,8 +42,7 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou size_t rlen = fread(buffer, 1, remain, in_file); if (rlen <= 0) { return writeLen; - } - else { + } else { fwrite(buffer, 1, remain, out_file); writeLen += remain; } @@ -54,7 +51,7 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou return writeLen; } -ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { - uError("not implemented yet"); +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) { + uError("taosSendFile not implemented yet"); return -1; } \ No newline at end of file diff --git a/src/os/src/detail/osFail.c b/src/os/src/detail/osFail.c index e0eb200851f976e1c9dad47afbc152bfb021e4e5..a99bcd01dbccd0290de1fc38a1220b573ab74b22 100644 --- a/src/os/src/detail/osFail.c +++ b/src/os/src/detail/osFail.c @@ -20,7 +20,7 @@ #ifdef TAOS_RANDOM_NETWORK_FAIL -ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags) { +int64_t taosSendRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags) { if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) { errno = ECONNRESET; return -1; @@ -29,8 +29,8 @@ ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags) { return send(sockfd, buf, len, flags); } -ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, - socklen_t addrlen) { +int64_t taosSendToRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags, + const struct sockaddr *dest_addr, socklen_t addrlen) { if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) { errno = ECONNRESET; return -1; @@ -39,7 +39,7 @@ ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, return sendto(sockfd, buf, len, flags, dest_addr, addrlen); } -ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count) { +int64_t taosReadSocketRandomFail(int32_t fd, void *buf, size_t count) { if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) { errno = ECONNRESET; return -1; @@ -48,7 +48,7 @@ ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count) { return read(fd, buf, count); } -ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count) { +int64_t taosWriteSocketRandomFail(int32_t fd, const void *buf, size_t count) { if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) { errno = EINTR; return -1; @@ -61,10 +61,10 @@ ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count) { #ifdef TAOS_RANDOM_FILE_FAIL -static int random_file_fail_factor = 20; +static int32_t random_file_fail_factor = 20; static FILE *fpRandomFileFailOutput = NULL; -void taosSetRandomFileFailFactor(int factor) { +void taosSetRandomFileFailFactor(int32_t factor) { random_file_fail_factor = factor; } @@ -77,7 +77,7 @@ static void close_random_file_fail_output() { } } -static void random_file_fail_output_sig(int sig) { +static void random_file_fail_output_sig(int32_t sig) { fprintf(fpRandomFileFailOutput, "signal %d received.\n", sig); struct sigaction act = {0}; @@ -105,7 +105,7 @@ void taosSetRandomFileFailOutput(const char *path) { sigaction(SIGILL, &act, NULL); } -ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) { +int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line) { if (random_file_fail_factor > 0) { if (rand() % random_file_fail_factor == 0) { errno = EIO; @@ -113,10 +113,10 @@ ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file } } - return taosTReadImp(fd, buf, count); + return taosReadImp(fd, buf, count); } -ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) { +int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line) { if (random_file_fail_factor > 0) { if (rand() % random_file_fail_factor == 0) { errno = EIO; @@ -124,10 +124,10 @@ ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *fil } } - return taosTWriteImp(fd, buf, count); + return taosWriteImp(fd, buf, count); } -off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line) { +int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line) { if (random_file_fail_factor > 0) { if (rand() % random_file_fail_factor == 0) { errno = EIO; @@ -135,7 +135,7 @@ off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, ui } } - return lseek(fd, offset, whence); + return taosLSeekImp(fd, offset, whence); } #endif //TAOS_RANDOM_FILE_FAIL diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 8f055dd8129f340f267e64cdd905505a7b675a2d..6eb4515f3098f89991640d2fdee2b0aca47c1703 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -15,16 +15,22 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "tglobal.h" #ifndef TAOS_OS_FUNC_FILE_GETTMPFILEPATH + void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { const char *tdengineTmpFileNamePrefix = "tdengine-"; char tmpPath[PATH_MAX]; - char *tmpDir = "/tmp/"; + int32_t len = strlen(tsTempDir); + memcpy(tmpPath, tsTempDir, len); + + if (tmpPath[len - 1] != '/') { + tmpPath[len++] = '/'; + } - strcpy(tmpPath, tmpDir); - strcat(tmpPath, tdengineTmpFileNamePrefix); + strcpy(tmpPath + len, tdengineTmpFileNamePrefix); if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { strcat(tmpPath, fileNamePrefix); strcat(tmpPath, "-%d-%s"); @@ -34,10 +40,10 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { taosRandStr(rand, tListLen(rand) - 1); snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); } + #endif -// rename file name -int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) { +int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath) { int32_t ts = taosGetTimestampSec(); char fname[PATH_MAX] = {0}; // max file name length must be less than 255 @@ -46,12 +52,13 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP if (delimiterPos == NULL) return -1; int32_t fileNameLen = 0; - if (suffix) + if (suffix) { fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix); - else + } else { fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts); + } - size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1); + int32_t len = (int32_t)((delimiterPos - fullPath) + fileNameLen + 1); if (*dstPath == NULL) { *dstPath = calloc(1, len + 1); if (*dstPath == NULL) return -1; @@ -64,9 +71,9 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP return rename(fullPath, *dstPath); } -ssize_t taosTReadImp(int fd, void *buf, size_t count) { - size_t leftbytes = count; - ssize_t readbytes; +int64_t taosReadImp(int32_t fd, void *buf, int64_t count) { + int64_t leftbytes = count; + int64_t readbytes; char * tbuf = (char *)buf; while (leftbytes > 0) { @@ -78,19 +85,19 @@ ssize_t taosTReadImp(int fd, void *buf, size_t count) { return -1; } } else if (readbytes == 0) { - return (ssize_t)(count - leftbytes); + return (int64_t)(count - leftbytes); } leftbytes -= readbytes; tbuf += readbytes; } - return (ssize_t)count; + return count; } -ssize_t taosTWriteImp(int fd, void *buf, size_t n) { - size_t nleft = n; - ssize_t nwritten = 0; +int64_t taosWriteImp(int32_t fd, void *buf, int64_t n) { + int64_t nleft = n; + int64_t nwritten = 0; char * tbuf = (char *)buf; while (nleft > 0) { @@ -105,13 +112,18 @@ ssize_t taosTWriteImp(int fd, void *buf, size_t n) { tbuf += nwritten; } - return (ssize_t)n; + return n; +} + +int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) { + return (int64_t)tlseek(fd, (long)offset, whence); } #ifndef TAOS_OS_FUNC_FILE_SENDIFLE -ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { - size_t leftbytes = size; - ssize_t sentbytes; + +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) { + int64_t leftbytes = size; + int64_t sentbytes; while (leftbytes > 0) { /* @@ -126,7 +138,7 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { return -1; } } else if (sentbytes == 0) { - return (ssize_t)(size - leftbytes); + return (int64_t)(size - leftbytes); } leftbytes -= sentbytes; @@ -134,4 +146,17 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { return size; } + +int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) { + return taosSendFile(fileno(outfile), fileno(infile), offset, size); +} + +#endif + +#ifndef TAOS_OS_FUNC_FILE_FTRUNCATE + +int32_t taosFtruncate(int32_t fd, int64_t length) { + return ftruncate(fd, length); +} + #endif \ No newline at end of file diff --git a/src/os/src/detail/osMemory.c b/src/os/src/detail/osMemory.c index dfd320be89a75a4700a4918f68d56ee66faa3135..53310d179c0090382e009de949e5158146dc282a 100644 --- a/src/os/src/detail/osMemory.c +++ b/src/os/src/detail/osMemory.c @@ -28,7 +28,7 @@ static FILE* fpAllocLog = NULL; extern int32_t taosGetTimestampSec(); static int32_t startTime = INT32_MAX; -static bool random_alloc_fail(size_t size, const char* file, uint32_t line) { +static bool taosRandomAllocFail(size_t size, const char* file, uint32_t line) { if (taosGetTimestampSec() < startTime) { return false; } @@ -48,33 +48,33 @@ static bool random_alloc_fail(size_t size, const char* file, uint32_t line) { return true; } -static void* malloc_random(size_t size, const char* file, uint32_t line) { - return random_alloc_fail(size, file, line) ? NULL : malloc(size); +static void* taosRandmoMalloc(size_t size, const char* file, uint32_t line) { + return taosRandomAllocFail(size, file, line) ? NULL : malloc(size); } -static void* calloc_random(size_t num, size_t size, const char* file, uint32_t line) { - return random_alloc_fail(num * size, file, line) ? NULL : calloc(num, size); +static void* taosRandomCalloc(size_t num, size_t size, const char* file, uint32_t line) { + return taosRandomAllocFail(num * size, file, line) ? NULL : calloc(num, size); } -static void* realloc_random(void* ptr, size_t size, const char* file, uint32_t line) { - return random_alloc_fail(size, file, line) ? NULL : realloc(ptr, size); +static void* taosRandomRealloc(void* ptr, size_t size, const char* file, uint32_t line) { + return taosRandomAllocFail(size, file, line) ? NULL : realloc(ptr, size); } -static char* strdup_random(const char* str, const char* file, uint32_t line) { +static char* taosRandomStrdup(const char* str, const char* file, uint32_t line) { size_t len = strlen(str); - return random_alloc_fail(len + 1, file, line) ? NULL : taosStrdupImp(str); + return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrdupImp(str); } -static char* strndup_random(const char* str, size_t size, const char* file, uint32_t line) { +static char* taosRandomStrndup(const char* str, size_t size, const char* file, uint32_t line) { size_t len = strlen(str); if (len > size) { len = size; } - return random_alloc_fail(len + 1, file, line) ? NULL : taosStrndupImp(str, len); + return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrndupImp(str, len); } -static ssize_t getline_random(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { - return random_alloc_fail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream); +static ssize_t taosRandomGetline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { + return taosRandomAllocFail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream); } //////////////////////////////////////////////////////////////////////////////// @@ -96,7 +96,7 @@ typedef struct SMemBlock { static SMemBlock *blocks = NULL; static uintptr_t lock = 0; -static void add_mem_block(SMemBlock* blk) { +static void taosAddMemBlock(SMemBlock* blk) { blk->prev = NULL; while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0); blk->next = blocks; @@ -107,7 +107,7 @@ static void add_mem_block(SMemBlock* blk) { atomic_store_ptr(&lock, 0); } -static void remove_mem_block(SMemBlock* blk) { +static void taosRemoveMemBlock(SMemBlock* blk) { while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0); if (blocks == blk) { @@ -126,7 +126,7 @@ static void remove_mem_block(SMemBlock* blk) { blk->next = NULL; } -static void free_detect_leak(void* ptr, const char* file, uint32_t line) { +static void taosFreeDetectLeak(void* ptr, const char* file, uint32_t line) { if (ptr == NULL) { return; } @@ -140,11 +140,11 @@ static void free_detect_leak(void* ptr, const char* file, uint32_t line) { return; } - remove_mem_block(blk); + taosRemoveMemBlock(blk); free(blk); } -static void* malloc_detect_leak(size_t size, const char* file, uint32_t line) { +static void* taosMallocDetectLeak(size_t size, const char* file, uint32_t line) { if (size == 0) { return NULL; } @@ -166,28 +166,28 @@ static void* malloc_detect_leak(size_t size, const char* file, uint32_t line) { blk->line = (uint16_t)line; blk->magic = MEMBLK_MAGIC; blk->size = size; - add_mem_block(blk); + taosAddMemBlock(blk); return blk->data; } -static void* calloc_detect_leak(size_t num, size_t size, const char* file, uint32_t line) { +static void* taosCallocDetectLeak(size_t num, size_t size, const char* file, uint32_t line) { size *= num; - void* p = malloc_detect_leak(size, file, line); + void* p = taosMallocDetectLeak(size, file, line); if (p != NULL) { memset(p, 0, size); } return p; } -static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint32_t line) { +static void* taosReallocDetectLeak(void* ptr, size_t size, const char* file, uint32_t line) { if (size == 0) { - free_detect_leak(ptr, file, line); + taosFreeDetectLeak(ptr, file, line); return NULL; } if (ptr == NULL) { - return malloc_detect_leak(size, file, line); + return taosMallocDetectLeak(size, file, line); } SMemBlock* blk = (SMemBlock *)((char*)ptr) - sizeof(SMemBlock); @@ -198,11 +198,11 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3 return realloc(ptr, size); } - remove_mem_block(blk); + taosRemoveMemBlock(blk); void* p = realloc(blk, size + sizeof(SMemBlock)); if (p == NULL) { - add_mem_block(blk); + taosAddMemBlock(blk); return NULL; } @@ -212,13 +212,13 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3 blk = (SMemBlock*)p; blk->size = size; - add_mem_block(blk); + taosAddMemBlock(blk); return blk->data; } -static char* strdup_detect_leak(const char* str, const char* file, uint32_t line) { +static char* taosStrdupDetectLeak(const char* str, const char* file, uint32_t line) { size_t len = strlen(str); - char *p = malloc_detect_leak(len + 1, file, line); + char *p = taosMallocDetectLeak(len + 1, file, line); if (p != NULL) { memcpy(p, str, len); p[len] = 0; @@ -226,12 +226,12 @@ static char* strdup_detect_leak(const char* str, const char* file, uint32_t line return p; } -static char* strndup_detect_leak(const char* str, size_t size, const char* file, uint32_t line) { +static char* taosStrndupDetectLeak(const char* str, size_t size, const char* file, uint32_t line) { size_t len = strlen(str); if (len > size) { len = size; } - char *p = malloc_detect_leak(len + 1, file, line); + char *p = taosMallocDetectLeak(len + 1, file, line); if (p != NULL) { memcpy(p, str, len); p[len] = 0; @@ -239,13 +239,13 @@ static char* strndup_detect_leak(const char* str, size_t size, const char* file, return p; } -static ssize_t getline_detect_leak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { +static ssize_t taosGetlineDetectLeak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { char* buf = NULL; size_t bufSize = 0; ssize_t size = taosGetlineImp(&buf, &bufSize, stream); if (size != -1) { if (*n < size + 1) { - void* p = realloc_detect_leak(*lineptr, size + 1, file, line); + void* p = taosReallocDetectLeak(*lineptr, size + 1, file, line); if (p == NULL) { free(buf); return -1; @@ -260,7 +260,7 @@ static ssize_t getline_detect_leak(char **lineptr, size_t *n, FILE *stream, cons return size; } -static void dump_memory_leak() { +static void taosDumpMemoryLeakImp() { const char* hex = "0123456789ABCDEF"; const char* fmt = ":%d: addr=%p, size=%d, content(first 16 bytes)="; size_t numOfBlk = 0, totalSize = 0; @@ -299,7 +299,7 @@ static void dump_memory_leak() { fflush(fpAllocLog); } -static void dump_memory_leak_on_sig(int sig) { +static void taosDumpMemoryLeakOnSig(int sig) { fprintf(fpAllocLog, "signal %d received.\n", sig); // restore default signal handler @@ -307,55 +307,55 @@ static void dump_memory_leak_on_sig(int sig) { act.sa_handler = SIG_DFL; sigaction(sig, &act, NULL); - dump_memory_leak(); + taosDumpMemoryLeakImp(); } //////////////////////////////////////////////////////////////////////////////// // interface functions -void* taos_malloc(size_t size, const char* file, uint32_t line) { +void* taosMallocMem(size_t size, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return malloc(size); case TAOS_ALLOC_MODE_RANDOM_FAIL: - return malloc_random(size, file, line); + return taosRandmoMalloc(size, file, line); case TAOS_ALLOC_MODE_DETECT_LEAK: - return malloc_detect_leak(size, file, line); + return taosMallocDetectLeak(size, file, line); } return malloc(size); } -void* taos_calloc(size_t num, size_t size, const char* file, uint32_t line) { +void* taosCallocMem(size_t num, size_t size, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return calloc(num, size); case TAOS_ALLOC_MODE_RANDOM_FAIL: - return calloc_random(num, size, file, line); + return taosRandomCalloc(num, size, file, line); case TAOS_ALLOC_MODE_DETECT_LEAK: - return calloc_detect_leak(num, size, file, line); + return taosCallocDetectLeak(num, size, file, line); } return calloc(num, size); } -void* taos_realloc(void* ptr, size_t size, const char* file, uint32_t line) { +void* taosReallocMem(void* ptr, size_t size, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return realloc(ptr, size); case TAOS_ALLOC_MODE_RANDOM_FAIL: - return realloc_random(ptr, size, file, line); + return taosRandomRealloc(ptr, size, file, line); case TAOS_ALLOC_MODE_DETECT_LEAK: - return realloc_detect_leak(ptr, size, file, line); + return taosReallocDetectLeak(ptr, size, file, line); } return realloc(ptr, size); } -void taos_free(void* ptr, const char* file, uint32_t line) { +void taosFreeMem(void* ptr, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return free(ptr); @@ -364,54 +364,54 @@ void taos_free(void* ptr, const char* file, uint32_t line) { return free(ptr); case TAOS_ALLOC_MODE_DETECT_LEAK: - return free_detect_leak(ptr, file, line); + return taosFreeDetectLeak(ptr, file, line); } return free(ptr); } -char* taos_strdup(const char* str, const char* file, uint32_t line) { +char* taosStrdupMem(const char* str, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return taosStrdupImp(str); case TAOS_ALLOC_MODE_RANDOM_FAIL: - return strdup_random(str, file, line); + return taosRandomStrdup(str, file, line); case TAOS_ALLOC_MODE_DETECT_LEAK: - return strdup_detect_leak(str, file, line); + return taosStrdupDetectLeak(str, file, line); } return taosStrdupImp(str); } -char* taos_strndup(const char* str, size_t size, const char* file, uint32_t line) { +char* taosStrndupMem(const char* str, size_t size, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return taosStrndupImp(str, size); case TAOS_ALLOC_MODE_RANDOM_FAIL: - return strndup_random(str, size, file, line); + return taosRandomStrndup(str, size, file, line); case TAOS_ALLOC_MODE_DETECT_LEAK: - return strndup_detect_leak(str, size, file, line); + return taosStrndupDetectLeak(str, size, file, line); } return taosStrndupImp(str, size); } -ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { +ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return taosGetlineImp(lineptr, n, stream); case TAOS_ALLOC_MODE_RANDOM_FAIL: - return getline_random(lineptr, n, stream, file, line); + return taosRandomGetline(lineptr, n, stream, file, line); case TAOS_ALLOC_MODE_DETECT_LEAK: - return getline_detect_leak(lineptr, n, stream, file, line); + return taosGetlineDetectLeak(lineptr, n, stream, file, line); } return taosGetlineImp(lineptr, n, stream); } -static void close_alloc_log() { +static void taosCloseAllocLog() { if (fpAllocLog != NULL) { if (fpAllocLog != stdout) { fclose(fpAllocLog); @@ -432,7 +432,7 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) { if (path == NULL || path[0] == 0) { fpAllocLog = stdout; } else if ((fpAllocLog = fopen(path, "w")) != NULL) { - atexit(close_alloc_log); + atexit(taosCloseAllocLog); } else { printf("failed to open memory allocation log file '%s', errno=%d\n", path, errno); return; @@ -446,10 +446,10 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) { } if (autoDump && mode == TAOS_ALLOC_MODE_DETECT_LEAK) { - atexit(dump_memory_leak); + atexit(taosDumpMemoryLeakImp); struct sigaction act = {0}; - act.sa_handler = dump_memory_leak_on_sig; + act.sa_handler = taosDumpMemoryLeakOnSig; sigaction(SIGFPE, &act, NULL); sigaction(SIGSEGV, &act, NULL); sigaction(SIGILL, &act, NULL); @@ -457,8 +457,8 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) { } void taosDumpMemoryLeak() { - dump_memory_leak(); - close_alloc_log(); + taosDumpMemoryLeakImp(); + taosCloseAllocLog(); } #else // 'TAOS_MEM_CHECK' not defined diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index 8a51c389e9f2efdcf3eaad97d434269ff3c0061f..c7c9d774271555ae5989aa18b7e00d325a0eddde 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -19,8 +19,8 @@ #ifndef TAOS_OS_FUNC_SOCKET -int taosSetNonblocking(SOCKET sock, int on) { - int flags = 0; +int32_t taosSetNonblocking(SOCKET sock, int32_t on) { + int32_t flags = 0; if ((flags = fcntl(sock, F_GETFL, 0)) < 0) { uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); return 1; @@ -43,7 +43,7 @@ void taosBlockSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); + int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); if (rc != 0) { uError("failed to block SIGPIPE"); } @@ -53,7 +53,7 @@ void taosBlockSIGPIPE() { #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT -int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) { +int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index 8df671f9c8974ecf77e5bf0943950772e3f9d192..b0ca6139ed9461b12ab2845d0060b51dc388bcab 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -61,7 +61,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { size_t len; char * line = NULL; while (!feof(fp)) { - taosTFree(line); + tfree(line); len = 0; getline(&line, &len, fp); if (line == NULL) { @@ -83,7 +83,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { sscanf(line, "%s %" PRId64, tmp, &memKB); *memoryUsedMB = (float)((double)memKB / 1024); - taosTFree(line); + tfree(line); fclose(fp); return true; } @@ -107,7 +107,7 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { char cpu[10] = {0}; sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); - taosTFree(line); + tfree(line); fclose(fp); return true; } @@ -136,7 +136,7 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - taosTFree(line); + tfree(line); fclose(fp); return true; } @@ -377,7 +377,7 @@ static bool taosGetCardInfo(int64_t *bytes) { *bytes += (rbytes + tbytes); } - taosTFree(line); + tfree(line); fclose(fp); return true; @@ -432,7 +432,7 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { int readIndex = 0; while (!feof(fp)) { - taosTFree(line); + tfree(line); len = 0; getline(&line, &len, fp); if (line == NULL) { @@ -450,7 +450,7 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { if (readIndex >= 2) break; } - taosTFree(line); + tfree(line); fclose(fp); if (readIndex < 2) { diff --git a/src/os/src/windows/wAtomic.c b/src/os/src/windows/wAtomic.c index a025cb8f0e67ff65ceb0cfadd420fd10cb99891b..b645893030b852428bff13744050b1f6f8ef1c42 100644 --- a/src/os/src/windows/wAtomic.c +++ b/src/os/src/windows/wAtomic.c @@ -44,7 +44,7 @@ long interlocked_add_fetch_32(long volatile* ptr, long val) { __int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) { //#ifdef _WIN64 - return _InterlockedExchangeAdd64(ptr, val) + val; + return InterlockedExchangeAdd64(ptr, val) + val; //#else // return _InterlockedExchangeAdd(ptr, val) + val; //#endif diff --git a/src/os/src/windows/wEnv.c b/src/os/src/windows/wEnv.c index 8110a194904bbbc1166ac57de187de284db34a6b..19351eb7c964a4c2a8a4d1d5d4d1c8ec669908dc 100644 --- a/src/os/src/windows/wEnv.c +++ b/src/os/src/windows/wEnv.c @@ -46,5 +46,16 @@ void osInit() { strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); strcpy(tsOsName, "Windows"); + + const char *tmpDir = getenv("tmp"); + if (tmpDir != NULL) { + tmpDir = getenv("temp"); + } + if (tmpDir != NULL) { + strcpy(tsTempDir, tmpDir); + } else { + strcpy(tsTempDir, "C:\\Windows\\Temp"); + } + taosWinSocketInit(); } diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index 5549c078a5220d3e493ce21e2c19e38fb9611cbb..2204135ae657eb813144dc2bd1ae637d9510842e 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -16,17 +16,20 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tulog.h" +#include "tglobal.h" void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { const char* tdengineTmpFileNamePrefix = "tdengine-"; - char tmpPath[PATH_MAX]; + char tmpPath[PATH_MAX]; - char *tmpDir = getenv("tmp"); - if (tmpDir == NULL) { - tmpDir = ""; + int32_t len = (int32_t)strlen(tsTempDir); + memcpy(tmpPath, tsTempDir, len); + + if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') { + tmpPath[len++] = '\\'; } - - strcpy(tmpPath, tmpDir); + + strcpy(tmpPath + len, tdengineTmpFileNamePrefix); strcat(tmpPath, tdengineTmpFileNamePrefix); if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { strcat(tmpPath, fileNamePrefix); @@ -40,19 +43,19 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { #define _SEND_FILE_STEP_ 1000 -int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) { +int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { fseek(in_file, (int32_t)(*offset), 0); - int writeLen = 0; + int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; - for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); if (rlen <= 0) { return writeLen; } else if (rlen < _SEND_FILE_STEP_) { fwrite(buffer, 1, rlen, out_file); - return (int)(writeLen + rlen); + return (int64_t)(writeLen + rlen); } else { fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); @@ -60,14 +63,14 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou } } - int remain = count - writeLen; + int64_t remain = count - writeLen; if (remain > 0) { - size_t rlen = fread(buffer, 1, remain, in_file); + size_t rlen = fread(buffer, 1, (size_t) remain, in_file); if (rlen <= 0) { return writeLen; } else { - fwrite(buffer, 1, remain, out_file); + fwrite(buffer, 1, (size_t) remain, out_file); writeLen += remain; } } @@ -75,12 +78,12 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou return writeLen; } -ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { - uError("taosTSendFileImp no implemented yet"); +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) { + uError("taosSendFile no implemented yet"); return 0; } -int taosFtruncate(int fd, int64_t length) { +int32_t taosFtruncate(int32_t fd, int64_t length) { uError("taosFtruncate no implemented yet"); return 0; } \ No newline at end of file diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c index da9242d6a3bff471c4e749f0faf7c95b6c3a9d8c..3b091b269931e644d9f8c2c01ba3c9cb9ddc520c 100644 --- a/src/os/src/windows/wSocket.c +++ b/src/os/src/windows/wSocket.c @@ -34,7 +34,7 @@ void taosWinSocketInit() { } } -int taosSetNonblocking(SOCKET sock, int on) { +int32_t taosSetNonblocking(SOCKET sock, int32_t on) { u_long mode; if (on) { mode = 1; @@ -48,7 +48,7 @@ int taosSetNonblocking(SOCKET sock, int on) { void taosBlockSIGPIPE() {} -int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) { +int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { return 0; } @@ -72,7 +72,7 @@ int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int op uint32_t taosInetAddr(char *ipAddr) { uint32_t value; - int ret = inet_pton(AF_INET, ipAddr, &value); + int32_t ret = inet_pton(AF_INET, ipAddr, &value); if (ret <= 0) { return INADDR_NONE; } else { diff --git a/src/plugins/http/inc/httpQueue.h b/src/plugins/http/inc/httpQueue.h index a4590719ff24d48eee875b2f2c4ff2f28a0a31f6..1ffbd5148164ab4d1f74bdac781c8fcf731ad772 100644 --- a/src/plugins/http/inc/httpQueue.h +++ b/src/plugins/http/inc/httpQueue.h @@ -22,9 +22,11 @@ extern "C" { #include +typedef void (*FHttpResultFp)(void *param, void *result, int32_t code, int32_t rows); + bool httpInitResultQueue(); void httpCleanupResultQueue(); -void httpDispatchToResultQueue(); +void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp); #ifdef __cplusplus } diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index ec60b984b2b0fcc72367295cef9d956165e678d5..22f464924e2c21cdeccb0c152457fad11a10294a 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -63,7 +63,7 @@ static void httpDestroyContext(void *data) { pContext->parser = NULL; } - taosTFree(pContext); + tfree(pContext); } bool httpInitContexts() { diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index 43a8ddbd1a0d207addf0576951b2af9b2be16555..1c039abb4d4cd546f6673648217a2410f57ae8e7 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -25,6 +25,7 @@ #include "httpResp.h" #include "httpAuth.h" #include "httpSession.h" +#include "httpQueue.h" typedef struct { pthread_t thread; @@ -37,42 +38,45 @@ typedef struct { } SHttpWorkerPool; typedef struct { - void *param; - void *result; - int32_t numOfRows; - void (*fp)(void *param, void *result, int32_t numOfRows); + void * param; + void * result; + int32_t code; + int32_t rows; + FHttpResultFp fp; } SHttpResult; static SHttpWorkerPool tsHttpPool; static taos_qset tsHttpQset; static taos_queue tsHttpQueue; -void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t numOfRows, void (*fp)(void *param, void *result, int32_t numOfRows)) { +void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp) { if (tsHttpQueue != NULL) { - SHttpResult *pMsg = (SHttpResult *)taosAllocateQitem(sizeof(SHttpResult)); + SHttpResult *pMsg = taosAllocateQitem(sizeof(SHttpResult)); pMsg->param = param; pMsg->result = result; - pMsg->numOfRows = numOfRows; + pMsg->code = code; + pMsg->rows = rows; pMsg->fp = fp; taosWriteQitem(tsHttpQueue, TAOS_QTYPE_RPC, pMsg); } else { - (*fp)(param, result, numOfRows); + (*fp)(param, result, code, rows); } } static void *httpProcessResultQueue(void *param) { SHttpResult *pMsg; - int32_t type; - void *unUsed; - + int32_t type; + void * unUsed; + while (1) { if (taosReadQitemFromQset(tsHttpQset, &type, (void **)&pMsg, &unUsed) == 0) { httpDebug("qset:%p, http queue got no message from qset, exiting", tsHttpQset); break; } - httpTrace("context:%p, res:%p will be processed in result queue", pMsg->param, pMsg->result); - (*pMsg->fp)(pMsg->param, pMsg->result, pMsg->numOfRows); + httpTrace("context:%p, res:%p will be processed in result queue, code:%d rows:%d", pMsg->param, pMsg->result, + pMsg->code, pMsg->rows); + (*pMsg->fp)(pMsg->param, pMsg->result, pMsg->code, pMsg->rows); taosFreeQitem(pMsg); } diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 70d644146cf5ebe15a42c8f00a2e6e4bd603d081..564f555c403b6f5b3deb5832da6505722104eff8 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -29,9 +29,9 @@ void httpProcessMultiSql(HttpContext *pContext); -void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows); +void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows); -void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int numOfRows) { +void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; @@ -43,7 +43,7 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n bool isContinue = false; - if (numOfRows > 0) { + if (code == TSDB_CODE_SUCCESS && numOfRows > 0) { if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->buildQueryJsonFp) { isContinue = (encode->buildQueryJsonFp)(pContext, singleCmd, result, numOfRows); } @@ -58,9 +58,9 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n httpDebug("context:%p, fd:%d, user:%s, process pos:%d, stop retrieve, numOfRows:%d, sql:%s", pContext, pContext->fd, pContext->user, multiCmds->pos, numOfRows, sql); - if (numOfRows < 0) { + if (code < 0) { httpError("context:%p, fd:%d, user:%s, process pos:%d, retrieve failed code:%s, sql:%s", pContext, pContext->fd, - pContext->user, multiCmds->pos, tstrerror(numOfRows), sql); + pContext->user, multiCmds->pos, tstrerror(code), sql); } taos_free_result(result); @@ -73,15 +73,15 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n } } -void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) { - httpDispatchToResultQueue(param, result, numOfRows, httpProcessMultiSqlRetrieveCallBackImp); +void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) { + int32_t code = taos_errno(result); + httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessMultiSqlRetrieveCallBackImp); } -void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) { +void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; - code = taos_errno(result); HttpSqlCmds *multiCmds = pContext->multiCmds; HttpEncodeMethod *encode = pContext->encodeMethod; @@ -94,7 +94,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) { return; } - if (code < 0) { + if (code != TSDB_CODE_SUCCESS) { if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code)) { singleCmd->code = code; httpDebug("context:%p, fd:%d, user:%s, process pos jump to:%d, last code:%s, last sql:%s", pContext, pContext->fd, @@ -119,7 +119,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) { bool isUpdate = tscIsUpdateQuery(result); if (isUpdate) { // not select or show commands - int affectRows = taos_affected_rows(result); + int32_t affectRows = taos_affected_rows(result); httpDebug("context:%p, fd:%d, user:%s, process pos:%d, affect rows:%d, sql:%s", pContext, pContext->fd, pContext->user, multiCmds->pos, affectRows, sql); @@ -156,8 +156,10 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) { } } -void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) { - httpDispatchToResultQueue(param, result, unUsedCode, httpProcessMultiSqlCallBackImp); +void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) { + int32_t code = taos_errno(result); + int32_t affectRows = taos_affected_rows(result); + httpDispatchToResultQueue(param, result, code, affectRows, httpProcessMultiSqlCallBackImp); } void httpProcessMultiSql(HttpContext *pContext) { @@ -202,9 +204,9 @@ void httpProcessMultiSqlCmd(HttpContext *pContext) { httpProcessMultiSql(pContext); } -void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows); +void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows); -void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int numOfRows) { +void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; @@ -212,7 +214,7 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int bool isContinue = false; - if (numOfRows > 0) { + if (code == TSDB_CODE_SUCCESS && numOfRows > 0) { if (encode->buildQueryJsonFp) { isContinue = (encode->buildQueryJsonFp)(pContext, &pContext->singleCmd, result, numOfRows); } @@ -227,9 +229,9 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int httpDebug("context:%p, fd:%d, user:%s, stop retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user, numOfRows); - if (numOfRows < 0) { + if (code < 0) { httpError("context:%p, fd:%d, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->user, - tstrerror(numOfRows)); + tstrerror(code)); } taos_free_result(result); @@ -242,30 +244,30 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int } } -void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) { - httpDispatchToResultQueue(param, result, numOfRows, httpProcessSingleSqlRetrieveCallBackImp); +void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) { + int32_t code = taos_errno(result); + httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessSingleSqlRetrieveCallBackImp); } -void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCode) { +void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; - int32_t code = taos_errno(result); - HttpEncodeMethod *encode = pContext->encodeMethod; if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { httpError("context:%p, fd:%d, user:%s, query error, code:%s:inprogress, sqlObj:%p", pContext, pContext->fd, - pContext->user, tstrerror(code), (SSqlObj *)result); + pContext->user, tstrerror(code), result); return; } - if (code < 0) { + if (code != TSDB_CODE_SUCCESS) { SSqlObj *pObj = (SSqlObj *)result; if (code == TSDB_CODE_TSC_INVALID_SQL) { - httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext, - pContext->fd, pContext->user, tstrerror(code), pObj, pObj->cmd.payload); - httpSendTaosdInvalidSqlErrorResp(pContext, pObj->cmd.payload); + terrno = code; + httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext, pContext->fd, + pContext->user, tstrerror(code), pObj, taos_errstr(pObj)); + httpSendTaosdInvalidSqlErrorResp(pContext, taos_errstr(pObj)); } else { httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd, pContext->user, tstrerror(code), pObj); @@ -278,7 +280,8 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo bool isUpdate = tscIsUpdateQuery(result); if (isUpdate) { // not select or show commands - int affectRows = taos_affected_rows(result); + int32_t affectRows = taos_affected_rows(result); + assert(affectRows == affectRowsInput); httpDebug("context:%p, fd:%d, user:%s, affect rows:%d, stop query, sqlObj:%p", pContext, pContext->fd, pContext->user, affectRows, result); @@ -308,8 +311,10 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo } } -void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) { - httpDispatchToResultQueue(param, result, unUsedCode, httpProcessSingleSqlCallBackImp); +void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) { + int32_t code = taos_errno(result); + int32_t affectRows = taos_affected_rows(result); + httpDispatchToResultQueue(param, result, code, affectRows, httpProcessSingleSqlCallBackImp); } void httpProcessSingleSqlCmd(HttpContext *pContext) { @@ -373,7 +378,7 @@ void httpExecCmd(HttpContext *pContext) { } } -void httpProcessRequestCb(void *param, TAOS_RES *result, int code) { +void httpProcessRequestCb(void *param, TAOS_RES *result, int32_t code) { HttpContext *pContext = param; taos_free_result(result); diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 8993b233dd1dde9283679d337845d42f5d649715..3b8858b62e1297f0db5ece60c59a26e376b546f3 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -107,7 +107,7 @@ void httpCleanUpSystem() { httpCleanupResultQueue(); pthread_mutex_destroy(&tsHttpServer.serverMutex); - taosTFree(tsHttpServer.pThreads); + tfree(tsHttpServer.pThreads); tsHttpServer.pThreads = NULL; tsHttpServer.status = HTTP_SERVER_CLOSED; diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c index 048f839b728fd7ed2a74d59744ebff5d1223cc33..de1e0e233c2b1441a00c226d7305f99a6db41109 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -27,12 +27,12 @@ #include "monitor.h" #include "taoserror.h" -#define monitorFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }} -#define monitorError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }} -#define monitorWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }} -#define monitorInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }} -#define monitorDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }} -#define monitorTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }} +#define mnFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }} +#define mnError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }} +#define mnWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }} +#define mnInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }} +#define mnDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }} +#define mnTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }} #define SQL_LENGTH 1030 #define LOG_LEN_STR 100 @@ -91,12 +91,12 @@ int32_t monitorInitSystem() { pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&tsMonitor.thread, &thAttr, monitorThreadFunc, NULL)) { - monitorError("failed to create thread to for monitor module, reason:%s", strerror(errno)); + mnError("failed to create thread to for monitor module, reason:%s", strerror(errno)); return -1; } pthread_attr_destroy(&thAttr); - monitorDebug("monitor thread is launched"); + mnDebug("monitor thread is launched"); monitorStartSystemFp = monitorStartSystem; monitorStopSystemFp = monitorStopSystem; @@ -107,12 +107,12 @@ int32_t monitorStartSystem() { taos_init(); tsMonitor.start = 1; monitorExecuteSQLFp = monitorExecuteSQL; - monitorInfo("monitor module start"); + mnInfo("monitor module start"); return 0; } static void *monitorThreadFunc(void *param) { - monitorDebug("starting to initialize monitor module ..."); + mnDebug("starting to initialize monitor module ..."); while (1) { static int32_t accessTimes = 0; @@ -121,7 +121,7 @@ static void *monitorThreadFunc(void *param) { if (tsMonitor.quiting) { tsMonitor.state = MON_STATE_NOT_INIT; - monitorInfo("monitor thread will quit, for taosd is quiting"); + mnInfo("monitor thread will quit, for taosd is quiting"); break; } else { taosGetDisk(); @@ -132,7 +132,7 @@ static void *monitorThreadFunc(void *param) { } if (dnodeGetDnodeId() <= 0) { - monitorDebug("dnode not initialized, waiting for 3000 ms to start monitor module"); + mnDebug("dnode not initialized, waiting for 3000 ms to start monitor module"); continue; } @@ -140,10 +140,10 @@ static void *monitorThreadFunc(void *param) { tsMonitor.state = MON_STATE_NOT_INIT; tsMonitor.conn = taos_connect(NULL, "monitor", tsInternalPass, "", 0); if (tsMonitor.conn == NULL) { - monitorError("failed to connect to database, reason:%s", tstrerror(terrno)); + mnError("failed to connect to database, reason:%s", tstrerror(terrno)); continue; } else { - monitorDebug("connect to database success"); + mnDebug("connect to database success"); } } @@ -155,10 +155,10 @@ static void *monitorThreadFunc(void *param) { taos_free_result(res); if (code != 0) { - monitorError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code)); + mnError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code)); break; } else { - monitorDebug("successfully to exec sql:%s", tsMonitor.sql); + mnDebug("successfully to exec sql:%s", tsMonitor.sql); } } @@ -174,7 +174,7 @@ static void *monitorThreadFunc(void *param) { } } - monitorInfo("monitor thread is stopped"); + mnInfo("monitor thread is stopped"); return NULL; } @@ -238,7 +238,7 @@ void monitorStopSystem() { tsMonitor.start = 0; tsMonitor.state = MON_STATE_NOT_INIT; monitorExecuteSQLFp = NULL; - monitorInfo("monitor module stopped"); + mnInfo("monitor module stopped"); } void monitorCleanUpSystem() { @@ -249,7 +249,7 @@ void monitorCleanUpSystem() { taos_close(tsMonitor.conn); tsMonitor.conn = NULL; } - monitorInfo("monitor module is cleaned up"); + mnInfo("monitor module is cleaned up"); } // unit is MB @@ -257,13 +257,13 @@ static int32_t monitorBuildMemorySql(char *sql) { float sysMemoryUsedMB = 0; bool suc = taosGetSysMemory(&sysMemoryUsedMB); if (!suc) { - monitorDebug("failed to get sys memory info"); + mnDebug("failed to get sys memory info"); } float procMemoryUsedMB = 0; suc = taosGetProcMemory(&procMemoryUsedMB); if (!suc) { - monitorDebug("failed to get proc memory info"); + mnDebug("failed to get proc memory info"); } return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB); @@ -274,7 +274,7 @@ static int32_t monitorBuildCpuSql(char *sql) { float sysCpuUsage = 0, procCpuUsage = 0; bool suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage); if (!suc) { - monitorDebug("failed to get cpu usage"); + mnDebug("failed to get cpu usage"); } if (sysCpuUsage <= procCpuUsage) { @@ -294,14 +294,14 @@ static int32_t monitorBuildBandSql(char *sql) { float bandSpeedKb = 0; bool suc = taosGetBandSpeed(&bandSpeedKb); if (!suc) { - monitorDebug("failed to get bandwidth speed"); + mnDebug("failed to get bandwidth speed"); } return sprintf(sql, ", %f", bandSpeedKb); } static int32_t monitorBuildReqSql(char *sql) { - SDnodeStatisInfo info = dnodeGetStatisInfo(); + SStatisInfo info = dnodeGetStatisInfo(); return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum); } @@ -309,7 +309,7 @@ static int32_t monitorBuildIoSql(char *sql) { float readKB = 0, writeKB = 0; bool suc = taosGetProcIO(&readKB, &writeKB); if (!suc) { - monitorDebug("failed to get io info"); + mnDebug("failed to get io info"); } return sprintf(sql, ", %f, %f", readKB, writeKB); @@ -332,19 +332,19 @@ static void monitorSaveSystemInfo() { taos_free_result(res); if (code != 0) { - monitorError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql); + mnError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql); } else { - monitorDebug("successfully to save system info, sql:%s", tsMonitor.sql); + mnDebug("successfully to save system info, sql:%s", tsMonitor.sql); } } static void montiorExecSqlCb(void *param, TAOS_RES *result, int32_t code) { int32_t c = taos_errno(result); if (c != TSDB_CODE_SUCCESS) { - monitorError("save %s failed, reason:%s", (char *)param, tstrerror(c)); + mnError("save %s failed, reason:%s", (char *)param, tstrerror(c)); } else { int32_t rows = taos_affected_rows(result); - monitorDebug("save %s succ, rows:%d", (char *)param, rows); + mnDebug("save %s succ, rows:%d", (char *)param, rows); } taos_free_result(result); @@ -380,7 +380,7 @@ void monitorSaveAcctLog(SAcctMonitorObj *pMon) { pMon->totalConns, pMon->maxConns, pMon->accessState); - monitorDebug("save account info, sql:%s", sql); + mnDebug("save account info, sql:%s", sql); taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "account info"); } @@ -401,13 +401,13 @@ void monitorSaveLog(int32_t level, const char *const format, ...) { len += sprintf(sql + len, "', '%s')", tsLocalEp); sql[len++] = 0; - monitorDebug("save log, sql: %s", sql); + mnDebug("save log, sql: %s", sql); taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "log"); } void monitorExecuteSQL(char *sql) { if (tsMonitor.state != MON_STATE_INITED) return; - monitorDebug("execute sql:%s", sql); + mnDebug("execute sql:%s", sql); taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "sql"); } diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index f392644e6736f528fa75f21ee2857c570c0c22c9..22397d0314c3081e8ed23d9671f4ccfcb7abc244 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -33,17 +33,26 @@ struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); -typedef struct SPosInfo { - int32_t pageId:20; - int32_t rowId:12; -} SPosInfo; - typedef struct SGroupResInfo { int32_t groupId; int32_t numOfDataPages; - SPosInfo pos; + int32_t pageId; + int32_t rowId; } SGroupResInfo; +typedef struct SResultRowPool { + int32_t elemSize; + int32_t blockSize; + int32_t numOfElemPerBlock; + + struct { + int32_t blockIndex; + int32_t pos; + } position; + + SArray* pData; // SArray +} SResultRowPool; + typedef struct SSqlGroupbyExpr { int16_t tableIndex; SArray* columnInfo; // SArray, group by columns information @@ -52,13 +61,14 @@ typedef struct SSqlGroupbyExpr { int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; -typedef struct SWindowResult { - SPosInfo pos; // Position of current result in disk-based output buffer +typedef struct SResultRow { + int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer + int32_t rowId:15; + bool closed:1; // this result status: closed or opened uint16_t numOfRows; // number of rows of current time window - bool closed; // this result status: closed or opened - SResultInfo* resultInfo; // For each result column, there is a resultInfo + SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo union {STimeWindow win; char* key;}; // start key of current time window -} SWindowResult; +} SResultRow; /** * If the number of generated results is greater than this value, @@ -72,16 +82,14 @@ typedef struct SResultRec { } SResultRec; typedef struct SWindowResInfo { - SWindowResult* pResult; // result list - SHashObj* hashList; // hash list for quick access - int16_t type; // data type for hash key + SResultRow** pResult; // result list + int16_t type:8; // data type for hash key + int32_t size:24; // number of result set + int32_t threshold; // threshold to halt query and return the generated results. int32_t capacity; // max capacity int32_t curIndex; // current start active index - int32_t size; // number of result set int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key - int64_t threshold; // threshold to halt query and return the generated results. - int64_t interval; // time window interval } SWindowResInfo; typedef struct SColumnFilterElem { @@ -97,7 +105,7 @@ typedef struct SSingleColumnFilterInfo { SColumnFilterElem* pFilters; } SSingleColumnFilterInfo; -typedef struct STableQueryInfo { // todo merge with the STableQueryInfo struct +typedef struct STableQueryInfo { TSKEY lastKey; int32_t groupIndex; // group id in table list int16_t queryRangeSet; // denote if the query range is set, only available for interval query @@ -125,7 +133,9 @@ typedef struct SQueryCostInfo { uint32_t discardBlocks; uint64_t elapsedTime; uint64_t firstStageMergeTime; - uint64_t internalSupSize; + uint64_t winInfoSize; + uint64_t tableInfoSize; + uint64_t hashSize; uint64_t numOfTimeWindows; } SQueryCostInfo; @@ -158,11 +168,11 @@ typedef struct SQuery { typedef struct SQueryRuntimeEnv { jmp_buf env; - SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo + SResultRow* pResultRow; // todo refactor to merge with SWindowResInfo SQuery* pQuery; SQLFunctionCtx* pCtx; int32_t numOfRowsPerPage; - int16_t offset[TSDB_MAX_COLUMNS]; + uint16_t offset[TSDB_MAX_COLUMNS]; uint16_t scanFlag; // denotes reversed scan of data or not SFillInfo* pFillInfo; SWindowResInfo windowResInfo; @@ -178,6 +188,11 @@ typedef struct SQueryRuntimeEnv { int32_t interBufSize; // intermediate buffer sizse int32_t prevGroupId; // previous executed group id SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file + SHashObj* pResultRowHashTable; // quick locate the window object for each result + char* keyBuf; // window key buffer + SResultRowPool* pool; // window result object pool + + int32_t* rowCellInfoOffset;// offset value for each row result cell info } SQueryRuntimeEnv; enum { @@ -185,12 +200,6 @@ enum { QUERY_RESULT_READY = 2, }; -typedef struct SMemRef { - int32_t ref; - void *mem; - void *imem; -} SMemRef; - typedef struct SQInfo { void* signature; int32_t code; // error code to returned to client diff --git a/src/query/inc/qFill.h b/src/query/inc/qFill.h index 6d44fee09557fa8b8d73527677a2c7b238ecb42c..1b5aca77c4d08bc2b51f1147279a7df64476c8e2 100644 --- a/src/query/inc/qFill.h +++ b/src/query/inc/qFill.h @@ -49,7 +49,6 @@ typedef struct SFillInfo { int32_t numOfTags; // number of tags int32_t numOfCols; // number of columns, including the tags columns int32_t rowSize; // size of each row -// char ** pTags; // tags value for current interpolation SFillTagColInfo* pTags; // tags value for filling gap SInterval interval; char * prevValues; // previous row of data, to generate the interpolation results @@ -71,7 +70,7 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_ void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp); -void* taosDestoryFillInfo(SFillInfo *pFillInfo); +void* taosDestroyFillInfo(SFillInfo *pFillInfo); void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); @@ -83,7 +82,7 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRo int32_t taosNumOfRemainRows(SFillInfo *pFillInfo); -int taosDoLinearInterpolation(int32_t type, SPoint *point1, SPoint *point2, SPoint *point); +int32_t taosGetLinearInterpolationVal(int32_t type, SPoint *point1, SPoint *point2, SPoint *point); int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity); diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index bc8f9a5e23df72f82bcb3bb5140bec42fe426268..f08e13db6463288fa2b9ec233c904e00d9b7dce8 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -129,6 +129,7 @@ typedef struct SCreateDBInfo { int32_t compressionLevel; SStrToken precision; bool ignoreExists; + int8_t update; tVariantList *keep; } SCreateDBInfo; @@ -223,13 +224,6 @@ typedef struct tSQLExprList { tSQLExprItem *a; /* One entry for each expression */ } tSQLExprList; -typedef struct tSQLExprListList { - int32_t nList; /* Number of expressions on the list */ - int32_t nAlloc; /* Number of entries allocated below */ - tSQLExprList **a; /* one entry for each row */ -} tSQLExprListList; - - /** * * @param yyp The parser diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 46e6f79014f32c9b3052824bdb702556f5c4f060..90bd64336fdeed91deb68b9b490224a7fb29bc80 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -26,7 +26,7 @@ extern "C" { #define MEM_BUF_SIZE (1 << 20) #define TS_COMP_FILE_MAGIC 0x87F5EC4C -#define TS_COMP_FILE_VNODE_MAX 512 +#define TS_COMP_FILE_GROUP_MAX 512 typedef struct STSList { char* rawBuf; @@ -35,17 +35,10 @@ typedef struct STSList { int32_t len; } STSList; -typedef struct STSRawBlock { - int32_t vnode; - int64_t tag; - TSKEY* ts; - int32_t len; -} STSRawBlock; - typedef struct STSElem { TSKEY ts; - tVariant tag; - int32_t vnode; + tVariant* tag; + int32_t id; } STSElem; typedef struct STSCursor { @@ -67,26 +60,27 @@ typedef struct STSBlock { * The size of buffer file should not be greater than 2G, * and the offset of int32_t type is enough */ -typedef struct STSVnodeBlockInfo { - int32_t vnode; // vnode id +typedef struct STSGroupBlockInfo { + int32_t id; // group id int32_t offset; // offset set value in file int32_t numOfBlocks; // number of total blocks int32_t compLen; // compressed size -} STSVnodeBlockInfo; +} STSGroupBlockInfo; -typedef struct STSVnodeBlockInfoEx { - STSVnodeBlockInfo info; +typedef struct STSGroupBlockInfoEx { + STSGroupBlockInfo info; int32_t len; // length before compress -} STSVnodeBlockInfoEx; +} STSGroupBlockInfoEx; typedef struct STSBuf { FILE* f; char path[PATH_MAX]; uint32_t fileSize; - STSVnodeBlockInfoEx* pData; + // todo use array + STSGroupBlockInfoEx* pData; uint32_t numOfAlloc; - uint32_t numOfVnodes; + uint32_t numOfGroups; char* assistBuf; int32_t bufSize; @@ -100,30 +94,31 @@ typedef struct STSBuf { typedef struct STSBufFileHeader { uint32_t magic; // file magic number - uint32_t numOfVnode; // number of vnode stored in current file + uint32_t numOfGroup; // number of group stored in current file int32_t tsOrder; // timestamp order in current file } STSBufFileHeader; STSBuf* tsBufCreate(bool autoDelete, int32_t order); STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete); -STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder); +STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t id); void* tsBufDestroy(STSBuf* pTSBuf); -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len); -int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeIdx); +void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len); +int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf); STSBuf* tsBufClone(STSBuf* pTSBuf); -STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId); +STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id); void tsBufFlush(STSBuf* pTSBuf); void tsBufResetPos(STSBuf* pTSBuf); STSElem tsBufGetElem(STSBuf* pTSBuf); + bool tsBufNextPos(STSBuf* pTSBuf); -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag); +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag); STSCursor tsBufGetCursor(STSBuf* pTSBuf); void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order); @@ -136,6 +131,16 @@ void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur); */ void tsBufDisplay(STSBuf* pTSBuf); +int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf); + +void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id); + +int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t id, void* buf, int32_t* len, int32_t* numOfBlocks); + +STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag); + +bool tsBufIsValidElem(STSElem* pElem); + #ifdef __cplusplus } #endif diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 5320e5622e70a93c06edb4a1e5a3fe568498ef21..5d649261a6e9fbf8b9eab44aab699ea792f86138 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -15,13 +15,22 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H +#define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ + do { \ + assert(sizeof(_uid) == sizeof(uint64_t)); \ + *(uint64_t *)(_k) = (_uid); \ + memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \ + } while (0) + +#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) + int32_t getOutputInterResultBufSize(SQuery* pQuery); -void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes); -void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, const SWindowResult* src); +void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow); +void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src); +SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); -int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size, - int32_t threshold, int16_t type); +int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type); void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo); void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo); @@ -33,9 +42,9 @@ void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot); void closeAllTimeWindow(SWindowResInfo* pWindowResInfo); void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order); -static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) { +static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size); - return &pWindowResInfo->pResult[slot]; + return pWindowResInfo->pResult[slot]; } #define curTimeWindowIndex(_winres) ((_winres)->curIndex) @@ -43,15 +52,15 @@ static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInf bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); -int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize); +int32_t initResultRow(SResultRow *pResultRow); -static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult, +static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SResultRow *pResult, tFilePage* page) { assert(pResult != NULL && pRuntimeEnv != NULL); SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t realRowId = (int32_t)(pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery)); + int32_t realRowId = (int32_t)(pResult->rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery)); return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage + pQuery->pSelectExpr[columnIndex].bytes * realRowId; } @@ -62,4 +71,14 @@ bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval); __filter_func_t *getRangeFilterFuncArray(int32_t type); __filter_func_t *getValueFilterFuncArray(int32_t type); +size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv); + +SResultRowPool* initResultRowPool(size_t size); +SResultRow* getNewResultRow(SResultRowPool* p); +int64_t getResultRowPoolMemSize(SResultRowPool* p); +void* destroyResultRowPool(SResultRowPool* p); +int32_t getNumOfAllocatedResultRows(SResultRowPool* p); +int32_t getNumOfUsedResultRows(SResultRowPool* p); + + #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index e5d1185330a47b79fec1becd74a01ed779db8c80..83386e79bbf7b77c0f84e8fa8970235d70b0401e 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -239,6 +239,7 @@ wal(Y) ::= WAL INTEGER(X). { Y = X; } fsync(Y) ::= FSYNC INTEGER(X). { Y = X; } comp(Y) ::= COMP INTEGER(X). { Y = X; } prec(Y) ::= PRECISION STRING(X). { Y = X; } +update(Y) ::= UPDATE INTEGER(X). { Y = X; } %type db_optr {SCreateDBInfo} db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);} @@ -256,6 +257,7 @@ db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; } db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } +db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } %type alter_db_optr {SCreateDBInfo} alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);} @@ -267,6 +269,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = s alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } +alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } %type typename {TAOS_FIELD} typename(A) ::= ids(X). { diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 28b9a60102d41a11fa02fee1ca51cbf8e263bf3b..84ca78d82249b88d68176b0a350a83fb56b8fda9 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -145,15 +145,14 @@ typedef struct SInterpInfoDetail { int8_t primaryCol; } SInterpInfoDetail; -typedef struct SResultInfo { +typedef struct SResultRowCellInfo { int8_t hasResult; // result generated, not NULL value - bool initialized; // output buffer has been initialized - bool complete; // query has completed - bool superTableQ; // is super table query - uint32_t bufLen; // buffer size - uint64_t numOfRes; // num of output result in current buffer - void* interResultBuf; // output result buffer -} SResultInfo; + bool initialized; // output buffer has been initialized + bool complete; // query has completed + uint32_t numOfRes; // num of output result in current buffer +} SResultRowCellInfo; + +#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo))) struct SQLFunctionCtx; @@ -175,9 +174,11 @@ typedef struct SQLFunctionCtx { int16_t inputBytes; int16_t outputType; - int16_t outputBytes; // size of results, determined by function and input column data type - bool hasNull; // null value exist in current block + int16_t outputBytes; // size of results, determined by function and input column data type + int32_t interBufBytes; // internal buffer size + bool hasNull; // null value exist in current block bool requireNull; // require null in some function + bool stableQuery; int16_t functionId; // function id void * aInputElemBuf; char * aOutputBuf; // final result output buffer, point to sdata->data @@ -189,7 +190,8 @@ typedef struct SQLFunctionCtx { void * ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ SQLPreAggVal preAggVals; tVariant tag; - SResultInfo *resultInfo; + + SResultRowCellInfo *resultInfo; SExtTagsInfo tagInfo; } SQLFunctionCtx; @@ -274,16 +276,16 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha (_r)->initialized = false; \ } while (0) -void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf); +//void setResultInfoBuf(SResultRowCellInfo *pResInfo, char* buf); -static FORCE_INLINE void initResultInfo(SResultInfo *pResInfo) { +static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, uint32_t bufLen) { pResInfo->initialized = true; // the this struct has been initialized flag pResInfo->complete = false; pResInfo->hasResult = false; pResInfo->numOfRes = 0; - memset(pResInfo->interResultBuf, 0, (size_t)pResInfo->bufLen); + memset(GET_ROWCELL_INTERBUF(pResInfo), 0, (size_t)bufLen); } #ifdef __cplusplus diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index 893105e44ac4eb82843514cda20928d3e0dcdaf9..f32ed7c39ceeb8f789951cbadefd613c79d709ff 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -32,7 +32,6 @@ #include "tstoken.h" #include "ttokendef.h" #include "tulog.h" -#include "tutil.h" /* * @@ -415,9 +414,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr } if (cond.start != NULL) { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC); } else { - iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->keyInfo.type, TSDB_ORDER_DESC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC); } if (cond.start != NULL) { @@ -432,7 +431,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr break; } - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); } } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal @@ -450,7 +449,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (ret == 0 && optr == TSDB_RELATION_GREATER) { continue; } else { - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); comp = false; } @@ -465,22 +464,22 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr continue; } - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); } tSkipListDestroyIter(iter); comp = true; - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC); while(tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); - if (comp) { + if (comp) { continue; } - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); } @@ -504,7 +503,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (ret == 0 && optr == TSDB_RELATION_LESS) { continue; } else { - STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); comp = false; // no need to compare anymore } @@ -518,7 +517,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { - STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); } } @@ -609,7 +608,7 @@ int32_t intersect(SArray *pLeft, SArray *pRight, SArray *pFinalRes) { /* * traverse the result and apply the function to each item to check if the item is qualified or not */ -static void tArrayTraverse(tExprNode *pExpr, __result_filter_fn_t fp, SArray *pResult) { +static UNUSED_FUNC void tArrayTraverse(tExprNode *pExpr, __result_filter_fn_t fp, SArray *pResult) { assert(pExpr->_node.pLeft->nodeType == TSQL_NODE_COL && pExpr->_node.pRight->nodeType == TSQL_NODE_VALUE && fp != NULL); // scan the result array list and check for each item in the list @@ -660,7 +659,7 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *p * @param pSchema tag schemas * @param fp filter callback function */ -static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) { +static UNUSED_FUNC void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) { size_t size = taosArrayGetSize(pResult); SArray* array = taosArrayInit(size, POINTER_BYTES); @@ -682,7 +681,7 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki while (tSkipListIterNext(iter)) { SSkipListNode *pNode = tSkipListIterGet(iter); if (filterItem(pExpr, pNode, param)) { - taosArrayPush(pResult, SL_GET_NODE_DATA(pNode)); + taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode))); } } tSkipListDestroyIter(iter); @@ -697,7 +696,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SSkipListNode *pNode = tSkipListIterGet(iter); char * pData = SL_GET_NODE_DATA(pNode); - tstr *name = (tstr*) tsdbGetTableName(*(void**) pData); + tstr *name = (tstr*) tsdbGetTableName((void*) pData); // todo speed up by using hash if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { @@ -711,7 +710,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, } if (addToResult) { - STableKeyInfo info = {.pTable = *(void**)pData, .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(res, &info); } } @@ -733,10 +732,6 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY)); param->setupInfoFn(pExpr, param->pExtInfo); - if (pSkipList == NULL) { - tArrayTraverse(pExpr, param->nodeFilterFn, result); - return; - } tQueryInfo *pQueryInfo = pExpr->_node.info; if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) { @@ -748,49 +743,14 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S return; } - // recursive traverse left child branch + // The value of hasPK is always 0. uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; + assert(weight == 0 && pSkipList != NULL && taosArrayGetSize(result) == 0); - if (weight == 0 ) { - if (taosArrayGetSize(result) > 0 && pSkipList == NULL) { - /** - * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. - * Since no index presented, the filter operation is done by scan all elements in the result set. - * - * if the query is a high selectivity filter, only small portion of meters are retrieved. - */ - exprTreeTraverseImpl(pExpr, result, param); - } else { - /** - * apply the hierarchical expression to every node in skiplist for find the qualified nodes - */ - assert(taosArrayGetSize(result) == 0); - tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); - } - - return; - } - - if (weight == 2 || (weight == 1 && pExpr->_node.optr == TSDB_RELATION_OR)) { - SArray* rLeft = taosArrayInit(10, POINTER_BYTES); - SArray* rRight = taosArrayInit(10, POINTER_BYTES); - - tExprTreeTraverse(pLeft, pSkipList, rLeft, param); - tExprTreeTraverse(pRight, pSkipList, rRight, param); - - if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS - intersect(rLeft, rRight, result); - } else if (pExpr->_node.optr == TSDB_RELATION_OR) { // or - merge(rLeft, rRight, result); - } else { - assert(false); - } - - taosArrayDestroy(rLeft); - taosArrayDestroy(rRight); - return; - } + //apply the hierarchical expression to every node in skiplist for find the qualified nodes + tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); + #if 0 /* * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here * @@ -819,6 +779,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S * So, we do not set the skip list index as a parameter */ tExprTreeTraverse(pSecond, NULL, result, param); +#endif } void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 5ad52ef29e6ba0a4b47129db891c0d1318e76c5f..98a4cf2c423dd7b997a3fb59718b93dfae9a3ebf 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -178,13 +178,13 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { // todo move to utility static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); -static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); -static void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); -static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo); +static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); +static void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); +static void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, - SDataStatis *pStatis, void *param, int32_t colIndex); + SDataStatis *pStatis, void *param, int32_t colIndex, int32_t vgId); static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); @@ -194,6 +194,8 @@ static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); +static int32_t checkForQueryBuf(size_t numOfTables); +static void releaseQueryBuf(size_t numOfTables); bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { @@ -253,7 +255,7 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { continue; } - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { maxOutput = pResInfo->numOfRes; } @@ -270,7 +272,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); int16_t functionId = pRuntimeEnv->pCtx[j].functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || @@ -445,11 +447,11 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis return true; } -static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, - int16_t bytes, bool masterscan) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - int32_t *p1 = (int32_t *) taosHashGet(pWindowResInfo->hashList, pData, bytes); +static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, + int16_t bytes, bool masterscan, uint64_t uid) { + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); + int32_t *p1 = + (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (p1 != NULL) { pWindowResInfo->curIndex = *p1; } else { @@ -459,41 +461,37 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin // more than the capacity, reallocate the resources if (pWindowResInfo->size >= pWindowResInfo->capacity) { - int64_t newCap = 0; + int64_t newCapacity = 0; if (pWindowResInfo->capacity > 10000) { - newCap = (int64_t)(pWindowResInfo->capacity * 1.25); + newCapacity = (int64_t)(pWindowResInfo->capacity * 1.25); } else { - newCap = (int64_t)(pWindowResInfo->capacity * 1.5); + newCapacity = (int64_t)(pWindowResInfo->capacity * 1.5); } - char *t = realloc(pWindowResInfo->pResult, (size_t)(newCap * sizeof(SWindowResult))); - pRuntimeEnv->summary.internalSupSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult); - pRuntimeEnv->summary.numOfTimeWindows += (newCap - pWindowResInfo->capacity); - + char *t = realloc(pWindowResInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pWindowResInfo->pResult = (SWindowResult *)t; + pWindowResInfo->pResult = (SResultRow **)t; - int32_t inc = (int32_t)newCap - pWindowResInfo->capacity; - memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc); + int32_t inc = (int32_t)newCapacity - pWindowResInfo->capacity; + memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, POINTER_BYTES * inc); - pRuntimeEnv->summary.internalSupSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; - - for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { - int32_t ret = createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } + pWindowResInfo->capacity = (int32_t)newCapacity; + } - pWindowResInfo->capacity = (int32_t)newCap; + SResultRow *pResult = getNewResultRow(pRuntimeEnv->pool); + pWindowResInfo->pResult[pWindowResInfo->size] = pResult; + int32_t ret = initResultRow(pResult); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } // add a new result set for a new group pWindowResInfo->curIndex = pWindowResInfo->size++; - taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); + taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), + (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); } // too many time window in query @@ -501,7 +499,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } - return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex); + return getResultRow(pWindowResInfo, pWindowResInfo->curIndex); } // get the correct time window according to the handled timestamp @@ -517,7 +515,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t } } else { int32_t slot = curTimeWindowIndex(pWindowResInfo); - SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot); + SResultRow* pWindowRes = getResultRow(pWindowResInfo, slot); w = pWindowRes->win; } @@ -553,9 +551,9 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t return w; } -static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t sid, +static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, int32_t numOfRowsPerPage) { - if (pWindowRes->pos.pageId != -1) { + if (pWindowRes->pageId != -1) { return 0; } @@ -563,10 +561,10 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult // in the first scan, new space needed for results int32_t pageId = -1; - SIDList list = getDataBufPagesIdList(pResultBuf, sid); + SIDList list = getDataBufPagesIdList(pResultBuf, tid); if (taosArrayGetSize(list) == 0) { - pData = getNewDataBuf(pResultBuf, sid, &pageId); + pData = getNewDataBuf(pResultBuf, tid, &pageId); } else { SPageInfo* pi = getLastPageInfo(list); pData = getResBufPage(pResultBuf, pi->pageId); @@ -576,7 +574,7 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult // release current page first, and prepare the next one releaseResBufPageInfo(pResultBuf, pi); - pData = getNewDataBuf(pResultBuf, sid, &pageId); + pData = getNewDataBuf(pResultBuf, tid, &pageId); if (pData != NULL) { assert(pData->num == 0); // number of elements must be 0 for new allocated buffer } @@ -588,24 +586,23 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult } // set the number of rows in current disk page - if (pWindowRes->pos.pageId == -1) { // not allocated yet, allocate new buffer - pWindowRes->pos.pageId = pageId; - pWindowRes->pos.rowId = (int32_t)(pData->num++); + if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer + pWindowRes->pageId = pageId; + pWindowRes->rowId = (int32_t)(pData->num++); - assert(pWindowRes->pos.pageId >= 0); + assert(pWindowRes->pageId >= 0); } return 0; } -static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t sid, +static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, SDataBlockInfo* pBockInfo, STimeWindow *win, bool masterscan, bool* newWind) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, - TSDB_KEYSIZE, masterscan); - if (pWindowRes == NULL) { + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, pBockInfo->uid); + if (pResultRow == NULL) { *newWind = false; return masterscan? -1:0; @@ -614,23 +611,22 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes *newWind = true; // not assign result buffer yet, add new result buffer - if (pWindowRes->pos.pageId == -1) { - int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, sid, pRuntimeEnv->numOfRowsPerPage); + if (pResultRow->pageId == -1) { + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, pBockInfo->tid, pRuntimeEnv->numOfRowsPerPage); if (ret != TSDB_CODE_SUCCESS) { return -1; } } // set time window for current result - pWindowRes->win = (*win); - - setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes); + pResultRow->win = (*win); + setWindowResOutputBufInitCtx(pRuntimeEnv, pResultRow); return TSDB_CODE_SUCCESS; } static bool getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(slot >= 0 && slot < pWindowResInfo->size); - return pWindowResInfo->pResult[slot].closed; + return pWindowResInfo->pResult[slot]->closed; } static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, @@ -689,7 +685,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe int64_t skey = TSKEY_INITIAL_VAL; for (i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *pResult = &pWindowResInfo->pResult[i]; + SResultRow *pResult = pWindowResInfo->pResult[i]; if (pResult->closed) { numOfClosed += 1; continue; @@ -713,7 +709,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe pWindowResInfo->curIndex = i; } - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].win.skey; + pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey; // the number of completed slots are larger than the threshold, return current generated results to client. if (numOfClosed > pWindowResInfo->threshold) { @@ -938,7 +934,6 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); if (sas->data == NULL) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1001,13 +996,13 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); if (sasArray == NULL) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } + SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k); + setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); @@ -1023,9 +1018,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * bool hasTimeWindow = false; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow) != - TSDB_CODE_SUCCESS) { - taosTFree(sasArray); + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { + tfree(sasArray); return; } @@ -1052,8 +1046,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * // null data, failed to allocate more memory buffer hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan, - &hasTimeWindow) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { break; } @@ -1088,13 +1081,13 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * continue; } - taosTFree(sasArray[i].data); + tfree(sasArray[i].data); } - taosTFree(sasArray); + tfree(sasArray); } -static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes) { +static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { if (isNull(pData, type)) { // ignore the null value return -1; } @@ -1111,13 +1104,14 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat len = varDataLen(pData); } else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - qError("QInfo:%p group by not supported on double/float/binary/nchar columns, abort", pQInfo); + qError("QInfo:%p group by not supported on double/float columns, abort", pQInfo); longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true); - if (pWindowRes == NULL) { + uint64_t uid = groupIndex; // uid is always set to be 0. + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); + if (pResultRow == NULL) { return -1; } @@ -1131,23 +1125,21 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - pWindowRes->key = malloc(varDataTLen(pData)); - varDataCopy(pWindowRes->key, pData); + pResultRow->key = malloc(varDataTLen(pData)); + varDataCopy(pResultRow->key, pData); } else { - pWindowRes->win.skey = v; - pWindowRes->win.ekey = v; + pResultRow->win.skey = v; + pResultRow->win.ekey = v; } - assert(pRuntimeEnv->windowResInfo.interval == 0); - - if (pWindowRes->pos.pageId == -1) { - int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); + if (pResultRow->pageId == -1) { + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); if (ret != 0) { return -1; } } - setWindowResOutputBuf(pRuntimeEnv, pWindowRes); + setResultOutputBuf(pRuntimeEnv, pResultRow); initCtxOutputBuf(pRuntimeEnv); return TSDB_CODE_SUCCESS; } @@ -1200,7 +1192,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; // compare tag first - if (tVariantCompare(&pCtx[0].tag, &elem.tag) != 0) { + if (tVariantCompare(&pCtx[0].tag, elem.tag) != 0) { return TS_JOIN_TAG_NOT_EQUALS; } @@ -1230,7 +1222,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { } static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SQuery* pQuery = pRuntimeEnv->pQuery; // in case of timestamp column, always generated results. @@ -1274,7 +1266,6 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); if (sasArray == NULL) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1286,9 +1277,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS groupbyColumnData = getGroupbyColumnData(pQuery, &type, &bytes, pDataBlock); } + SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k); + setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); } // set the input column data @@ -1303,7 +1295,6 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // from top to bottom in desc // from bottom to top in asc order if (pRuntimeEnv->pTSBuf != NULL) { - SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pRuntimeEnv); qDebug("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->rows, pQuery->order.order, pRuntimeEnv->pTSBuf->cur.order); } @@ -1335,7 +1326,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); bool hasTimeWindow = false; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow); + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1363,7 +1354,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // null data, failed to allocate more memory buffer hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { break; } @@ -1379,7 +1370,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS if (groupbyColumnValue) { char *val = groupbyColumnData + bytes * offset; - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes); + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1409,13 +1400,17 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS item->lastKey = (QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey) + step; } + if (pRuntimeEnv->pTSBuf != NULL) { + item->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + } + // todo refactor: extract method for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { if (pQuery->pSelectExpr[i].base.functionId != TSDB_FUNC_ARITHM) { continue; } - taosTFree(sasArray[i].data); + tfree(sasArray[i].data); } free(sasArray); @@ -1469,7 +1464,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl } void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, - SDataStatis *pStatis, void *param, int32_t colIndex) { + SDataStatis *pStatis, void *param, int32_t colIndex, int32_t vgId) { int32_t functionId = pQuery->pSelectExpr[colIndex].base.functionId; int32_t colId = pQuery->pSelectExpr[colIndex].base.colInfo.colId; @@ -1514,7 +1509,8 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY * top/bottom values emerge, so does diff function */ if (functionId == TSDB_FUNC_TWA) { - STwaInfo *pTWAInfo = GET_RES_INFO(pCtx)->interResultBuf; + SResultRowCellInfo* pInfo = GET_RES_INFO(pCtx); + STwaInfo *pTWAInfo = (STwaInfo*) GET_ROWCELL_INTERBUF(pInfo); pTWAInfo->SKey = pQuery->window.skey; pTWAInfo->EKey = pQuery->window.ekey; } @@ -1528,7 +1524,9 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY pCtx->preAggVals.statis.max = pBlockInfo->window.ekey; } } else if (functionId == TSDB_FUNC_INTERP) { - SInterpInfoDetail *pInterpInfo = GET_RES_INFO(pCtx)->interResultBuf; + SResultRowCellInfo* pInfo = GET_RES_INFO(pCtx); + + SInterpInfoDetail *pInterpInfo = (SInterpInfoDetail *)GET_ROWCELL_INTERBUF(pInfo); pInterpInfo->type = (int8_t)pQuery->fillType; pInterpInfo->ts = pQuery->window.skey; pInterpInfo->primaryCol = (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); @@ -1542,6 +1540,9 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY } } } + } else if (functionId == TSDB_FUNC_TS_COMP) { + pCtx->param[0].i64Key = vgId; + pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; } #if defined(_DEBUG_VIEW) @@ -1595,33 +1596,22 @@ static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx p->tagInfo.numOfTagCols = num; p->tagInfo.tagsLen = tagLen; } else { - taosTFree(pTagCtx); + tfree(pTagCtx); } } return TSDB_CODE_SUCCESS; } -static FORCE_INLINE void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery, char* buf) { - char* p = buf; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t size = pQuery->pSelectExpr[i].interBytes; - setResultInfoBuf(&pResultInfo[i], size, isStableQuery, p); - - p += size; - } -} - static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order) { qDebug("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); SQuery *pQuery = pRuntimeEnv->pQuery; - size_t size = pRuntimeEnv->interBufSize + pQuery->numOfOutput * sizeof(SResultInfo); - - pRuntimeEnv->resultInfo = calloc(1, size); pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); + pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t)); + pRuntimeEnv->pResultRow = getNewResultRow(pRuntimeEnv->pool); - if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) { + if (pRuntimeEnv->pResultRow == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) { goto _clean; } @@ -1658,7 +1648,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->inputType = pQuery->colList[index].type; } - assert(isValidDataType(pCtx->inputType)); pCtx->ptsOutputBuf = NULL; @@ -1667,6 +1656,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->order = pQuery->order.order; pCtx->functionId = pSqlFuncMsg->functionId; + pCtx->stableQuery = pRuntimeEnv->stableQuery; + pCtx->interBufBytes = pQuery->pSelectExpr[i].interBytes; pCtx->numOfParams = pSqlFuncMsg->numOfParams; for (int32_t j = 0; j < pCtx->numOfParams; ++j) { @@ -1696,16 +1687,14 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order if (i > 0) { pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes; + pRuntimeEnv->rowCellInfoOffset[i] = pRuntimeEnv->rowCellInfoOffset[i - 1] + sizeof(SResultRowCellInfo) + pQuery->pSelectExpr[i - 1].interBytes; } - } - - char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultInfo) * pQuery->numOfOutput; - // set the intermediate result output buffer - setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf); + } // if it is group by normal column, do not set output buffer, the output buffer is pResult - if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery) { + // fixed output query/multi-output query for normal table + if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { resetCtxOutputBuf(pRuntimeEnv); } @@ -1717,12 +1706,24 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order return TSDB_CODE_SUCCESS; _clean: - taosTFree(pRuntimeEnv->resultInfo); - taosTFree(pRuntimeEnv->pCtx); + tfree(pRuntimeEnv->pCtx); return TSDB_CODE_QRY_OUT_OF_MEMORY; } +static void doFreeQueryHandle(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + + tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); + tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); + + pRuntimeEnv->pQueryHandle = NULL; + pRuntimeEnv->pSecQueryHandle = NULL; + + SMemRef* pMemRef = &pQInfo->memRef; + assert(pMemRef->ref == 0 && pMemRef->imem == NULL && pMemRef->mem == NULL); +} + static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { if (pRuntimeEnv->pQuery == NULL) { return; @@ -1743,20 +1744,25 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { } tVariantDestroy(&pCtx->tag); - taosTFree(pCtx->tagInfo.pTagCtxList); + tfree(pCtx->tagInfo.pTagCtxList); } - taosTFree(pRuntimeEnv->resultInfo); - taosTFree(pRuntimeEnv->pCtx); + tfree(pRuntimeEnv->pCtx); } - pRuntimeEnv->pFillInfo = taosDestoryFillInfo(pRuntimeEnv->pFillInfo); + pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); - tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); + doFreeQueryHandle(pQInfo); pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf); + tfree(pRuntimeEnv->keyBuf); + tfree(pRuntimeEnv->rowCellInfoOffset); + + taosHashCleanup(pRuntimeEnv->pResultRowHashTable); + pRuntimeEnv->pResultRowHashTable = NULL; + + pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); } #define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) @@ -1847,8 +1853,11 @@ static bool needReverseScan(SQuery *pQuery) { } if (functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) { + // the scan order to acquire the last result of the specified column int32_t order = (int32_t)pQuery->pSelectExpr[i].base.arg->argValue.i64; - return order != pQuery->order.order; + if (order != pQuery->order.order) { + return true; + } } } @@ -2252,7 +2261,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo * pW TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey; STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo->tid, &win, masterScan, &hasTimeWindow) != + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { // todo handle error in set result for timewindow } @@ -2621,21 +2630,28 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { assert(pFuncMsg->numOfParams == 1); - int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64; - SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); + int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64; + SColumnInfo *pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); - qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%"PRId64, pQInfo, pExprInfo->base.arg->argValue.i64, - pRuntimeEnv->pCtx[0].tag.i64Key) + + int16_t tagType = pRuntimeEnv->pCtx[0].tag.nType; + if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%s", pQInfo, + pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.pz); + } else { + qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, pQInfo, + pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.i64Key); + } } } } -static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowResult *pWindowRes, bool mergeFlag) { +static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SResultRow *pWindowRes, bool mergeFlag) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].base.functionId; @@ -2805,15 +2821,15 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) } SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; - SWindowResult * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos); - tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pos.pageId); + SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); + tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId); char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1); TSKEY leftTimestamp = GET_INT64_VAL(b1); SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; - SWindowResult * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos); - tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pos.pageId); + SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); + tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId); char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2); TSKEY rightTimestamp = GET_INT64_VAL(b2); @@ -2850,7 +2866,7 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { } SGroupResInfo* info = &pQInfo->groupResInfo; - if (pQInfo->groupIndex == numOfGroups && info->pos.pageId == info->numOfDataPages) { + if (pQInfo->groupIndex == numOfGroups && info->pageId == info->numOfDataPages) { SET_STABLE_QUERY_OVER(pQInfo); } @@ -2866,10 +2882,10 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; // all results have been return to client, try next group - if (pGroupResInfo->pos.pageId == pGroupResInfo->numOfDataPages) { + if (pGroupResInfo->pageId == pGroupResInfo->numOfDataPages) { pGroupResInfo->numOfDataPages = 0; - pGroupResInfo->pos.pageId = 0; - pGroupResInfo->pos.rowId = 0; + pGroupResInfo->pageId = 0; + pGroupResInfo->rowId = 0; // current results of group has been sent to client, try next group if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { @@ -2897,22 +2913,22 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { assert(size == pGroupResInfo->numOfDataPages); bool done = false; - for (int32_t j = pGroupResInfo->pos.pageId; j < size; ++j) { + for (int32_t j = pGroupResInfo->pageId; j < size; ++j) { SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); - assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->pos.rowId < pData->num); - int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->pos.rowId); + assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->rowId < pData->num); + int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->rowId); if (numOfRes > pQuery->rec.capacity - offset) { numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset); - pGroupResInfo->pos.rowId += numOfCopiedRows; + pGroupResInfo->rowId += numOfCopiedRows; done = true; } else { numOfCopiedRows = (int32_t)pData->num; - pGroupResInfo->pos.pageId += 1; - pGroupResInfo->pos.rowId = 0; + pGroupResInfo->pageId += 1; + pGroupResInfo->rowId = 0; } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { @@ -2933,7 +2949,9 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { pQuery->rec.rows += offset; } -int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) { +int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) { + SQuery* pQuery = pRuntimeEnv->pQuery; + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].base.functionId; @@ -2945,7 +2963,7 @@ int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) { continue; } - SResultInfo *pResultInfo = &pWindowRes->resultInfo[j]; + SResultRowCellInfo *pResultInfo = getResultCell(pRuntimeEnv, pResultRow, j); assert(pResultInfo != NULL); if (pResultInfo->numOfRes > 0) { @@ -2967,8 +2985,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { STableQueryInfo **pTableList = malloc(POINTER_BYTES * size); if (pTableList == NULL || posList == NULL) { - taosTFree(posList); - taosTFree(pTableList); + tfree(posList); + tfree(pTableList); qError("QInfo:%p failed alloc memory", pQInfo); longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -2992,19 +3010,19 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { // there is no data in current group if (numOfTables == 0) { - taosTFree(posList); - taosTFree(pTableList); + tfree(posList); + tfree(pTableList); return 0; } else if (numOfTables == 1) { // no need to merge results since only one table in each group - taosTFree(posList); - taosTFree(pTableList); + tfree(posList); + tfree(pTableList); SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList); pGroupResInfo->groupId = tid; - pGroupResInfo->pos.pageId = 0; - pGroupResInfo->pos.rowId = 0; + pGroupResInfo->pageId = 0; + pGroupResInfo->rowId = 0; return pGroupResInfo->numOfDataPages; } @@ -3014,18 +3032,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SLoserTreeInfo *pTree = NULL; tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); - SResultInfo *pResultInfo = calloc(pQuery->numOfOutput, sizeof(SResultInfo)); - if (pResultInfo == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - char* buf = calloc(1, pRuntimeEnv->interBufSize); - if (buf == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf); - resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + SResultRow* pRow = getNewResultRow(pRuntimeEnv->pool); + resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex); @@ -3037,26 +3045,23 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); - taosTFree(pTableList); - taosTFree(posList); - taosTFree(pTree); - taosTFree(pResultInfo); - taosTFree(buf); - + tfree(pTableList); + tfree(posList); + tfree(pTree); longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } int32_t pos = pTree->pNode[0].index; SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; - SWindowResult *pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId); + SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); TSKEY ts = GET_INT64_VAL(b); assert(ts == pWindowRes->win.skey); - int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes); + int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes); if (num <= 0) { cs.position[pos] += 1; @@ -3077,7 +3082,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { return -1; } - resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); } doMerge(pRuntimeEnv, ts, pWindowRes, false); @@ -3087,7 +3092,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { lastTimestamp = ts; // move to the next element of current entry - int32_t currentPageId = pWindowRes->pos.pageId; + int32_t currentPageId = pWindowRes->pageId; cs.position[pos] += 1; if (cs.position[pos] >= pWindowResInfo->size) { @@ -3099,8 +3104,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } } else { // current page is not needed anymore - SWindowResult *pNextWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); - if (pNextWindowRes->pos.pageId != currentPageId) { + SResultRow *pNextWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); + if (pNextWindowRes->pageId != currentPageId) { releaseResBufPage(pRuntimeEnv->pResultBuf, page); } } @@ -3113,11 +3118,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); - taosTFree(pTree); - taosTFree(pTableList); - taosTFree(posList); - taosTFree(pResultInfo); - + tfree(pTree); + tfree(pTableList); + tfree(posList); return -1; } } @@ -3130,12 +3133,12 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt); - taosTFree(pTableList); - taosTFree(posList); - taosTFree(pTree); + tfree(pTableList); + tfree(posList); + tfree(pTree); - taosTFree(pResultInfo); - taosTFree(buf); +// tfree(pResultInfo); +// tfree(buf); return pQInfo->groupResInfo.numOfDataPages; } @@ -3178,12 +3181,14 @@ int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupR return TSDB_CODE_SUCCESS; } -void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo) { +void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow) { + SQuery* pQuery = pRuntimeEnv->pQuery; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; pCtx[k].size = 1; pCtx[k].startOffset = 0; - pCtx[k].resultInfo = &pResultInfo[k]; + pCtx[k].resultInfo = getResultCell(pRuntimeEnv, pRow, k); pQuery->sdata[k]->num = 0; } @@ -3220,8 +3225,8 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * pTableQueryInfo->windowResInfo.curIndex = pTableQueryInfo->windowResInfo.size - 1; } -static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindowResInfo, int32_t order) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; +static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t order) { + SQuery* pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { bool closed = getTimeWindowResStatus(pWindowResInfo, i); @@ -3229,17 +3234,18 @@ static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindow continue; } - SWindowResult *buf = getWindowResult(pWindowResInfo, i); + SResultRow *pRow = getResultRow(pWindowResInfo, i); // open/close the specified query for each group result for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functId = pQuery->pSelectExpr[j].base.functionId; + SResultRowCellInfo* pInfo = getResultCell(pRuntimeEnv, pRow, j); if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { - buf->resultInfo[j].complete = false; + pInfo->complete = false; } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { - buf->resultInfo[j].complete = true; + pInfo->complete = true; } } } @@ -3253,7 +3259,7 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { // group by normal columns and interval query on normal table SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { - disableFuncInReverseScanImpl(pQInfo, pWindowResInfo, order); + disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order); } else { // for simple result of table query, for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor int32_t functId = pQuery->pSelectExpr[j].base.functionId; @@ -3303,26 +3309,16 @@ void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { } } -int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize) { - int32_t numOfCols = pQuery->numOfOutput; - - size_t size = numOfCols * sizeof(SResultInfo) + interBufSize; - pResultRow->resultInfo = calloc(1, size); - if (pResultRow->resultInfo == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - pResultRow->pos = (SPosInfo) {-1, -1}; - - char* buf = (char*) pResultRow->resultInfo + numOfCols * sizeof(SResultInfo); - - // set the intermediate result output buffer - setWindowResultInfo(pResultRow->resultInfo, pQuery, isSTableQuery, buf); +int32_t initResultRow(SResultRow *pResultRow) { + pResultRow->pCellInfo = (SResultRowCellInfo*)((char*)pResultRow + sizeof(SResultRow)); + pResultRow->pageId = -1; + pResultRow->rowId = -1; return TSDB_CODE_SUCCESS; } void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; + SResultRow* pRow = pRuntimeEnv->pResultRow; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; @@ -3332,8 +3328,9 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { * set the output buffer information and intermediate buffer * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. */ - RESET_RESULT_INFO(&pRuntimeEnv->resultInfo[i]); - pCtx->resultInfo = &pRuntimeEnv->resultInfo[i]; + SResultRowCellInfo* pCellInfo = getResultCell(pRuntimeEnv, pRow, i); + RESET_RESULT_INFO(pCellInfo); + pCtx->resultInfo = pCellInfo; // set the timestamp output buffer for top/bottom/diff query int32_t functionId = pQuery->pSelectExpr[i].base.functionId; @@ -3382,7 +3379,7 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { int32_t functionId = pQuery->pSelectExpr[j].base.functionId; pRuntimeEnv->pCtx[j].currentStage = 0; - SResultInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); if (pResInfo->initialized) { continue; } @@ -3451,12 +3448,12 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *pResult = getWindowResult(pWindowResInfo, i); + SResultRow *pResult = getResultRow(pWindowResInfo, i); if (!pResult->closed) { continue; } - setWindowResOutputBuf(pRuntimeEnv, pResult); + setResultOutputBuf(pRuntimeEnv, pResult); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int16_t functId = pQuery->pSelectExpr[j].base.functionId; @@ -3465,7 +3462,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { } aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); toContinue |= (!pResInfo->complete); } @@ -3478,7 +3475,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { } aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); toContinue |= (!pResInfo->complete); } @@ -3551,7 +3548,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); if (pRuntimeEnv->pSecQueryHandle == NULL) { longjmp(pRuntimeEnv->env, terrno); } @@ -3635,7 +3632,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { } restoreTimeWindow(&pQInfo->tableGroupInfo, &cond); - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); if (pRuntimeEnv->pSecQueryHandle == NULL) { longjmp(pRuntimeEnv->env, terrno); } @@ -3649,7 +3646,6 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { // check if query is killed or not if (IS_QUERY_KILLED(pQInfo)) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } } @@ -3678,12 +3674,12 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *buf = &pWindowResInfo->pResult[i]; + SResultRow *buf = pWindowResInfo->pResult[i]; if (!isWindowResClosed(pWindowResInfo, i)) { continue; } - setWindowResOutputBuf(pRuntimeEnv, buf); + setResultOutputBuf(pRuntimeEnv, buf); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { aAggs[pQuery->pSelectExpr[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); @@ -3730,7 +3726,7 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { int32_t initialSize = 16; int32_t initialThreshold = 100; - int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, initialSize, initialThreshold, TSDB_DATA_TYPE_INT); + int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, initialSize, initialThreshold, TSDB_DATA_TYPE_INT); if (code != TSDB_CODE_SUCCESS) { return NULL; } @@ -3745,6 +3741,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { return; } + tVariantDestroy(&pTableQueryInfo->tag); cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo); } @@ -3769,9 +3766,10 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { return; } - SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, - sizeof(groupIndex), true); - if (pWindowRes == NULL) { + uint64_t uid = 0; // uid is always set to be 0 + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, + sizeof(groupIndex), true, uid); + if (pResultRow == NULL) { return; } @@ -3779,8 +3777,8 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { * not assign result buffer yet, add new result buffer * all group belong to one result set, and each group result has different group id so set the id to be one */ - if (pWindowRes->pos.pageId == -1) { - if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) != + if (pResultRow->pageId == -1) { + if (addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) != TSDB_CODE_SUCCESS) { return; } @@ -3788,15 +3786,15 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { // record the current active group id pRuntimeEnv->prevGroupId = groupIndex; - setWindowResOutputBuf(pRuntimeEnv, pWindowRes); + setResultOutputBuf(pRuntimeEnv, pResultRow); initCtxOutputBuf(pRuntimeEnv); } -void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) { +void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; @@ -3808,27 +3806,23 @@ void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult } /* - * set the output buffer information and intermediate buffer + * set the output buffer information and intermediate buffer, * not all queries require the interResultBuf, such as COUNT */ - pCtx->resultInfo = &pResult->resultInfo[i]; - - // set super table query flag - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->superTableQ = pRuntimeEnv->stableQuery; + pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i); } } -void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) { +void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - pCtx->resultInfo = &pResult->resultInfo[i]; + pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i); if (pCtx->resultInfo->initialized && pCtx->resultInfo->complete) { continue; } @@ -3841,12 +3835,6 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - /* - * set the output buffer information and intermediate buffer - * not all queries require the interResultBuf, such as COUNT - */ - pCtx->resultInfo->superTableQ = pRuntimeEnv->stableQuery; // set super table query flag - if (!pCtx->resultInfo->initialized) { aAggs[functionId].init(pCtx); } @@ -3860,14 +3848,40 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ // both the master and supplement scan needs to set the correct ts comp start position if (pRuntimeEnv->pTSBuf != NULL) { + tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; + if (pTableQueryInfo->cur.vgroupIndex == -1) { - tVariantAssign(&pTableQueryInfo->tag, &pRuntimeEnv->pCtx[0].tag); - tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pTableQueryInfo->tag); + tVariantAssign(&pTableQueryInfo->tag, pTag); + + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pTableQueryInfo->tag); + + // failed to find data with the specified tag value and vnodeId + if (!tsBufIsValidElem(&elem)) { + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); + } else { + qError("QInfo:%p failed to find tag:%" PRId64 " in ts_comp", pQInfo, pTag->i64Key); + } + + return false; + } // keep the cursor info of current meter - pTableQueryInfo->cur = pRuntimeEnv->pTSBuf->cur; + pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } else { + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } + } else { tsBufSetCursor(pRuntimeEnv->pTSBuf, &pTableQueryInfo->cur); + + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } else { + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } } } @@ -3961,7 +3975,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ qDebug("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo); int32_t totalSet = numOfClosedTimeWindow(pResultInfo); - SWindowResult* result = pResultInfo->pResult; + SResultRow** result = pResultInfo->pResult; if (orderType == TSDB_ORDER_ASC) { startIdx = pQInfo->groupIndex; @@ -3974,14 +3988,14 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { - if (result[i].numOfRows == 0) { + if (result[i]->numOfRows == 0) { pQInfo->groupIndex += 1; - pGroupResInfo->pos.rowId = 0; + pGroupResInfo->rowId = 0; continue; } - int32_t numOfRowsToCopy = result[i].numOfRows - pGroupResInfo->pos.rowId; - int32_t oldOffset = pGroupResInfo->pos.rowId; + int32_t numOfRowsToCopy = result[i]->numOfRows - pGroupResInfo->rowId; + int32_t oldOffset = pGroupResInfo->rowId; /* * current output space is not enough to accommodate all data of this page, only partial results @@ -3989,19 +4003,19 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ */ if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) { numOfRowsToCopy = (int32_t) pQuery->rec.capacity - numOfResult; - pGroupResInfo->pos.rowId += numOfRowsToCopy; + pGroupResInfo->rowId += numOfRowsToCopy; } else { - pGroupResInfo->pos.rowId = 0; + pGroupResInfo->rowId = 0; pQInfo->groupIndex += 1; } - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i].pos.pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i]->pageId); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; char *out = pQuery->sdata[j]->data + numOfResult * size; - char *in = getPosInResultPage(pRuntimeEnv, j, &result[i], page); + char *in = getPosInResultPage(pRuntimeEnv, j, result[i], page); memcpy(out, in + oldOffset * size, size * numOfRowsToCopy); } @@ -4048,7 +4062,7 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { } for (int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { - SWindowResult *pResult = &pRuntimeEnv->windowResInfo.pResult[i]; + SResultRow *pResult = pRuntimeEnv->windowResInfo.pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pRuntimeEnv->pCtx[j].functionId; @@ -4056,7 +4070,8 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { continue; } - pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); + SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); } } } @@ -4202,16 +4217,24 @@ static void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryCostInfo *pSummary = &pRuntimeEnv->summary; + uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); + hashSize += taosHashGetMemSize(pQInfo->tableqinfoGroupInfo.map); + pSummary->hashSize = hashSize; + // add the merge time pSummary->elapsedTime += pSummary->firstStageMergeTime; + SResultRowPool* p = pQInfo->runtimeEnv.pool; + pSummary->winInfoSize = getResultRowPoolMemSize(p); + pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); + qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: internal size:%"PRId64"B, numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, - pSummary->numOfTimeWindows); + qDebug("QInfo:%p :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo, pSummary->winInfoSize/1024.0, + pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); } static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { @@ -4266,7 +4289,6 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; while (tsdbNextDataBlock(pQueryHandle)) { if (IS_QUERY_KILLED(GET_QINFO_ADDR(pRuntimeEnv))) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -4452,7 +4474,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); + pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); // update the query time window pQuery->window = cond.twindow; @@ -4474,9 +4496,9 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) } } } else if (isPointInterpoQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); + pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); } else { - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); + pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); } return terrno; @@ -4570,7 +4592,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo } } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 8, threshold, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, 8, threshold, type); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4590,7 +4612,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo type = TSDB_DATA_TYPE_TIMESTAMP; } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, numOfResultRows, 4096, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, 1024, type); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4617,7 +4639,7 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); if (pResInfo != NULL) { pResInfo->complete = false; } @@ -4755,7 +4777,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { pRuntimeEnv->pQueryHandle = NULL; } - pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo); + pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef); taosArrayDestroy(tx); taosArrayDestroy(g1); if (pRuntimeEnv->pQueryHandle == NULL) { @@ -4763,15 +4785,62 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { } if (pRuntimeEnv->pTSBuf != NULL) { + tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; + if (pRuntimeEnv->cur.vgroupIndex == -1) { - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pRuntimeEnv->pCtx[0].tag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); + // failed to find data with the specified tag value and vnodeId + if (!tsBufIsValidElem(&elem)) { + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); + } else { + qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); + } - // failed to find data with the specified tag value - if (elem.vnode < 0) { return false; + } else { + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, + cur.blockIndex, cur.tsIndex); + } else { + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, + cur.blockIndex, cur.tsIndex); + } } } else { - tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); + STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); + if (tVariantCompare(elem.tag, &pRuntimeEnv->pCtx[0].tag) != 0) { + + STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); + // failed to find data with the specified tag value and vnodeId + if (!tsBufIsValidElem(&elem1)) { + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); + } else { + qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); + } + + return false; + } else { + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); + } else { + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); + } + } + + } else { + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p continue scan ts_comp file, tag:%s blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); + } else { + qDebug("QInfo:%p continue scan ts_comp file, tag:%"PRId64" blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); + } + } } } @@ -4793,7 +4862,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); - if (isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery)) { + if (isPointInterpoQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); @@ -4823,11 +4892,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pRuntimeEnv->pQueryHandle = NULL; } - if (isFirstLastRowQuery(pQuery)) { - assert(0); // last_row query switch to other routine to handle - } else { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo); - } + pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef); taosArrayDestroy(tx); taosArrayDestroy(g1); @@ -4841,10 +4906,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) { assert(taosArrayGetSize(s) >= 1); setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); - if (isFirstLastRowQuery(pQuery)) { - assert(taosArrayGetSize(s) == 1); - } - taosArrayDestroy(s); // here we simply set the first table as current table @@ -4897,7 +4958,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } // no need to update the lastkey for each table - pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo); + pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef); taosArrayDestroy(g1); taosArrayDestroy(tx); @@ -4923,11 +4984,12 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - pWindowResInfo->pResult[i].closed = true; // enable return all results for group by normal columns + pWindowResInfo->pResult[i]->closed = true; // enable return all results for group by normal columns - SWindowResult *pResult = &pWindowResInfo->pResult[i]; + SResultRow *pResult = pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); + SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); } } @@ -5027,6 +5089,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { break; } + if (pRuntimeEnv->pTSBuf != NULL) { + pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; + } + } else { // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter if (pQuery->rec.rows == 0) { @@ -5079,7 +5145,7 @@ static void doSaveContext(SQInfo *pQInfo) { SWITCH_ORDER(pQuery->order.order); if (pRuntimeEnv->pTSBuf != NULL) { - pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order; + SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); } STsdbQueryCond cond = { @@ -5101,7 +5167,7 @@ static void doSaveContext(SQInfo *pQInfo) { setupQueryRangeForReverseScan(pQInfo); pRuntimeEnv->prevGroupId = INT32_MIN; - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); if (pRuntimeEnv->pSecQueryHandle == NULL) { longjmp(pRuntimeEnv->env, terrno); } @@ -5172,7 +5238,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { // query error occurred or query is killed, abort current execution if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -5194,7 +5259,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query + //TODO finalizeQueryResult may cause SEGSEV, since the memory may not allocated yet, add a cleanup function instead longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -5234,7 +5299,6 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) finalizeQueryResult(pRuntimeEnv); if (IS_QUERY_KILLED(pQInfo)) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -5816,13 +5880,13 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, return TSDB_CODE_SUCCESS; _cleanup: - taosTFree(*pExpr); + tfree(*pExpr); taosArrayDestroy(*pTableIdList); *pTableIdList = NULL; - taosTFree(*tbnameCond); - taosTFree(*groupbyCols); - taosTFree(*tagCols); - taosTFree(*tagCond); + tfree(*tbnameCond); + tfree(*groupbyCols); + tfree(*tagCols); + tfree(*tagCond); return code; } @@ -5873,7 +5937,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg); if (code != TSDB_CODE_SUCCESS) { - taosTFree(pExprs); + tfree(pExprs); return code; } @@ -5912,7 +5976,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * int32_t param = (int32_t)pExprs[i].base.arg[0].argValue.i64; if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) { - taosTFree(pExprs); + tfree(pExprs); return TSDB_CODE_QRY_INVALID_MSG; } @@ -6186,8 +6250,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou calResultBufSize(pQuery); for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - assert(pExprs[col].interBytes >= pExprs[col].bytes); - // allocate additional memory for interResults that are usually larger then final results size_t size = (size_t)((pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interBytes + sizeof(tFilePage)); pQuery->sdata[col] = (tFilePage *)calloc(1, size); @@ -6213,12 +6275,18 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->tableqinfoGroupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); pQInfo->tableqinfoGroupInfo.numOfTables = pTableGroupInfo->numOfTables; pQInfo->tableqinfoGroupInfo.map = taosHashInit(pTableGroupInfo->numOfTables, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); } int tableIndex = 0; pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); + pQInfo->runtimeEnv.summary.tableInfoSize += (pTableGroupInfo->numOfTables * sizeof(STableQueryInfo)); + + pQInfo->runtimeEnv.pResultRowHashTable = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pQInfo->runtimeEnv.keyBuf = malloc(TSDB_MAX_BYTES_PER_ROW); + pQInfo->runtimeEnv.pool = initResultRowPool(getWindowResultSize(&pQInfo->runtimeEnv)); + pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); if (pQInfo->pBuf == NULL) { goto _cleanup; @@ -6253,7 +6321,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou for(int32_t j = 0; j < s; ++j) { STableKeyInfo* info = taosArrayGet(pa, j); - void* buf = (char*)pQInfo->pBuf + index * sizeof(STableQueryInfo); + void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); window.skey = info->lastKey; STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); @@ -6284,7 +6352,7 @@ _cleanup_query: free(pGroupbyExpr); } - taosTFree(pTagCols); + tfree(pTagCols); for (int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExprInfo = &pExprs[i]; if (pExprInfo->pExpr != NULL) { @@ -6292,7 +6360,7 @@ _cleanup_query: } } - taosTFree(pExprs); + tfree(pExprs); _cleanup: freeQInfo(pQInfo); @@ -6318,12 +6386,13 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ SQuery *pQuery = pQInfo->runtimeEnv.pQuery; STSBuf *pTSBuf = NULL; - if (pQueryMsg->tsLen > 0) { // open new file to save the result + if (pQueryMsg->tsLen > 0) { // open new file to save the result char *tsBlock = (char *) pQueryMsg + pQueryMsg->tsOffset; - pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); + pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder, vgId); tsBufResetPos(pTSBuf); bool ret = tsBufNextPos(pTSBuf); + UNUSED(ret); } @@ -6402,25 +6471,27 @@ static void freeQInfo(SQInfo *pQInfo) { qDebug("QInfo:%p start to free QInfo", pQInfo); + releaseQueryBuf(pQInfo->tableqinfoGroupInfo.numOfTables); + teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (pQuery != NULL) { if (pQuery->sdata != NULL) { for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - taosTFree(pQuery->sdata[col]); + tfree(pQuery->sdata[col]); } - taosTFree(pQuery->sdata); + tfree(pQuery->sdata); } if (pQuery->fillVal != NULL) { - taosTFree(pQuery->fillVal); + tfree(pQuery->fillVal); } for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; if (pColFilter->numOfFilters > 0) { - taosTFree(pColFilter->pFilters); + tfree(pColFilter->pFilters); } } @@ -6433,31 +6504,31 @@ static void freeQInfo(SQInfo *pQInfo) { } } - taosTFree(pQuery->pSelectExpr); + tfree(pQuery->pSelectExpr); } if (pQuery->pGroupbyExpr != NULL) { taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); - taosTFree(pQuery->pGroupbyExpr); + tfree(pQuery->pGroupbyExpr); } - taosTFree(pQuery->tagColList); - taosTFree(pQuery->pFilterInfo); + tfree(pQuery->tagColList); + tfree(pQuery->pFilterInfo); if (pQuery->colList != NULL) { for (int32_t i = 0; i < pQuery->numOfCols; i++) { SColumnInfo *column = pQuery->colList + i; freeColumnFilterInfo(column->filters, column->numOfFilters); } - taosTFree(pQuery->colList); + tfree(pQuery->colList); } - taosTFree(pQuery); + tfree(pQuery); } doDestroyTableQueryInfo(&pQInfo->tableqinfoGroupInfo); - taosTFree(pQInfo->pBuf); + tfree(pQInfo->pBuf); tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosArrayDestroy(pQInfo->arrTableIdInfo); @@ -6465,7 +6536,7 @@ static void freeQInfo(SQInfo *pQInfo) { qDebug("QInfo:%p QInfo is freed", pQInfo); - taosTFree(pQInfo); + tfree(pQInfo); } static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { @@ -6636,6 +6707,11 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi assert(0); } + code = checkForQueryBuf(tableGroupInfo.numOfTables); + if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort + goto _over; + } + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &tableGroupInfo, pTagColumnInfo, isSTableQuery); pExprs = NULL; pGroupbyExpr = NULL; @@ -6777,7 +6853,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex int32_t code = TSDB_CODE_SUCCESS; -#if 0 +#if _NON_BLOCKING_RETRIEVE SQuery *pQuery = pQInfo->runtimeEnv.pQuery; pthread_mutex_lock(&pQInfo->lock); @@ -6849,6 +6925,8 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co pQInfo->dataReady = QUERY_RESULT_NOT_READY; if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { + // here current thread hold the refcount, so it is safe to free tsdbQueryHandle. + doFreeQueryHandle(pQInfo); *continueExec = false; (*pRsp)->completed = 1; // notify no more result to client } else { @@ -7037,6 +7115,48 @@ static void buildTagQueryResult(SQInfo* pQInfo) { setQueryStatus(pQuery, QUERY_COMPLETED); } +static int64_t getQuerySupportBufSize(size_t numOfTables) { + size_t s1 = sizeof(STableQueryInfo); + size_t s2 = sizeof(SHashNode); + +// size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb + return (int64_t)((s1 + s2) * 1.5 * numOfTables); +} + +int32_t checkForQueryBuf(size_t numOfTables) { + int64_t t = getQuerySupportBufSize(numOfTables); + if (tsQueryBufferSize < 0) { + return TSDB_CODE_SUCCESS; + } else if (tsQueryBufferSize > 0) { + + while(1) { + int64_t s = tsQueryBufferSize; + int64_t remain = s - t; + if (remain >= 0) { + if (atomic_val_compare_exchange_64(&tsQueryBufferSize, s, remain) == s) { + return TSDB_CODE_SUCCESS; + } + } else { + return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; + } + } + } + + // disable query processing if the value of tsQueryBufferSize is zero. + return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; +} + +void releaseQueryBuf(size_t numOfTables) { + if (tsQueryBufferSize <= 0) { + return; + } + + int64_t t = getQuerySupportBufSize(numOfTables); + + // restore value is not enough buffer available + atomic_add_fetch_64(&tsQueryBufferSize, t); +} + void* qGetResultRetrieveMsg(qinfo_t qinfo) { SQInfo* pQInfo = (SQInfo*) qinfo; assert(pQInfo != NULL); @@ -7111,9 +7231,9 @@ void qCleanupQueryMgmt(void* pQMgmt) { taosCacheCleanup(pqinfoPool); pthread_mutex_destroy(&pQueryMgmt->lock); - taosTFree(pQueryMgmt); + tfree(pQueryMgmt); - qDebug("vgId:%d queryMgmt cleanup completed", vgId); + qDebug("vgId:%d, queryMgmt cleanup completed", vgId); } void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index fc9c60b39b0cfa4b591cc77c1efcdac4e6647ce9..fa3fe285a857cc0d146e25f52710d0ebc0a69193 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -64,7 +64,7 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) { // release flush out info link SExtFileInfo *pFileMeta = &pMemBuffer->fileMeta; if (pFileMeta->flushoutData.nAllocSize != 0 && pFileMeta->flushoutData.pFlushoutInfo != NULL) { - taosTFree(pFileMeta->flushoutData.pFlushoutInfo); + tfree(pFileMeta->flushoutData.pFlushoutInfo); } // release all in-memory buffer pages @@ -72,7 +72,7 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) { while (pFilePages != NULL) { tFilePagesItem *pTmp = pFilePages; pFilePages = pFilePages->pNext; - taosTFree(pTmp); + tfree(pTmp); } // close temp file @@ -87,8 +87,8 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) { destroyColumnModel(pMemBuffer->pColumnModel); - taosTFree(pMemBuffer->path); - taosTFree(pMemBuffer); + tfree(pMemBuffer->path); + tfree(pMemBuffer); return NULL; } @@ -275,7 +275,7 @@ int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) { tFilePagesItem *ptmp = first; first = first->pNext; - taosTFree(ptmp); // release all data in memory buffer + tfree(ptmp); // release all data in memory buffer } fflush(pMemBuffer->file); // flush to disk @@ -300,7 +300,7 @@ void tExtMemBufferClear(tExtMemBuffer *pMemBuffer) { while (first != NULL) { tFilePagesItem *ptmp = first; first = first->pNext; - taosTFree(ptmp); + tfree(ptmp); } pMemBuffer->fileMeta.numOfElemsInFile = 0; @@ -344,8 +344,6 @@ static FORCE_INLINE int32_t primaryKeyComparator(int64_t f1, int64_t f2, int32_t return 0; } - assert(colIdx == 0); - if (tsOrder == TSDB_ORDER_DESC) { // primary column desc order return (f1 < f2) ? 1 : -1; } else { // asc @@ -804,7 +802,7 @@ void destroyColumnModel(SColumnModel *pModel) { return; } - taosTFree(pModel); + tfree(pModel); } static void printBinaryData(char *data, int32_t len) { @@ -1089,5 +1087,5 @@ void tOrderDescDestroy(tOrderDescriptor *pDesc) { } destroyColumnModel(pDesc->pColumnModel); - taosTFree(pDesc); + tfree(pDesc); } diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index f186726c0120f2e2e34580fec0da00c2083e5da9..33d627c83973cf40b378573e6aa9ddbf2be1a258 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -91,23 +91,23 @@ void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) { pFillInfo->numOfTotal = 0; } -void* taosDestoryFillInfo(SFillInfo* pFillInfo) { +void* taosDestroyFillInfo(SFillInfo* pFillInfo) { if (pFillInfo == NULL) { return NULL; } - taosTFree(pFillInfo->prevValues); - taosTFree(pFillInfo->nextValues); - taosTFree(pFillInfo->pTags); + tfree(pFillInfo->prevValues); + tfree(pFillInfo->nextValues); + tfree(pFillInfo->pTags); for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - taosTFree(pFillInfo->pData[i]); + tfree(pFillInfo->pData[i]); } - taosTFree(pFillInfo->pData); - taosTFree(pFillInfo->pFillCol); + tfree(pFillInfo->pData); + tfree(pFillInfo->pFillCol); - taosTFree(pFillInfo); + tfree(pFillInfo); return NULL; } @@ -170,7 +170,7 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows int32_t numOfRows = taosNumOfRemainRows(pFillInfo); TSKEY ekey1 = ekey; - if (pFillInfo->order != TSDB_ORDER_ASC) { + if (!FILL_IS_ASC_FILL(pFillInfo)) { pFillInfo->endKey = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision); } @@ -215,7 +215,7 @@ static double linearInterpolationImpl(double v1, double v2, double k1, double k2 return v1 + (v2 - v1) * (k - k1) / (k2 - k1); } -int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) { +int32_t taosGetLinearInterpolationVal(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) { switch (type) { case TSDB_DATA_TYPE_INT: { *(int32_t*)point->val = (int32_t)linearInterpolationImpl(*(int32_t*)point1->val, *(int32_t*)point2->val, (double)point1->key, @@ -343,7 +343,7 @@ static void doFillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t* nu point1 = (SPoint){.key = *(TSKEY*)(prevValues), .val = prevValues + pCol->col.offset}; point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->rowIdx * bytes}; point = (SPoint){.key = pFillInfo->start, .val = val1}; - taosDoLinearInterpolation(type, &point1, &point2, &point); + taosGetLinearInterpolationVal(type, &point1, &point2, &point); } setTagsValue(pFillInfo, data, *num); @@ -496,7 +496,7 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu pFillInfo->numOfRows = 0; /* the raw data block is exhausted, next value does not exists */ - taosTFree(*nextValues); + tfree(*nextValues); } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index 703ee2c52129a067d1070cfbe567087ec48a7a6e..35e5906d1f797c2f53f3906bf93c6797ac766c3e 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -168,7 +168,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { (*pHisto)->numOfEntries += 1; } } else { /* insert a new slot */ - if ((*pHisto)->numOfElems > 1 && idx < (*pHisto)->numOfEntries) { + if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) { if (idx > 0) { assert((*pHisto)->elems[idx - 1].val <= val); } @@ -661,4 +661,4 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2 free(pHistoBins); return pResHistogram; -} \ No newline at end of file +} diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 7e8128f200658af282bab89bd73dc2c27ddb25c1..02a7012b0ef40d9b8635cb0bc502e791379581f7 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -571,7 +571,7 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) { doDestroyQuerySql(pQuerySql); } - taosTFree(pClause->pClause); + tfree(pClause->pClause); } SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pStableName, @@ -641,12 +641,12 @@ void SQLInfoDestroy(SSqlInfo *pInfo) { tFieldListDestroy(pCreateTableInfo->colInfo.pTagColumns); tVariantListDestroy(pCreateTableInfo->usingInfo.pTagVals); - taosTFree(pInfo->pCreateTableInfo); + tfree(pInfo->pCreateTableInfo); } else if (pInfo->type == TSDB_SQL_ALTER_TABLE) { tVariantListDestroy(pInfo->pAlterInfo->varList); tFieldListDestroy(pInfo->pAlterInfo->pAddColumns); - taosTFree(pInfo->pAlterInfo); + tfree(pInfo->pAlterInfo); } else { if (pInfo->pDCLInfo != NULL && pInfo->pDCLInfo->nAlloc > 0) { free(pInfo->pDCLInfo->a); @@ -656,7 +656,7 @@ void SQLInfoDestroy(SSqlInfo *pInfo) { tVariantListDestroy(pInfo->pDCLInfo->dbOpt.keep); } - taosTFree(pInfo->pDCLInfo); + tfree(pInfo->pDCLInfo); } } @@ -872,5 +872,6 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { pDBInfo->quorum = -1; pDBInfo->keep = NULL; + pDBInfo->update = -1; memset(&pDBInfo->precision, 0, sizeof(SStrToken)); } diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index ab9ffb7bcb458129b7f170e7020cee904d2dfda6..3bdc0d477f777faa71a8e5d2264021932dddeea4 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -361,8 +361,8 @@ void tMemBucketDestroy(tMemBucket *pBucket) { } destroyResultBuf(pBucket->pBuffer); - taosTFree(pBucket->pSlots); - taosTFree(pBucket); + tfree(pBucket->pSlots); + tfree(pBucket); } void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) { @@ -680,7 +680,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) } double val = (1 - fraction) * td + fraction * nd; - taosTFree(buffer); + tfree(buffer); return val; } else { // incur a second round bucket split diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index b3e97459d36c68d48c04447b159473bd179a5dbb..2645cff678b7fe91f62dda0f3fe65f3a968f9c11 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -267,7 +267,7 @@ static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) { assert(d->pn == pn); d->pn = NULL; - taosTFree(pn); + tfree(pn); bufPage = flushPageToDisk(pResultBuf, d); } @@ -407,18 +407,18 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } if (pResultBuf->file != NULL) { - qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, inmem size:%dbytes, file size:%"PRId64" bytes", - pResultBuf->handle, pResultBuf->totalBufSize, listNEles(pResultBuf->lruList) * pResultBuf->pageSize, - pResultBuf->fileSize); + qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f", + pResultBuf->handle, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0, + pResultBuf->fileSize/1024.0); fclose(pResultBuf->file); } else { - qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle, - pResultBuf->totalBufSize); + qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, no file created", pResultBuf->handle, + pResultBuf->totalBufSize/1024.0); } unlink(pResultBuf->path); - taosTFree(pResultBuf->path); + tfree(pResultBuf->path); SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet); while(taosHashIterNext(iter)) { @@ -426,8 +426,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { size_t n = taosArrayGetSize(*p); for(int32_t i = 0; i < n; ++i) { SPageInfo* pi = taosArrayGetP(*p, i); - taosTFree(pi->pData); - taosTFree(pi); + tfree(pi->pData); + tfree(pi); } taosArrayDestroy(*p); @@ -440,8 +440,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { taosHashCleanup(pResultBuf->groupSet); taosHashCleanup(pResultBuf->all); - taosTFree(pResultBuf->assistBuf); - taosTFree(pResultBuf); + tfree(pResultBuf->assistBuf); + tfree(pResultBuf); } SPageInfo* getLastPageInfo(SIDList pList) { diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 0c9f92786fdf837e2453fe70f594c10e7a093421..98545c8ef3930587b6f7bd2c5b8d40ded0066d52 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -155,6 +155,7 @@ static SKeyword keywordTable[] = { {"INSERT", TK_INSERT}, {"INTO", TK_INTO}, {"VALUES", TK_VALUES}, + {"UPDATE", TK_UPDATE}, {"RESET", TK_RESET}, {"QUERY", TK_QUERY}, {"ADD", TK_ADD}, @@ -251,16 +252,16 @@ static const char isIdChar[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ }; -static void* KeywordHashTable = NULL; +static void* keywordHashTable = NULL; static void doInitKeywordsTable(void) { int numOfEntries = tListLen(keywordTable); - KeywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false); + keywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false); for (int32_t i = 0; i < numOfEntries; i++) { keywordTable[i].len = (uint8_t)strlen(keywordTable[i].name); void* ptr = &keywordTable[i]; - taosHashPut(KeywordHashTable, keywordTable[i].name, keywordTable[i].len, (void*)&ptr, POINTER_BYTES); + taosHashPut(keywordHashTable, keywordTable[i].name, keywordTable[i].len, (void*)&ptr, POINTER_BYTES); } } @@ -282,7 +283,7 @@ int tSQLKeywordCode(const char* z, int n) { } } - SKeyword** pKey = (SKeyword**)taosHashGet(KeywordHashTable, key, n); + SKeyword** pKey = (SKeyword**)taosHashGet(keywordHashTable, key, n); return (pKey != NULL)? (*pKey)->type:TK_ID; } @@ -660,5 +661,8 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); } void taosCleanupKeywordsTable() { - taosHashCleanup(KeywordHashTable); -} \ No newline at end of file + void* m = keywordHashTable; + if (m != NULL && atomic_val_compare_exchange_ptr(&keywordHashTable, m, 0) == m) { + taosHashCleanup(m); + } +} diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index b264f6cdc9d815a12cc5a3ab0e5c09c0d670bcdb..8a5a87baab5abbfd04f1c97dbe91a7226606263e 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -4,7 +4,7 @@ #include "tutil.h" static int32_t getDataStartOffset(); -static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo); +static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); static STSBuf* allocResForTSBuf(STSBuf* pTSBuf); static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); @@ -32,7 +32,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { } // update the header info - STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = TSDB_ORDER_ASC}; + STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = TSDB_ORDER_ASC}; STSBufUpdateHeader(pTSBuf, &header); tsBufResetPos(pTSBuf); @@ -75,9 +75,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return NULL; } - if (header.numOfVnode > pTSBuf->numOfAlloc) { - pTSBuf->numOfAlloc = header.numOfVnode; - STSVnodeBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * pTSBuf->numOfAlloc); + if (header.numOfGroup > pTSBuf->numOfAlloc) { + pTSBuf->numOfAlloc = header.numOfGroup; + STSGroupBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc); if (tmp == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -86,7 +86,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { pTSBuf->pData = tmp; } - pTSBuf->numOfVnodes = header.numOfVnode; + pTSBuf->numOfGroups = header.numOfGroup; // check the ts order pTSBuf->tsOrder = header.tsOrder; @@ -96,9 +96,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return NULL; } - size_t infoSize = sizeof(STSVnodeBlockInfo) * pTSBuf->numOfVnodes; + size_t infoSize = sizeof(STSGroupBlockInfo) * pTSBuf->numOfGroups; - STSVnodeBlockInfo* buf = (STSVnodeBlockInfo*)calloc(1, infoSize); + STSGroupBlockInfo* buf = (STSGroupBlockInfo*)calloc(1, infoSize); if (buf == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -109,9 +109,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { UNUSED(sz); // the length value for each vnode is not kept in file, so does not set the length value - for (int32_t i = 0; i < pTSBuf->numOfVnodes; ++i) { - STSVnodeBlockInfoEx* pBlockList = &pTSBuf->pData[i]; - memcpy(&pBlockList->info, &buf[i], sizeof(STSVnodeBlockInfo)); + for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) { + STSGroupBlockInfoEx* pBlockList = &pTSBuf->pData[i]; + memcpy(&pBlockList->info, &buf[i], sizeof(STSGroupBlockInfo)); } free(buf); @@ -131,8 +131,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { pTSBuf->cur.order = TSDB_ORDER_ASC; pTSBuf->autoDelete = autoDelete; -// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfVnode:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f), -// pTSBuf->fileSize, pTSBuf->numOfVnodes, pTSBuf->autoDelete); +// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f), +// pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); return pTSBuf; } @@ -142,11 +142,11 @@ void* tsBufDestroy(STSBuf* pTSBuf) { return NULL; } - taosTFree(pTSBuf->assistBuf); - taosTFree(pTSBuf->tsData.rawBuf); + tfree(pTSBuf->assistBuf); + tfree(pTSBuf->tsData.rawBuf); - taosTFree(pTSBuf->pData); - taosTFree(pTSBuf->block.payload); + tfree(pTSBuf->pData); + tfree(pTSBuf->block.payload); fclose(pTSBuf->f); @@ -156,58 +156,59 @@ void* tsBufDestroy(STSBuf* pTSBuf) { } else { // tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); } - + + tVariantDestroy(&pTSBuf->block.tag); free(pTSBuf); return NULL; } -static STSVnodeBlockInfoEx* tsBufGetLastVnodeInfo(STSBuf* pTSBuf) { - int32_t last = pTSBuf->numOfVnodes - 1; +static STSGroupBlockInfoEx* tsBufGetLastGroupInfo(STSBuf* pTSBuf) { + int32_t last = pTSBuf->numOfGroups - 1; assert(last >= 0); return &pTSBuf->pData[last]; } -static STSVnodeBlockInfoEx* addOneVnodeInfo(STSBuf* pTSBuf, int32_t vnodeId) { - if (pTSBuf->numOfAlloc <= pTSBuf->numOfVnodes) { +static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { + if (pTSBuf->numOfAlloc <= pTSBuf->numOfGroups) { uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5); assert((int32_t)newSize > pTSBuf->numOfAlloc); - STSVnodeBlockInfoEx* tmp = (STSVnodeBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * newSize); + STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return NULL; } pTSBuf->pData = tmp; pTSBuf->numOfAlloc = newSize; - memset(&pTSBuf->pData[pTSBuf->numOfVnodes], 0, sizeof(STSVnodeBlockInfoEx) * (newSize - pTSBuf->numOfVnodes)); + memset(&pTSBuf->pData[pTSBuf->numOfGroups], 0, sizeof(STSGroupBlockInfoEx) * (newSize - pTSBuf->numOfGroups)); } - if (pTSBuf->numOfVnodes > 0) { - STSVnodeBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastVnodeInfo(pTSBuf); + if (pTSBuf->numOfGroups > 0) { + STSGroupBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); // update prev vnode length info in file - TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, &pPrevBlockInfoEx->info); + TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pPrevBlockInfoEx->info); } // set initial value for vnode block - STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfVnodes].info; - pBlockInfo->vnode = vnodeId; + STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfGroups].info; + pBlockInfo->id = id; pBlockInfo->offset = pTSBuf->fileSize; assert(pBlockInfo->offset >= getDataStartOffset()); // update vnode info in file - TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes, pBlockInfo); + TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups, pBlockInfo); // add one vnode info - pTSBuf->numOfVnodes += 1; + pTSBuf->numOfGroups += 1; // update the header info STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder}; + .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; STSBufUpdateHeader(pTSBuf, &header); - return tsBufGetLastVnodeInfo(pTSBuf); + return tsBufGetLastGroupInfo(pTSBuf); } static void shrinkBuffer(STSList* ptsData) { @@ -218,6 +219,15 @@ static void shrinkBuffer(STSList* ptsData) { } } +static int32_t getTagAreaLength(tVariant* pa) { + int32_t t = sizeof(pa->nLen) * 2 + sizeof(pa->nType); + if (pa->nType != TSDB_DATA_TYPE_NULL) { + t += pa->nLen; + } + + return t; +} + static void writeDataToDisk(STSBuf* pTSBuf) { if (pTSBuf->tsData.len == 0) { return; @@ -243,28 +253,36 @@ static void writeDataToDisk(STSBuf* pTSBuf) { */ int32_t metaLen = 0; metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + int32_t trueLen = pBlock->tag.nLen; if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + } else { + trueLen = 0; + metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); } fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - + + metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + assert(metaLen == getTagAreaLength(&pBlock->tag)); + int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; pTSBuf->fileSize += blockSize; pTSBuf->tsData.len = 0; - STSVnodeBlockInfoEx* pVnodeBlockInfoEx = tsBufGetLastVnodeInfo(pTSBuf); + STSGroupBlockInfoEx* pGroupBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - pVnodeBlockInfoEx->info.compLen += blockSize; - pVnodeBlockInfoEx->info.numOfBlocks += 1; + pGroupBlockInfoEx->info.compLen += blockSize; + pGroupBlockInfoEx->info.numOfBlocks += 1; shrinkBuffer(&pTSBuf->tsData); } @@ -284,23 +302,28 @@ static void expandBuffer(STSList* ptsData, int32_t inputSize) { STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { STSBlock* pBlock = &pTSBuf->block; - + // clear the memory buffer - void* tmp = pBlock->payload; - memset(pBlock, 0, sizeof(STSBlock)); - pBlock->payload = tmp; - + pBlock->compLen = 0; + pBlock->padding = 0; + pBlock->numOfElem = 0; + + int32_t offset = -1; + if (order == TSDB_ORDER_DESC) { /* * set the right position for the reversed traverse, the reversed traverse is started from * the end of each comp data block */ - int32_t ret = fseek(pTSBuf->f, -(int32_t)(sizeof(pBlock->padding)), SEEK_CUR); - size_t sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); + int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); + int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR); + size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f); + sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); UNUSED(sz); - + pBlock->compLen = pBlock->padding; - int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); + + offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); ret = fseek(pTSBuf->f, -offset, SEEK_CUR); UNUSED(ret); } @@ -319,7 +342,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - sz = fread(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f); + sz = fread(&pBlock->tag.i64Key, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); } sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); @@ -327,8 +350,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); UNUSED(sz); sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - UNUSED(sz); - + if (decomp) { pTSBuf->tsData.len = tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, @@ -337,11 +359,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // read the comp length at the length of comp block sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); + assert(pBlock->padding == pBlock->compLen); + + int32_t n = 0; + sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { + assert(n == 0); + } else { + assert(n == pBlock->tag.nLen); + } + UNUSED(sz); // for backwards traverse, set the start position at the end of previous block if (order == TSDB_ORDER_DESC) { - int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); UNUSED(r); } @@ -382,20 +413,20 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { return TSDB_CODE_SUCCESS; } -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len) { - STSVnodeBlockInfoEx* pBlockInfo = NULL; +void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len) { + STSGroupBlockInfoEx* pBlockInfo = NULL; STSList* ptsData = &pTSBuf->tsData; - if (pTSBuf->numOfVnodes == 0 || tsBufGetLastVnodeInfo(pTSBuf)->info.vnode != vnodeId) { + if (pTSBuf->numOfGroups == 0 || tsBufGetLastGroupInfo(pTSBuf)->info.id != id) { writeDataToDisk(pTSBuf); shrinkBuffer(ptsData); - pBlockInfo = addOneVnodeInfo(pTSBuf, vnodeId); + pBlockInfo = addOneGroupInfo(pTSBuf, id); } else { - pBlockInfo = tsBufGetLastVnodeInfo(pTSBuf); + pBlockInfo = tsBufGetLastGroupInfo(pTSBuf); } - assert(pBlockInfo->info.vnode == vnodeId); + assert(pBlockInfo->info.id == id); if ((tVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { // new arrived data with different tags value, save current value into disk first @@ -403,7 +434,7 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pDa } else { expandBuffer(ptsData, len); } - + tVariantAssign(&pTSBuf->block.tag, tag); memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); @@ -433,23 +464,23 @@ void tsBufFlush(STSBuf* pTSBuf) { writeDataToDisk(pTSBuf); shrinkBuffer(&pTSBuf->tsData); - STSVnodeBlockInfoEx* pBlockInfoEx = tsBufGetLastVnodeInfo(pTSBuf); + STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); // update prev vnode length info in file - TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, &pBlockInfoEx->info); + TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pBlockInfoEx->info); // save the ts order into header STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder}; + .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; STSBufUpdateHeader(pTSBuf, &header); fsync(fileno(pTSBuf->f)); } -static int32_t tsBufFindVnodeIndexFromId(STSVnodeBlockInfoEx* pVnodeInfoEx, int32_t numOfVnodes, int32_t vnodeId) { +static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t numOfGroups, int32_t id) { int32_t j = -1; - for (int32_t i = 0; i < numOfVnodes; ++i) { - if (pVnodeInfoEx[i].info.vnode == vnodeId) { + for (int32_t i = 0; i < numOfGroups; ++i) { + if (pGroupInfoEx[i].info.id == id) { j = i; break; } @@ -459,7 +490,7 @@ static int32_t tsBufFindVnodeIndexFromId(STSVnodeBlockInfoEx* pVnodeInfoEx, int3 } // todo opt performance by cache blocks info -static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int32_t blockIndex) { +static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) { if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { return -1; } @@ -478,7 +509,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int if (pTSBuf->cur.order == TSDB_ORDER_DESC) { STSBlock* pBlock = &pTSBuf->block; int32_t compBlockSize = - pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); + pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR); UNUSED(ret); } @@ -486,7 +517,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int return 0; } -static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, tVariant* tag) { +static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, tVariant* tag) { bool decomp = false; int64_t offset = 0; @@ -506,21 +537,21 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo } if (tVariantCompare(&pTSBuf->block.tag, tag) == 0) { - return i; + return (pTSBuf->cur.order == TSDB_ORDER_ASC)? i: (pBlockInfo->numOfBlocks - (i + 1)); } } return -1; } -static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex) { - STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[vnodeIndex].info; +static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex) { + STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info; if (pBlockInfo->numOfBlocks <= blockIndex) { assert(false); } STSCursor* pCur = &pTSBuf->cur; - if (pCur->vgroupIndex == vnodeIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) || + if (pCur->vgroupIndex == groupIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) || (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) { int32_t i = 0; bool decomp = false; @@ -555,14 +586,27 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex assert((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len)); - pCur->vgroupIndex = vnodeIndex; + pCur->vgroupIndex = groupIndex; pCur->blockIndex = blockIndex; pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1; } -STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) { - int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); +static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockInfo* pVInfo) { + if (offset < 0 || offset >= getDataStartOffset()) { + return -1; + } + + if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { + return -1; + } + + fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f); + return 0; +} + +STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) { + int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id); if (j == -1) { return NULL; } @@ -571,7 +615,7 @@ STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) { } int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { - if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfVnode == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { + if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { return -1; } @@ -587,7 +631,7 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { } bool tsBufNextPos(STSBuf* pTSBuf) { - if (pTSBuf == NULL || pTSBuf->numOfVnodes == 0) { + if (pTSBuf == NULL || pTSBuf->numOfGroups == 0) { return false; } @@ -606,16 +650,16 @@ bool tsBufNextPos(STSBuf* pTSBuf) { } } else { // get the last timestamp record in the last block of the last vnode - assert(pTSBuf->numOfVnodes > 0); + assert(pTSBuf->numOfGroups > 0); - int32_t vnodeIndex = pTSBuf->numOfVnodes - 1; - pCur->vgroupIndex = vnodeIndex; + int32_t groupIndex = pTSBuf->numOfGroups - 1; + pCur->vgroupIndex = groupIndex; - int32_t vnodeId = pTSBuf->pData[pCur->vgroupIndex].info.vnode; - STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId); + int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; + STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); int32_t blockIndex = pBlockInfo->numOfBlocks - 1; - tsBufGetBlock(pTSBuf, vnodeIndex, blockIndex); + tsBufGetBlock(pTSBuf, groupIndex, blockIndex); pCur->tsIndex = pTSBuf->block.numOfElem - 1; if (pTSBuf->block.numOfElem == 0) { @@ -634,12 +678,12 @@ bool tsBufNextPos(STSBuf* pTSBuf) { if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) { - int32_t vnodeId = pTSBuf->pData[pCur->vgroupIndex].info.vnode; + int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; - STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId); + STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) || (pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { - if ((pCur->vgroupIndex >= pTSBuf->numOfVnodes - 1 && pCur->order == TSDB_ORDER_ASC) || + if ((pCur->vgroupIndex >= pTSBuf->numOfGroups - 1 && pCur->order == TSDB_ORDER_ASC) || (pCur->vgroupIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { pCur->vgroupIndex = -1; return false; @@ -649,7 +693,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) { return false; } - int32_t blockIndex = pCur->order == TSDB_ORDER_ASC ? 0 : pBlockInfo->numOfBlocks - 1; + int32_t blockIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : (pBlockInfo->numOfBlocks - 1); tsBufGetBlock(pTSBuf, pCur->vgroupIndex + step, blockIndex); break; @@ -675,8 +719,7 @@ void tsBufResetPos(STSBuf* pTSBuf) { } STSElem tsBufGetElem(STSBuf* pTSBuf) { - STSElem elem1 = {.vnode = -1}; - + STSElem elem1 = {.id = -1}; if (pTSBuf == NULL) { return elem1; } @@ -688,9 +731,9 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { STSBlock* pBlock = &pTSBuf->block; - elem1.vnode = pTSBuf->pData[pCur->vgroupIndex].info.vnode; + elem1.id = pTSBuf->pData[pCur->vgroupIndex].info.id; elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); - tVariantAssign(&elem1.tag, &pBlock->tag); + elem1.tag = &pBlock->tag; return elem1; } @@ -699,35 +742,34 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { * current only support ts comp data from two vnode merge * @param pDestBuf * @param pSrcBuf - * @param vnodeId + * @param id * @return */ -int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) { - if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfVnodes <= 0) { +int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { + if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfGroups <= 0) { return 0; } - if (pDestBuf->numOfVnodes + pSrcBuf->numOfVnodes > TS_COMP_FILE_VNODE_MAX) { + if (pDestBuf->numOfGroups + pSrcBuf->numOfGroups > TS_COMP_FILE_GROUP_MAX) { return -1; } // src can only have one vnode index - if (pSrcBuf->numOfVnodes > 1) { - return -1; - } - + assert(pSrcBuf->numOfGroups == 1); + // there are data in buffer, flush to disk first tsBufFlush(pDestBuf); // compared with the last vnode id - if (vnodeId != tsBufGetLastVnodeInfo(pDestBuf)->info.vnode) { - int32_t oldSize = pDestBuf->numOfVnodes; - int32_t newSize = oldSize + pSrcBuf->numOfVnodes; + int32_t id = tsBufGetLastGroupInfo((STSBuf*) pSrcBuf)->info.id; + if (id != tsBufGetLastGroupInfo(pDestBuf)->info.id) { + int32_t oldSize = pDestBuf->numOfGroups; + int32_t newSize = oldSize + pSrcBuf->numOfGroups; if (pDestBuf->numOfAlloc < newSize) { pDestBuf->numOfAlloc = newSize; - STSVnodeBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSVnodeBlockInfoEx) * newSize); + STSGroupBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return -1; } @@ -736,23 +778,23 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) { } // directly copy the vnode index information - memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfVnodes * sizeof(STSVnodeBlockInfoEx)); + memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfGroups * sizeof(STSGroupBlockInfoEx)); // set the new offset value - for (int32_t i = 0; i < pSrcBuf->numOfVnodes; ++i) { - STSVnodeBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize]; + for (int32_t i = 0; i < pSrcBuf->numOfGroups; ++i) { + STSGroupBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize]; pBlockInfoEx->info.offset = (pSrcBuf->pData[i].info.offset - getDataStartOffset()) + pDestBuf->fileSize; - pBlockInfoEx->info.vnode = vnodeId; + pBlockInfoEx->info.id = id; } - pDestBuf->numOfVnodes = newSize; + pDestBuf->numOfGroups = newSize; } else { - STSVnodeBlockInfoEx* pBlockInfoEx = tsBufGetLastVnodeInfo(pDestBuf); + STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pDestBuf); pBlockInfoEx->len += pSrcBuf->pData[0].len; pBlockInfoEx->info.numOfBlocks += pSrcBuf->pData[0].info.numOfBlocks; pBlockInfoEx->info.compLen += pSrcBuf->pData[0].info.compLen; - pBlockInfoEx->info.vnode = vnodeId; + pBlockInfoEx->info.id = id; } int32_t r = fseek(pDestBuf->f, 0, SEEK_END); @@ -761,7 +803,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) { int64_t offset = getDataStartOffset(); int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; - ssize_t rc = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size); + int64_t rc = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size); if (rc == -1) { // tscError("failed to merge tsBuf from:%s to %s, reason:%s\n", pSrcBuf->path, pDestBuf->path, strerror(errno)); @@ -785,23 +827,23 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) { assert(pDestBuf->fileSize == oldSize + size); -// tscDebug("tsBuf merge success, %p, path:%s, fd:%d, file size:%d, numOfVnode:%d, autoDelete:%d", pDestBuf, -// pDestBuf->path, fileno(pDestBuf->f), pDestBuf->fileSize, pDestBuf->numOfVnodes, pDestBuf->autoDelete); +// tscDebug("tsBuf merge success, %p, path:%s, fd:%d, file size:%d, numOfGroups:%d, autoDelete:%d", pDestBuf, +// pDestBuf->path, fileno(pDestBuf->f), pDestBuf->fileSize, pDestBuf->numOfGroups, pDestBuf->autoDelete); return 0; } -STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order) { +STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order, int32_t id) { STSBuf* pTSBuf = tsBufCreate(true, order); - STSVnodeBlockInfo* pBlockInfo = &(addOneVnodeInfo(pTSBuf, 0)->info); + STSGroupBlockInfo* pBlockInfo = &(addOneGroupInfo(pTSBuf, 0)->info); pBlockInfo->numOfBlocks = numOfBlocks; pBlockInfo->compLen = len; pBlockInfo->offset = getDataStartOffset(); - pBlockInfo->vnode = 0; + pBlockInfo->id = id; // update prev vnode length info in file - TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, pBlockInfo); + TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET); UNUSED(ret); @@ -813,7 +855,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); STSBufFileHeader header = { - .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder}; + .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; STSBufUpdateHeader(pTSBuf, &header); fsync(fileno(pTSBuf->f)); @@ -821,14 +863,14 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ return pTSBuf; } -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) { - STSElem elem = {.vnode = -1}; +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag) { + STSElem elem = {.id = -1}; if (pTSBuf == NULL) { return elem; } - int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); + int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id); if (j == -1) { return elem; } @@ -837,7 +879,7 @@ STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) { // tsBufDisplay(pTSBuf); STSCursor* pCur = &pTSBuf->cur; - STSVnodeBlockInfo* pBlockInfo = &pTSBuf->pData[j].info; + STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[j].info; int32_t blockIndex = tsBufFindBlockByTag(pTSBuf, pBlockInfo, tag); if (blockIndex < 0) { @@ -893,7 +935,7 @@ STSBuf* tsBufClone(STSBuf* pTSBuf) { void tsBufDisplay(STSBuf* pTSBuf) { printf("-------start of ts comp file-------\n"); - printf("number of vnode:%d\n", pTSBuf->numOfVnodes); + printf("number of vnode:%d\n", pTSBuf->numOfGroups); int32_t old = pTSBuf->cur.order; pTSBuf->cur.order = TSDB_ORDER_ASC; @@ -902,8 +944,8 @@ void tsBufDisplay(STSBuf* pTSBuf) { while (tsBufNextPos(pTSBuf)) { STSElem elem = tsBufGetElem(pTSBuf); - if (elem.tag.nType == TSDB_DATA_TYPE_BIGINT) { - printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); + if (elem.tag->nType == TSDB_DATA_TYPE_BIGINT) { + printf("%d-%" PRId64 "-%" PRId64 "\n", elem.id, elem.tag->i64Key, elem.ts); } } @@ -912,33 +954,20 @@ void tsBufDisplay(STSBuf* pTSBuf) { } static int32_t getDataStartOffset() { - return sizeof(STSBufFileHeader) + TS_COMP_FILE_VNODE_MAX * sizeof(STSVnodeBlockInfo); -} - -static int32_t doUpdateVnodeInfo(STSBuf* pTSBuf, int64_t offset, STSVnodeBlockInfo* pVInfo) { - if (offset < 0 || offset >= getDataStartOffset()) { - return -1; - } - - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { - return -1; - } - - fwrite(pVInfo, sizeof(STSVnodeBlockInfo), 1, pTSBuf->f); - return 0; + return sizeof(STSBufFileHeader) + TS_COMP_FILE_GROUP_MAX * sizeof(STSGroupBlockInfo); } // update prev vnode length info in file -static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo) { - int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSVnodeBlockInfo); - doUpdateVnodeInfo(pTSBuf, offset, pBlockInfo); +static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo) { + int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSGroupBlockInfo); + doUpdateGroupInfo(pTSBuf, offset, pBlockInfo); } static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { - const int32_t INITIAL_VNODEINFO_SIZE = 4; + const int32_t INITIAL_GROUPINFO_SIZE = 4; - pTSBuf->numOfAlloc = INITIAL_VNODEINFO_SIZE; - pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSVnodeBlockInfoEx)); + pTSBuf->numOfAlloc = INITIAL_GROUPINFO_SIZE; + pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx)); if (pTSBuf->pData == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -969,3 +998,72 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { pTSBuf->fileSize += getDataStartOffset(); return pTSBuf; } + +int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf) { + if (pTSBuf == NULL) { + return 0; + } + + return pTSBuf->numOfGroups; +} + +void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) { + int32_t size = tsBufGetNumOfGroup(pTSBuf); + if (num != NULL) { + *num = size; + } + + *id = NULL; + if (size == 0) { + return; + } + + (*id) = malloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t)); + + for(int32_t i = 0; i < size; ++i) { + (*id)[i] = pTSBuf->pData[i].info.id; + } +} + +int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, int32_t* len, int32_t* numOfBlocks) { + assert(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups); + STSGroupBlockInfo *pBlockInfo = &pTSBuf->pData[groupIndex].info; + + *len = 0; + *numOfBlocks = 0; + + if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { + int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); +// qError("%p: fseek failed: %s", pSql, tstrerror(code)); + return code; + } + + size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f); + if (s != pBlockInfo->compLen) { + int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); +// tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); + return code; + } + + *len = pBlockInfo->compLen; + *numOfBlocks = pBlockInfo->numOfBlocks; + + return TSDB_CODE_SUCCESS; +} + +STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag) { + STSElem el = {.id = -1}; + + for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) { + el = tsBufGetElemStartPos(pTSBuf, pTSBuf->pData[i].info.id, pTag); + if (el.id == pTSBuf->pData[i].info.id) { + return el; + } + } + + return el; +} + +bool tsBufIsValidElem(STSElem* pElem) { + return pElem->id >= 0; +} diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 2bd92c74a456c16fe946288968d25d797907a390..f057dc1a497f63b6a91aa69c83a531230c2ae8d4 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -14,8 +14,9 @@ */ #include "os.h" -#include "hash.h" #include "taosmsg.h" +#include "hash.h" + #include "qExecutor.h" #include "qUtil.h" @@ -26,75 +27,37 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { size += pQuery->pSelectExpr[i].interBytes; } - assert(size > 0); + assert(size >= 0); return size; } -int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t size, - int32_t threshold, int16_t type) { +int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int32_t threshold, int16_t type) { pWindowResInfo->capacity = size; pWindowResInfo->threshold = threshold; pWindowResInfo->type = type; - _hash_fn_t fn = taosGetDefaultHashFunction(type); - pWindowResInfo->hashList = taosHashInit(threshold, fn, true, false); - if (pWindowResInfo->hashList == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - pWindowResInfo->curIndex = -1; pWindowResInfo->size = 0; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; - SQueryCostInfo* pSummary = &pRuntimeEnv->summary; - - // use the pointer arraylist - pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); + pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, POINTER_BYTES); if (pWindowResInfo->pResult == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - pWindowResInfo->interval = pRuntimeEnv->pQuery->interval.interval; - - pSummary->internalSupSize += sizeof(SWindowResult) * threshold; - pSummary->internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; - pSummary->numOfTimeWindows = threshold; - - for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { - int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - return TSDB_CODE_SUCCESS; } -void destroyTimeWindowRes(SWindowResult *pWindowRes) { - if (pWindowRes == NULL) { - return; - } - - free(pWindowRes->resultInfo); -} - void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { if (pWindowResInfo == NULL) { return; } if (pWindowResInfo->capacity == 0) { - assert(pWindowResInfo->hashList == NULL && pWindowResInfo->pResult == NULL); + assert(/*pWindowResInfo->hashList == NULL && */pWindowResInfo->pResult == NULL); return; } - if (pWindowResInfo->pResult != NULL) { - for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { - destroyTimeWindowRes(&pWindowResInfo->pResult[i]); - } - } - - taosHashCleanup(pWindowResInfo->hashList); - taosTFree(pWindowResInfo->pResult); + tfree(pWindowResInfo->pResult); } void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo) { @@ -103,17 +66,13 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *pWindowRes = &pWindowResInfo->pResult[i]; - clearTimeWindowResBuf(pRuntimeEnv, pWindowRes); + SResultRow *pWindowRes = pWindowResInfo->pResult[i]; + clearResultRow(pRuntimeEnv, pWindowRes); } pWindowResInfo->curIndex = -1; - taosHashCleanup(pWindowResInfo->hashList); pWindowResInfo->size = 0; - _hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type); - pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, true, false); - pWindowResInfo->startTime = TSKEY_INITIAL_VAL; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; } @@ -127,13 +86,13 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); assert(num >= 0 && num <= numOfClosed); - int16_t type = pWindowResInfo->type; - - char *key = NULL; - int16_t bytes = -1; + int16_t type = pWindowResInfo->type; + STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); // uid is always set to be 0. + char *key = NULL; + int16_t bytes = -1; for (int32_t i = 0; i < num; ++i) { - SWindowResult *pResult = &pWindowResInfo->pResult[i]; + SResultRow *pResult = pWindowResInfo->pResult[i]; if (pResult->closed) { // remove the window slot from hash table // todo refactor @@ -145,7 +104,8 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { bytes = tDataTypeDesc[pWindowResInfo->type].nSize; } - taosHashRemove(pWindowResInfo->hashList, (const char *)key, bytes); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid); + taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); } else { break; } @@ -155,19 +115,19 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { // clear all the closed windows from the window list for (int32_t k = 0; k < remain; ++k) { - copyTimeWindowResBuf(pRuntimeEnv, &pWindowResInfo->pResult[k], &pWindowResInfo->pResult[num + k]); + copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k]); } // move the unclosed window in the front of the window list for (int32_t k = remain; k < pWindowResInfo->size; ++k) { - SWindowResult *pWindowRes = &pWindowResInfo->pResult[k]; - clearTimeWindowResBuf(pRuntimeEnv, pWindowRes); + SResultRow *pWindowRes = pWindowResInfo->pResult[k]; + clearResultRow(pRuntimeEnv, pWindowRes); } pWindowResInfo->size = remain; for (int32_t k = 0; k < pWindowResInfo->size; ++k) { - SWindowResult *pResult = &pWindowResInfo->pResult[k]; + SResultRow *pResult = pWindowResInfo->pResult[k]; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { key = varDataVal(pResult->key); @@ -177,12 +137,15 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { bytes = tDataTypeDesc[pWindowResInfo->type].nSize; } - int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)key, bytes); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid); + int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); assert(p != NULL); int32_t v = (*p - num); assert(v >= 0 && v <= pWindowResInfo->size); - taosHashPut(pWindowResInfo->hashList, (char *)key, bytes, (char *)&v, sizeof(int32_t)); + + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid); + taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t)); } pWindowResInfo->curIndex = -1; @@ -200,7 +163,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { int32_t i = 0; - while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].closed) { + while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) { ++i; } @@ -211,11 +174,11 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - if (pWindowResInfo->pResult[i].closed) { + if (pWindowResInfo->pResult[i]->closed) { continue; } - pWindowResInfo->pResult[i].closed = true; + pWindowResInfo->pResult[i]->closed = true; } } @@ -231,19 +194,19 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } // get the result order - int32_t resultOrder = (pWindowResInfo->pResult[0].win.skey < pWindowResInfo->pResult[1].win.skey)? 1:-1; + int32_t resultOrder = (pWindowResInfo->pResult[0]->win.skey < pWindowResInfo->pResult[1]->win.skey)? 1:-1; if (order != resultOrder) { return; } int32_t i = 0; if (order == QUERY_ASC_FORWARD_STEP) { - TSKEY ekey = pWindowResInfo->pResult[i].win.ekey; + TSKEY ekey = pWindowResInfo->pResult[i]->win.ekey; while (i < pWindowResInfo->size && (ekey < lastKey)) { ++i; } } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].win.skey > lastKey)) { + while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i]->win.skey > lastKey)) { ++i; } } @@ -254,32 +217,36 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { - return (getWindowResult(pWindowResInfo, slot)->closed == true); + return (getResultRow(pWindowResInfo, slot)->closed == true); } void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { - getWindowResult(pWindowResInfo, slot)->closed = true; + getResultRow(pWindowResInfo, slot)->closed = true; } -void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) { +void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) { if (pWindowRes == NULL) { return; } - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId); + // the result does not put into the SDiskbasedResultBuf, ignore it. + if (pWindowRes->pageId >= 0) { + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); - for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { - SResultInfo *pResultInfo = &pWindowRes->resultInfo[i]; - - char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); - size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; - memset(s, 0, size); - - RESET_RESULT_INFO(pResultInfo); + for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { + SResultRowCellInfo *pResultInfo = &pWindowRes->pCellInfo[i]; + + char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); + size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; + memset(s, 0, size); + + RESET_RESULT_INFO(pResultInfo); + } } - + pWindowRes->numOfRows = 0; - pWindowRes->pos = (SPosInfo){-1, -1}; + pWindowRes->pageId = -1; + pWindowRes->rowId = -1; pWindowRes->closed = false; pWindowRes->win = TSWINDOW_INITIALIZER; } @@ -289,7 +256,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow * since the attribute of "Pos" is bound to each window result when the window result is created in the * disk-based result buffer. */ -void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) { +void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src) { dst->numOfRows = src->numOfRows; dst->win = src->win; dst->closed = src->closed; @@ -297,25 +264,105 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; for (int32_t i = 0; i < nOutputCols; ++i) { - SResultInfo *pDst = &dst->resultInfo[i]; - SResultInfo *pSrc = &src->resultInfo[i]; + SResultRowCellInfo *pDst = getResultCell(pRuntimeEnv, dst, i); + SResultRowCellInfo *pSrc = getResultCell(pRuntimeEnv, src, i); - char *buf = pDst->interResultBuf; - memcpy(pDst, pSrc, sizeof(SResultInfo)); - pDst->interResultBuf = buf; // restore the allocated buffer +// char *buf = pDst->interResultBuf; + memcpy(pDst, pSrc, sizeof(SResultRowCellInfo) + pRuntimeEnv->pCtx[i].interBufBytes); +// pDst->interResultBuf = buf; // restore the allocated buffer // copy the result info struct - memcpy(pDst->interResultBuf, pSrc->interResultBuf, pDst->bufLen); +// memcpy(pDst->interResultBuf, pSrc->interResultBuf, pRuntimeEnv->pCtx[i].interBufBytes); // copy the output buffer data from src to dst, the position info keep unchanged - tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pos.pageId); + tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pageId); char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage); - tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pos.pageId); - char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src, srcpage); + tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pageId); + char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SResultRow *)src, srcpage); size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memcpy(dstBuf, srcBuf, s); } } +SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index) { + assert(index >= 0 && index < pRuntimeEnv->pQuery->numOfOutput); + return (SResultRowCellInfo*)((char*) pRow->pCellInfo + pRuntimeEnv->rowCellInfoOffset[index]); +} + +size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv) { + return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pRuntimeEnv->interBufSize + sizeof(SResultRow); +} + +SResultRowPool* initResultRowPool(size_t size) { + SResultRowPool* p = calloc(1, sizeof(SResultRowPool)); + if (p == NULL) { + return NULL; + } + + p->numOfElemPerBlock = 128; + + p->elemSize = (int32_t) size; + p->blockSize = p->numOfElemPerBlock * p->elemSize; + p->position.pos = 0; + + p->pData = taosArrayInit(8, POINTER_BYTES); + return p; +} + +SResultRow* getNewResultRow(SResultRowPool* p) { + if (p == NULL) { + return NULL; + } + + void* ptr = NULL; + if (p->position.pos == 0) { + ptr = calloc(1, p->blockSize); + taosArrayPush(p->pData, &ptr); + + } else { + size_t last = taosArrayGetSize(p->pData); + + void** pBlock = taosArrayGet(p->pData, last - 1); + ptr = ((char*) (*pBlock)) + p->elemSize * p->position.pos; + } + + p->position.pos = (p->position.pos + 1)%p->numOfElemPerBlock; + initResultRow(ptr); + + return ptr; +} + +int64_t getResultRowPoolMemSize(SResultRowPool* p) { + if (p == NULL) { + return 0; + } + + return taosArrayGetSize(p->pData) * p->blockSize; +} + +int32_t getNumOfAllocatedResultRows(SResultRowPool* p) { + return (int32_t) taosArrayGetSize(p->pData) * p->numOfElemPerBlock; +} + +int32_t getNumOfUsedResultRows(SResultRowPool* p) { + return getNumOfAllocatedResultRows(p) - p->numOfElemPerBlock + p->position.pos; +} + +void* destroyResultRowPool(SResultRowPool* p) { + if (p == NULL) { + return NULL; + } + + size_t size = taosArrayGetSize(p->pData); + for(int32_t i = 0; i < size; ++i) { + void** ptr = taosArrayGet(p->pData, i); + tfree(*ptr); + } + + taosArrayDestroy(p->pData); + + tfree(p); + return NULL; +} diff --git a/src/query/src/sql.c b/src/query/src/sql.c index a18efdeb744ba039c685d5fc1067c95e3ca15d86..93bb9b2ee75af6bc46f49ab7535fbf081fe6d8cd 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -97,27 +97,27 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 272 +#define YYNOCODE 274 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SSubclauseInfo* yy25; - tSQLExpr* yy66; - SCreateAcctSQL yy73; - int yy82; - SQuerySQL* yy150; - SCreateDBInfo yy158; - TAOS_FIELD yy181; - SLimitVal yy188; - tSQLExprList* yy224; - int64_t yy271; - tVariant yy312; - SIntervalVal yy314; - SCreateTableSQL* yy374; - tFieldList* yy449; - tVariantList* yy494; + int yy46; + tSQLExpr* yy64; + tVariant yy134; + SCreateAcctSQL yy149; + int64_t yy207; + SLimitVal yy216; + TAOS_FIELD yy223; + SSubclauseInfo* yy231; + SCreateDBInfo yy268; + tSQLExprList* yy290; + SQuerySQL* yy414; + SCreateTableSQL* yy470; + tVariantList* yy498; + tFieldList* yy523; + SIntervalVal yy532; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -127,17 +127,17 @@ typedef union { #define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo #define ParseARG_STORE yypParser->pInfo = pInfo #define YYFALLBACK 1 -#define YYNSTATE 252 -#define YYNRULE 230 -#define YYNTOKEN 206 -#define YY_MAX_SHIFT 251 -#define YY_MIN_SHIFTREDUCE 416 -#define YY_MAX_SHIFTREDUCE 645 -#define YY_ERROR_ACTION 646 -#define YY_ACCEPT_ACTION 647 -#define YY_NO_ACTION 648 -#define YY_MIN_REDUCE 649 -#define YY_MAX_REDUCE 878 +#define YYNSTATE 253 +#define YYNRULE 233 +#define YYNTOKEN 207 +#define YY_MAX_SHIFT 252 +#define YY_MIN_SHIFTREDUCE 420 +#define YY_MAX_SHIFTREDUCE 652 +#define YY_ERROR_ACTION 653 +#define YY_ACCEPT_ACTION 654 +#define YY_NO_ACTION 655 +#define YY_MIN_REDUCE 656 +#define YY_MAX_REDUCE 888 /************* End control #defines *******************************************/ /* Define the yytestcase() macro to be a no-op if is not already defined @@ -203,223 +203,224 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (566) +#define YY_ACTTAB_COUNT (571) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 751, 459, 11, 749, 750, 647, 251, 459, 752, 460, - /* 10 */ 754, 755, 753, 35, 36, 460, 37, 38, 159, 249, - /* 20 */ 170, 29, 141, 459, 206, 41, 39, 43, 40, 140, - /* 30 */ 145, 460, 865, 34, 33, 862, 141, 32, 31, 30, - /* 40 */ 35, 36, 781, 37, 38, 165, 866, 170, 29, 141, - /* 50 */ 62, 206, 41, 39, 43, 40, 191, 525, 164, 866, - /* 60 */ 34, 33, 27, 21, 32, 31, 30, 417, 418, 419, - /* 70 */ 420, 421, 422, 423, 424, 425, 426, 427, 428, 250, - /* 80 */ 35, 36, 181, 37, 38, 227, 226, 170, 29, 781, - /* 90 */ 176, 206, 41, 39, 43, 40, 174, 162, 767, 792, - /* 100 */ 34, 33, 56, 160, 32, 31, 30, 21, 36, 8, - /* 110 */ 37, 38, 63, 118, 170, 29, 770, 108, 206, 41, - /* 120 */ 39, 43, 40, 32, 31, 30, 599, 34, 33, 78, - /* 130 */ 875, 32, 31, 30, 238, 37, 38, 108, 238, 170, - /* 140 */ 29, 184, 766, 206, 41, 39, 43, 40, 188, 187, - /* 150 */ 789, 177, 34, 33, 224, 223, 32, 31, 30, 16, - /* 160 */ 218, 244, 243, 217, 216, 215, 242, 214, 241, 240, - /* 170 */ 239, 213, 747, 818, 735, 736, 737, 738, 739, 740, - /* 180 */ 741, 742, 743, 744, 745, 746, 169, 612, 103, 12, - /* 190 */ 603, 17, 606, 819, 609, 201, 169, 612, 26, 108, - /* 200 */ 603, 108, 606, 861, 609, 153, 169, 612, 173, 567, - /* 210 */ 603, 154, 606, 105, 609, 90, 89, 148, 166, 167, - /* 220 */ 34, 33, 205, 102, 32, 31, 30, 770, 166, 167, - /* 230 */ 26, 21, 557, 41, 39, 43, 40, 549, 166, 167, - /* 240 */ 194, 34, 33, 17, 193, 32, 31, 30, 860, 16, - /* 250 */ 26, 244, 243, 203, 21, 60, 242, 61, 241, 240, - /* 260 */ 239, 248, 247, 96, 175, 229, 767, 76, 80, 245, - /* 270 */ 190, 554, 21, 85, 88, 79, 18, 156, 121, 122, - /* 280 */ 605, 82, 608, 42, 70, 66, 69, 225, 770, 767, - /* 290 */ 135, 133, 601, 42, 611, 768, 93, 92, 91, 690, - /* 300 */ 168, 207, 131, 42, 611, 230, 545, 767, 546, 610, - /* 310 */ 699, 157, 691, 131, 611, 131, 604, 541, 607, 610, - /* 320 */ 538, 571, 539, 47, 540, 46, 580, 581, 602, 610, - /* 330 */ 572, 631, 613, 50, 14, 13, 13, 531, 543, 3, - /* 340 */ 544, 46, 48, 530, 75, 74, 811, 22, 178, 179, - /* 350 */ 51, 211, 10, 9, 829, 22, 87, 86, 101, 99, - /* 360 */ 158, 143, 144, 146, 147, 151, 152, 150, 139, 149, - /* 370 */ 769, 142, 828, 171, 825, 824, 172, 791, 761, 796, - /* 380 */ 228, 783, 798, 104, 810, 119, 120, 701, 117, 212, - /* 390 */ 615, 137, 24, 221, 698, 26, 222, 192, 874, 72, - /* 400 */ 873, 871, 123, 719, 25, 100, 23, 138, 566, 688, - /* 410 */ 81, 686, 83, 84, 684, 195, 780, 683, 161, 542, - /* 420 */ 180, 199, 132, 681, 680, 679, 52, 49, 678, 677, - /* 430 */ 109, 134, 44, 675, 204, 673, 671, 669, 667, 202, - /* 440 */ 200, 198, 196, 28, 136, 220, 57, 58, 812, 77, - /* 450 */ 231, 232, 233, 234, 235, 236, 237, 246, 209, 645, - /* 460 */ 53, 182, 183, 644, 110, 64, 67, 155, 186, 185, - /* 470 */ 682, 643, 94, 636, 676, 189, 126, 125, 720, 124, - /* 480 */ 127, 128, 130, 129, 95, 668, 1, 551, 193, 765, - /* 490 */ 2, 55, 113, 111, 114, 112, 115, 116, 59, 568, - /* 500 */ 163, 106, 197, 5, 573, 107, 6, 65, 614, 19, - /* 510 */ 4, 20, 15, 208, 616, 7, 210, 500, 496, 494, - /* 520 */ 493, 492, 489, 463, 219, 68, 45, 71, 73, 22, - /* 530 */ 527, 526, 524, 54, 484, 482, 474, 480, 476, 478, - /* 540 */ 472, 470, 499, 498, 497, 495, 491, 490, 46, 461, - /* 550 */ 432, 430, 649, 648, 648, 648, 648, 648, 648, 648, - /* 560 */ 648, 648, 648, 648, 97, 98, + /* 0 */ 108, 463, 141, 11, 654, 252, 802, 463, 140, 464, + /* 10 */ 162, 165, 876, 35, 36, 464, 37, 38, 159, 250, + /* 20 */ 170, 29, 141, 463, 206, 41, 39, 43, 40, 173, + /* 30 */ 780, 464, 875, 34, 33, 145, 141, 32, 31, 30, + /* 40 */ 35, 36, 791, 37, 38, 164, 876, 170, 29, 780, + /* 50 */ 21, 206, 41, 39, 43, 40, 191, 829, 799, 201, + /* 60 */ 34, 33, 21, 21, 32, 31, 30, 421, 422, 423, + /* 70 */ 424, 425, 426, 427, 428, 429, 430, 431, 432, 251, + /* 80 */ 35, 36, 181, 37, 38, 532, 776, 170, 29, 238, + /* 90 */ 246, 206, 41, 39, 43, 40, 174, 175, 777, 777, + /* 100 */ 34, 33, 872, 56, 32, 31, 30, 176, 871, 36, + /* 110 */ 780, 37, 38, 227, 226, 170, 29, 791, 17, 206, + /* 120 */ 41, 39, 43, 40, 108, 26, 870, 606, 34, 33, + /* 130 */ 78, 160, 32, 31, 30, 238, 157, 16, 218, 245, + /* 140 */ 244, 217, 216, 215, 243, 214, 242, 241, 240, 213, + /* 150 */ 239, 755, 103, 743, 744, 745, 746, 747, 748, 749, + /* 160 */ 750, 751, 752, 753, 754, 756, 37, 38, 229, 177, + /* 170 */ 170, 29, 224, 223, 206, 41, 39, 43, 40, 203, + /* 180 */ 62, 60, 8, 34, 33, 63, 118, 32, 31, 30, + /* 190 */ 169, 619, 27, 12, 610, 184, 613, 158, 616, 778, + /* 200 */ 169, 619, 188, 187, 610, 194, 613, 108, 616, 153, + /* 210 */ 169, 619, 561, 108, 610, 154, 613, 18, 616, 90, + /* 220 */ 89, 148, 166, 167, 34, 33, 205, 143, 32, 31, + /* 230 */ 30, 697, 166, 167, 131, 144, 564, 41, 39, 43, + /* 240 */ 40, 706, 166, 167, 131, 34, 33, 146, 17, 32, + /* 250 */ 31, 30, 32, 31, 30, 26, 16, 207, 245, 244, + /* 260 */ 21, 587, 588, 243, 828, 242, 241, 240, 698, 239, + /* 270 */ 61, 131, 76, 80, 147, 190, 102, 151, 85, 88, + /* 280 */ 79, 760, 156, 26, 758, 759, 82, 21, 42, 761, + /* 290 */ 556, 763, 764, 762, 225, 765, 777, 193, 42, 618, + /* 300 */ 249, 248, 96, 574, 121, 122, 608, 105, 42, 618, + /* 310 */ 70, 66, 69, 578, 617, 168, 579, 46, 152, 618, + /* 320 */ 14, 230, 548, 777, 617, 545, 638, 546, 150, 547, + /* 330 */ 13, 135, 133, 612, 617, 615, 139, 93, 92, 91, + /* 340 */ 620, 611, 609, 614, 13, 47, 538, 622, 50, 552, + /* 350 */ 46, 553, 537, 178, 179, 3, 22, 211, 75, 74, + /* 360 */ 149, 22, 10, 9, 48, 51, 142, 550, 885, 551, + /* 370 */ 87, 86, 101, 99, 779, 839, 838, 171, 835, 834, + /* 380 */ 172, 801, 771, 228, 806, 793, 808, 104, 821, 119, + /* 390 */ 820, 117, 120, 708, 212, 137, 24, 221, 705, 222, + /* 400 */ 26, 192, 100, 884, 72, 883, 881, 123, 726, 25, + /* 410 */ 573, 23, 138, 695, 49, 81, 693, 83, 84, 691, + /* 420 */ 790, 690, 195, 161, 199, 549, 57, 52, 180, 132, + /* 430 */ 688, 687, 686, 685, 684, 134, 682, 109, 680, 678, + /* 440 */ 44, 676, 674, 136, 204, 202, 58, 822, 200, 198, + /* 450 */ 196, 220, 77, 28, 231, 232, 233, 235, 652, 234, + /* 460 */ 236, 237, 247, 209, 183, 53, 651, 182, 185, 186, + /* 470 */ 64, 67, 155, 650, 643, 189, 193, 689, 558, 94, + /* 480 */ 683, 675, 126, 125, 727, 129, 124, 127, 128, 95, + /* 490 */ 130, 1, 114, 110, 111, 775, 2, 55, 59, 116, + /* 500 */ 112, 113, 115, 575, 106, 163, 197, 5, 580, 107, + /* 510 */ 6, 65, 621, 19, 4, 20, 15, 208, 623, 7, + /* 520 */ 210, 504, 500, 498, 497, 496, 493, 467, 219, 68, + /* 530 */ 45, 71, 73, 22, 534, 533, 531, 488, 54, 486, + /* 540 */ 478, 484, 480, 482, 476, 474, 505, 503, 502, 501, + /* 550 */ 499, 495, 494, 46, 465, 436, 434, 656, 655, 655, + /* 560 */ 655, 655, 655, 655, 655, 655, 655, 655, 655, 97, + /* 570 */ 98, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 226, 1, 260, 229, 230, 207, 208, 1, 234, 9, - /* 10 */ 236, 237, 238, 13, 14, 9, 16, 17, 209, 210, - /* 20 */ 20, 21, 260, 1, 24, 25, 26, 27, 28, 260, - /* 30 */ 260, 9, 270, 33, 34, 260, 260, 37, 38, 39, - /* 40 */ 13, 14, 244, 16, 17, 269, 270, 20, 21, 260, - /* 50 */ 247, 24, 25, 26, 27, 28, 258, 5, 269, 270, - /* 60 */ 33, 34, 259, 210, 37, 38, 39, 45, 46, 47, + /* 0 */ 211, 1, 262, 262, 208, 209, 211, 1, 262, 9, + /* 10 */ 228, 271, 272, 13, 14, 9, 16, 17, 210, 211, + /* 20 */ 20, 21, 262, 1, 24, 25, 26, 27, 28, 228, + /* 30 */ 248, 9, 272, 33, 34, 262, 262, 37, 38, 39, + /* 40 */ 13, 14, 246, 16, 17, 271, 272, 20, 21, 248, + /* 50 */ 211, 24, 25, 26, 27, 28, 260, 268, 263, 270, + /* 60 */ 33, 34, 211, 211, 37, 38, 39, 45, 46, 47, /* 70 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 80 */ 13, 14, 60, 16, 17, 33, 34, 20, 21, 244, - /* 90 */ 66, 24, 25, 26, 27, 28, 243, 227, 245, 210, - /* 100 */ 33, 34, 102, 258, 37, 38, 39, 210, 14, 98, - /* 110 */ 16, 17, 101, 102, 20, 21, 246, 210, 24, 25, - /* 120 */ 26, 27, 28, 37, 38, 39, 99, 33, 34, 73, - /* 130 */ 246, 37, 38, 39, 78, 16, 17, 210, 78, 20, - /* 140 */ 21, 126, 245, 24, 25, 26, 27, 28, 133, 134, - /* 150 */ 261, 127, 33, 34, 130, 131, 37, 38, 39, 85, - /* 160 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 170 */ 96, 97, 226, 266, 228, 229, 230, 231, 232, 233, - /* 180 */ 234, 235, 236, 237, 238, 239, 1, 2, 210, 44, - /* 190 */ 5, 98, 7, 266, 9, 268, 1, 2, 105, 210, - /* 200 */ 5, 210, 7, 260, 9, 60, 1, 2, 227, 99, - /* 210 */ 5, 66, 7, 103, 9, 70, 71, 72, 33, 34, - /* 220 */ 33, 34, 37, 98, 37, 38, 39, 246, 33, 34, - /* 230 */ 105, 210, 37, 25, 26, 27, 28, 99, 33, 34, - /* 240 */ 262, 33, 34, 98, 106, 37, 38, 39, 260, 85, - /* 250 */ 105, 87, 88, 264, 210, 266, 92, 266, 94, 95, - /* 260 */ 96, 63, 64, 65, 243, 210, 245, 61, 62, 227, - /* 270 */ 125, 103, 210, 67, 68, 69, 108, 132, 61, 62, - /* 280 */ 5, 75, 7, 98, 67, 68, 69, 243, 246, 245, - /* 290 */ 61, 62, 1, 98, 109, 240, 67, 68, 69, 214, - /* 300 */ 59, 15, 217, 98, 109, 243, 5, 245, 7, 124, - /* 310 */ 214, 260, 214, 217, 109, 217, 5, 2, 7, 124, - /* 320 */ 5, 99, 7, 103, 9, 103, 115, 116, 37, 124, - /* 330 */ 99, 99, 99, 103, 103, 103, 103, 99, 5, 98, - /* 340 */ 7, 103, 122, 99, 128, 129, 267, 103, 33, 34, - /* 350 */ 120, 99, 128, 129, 241, 103, 73, 74, 61, 62, - /* 360 */ 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, - /* 370 */ 246, 260, 241, 241, 241, 241, 241, 210, 242, 210, - /* 380 */ 241, 244, 210, 210, 267, 210, 210, 210, 248, 210, - /* 390 */ 104, 210, 210, 210, 210, 105, 210, 244, 210, 210, - /* 400 */ 210, 210, 210, 210, 210, 59, 210, 210, 109, 210, - /* 410 */ 210, 210, 210, 210, 210, 263, 257, 210, 263, 104, - /* 420 */ 210, 263, 210, 210, 210, 210, 119, 121, 210, 210, - /* 430 */ 256, 210, 118, 210, 113, 210, 210, 210, 210, 117, - /* 440 */ 112, 111, 110, 123, 210, 76, 211, 211, 211, 84, - /* 450 */ 83, 49, 80, 82, 53, 81, 79, 76, 211, 5, - /* 460 */ 211, 135, 5, 5, 255, 215, 215, 211, 5, 135, - /* 470 */ 211, 5, 212, 86, 211, 126, 219, 223, 225, 224, - /* 480 */ 222, 220, 218, 221, 212, 211, 216, 99, 106, 244, - /* 490 */ 213, 107, 252, 254, 251, 253, 250, 249, 103, 99, - /* 500 */ 1, 98, 98, 114, 99, 98, 114, 73, 99, 103, - /* 510 */ 98, 103, 98, 100, 104, 98, 100, 9, 5, 5, - /* 520 */ 5, 5, 5, 77, 15, 73, 16, 129, 129, 103, - /* 530 */ 5, 5, 99, 98, 5, 5, 5, 5, 5, 5, - /* 540 */ 5, 5, 5, 5, 5, 5, 5, 5, 103, 77, - /* 550 */ 59, 58, 0, 271, 271, 271, 271, 271, 271, 271, - /* 560 */ 271, 271, 271, 271, 21, 21, 271, 271, 271, 271, - /* 570 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 580 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 590 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 600 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 610 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 620 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 630 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 640 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 650 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 660 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 670 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 680 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 690 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 700 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 710 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 720 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 730 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 740 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 750 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 760 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 770 */ 271, 271, + /* 80 */ 13, 14, 60, 16, 17, 5, 247, 20, 21, 78, + /* 90 */ 228, 24, 25, 26, 27, 28, 245, 245, 247, 247, + /* 100 */ 33, 34, 262, 103, 37, 38, 39, 66, 262, 14, + /* 110 */ 248, 16, 17, 33, 34, 20, 21, 246, 99, 24, + /* 120 */ 25, 26, 27, 28, 211, 106, 262, 100, 33, 34, + /* 130 */ 73, 260, 37, 38, 39, 78, 262, 85, 86, 87, + /* 140 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, + /* 150 */ 98, 227, 211, 229, 230, 231, 232, 233, 234, 235, + /* 160 */ 236, 237, 238, 239, 240, 241, 16, 17, 211, 128, + /* 170 */ 20, 21, 131, 132, 24, 25, 26, 27, 28, 266, + /* 180 */ 249, 268, 99, 33, 34, 102, 103, 37, 38, 39, + /* 190 */ 1, 2, 261, 44, 5, 127, 7, 262, 9, 242, + /* 200 */ 1, 2, 134, 135, 5, 264, 7, 211, 9, 60, + /* 210 */ 1, 2, 104, 211, 5, 66, 7, 109, 9, 70, + /* 220 */ 71, 72, 33, 34, 33, 34, 37, 262, 37, 38, + /* 230 */ 39, 215, 33, 34, 218, 262, 37, 25, 26, 27, + /* 240 */ 28, 215, 33, 34, 218, 33, 34, 262, 99, 37, + /* 250 */ 38, 39, 37, 38, 39, 106, 85, 15, 87, 88, + /* 260 */ 211, 116, 117, 92, 268, 94, 95, 96, 215, 98, + /* 270 */ 268, 218, 61, 62, 262, 126, 99, 262, 67, 68, + /* 280 */ 69, 227, 133, 106, 230, 231, 75, 211, 99, 235, + /* 290 */ 100, 237, 238, 239, 245, 241, 247, 107, 99, 110, + /* 300 */ 63, 64, 65, 100, 61, 62, 1, 104, 99, 110, + /* 310 */ 67, 68, 69, 100, 125, 59, 100, 104, 262, 110, + /* 320 */ 104, 245, 2, 247, 125, 5, 100, 7, 262, 9, + /* 330 */ 104, 61, 62, 5, 125, 7, 262, 67, 68, 69, + /* 340 */ 100, 5, 37, 7, 104, 104, 100, 105, 104, 5, + /* 350 */ 104, 7, 100, 33, 34, 99, 104, 100, 129, 130, + /* 360 */ 262, 104, 129, 130, 123, 121, 262, 5, 248, 7, + /* 370 */ 73, 74, 61, 62, 248, 243, 243, 243, 243, 243, + /* 380 */ 243, 211, 244, 243, 211, 246, 211, 211, 269, 211, + /* 390 */ 269, 250, 211, 211, 211, 211, 211, 211, 211, 211, + /* 400 */ 106, 246, 59, 211, 211, 211, 211, 211, 211, 211, + /* 410 */ 110, 211, 211, 211, 122, 211, 211, 211, 211, 211, + /* 420 */ 259, 211, 265, 265, 265, 105, 212, 120, 211, 211, + /* 430 */ 211, 211, 211, 211, 211, 211, 211, 258, 211, 211, + /* 440 */ 119, 211, 211, 211, 114, 118, 212, 212, 113, 112, + /* 450 */ 111, 76, 84, 124, 83, 49, 80, 53, 5, 82, + /* 460 */ 81, 79, 76, 212, 5, 212, 5, 136, 136, 5, + /* 470 */ 216, 216, 212, 5, 86, 127, 107, 212, 100, 213, + /* 480 */ 212, 212, 220, 224, 226, 222, 225, 223, 221, 213, + /* 490 */ 219, 217, 253, 257, 256, 246, 214, 108, 104, 251, + /* 500 */ 255, 254, 252, 100, 99, 1, 99, 115, 100, 99, + /* 510 */ 115, 73, 100, 104, 99, 104, 99, 101, 105, 99, + /* 520 */ 101, 9, 5, 5, 5, 5, 5, 77, 15, 73, + /* 530 */ 16, 130, 130, 104, 5, 5, 100, 5, 99, 5, + /* 540 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + /* 550 */ 5, 5, 5, 104, 77, 59, 58, 0, 273, 273, + /* 560 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 21, + /* 570 */ 21, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 580 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 590 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 600 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 610 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 620 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 630 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 640 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 650 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 660 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 670 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 680 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 690 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 700 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 710 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 720 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 730 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 740 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 750 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 760 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 770 */ 273, 273, 273, 273, 273, 273, 273, 273, }; -#define YY_SHIFT_COUNT (251) +#define YY_SHIFT_COUNT (252) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (552) +#define YY_SHIFT_MAX (557) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 145, 74, 164, 185, 205, 6, 6, 6, 6, 6, - /* 10 */ 6, 0, 22, 205, 315, 315, 315, 93, 6, 6, - /* 20 */ 6, 6, 6, 56, 60, 60, 566, 195, 205, 205, - /* 30 */ 205, 205, 205, 205, 205, 205, 205, 205, 205, 205, - /* 40 */ 205, 205, 205, 205, 205, 315, 315, 52, 52, 52, - /* 50 */ 52, 52, 52, 11, 52, 125, 6, 6, 6, 6, - /* 60 */ 211, 211, 168, 6, 6, 6, 6, 6, 6, 6, + /* 0 */ 149, 52, 171, 189, 209, 6, 6, 6, 6, 6, + /* 10 */ 6, 0, 22, 209, 320, 320, 320, 19, 6, 6, + /* 20 */ 6, 6, 6, 57, 11, 11, 571, 199, 209, 209, + /* 30 */ 209, 209, 209, 209, 209, 209, 209, 209, 209, 209, + /* 40 */ 209, 209, 209, 209, 209, 320, 320, 80, 80, 80, + /* 50 */ 80, 80, 80, 83, 80, 177, 6, 6, 6, 6, + /* 60 */ 145, 145, 108, 6, 6, 6, 6, 6, 6, 6, /* 70 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, /* 80 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, /* 90 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 100 */ 6, 6, 290, 346, 346, 299, 299, 299, 346, 307, - /* 110 */ 306, 314, 321, 322, 328, 330, 332, 320, 290, 346, - /* 120 */ 346, 369, 369, 346, 365, 367, 402, 372, 371, 401, - /* 130 */ 374, 377, 346, 381, 346, 381, 346, 566, 566, 27, - /* 140 */ 67, 67, 67, 94, 119, 208, 208, 208, 206, 187, - /* 150 */ 187, 187, 187, 217, 229, 24, 15, 86, 86, 198, - /* 160 */ 138, 110, 222, 231, 232, 233, 275, 311, 291, 241, - /* 170 */ 286, 220, 230, 238, 244, 252, 216, 224, 301, 333, - /* 180 */ 283, 297, 454, 326, 457, 458, 334, 463, 466, 387, - /* 190 */ 349, 382, 388, 384, 395, 400, 403, 499, 404, 405, - /* 200 */ 407, 406, 389, 408, 392, 409, 412, 410, 414, 413, - /* 210 */ 417, 416, 434, 508, 513, 514, 515, 516, 517, 446, - /* 220 */ 509, 452, 510, 398, 399, 426, 525, 526, 433, 435, - /* 230 */ 426, 529, 530, 531, 532, 533, 534, 535, 536, 537, - /* 240 */ 538, 539, 540, 541, 542, 445, 472, 543, 544, 491, - /* 250 */ 493, 552, + /* 100 */ 6, 6, 294, 343, 343, 300, 300, 300, 343, 307, + /* 110 */ 292, 321, 330, 327, 335, 337, 339, 329, 294, 343, + /* 120 */ 343, 375, 375, 343, 368, 371, 406, 376, 377, 404, + /* 130 */ 379, 382, 343, 386, 343, 386, 343, 571, 571, 27, + /* 140 */ 67, 67, 67, 95, 150, 212, 212, 212, 211, 191, + /* 150 */ 191, 191, 191, 243, 270, 41, 68, 215, 215, 237, + /* 160 */ 190, 203, 213, 216, 226, 240, 328, 336, 305, 256, + /* 170 */ 242, 241, 244, 246, 252, 257, 229, 233, 344, 362, + /* 180 */ 297, 311, 453, 331, 459, 461, 332, 464, 468, 388, + /* 190 */ 348, 369, 378, 389, 394, 403, 405, 504, 407, 408, + /* 200 */ 410, 409, 392, 411, 395, 412, 415, 413, 417, 416, + /* 210 */ 420, 419, 438, 512, 517, 518, 519, 520, 521, 450, + /* 220 */ 513, 456, 514, 401, 402, 429, 529, 530, 436, 439, + /* 230 */ 429, 532, 534, 535, 536, 537, 538, 539, 540, 541, + /* 240 */ 542, 543, 544, 545, 546, 547, 449, 477, 548, 549, + /* 250 */ 496, 498, 557, }; #define YY_REDUCE_COUNT (138) -#define YY_REDUCE_MIN (-258) -#define YY_REDUCE_MAX (277) +#define YY_REDUCE_MIN (-260) +#define YY_REDUCE_MAX (282) static const short yy_reduce_ofst[] = { - /* 0 */ -202, -54, -226, -224, -211, -73, -11, -147, 21, 44, - /* 10 */ 62, -111, -191, -238, -130, -19, 42, -155, -22, -93, - /* 20 */ -9, 55, -103, 85, 96, 98, -197, -258, -231, -230, - /* 30 */ -225, -57, -12, 51, 100, 101, 102, 103, 104, 105, - /* 40 */ 106, 107, 108, 109, 111, -116, 124, 113, 131, 132, - /* 50 */ 133, 134, 135, 136, 139, 137, 167, 169, 172, 173, - /* 60 */ 79, 117, 140, 175, 176, 177, 179, 181, 182, 183, - /* 70 */ 184, 186, 188, 189, 190, 191, 192, 193, 194, 196, - /* 80 */ 197, 199, 200, 201, 202, 203, 204, 207, 210, 212, - /* 90 */ 213, 214, 215, 218, 219, 221, 223, 225, 226, 227, - /* 100 */ 228, 234, 153, 235, 236, 152, 155, 158, 237, 159, - /* 110 */ 174, 209, 239, 242, 240, 243, 246, 248, 245, 247, - /* 120 */ 249, 250, 251, 256, 253, 255, 254, 257, 258, 261, - /* 130 */ 262, 264, 259, 260, 263, 272, 274, 270, 277, + /* 0 */ -204, -76, 54, -260, -226, -211, -87, -149, -148, 49, + /* 10 */ 76, -205, -192, -240, -218, -199, -138, -129, -59, -4, + /* 20 */ 2, -43, -161, 16, 26, 53, -69, -259, -254, -227, + /* 30 */ -160, -154, -136, -126, -65, -35, -27, -15, 12, 15, + /* 40 */ 56, 66, 74, 98, 104, 120, 126, 132, 133, 134, + /* 50 */ 135, 136, 137, 138, 140, 139, 170, 173, 175, 176, + /* 60 */ 119, 121, 141, 178, 181, 182, 183, 184, 185, 186, + /* 70 */ 187, 188, 192, 193, 194, 195, 196, 197, 198, 200, + /* 80 */ 201, 202, 204, 205, 206, 207, 208, 210, 217, 218, + /* 90 */ 219, 220, 221, 222, 223, 224, 225, 227, 228, 230, + /* 100 */ 231, 232, 155, 214, 234, 157, 158, 159, 235, 161, + /* 110 */ 179, 236, 238, 245, 247, 239, 250, 248, 249, 251, + /* 120 */ 253, 254, 255, 260, 258, 261, 259, 262, 264, 267, + /* 130 */ 263, 271, 265, 266, 268, 276, 269, 274, 282, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 646, 700, 689, 868, 868, 646, 646, 646, 646, 646, - /* 10 */ 646, 793, 664, 868, 646, 646, 646, 646, 646, 646, - /* 20 */ 646, 646, 646, 702, 702, 702, 788, 646, 646, 646, - /* 30 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 646, - /* 40 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 646, - /* 50 */ 646, 646, 646, 646, 646, 646, 646, 795, 797, 646, - /* 60 */ 815, 815, 786, 646, 646, 646, 646, 646, 646, 646, - /* 70 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 646, - /* 80 */ 646, 687, 646, 685, 646, 646, 646, 646, 646, 646, - /* 90 */ 646, 646, 646, 646, 646, 646, 674, 646, 646, 646, - /* 100 */ 646, 646, 646, 666, 666, 646, 646, 646, 666, 822, - /* 110 */ 826, 820, 808, 816, 807, 803, 802, 830, 646, 666, - /* 120 */ 666, 697, 697, 666, 718, 716, 714, 706, 712, 708, - /* 130 */ 710, 704, 666, 695, 666, 695, 666, 734, 748, 646, - /* 140 */ 831, 867, 821, 857, 856, 863, 855, 854, 646, 850, - /* 150 */ 851, 853, 852, 646, 646, 646, 646, 859, 858, 646, - /* 160 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 833, - /* 170 */ 646, 827, 823, 646, 646, 646, 646, 646, 646, 646, - /* 180 */ 646, 646, 646, 646, 646, 646, 646, 646, 646, 646, - /* 190 */ 646, 785, 646, 646, 794, 646, 646, 646, 646, 646, - /* 200 */ 646, 817, 646, 809, 646, 646, 646, 646, 646, 646, - /* 210 */ 646, 762, 646, 646, 646, 646, 646, 646, 646, 646, - /* 220 */ 646, 646, 646, 646, 646, 872, 646, 646, 646, 756, - /* 230 */ 870, 646, 646, 646, 646, 646, 646, 646, 646, 646, - /* 240 */ 646, 646, 646, 646, 646, 721, 646, 672, 670, 646, - /* 250 */ 662, 646, + /* 0 */ 653, 707, 696, 878, 878, 653, 653, 653, 653, 653, + /* 10 */ 653, 803, 671, 878, 653, 653, 653, 653, 653, 653, + /* 20 */ 653, 653, 653, 709, 709, 709, 798, 653, 653, 653, + /* 30 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653, + /* 40 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653, + /* 50 */ 653, 653, 653, 653, 653, 653, 653, 805, 807, 653, + /* 60 */ 825, 825, 796, 653, 653, 653, 653, 653, 653, 653, + /* 70 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653, + /* 80 */ 653, 694, 653, 692, 653, 653, 653, 653, 653, 653, + /* 90 */ 653, 653, 653, 653, 653, 653, 681, 653, 653, 653, + /* 100 */ 653, 653, 653, 673, 673, 653, 653, 653, 673, 832, + /* 110 */ 836, 830, 818, 826, 817, 813, 812, 840, 653, 673, + /* 120 */ 673, 704, 704, 673, 725, 723, 721, 713, 719, 715, + /* 130 */ 717, 711, 673, 702, 673, 702, 673, 742, 757, 653, + /* 140 */ 841, 877, 831, 867, 866, 873, 865, 864, 653, 860, + /* 150 */ 861, 863, 862, 653, 653, 653, 653, 869, 868, 653, + /* 160 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 843, + /* 170 */ 653, 837, 833, 653, 653, 653, 653, 653, 653, 653, + /* 180 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653, + /* 190 */ 653, 795, 653, 653, 804, 653, 653, 653, 653, 653, + /* 200 */ 653, 827, 653, 819, 653, 653, 653, 653, 653, 653, + /* 210 */ 653, 772, 653, 653, 653, 653, 653, 653, 653, 653, + /* 220 */ 653, 653, 653, 653, 653, 882, 653, 653, 653, 766, + /* 230 */ 880, 653, 653, 653, 653, 653, 653, 653, 653, 653, + /* 240 */ 653, 653, 653, 653, 653, 653, 728, 653, 679, 677, + /* 250 */ 653, 669, 653, }; /********** End of lemon-generated parsing tables *****************************/ @@ -537,6 +538,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* FSYNC => nothing */ 0, /* COMP => nothing */ 0, /* PRECISION => nothing */ + 0, /* UPDATE => nothing */ 0, /* LP => nothing */ 0, /* RP => nothing */ 0, /* TAGS => nothing */ @@ -829,179 +831,181 @@ static const char *const yyTokenName[] = { /* 95 */ "FSYNC", /* 96 */ "COMP", /* 97 */ "PRECISION", - /* 98 */ "LP", - /* 99 */ "RP", - /* 100 */ "TAGS", - /* 101 */ "USING", - /* 102 */ "AS", - /* 103 */ "COMMA", - /* 104 */ "NULL", - /* 105 */ "SELECT", - /* 106 */ "UNION", - /* 107 */ "ALL", - /* 108 */ "FROM", - /* 109 */ "VARIABLE", - /* 110 */ "INTERVAL", - /* 111 */ "FILL", - /* 112 */ "SLIDING", - /* 113 */ "ORDER", - /* 114 */ "BY", - /* 115 */ "ASC", - /* 116 */ "DESC", - /* 117 */ "GROUP", - /* 118 */ "HAVING", - /* 119 */ "LIMIT", - /* 120 */ "OFFSET", - /* 121 */ "SLIMIT", - /* 122 */ "SOFFSET", - /* 123 */ "WHERE", - /* 124 */ "NOW", - /* 125 */ "RESET", - /* 126 */ "QUERY", - /* 127 */ "ADD", - /* 128 */ "COLUMN", - /* 129 */ "TAG", - /* 130 */ "CHANGE", - /* 131 */ "SET", - /* 132 */ "KILL", - /* 133 */ "CONNECTION", - /* 134 */ "STREAM", - /* 135 */ "COLON", - /* 136 */ "ABORT", - /* 137 */ "AFTER", - /* 138 */ "ATTACH", - /* 139 */ "BEFORE", - /* 140 */ "BEGIN", - /* 141 */ "CASCADE", - /* 142 */ "CLUSTER", - /* 143 */ "CONFLICT", - /* 144 */ "COPY", - /* 145 */ "DEFERRED", - /* 146 */ "DELIMITERS", - /* 147 */ "DETACH", - /* 148 */ "EACH", - /* 149 */ "END", - /* 150 */ "EXPLAIN", - /* 151 */ "FAIL", - /* 152 */ "FOR", - /* 153 */ "IGNORE", - /* 154 */ "IMMEDIATE", - /* 155 */ "INITIALLY", - /* 156 */ "INSTEAD", - /* 157 */ "MATCH", - /* 158 */ "KEY", - /* 159 */ "OF", - /* 160 */ "RAISE", - /* 161 */ "REPLACE", - /* 162 */ "RESTRICT", - /* 163 */ "ROW", - /* 164 */ "STATEMENT", - /* 165 */ "TRIGGER", - /* 166 */ "VIEW", - /* 167 */ "COUNT", - /* 168 */ "SUM", - /* 169 */ "AVG", - /* 170 */ "MIN", - /* 171 */ "MAX", - /* 172 */ "FIRST", - /* 173 */ "LAST", - /* 174 */ "TOP", - /* 175 */ "BOTTOM", - /* 176 */ "STDDEV", - /* 177 */ "PERCENTILE", - /* 178 */ "APERCENTILE", - /* 179 */ "LEASTSQUARES", - /* 180 */ "HISTOGRAM", - /* 181 */ "DIFF", - /* 182 */ "SPREAD", - /* 183 */ "TWA", - /* 184 */ "INTERP", - /* 185 */ "LAST_ROW", - /* 186 */ "RATE", - /* 187 */ "IRATE", - /* 188 */ "SUM_RATE", - /* 189 */ "SUM_IRATE", - /* 190 */ "AVG_RATE", - /* 191 */ "AVG_IRATE", - /* 192 */ "TBID", - /* 193 */ "SEMI", - /* 194 */ "NONE", - /* 195 */ "PREV", - /* 196 */ "LINEAR", - /* 197 */ "IMPORT", - /* 198 */ "METRIC", - /* 199 */ "TBNAME", - /* 200 */ "JOIN", - /* 201 */ "METRICS", - /* 202 */ "STABLE", - /* 203 */ "INSERT", - /* 204 */ "INTO", - /* 205 */ "VALUES", - /* 206 */ "error", - /* 207 */ "program", - /* 208 */ "cmd", - /* 209 */ "dbPrefix", - /* 210 */ "ids", - /* 211 */ "cpxName", - /* 212 */ "ifexists", - /* 213 */ "alter_db_optr", - /* 214 */ "acct_optr", - /* 215 */ "ifnotexists", - /* 216 */ "db_optr", - /* 217 */ "pps", - /* 218 */ "tseries", - /* 219 */ "dbs", - /* 220 */ "streams", - /* 221 */ "storage", - /* 222 */ "qtime", - /* 223 */ "users", - /* 224 */ "conns", - /* 225 */ "state", - /* 226 */ "keep", - /* 227 */ "tagitemlist", - /* 228 */ "cache", - /* 229 */ "replica", - /* 230 */ "quorum", - /* 231 */ "days", - /* 232 */ "minrows", - /* 233 */ "maxrows", - /* 234 */ "blocks", - /* 235 */ "ctime", - /* 236 */ "wal", - /* 237 */ "fsync", - /* 238 */ "comp", - /* 239 */ "prec", - /* 240 */ "typename", - /* 241 */ "signed", - /* 242 */ "create_table_args", - /* 243 */ "columnlist", - /* 244 */ "select", - /* 245 */ "column", - /* 246 */ "tagitem", - /* 247 */ "selcollist", - /* 248 */ "from", - /* 249 */ "where_opt", - /* 250 */ "interval_opt", - /* 251 */ "fill_opt", - /* 252 */ "sliding_opt", - /* 253 */ "groupby_opt", - /* 254 */ "orderby_opt", - /* 255 */ "having_opt", - /* 256 */ "slimit_opt", - /* 257 */ "limit_opt", - /* 258 */ "union", - /* 259 */ "sclp", - /* 260 */ "expr", - /* 261 */ "as", - /* 262 */ "tablelist", - /* 263 */ "tmvar", - /* 264 */ "sortlist", - /* 265 */ "sortitem", - /* 266 */ "item", - /* 267 */ "sortorder", - /* 268 */ "grouplist", - /* 269 */ "exprlist", - /* 270 */ "expritem", + /* 98 */ "UPDATE", + /* 99 */ "LP", + /* 100 */ "RP", + /* 101 */ "TAGS", + /* 102 */ "USING", + /* 103 */ "AS", + /* 104 */ "COMMA", + /* 105 */ "NULL", + /* 106 */ "SELECT", + /* 107 */ "UNION", + /* 108 */ "ALL", + /* 109 */ "FROM", + /* 110 */ "VARIABLE", + /* 111 */ "INTERVAL", + /* 112 */ "FILL", + /* 113 */ "SLIDING", + /* 114 */ "ORDER", + /* 115 */ "BY", + /* 116 */ "ASC", + /* 117 */ "DESC", + /* 118 */ "GROUP", + /* 119 */ "HAVING", + /* 120 */ "LIMIT", + /* 121 */ "OFFSET", + /* 122 */ "SLIMIT", + /* 123 */ "SOFFSET", + /* 124 */ "WHERE", + /* 125 */ "NOW", + /* 126 */ "RESET", + /* 127 */ "QUERY", + /* 128 */ "ADD", + /* 129 */ "COLUMN", + /* 130 */ "TAG", + /* 131 */ "CHANGE", + /* 132 */ "SET", + /* 133 */ "KILL", + /* 134 */ "CONNECTION", + /* 135 */ "STREAM", + /* 136 */ "COLON", + /* 137 */ "ABORT", + /* 138 */ "AFTER", + /* 139 */ "ATTACH", + /* 140 */ "BEFORE", + /* 141 */ "BEGIN", + /* 142 */ "CASCADE", + /* 143 */ "CLUSTER", + /* 144 */ "CONFLICT", + /* 145 */ "COPY", + /* 146 */ "DEFERRED", + /* 147 */ "DELIMITERS", + /* 148 */ "DETACH", + /* 149 */ "EACH", + /* 150 */ "END", + /* 151 */ "EXPLAIN", + /* 152 */ "FAIL", + /* 153 */ "FOR", + /* 154 */ "IGNORE", + /* 155 */ "IMMEDIATE", + /* 156 */ "INITIALLY", + /* 157 */ "INSTEAD", + /* 158 */ "MATCH", + /* 159 */ "KEY", + /* 160 */ "OF", + /* 161 */ "RAISE", + /* 162 */ "REPLACE", + /* 163 */ "RESTRICT", + /* 164 */ "ROW", + /* 165 */ "STATEMENT", + /* 166 */ "TRIGGER", + /* 167 */ "VIEW", + /* 168 */ "COUNT", + /* 169 */ "SUM", + /* 170 */ "AVG", + /* 171 */ "MIN", + /* 172 */ "MAX", + /* 173 */ "FIRST", + /* 174 */ "LAST", + /* 175 */ "TOP", + /* 176 */ "BOTTOM", + /* 177 */ "STDDEV", + /* 178 */ "PERCENTILE", + /* 179 */ "APERCENTILE", + /* 180 */ "LEASTSQUARES", + /* 181 */ "HISTOGRAM", + /* 182 */ "DIFF", + /* 183 */ "SPREAD", + /* 184 */ "TWA", + /* 185 */ "INTERP", + /* 186 */ "LAST_ROW", + /* 187 */ "RATE", + /* 188 */ "IRATE", + /* 189 */ "SUM_RATE", + /* 190 */ "SUM_IRATE", + /* 191 */ "AVG_RATE", + /* 192 */ "AVG_IRATE", + /* 193 */ "TBID", + /* 194 */ "SEMI", + /* 195 */ "NONE", + /* 196 */ "PREV", + /* 197 */ "LINEAR", + /* 198 */ "IMPORT", + /* 199 */ "METRIC", + /* 200 */ "TBNAME", + /* 201 */ "JOIN", + /* 202 */ "METRICS", + /* 203 */ "STABLE", + /* 204 */ "INSERT", + /* 205 */ "INTO", + /* 206 */ "VALUES", + /* 207 */ "error", + /* 208 */ "program", + /* 209 */ "cmd", + /* 210 */ "dbPrefix", + /* 211 */ "ids", + /* 212 */ "cpxName", + /* 213 */ "ifexists", + /* 214 */ "alter_db_optr", + /* 215 */ "acct_optr", + /* 216 */ "ifnotexists", + /* 217 */ "db_optr", + /* 218 */ "pps", + /* 219 */ "tseries", + /* 220 */ "dbs", + /* 221 */ "streams", + /* 222 */ "storage", + /* 223 */ "qtime", + /* 224 */ "users", + /* 225 */ "conns", + /* 226 */ "state", + /* 227 */ "keep", + /* 228 */ "tagitemlist", + /* 229 */ "cache", + /* 230 */ "replica", + /* 231 */ "quorum", + /* 232 */ "days", + /* 233 */ "minrows", + /* 234 */ "maxrows", + /* 235 */ "blocks", + /* 236 */ "ctime", + /* 237 */ "wal", + /* 238 */ "fsync", + /* 239 */ "comp", + /* 240 */ "prec", + /* 241 */ "update", + /* 242 */ "typename", + /* 243 */ "signed", + /* 244 */ "create_table_args", + /* 245 */ "columnlist", + /* 246 */ "select", + /* 247 */ "column", + /* 248 */ "tagitem", + /* 249 */ "selcollist", + /* 250 */ "from", + /* 251 */ "where_opt", + /* 252 */ "interval_opt", + /* 253 */ "fill_opt", + /* 254 */ "sliding_opt", + /* 255 */ "groupby_opt", + /* 256 */ "orderby_opt", + /* 257 */ "having_opt", + /* 258 */ "slimit_opt", + /* 259 */ "limit_opt", + /* 260 */ "union", + /* 261 */ "sclp", + /* 262 */ "expr", + /* 263 */ "as", + /* 264 */ "tablelist", + /* 265 */ "tmvar", + /* 266 */ "sortlist", + /* 267 */ "sortitem", + /* 268 */ "item", + /* 269 */ "sortorder", + /* 270 */ "grouplist", + /* 271 */ "exprlist", + /* 272 */ "expritem", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1094,151 +1098,154 @@ static const char *const yyRuleName[] = { /* 82 */ "fsync ::= FSYNC INTEGER", /* 83 */ "comp ::= COMP INTEGER", /* 84 */ "prec ::= PRECISION STRING", - /* 85 */ "db_optr ::=", - /* 86 */ "db_optr ::= db_optr cache", - /* 87 */ "db_optr ::= db_optr replica", - /* 88 */ "db_optr ::= db_optr quorum", - /* 89 */ "db_optr ::= db_optr days", - /* 90 */ "db_optr ::= db_optr minrows", - /* 91 */ "db_optr ::= db_optr maxrows", - /* 92 */ "db_optr ::= db_optr blocks", - /* 93 */ "db_optr ::= db_optr ctime", - /* 94 */ "db_optr ::= db_optr wal", - /* 95 */ "db_optr ::= db_optr fsync", - /* 96 */ "db_optr ::= db_optr comp", - /* 97 */ "db_optr ::= db_optr prec", - /* 98 */ "db_optr ::= db_optr keep", - /* 99 */ "alter_db_optr ::=", - /* 100 */ "alter_db_optr ::= alter_db_optr replica", - /* 101 */ "alter_db_optr ::= alter_db_optr quorum", - /* 102 */ "alter_db_optr ::= alter_db_optr keep", - /* 103 */ "alter_db_optr ::= alter_db_optr blocks", - /* 104 */ "alter_db_optr ::= alter_db_optr comp", - /* 105 */ "alter_db_optr ::= alter_db_optr wal", - /* 106 */ "alter_db_optr ::= alter_db_optr fsync", - /* 107 */ "typename ::= ids", - /* 108 */ "typename ::= ids LP signed RP", - /* 109 */ "signed ::= INTEGER", - /* 110 */ "signed ::= PLUS INTEGER", - /* 111 */ "signed ::= MINUS INTEGER", - /* 112 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args", - /* 113 */ "create_table_args ::= LP columnlist RP", - /* 114 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP", - /* 115 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP", - /* 116 */ "create_table_args ::= AS select", - /* 117 */ "columnlist ::= columnlist COMMA column", - /* 118 */ "columnlist ::= column", - /* 119 */ "column ::= ids typename", - /* 120 */ "tagitemlist ::= tagitemlist COMMA tagitem", - /* 121 */ "tagitemlist ::= tagitem", - /* 122 */ "tagitem ::= INTEGER", - /* 123 */ "tagitem ::= FLOAT", - /* 124 */ "tagitem ::= STRING", - /* 125 */ "tagitem ::= BOOL", - /* 126 */ "tagitem ::= NULL", - /* 127 */ "tagitem ::= MINUS INTEGER", - /* 128 */ "tagitem ::= MINUS FLOAT", - /* 129 */ "tagitem ::= PLUS INTEGER", - /* 130 */ "tagitem ::= PLUS FLOAT", - /* 131 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", - /* 132 */ "union ::= select", - /* 133 */ "union ::= LP union RP", - /* 134 */ "union ::= union UNION ALL select", - /* 135 */ "union ::= union UNION ALL LP select RP", - /* 136 */ "cmd ::= union", - /* 137 */ "select ::= SELECT selcollist", - /* 138 */ "sclp ::= selcollist COMMA", - /* 139 */ "sclp ::=", - /* 140 */ "selcollist ::= sclp expr as", - /* 141 */ "selcollist ::= sclp STAR", - /* 142 */ "as ::= AS ids", - /* 143 */ "as ::= ids", - /* 144 */ "as ::=", - /* 145 */ "from ::= FROM tablelist", - /* 146 */ "tablelist ::= ids cpxName", - /* 147 */ "tablelist ::= ids cpxName ids", - /* 148 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 149 */ "tablelist ::= tablelist COMMA ids cpxName ids", - /* 150 */ "tmvar ::= VARIABLE", - /* 151 */ "interval_opt ::= INTERVAL LP tmvar RP", - /* 152 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", - /* 153 */ "interval_opt ::=", - /* 154 */ "fill_opt ::=", - /* 155 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 156 */ "fill_opt ::= FILL LP ID RP", - /* 157 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 158 */ "sliding_opt ::=", - /* 159 */ "orderby_opt ::=", - /* 160 */ "orderby_opt ::= ORDER BY sortlist", - /* 161 */ "sortlist ::= sortlist COMMA item sortorder", - /* 162 */ "sortlist ::= item sortorder", - /* 163 */ "item ::= ids cpxName", - /* 164 */ "sortorder ::= ASC", - /* 165 */ "sortorder ::= DESC", - /* 166 */ "sortorder ::=", - /* 167 */ "groupby_opt ::=", - /* 168 */ "groupby_opt ::= GROUP BY grouplist", - /* 169 */ "grouplist ::= grouplist COMMA item", - /* 170 */ "grouplist ::= item", - /* 171 */ "having_opt ::=", - /* 172 */ "having_opt ::= HAVING expr", - /* 173 */ "limit_opt ::=", - /* 174 */ "limit_opt ::= LIMIT signed", - /* 175 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 176 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 177 */ "slimit_opt ::=", - /* 178 */ "slimit_opt ::= SLIMIT signed", - /* 179 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 180 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 181 */ "where_opt ::=", - /* 182 */ "where_opt ::= WHERE expr", - /* 183 */ "expr ::= LP expr RP", - /* 184 */ "expr ::= ID", - /* 185 */ "expr ::= ID DOT ID", - /* 186 */ "expr ::= ID DOT STAR", - /* 187 */ "expr ::= INTEGER", - /* 188 */ "expr ::= MINUS INTEGER", - /* 189 */ "expr ::= PLUS INTEGER", - /* 190 */ "expr ::= FLOAT", - /* 191 */ "expr ::= MINUS FLOAT", - /* 192 */ "expr ::= PLUS FLOAT", - /* 193 */ "expr ::= STRING", - /* 194 */ "expr ::= NOW", - /* 195 */ "expr ::= VARIABLE", - /* 196 */ "expr ::= BOOL", - /* 197 */ "expr ::= ID LP exprlist RP", - /* 198 */ "expr ::= ID LP STAR RP", - /* 199 */ "expr ::= expr IS NULL", - /* 200 */ "expr ::= expr IS NOT NULL", - /* 201 */ "expr ::= expr LT expr", - /* 202 */ "expr ::= expr GT expr", - /* 203 */ "expr ::= expr LE expr", - /* 204 */ "expr ::= expr GE expr", - /* 205 */ "expr ::= expr NE expr", - /* 206 */ "expr ::= expr EQ expr", - /* 207 */ "expr ::= expr AND expr", - /* 208 */ "expr ::= expr OR expr", - /* 209 */ "expr ::= expr PLUS expr", - /* 210 */ "expr ::= expr MINUS expr", - /* 211 */ "expr ::= expr STAR expr", - /* 212 */ "expr ::= expr SLASH expr", - /* 213 */ "expr ::= expr REM expr", - /* 214 */ "expr ::= expr LIKE expr", - /* 215 */ "expr ::= expr IN LP exprlist RP", - /* 216 */ "exprlist ::= exprlist COMMA expritem", - /* 217 */ "exprlist ::= expritem", - /* 218 */ "expritem ::= expr", - /* 219 */ "expritem ::=", - /* 220 */ "cmd ::= RESET QUERY CACHE", - /* 221 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 222 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 223 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 224 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 225 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 226 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 227 */ "cmd ::= KILL CONNECTION INTEGER", - /* 228 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 229 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 85 */ "update ::= UPDATE INTEGER", + /* 86 */ "db_optr ::=", + /* 87 */ "db_optr ::= db_optr cache", + /* 88 */ "db_optr ::= db_optr replica", + /* 89 */ "db_optr ::= db_optr quorum", + /* 90 */ "db_optr ::= db_optr days", + /* 91 */ "db_optr ::= db_optr minrows", + /* 92 */ "db_optr ::= db_optr maxrows", + /* 93 */ "db_optr ::= db_optr blocks", + /* 94 */ "db_optr ::= db_optr ctime", + /* 95 */ "db_optr ::= db_optr wal", + /* 96 */ "db_optr ::= db_optr fsync", + /* 97 */ "db_optr ::= db_optr comp", + /* 98 */ "db_optr ::= db_optr prec", + /* 99 */ "db_optr ::= db_optr keep", + /* 100 */ "db_optr ::= db_optr update", + /* 101 */ "alter_db_optr ::=", + /* 102 */ "alter_db_optr ::= alter_db_optr replica", + /* 103 */ "alter_db_optr ::= alter_db_optr quorum", + /* 104 */ "alter_db_optr ::= alter_db_optr keep", + /* 105 */ "alter_db_optr ::= alter_db_optr blocks", + /* 106 */ "alter_db_optr ::= alter_db_optr comp", + /* 107 */ "alter_db_optr ::= alter_db_optr wal", + /* 108 */ "alter_db_optr ::= alter_db_optr fsync", + /* 109 */ "alter_db_optr ::= alter_db_optr update", + /* 110 */ "typename ::= ids", + /* 111 */ "typename ::= ids LP signed RP", + /* 112 */ "signed ::= INTEGER", + /* 113 */ "signed ::= PLUS INTEGER", + /* 114 */ "signed ::= MINUS INTEGER", + /* 115 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args", + /* 116 */ "create_table_args ::= LP columnlist RP", + /* 117 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP", + /* 118 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP", + /* 119 */ "create_table_args ::= AS select", + /* 120 */ "columnlist ::= columnlist COMMA column", + /* 121 */ "columnlist ::= column", + /* 122 */ "column ::= ids typename", + /* 123 */ "tagitemlist ::= tagitemlist COMMA tagitem", + /* 124 */ "tagitemlist ::= tagitem", + /* 125 */ "tagitem ::= INTEGER", + /* 126 */ "tagitem ::= FLOAT", + /* 127 */ "tagitem ::= STRING", + /* 128 */ "tagitem ::= BOOL", + /* 129 */ "tagitem ::= NULL", + /* 130 */ "tagitem ::= MINUS INTEGER", + /* 131 */ "tagitem ::= MINUS FLOAT", + /* 132 */ "tagitem ::= PLUS INTEGER", + /* 133 */ "tagitem ::= PLUS FLOAT", + /* 134 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", + /* 135 */ "union ::= select", + /* 136 */ "union ::= LP union RP", + /* 137 */ "union ::= union UNION ALL select", + /* 138 */ "union ::= union UNION ALL LP select RP", + /* 139 */ "cmd ::= union", + /* 140 */ "select ::= SELECT selcollist", + /* 141 */ "sclp ::= selcollist COMMA", + /* 142 */ "sclp ::=", + /* 143 */ "selcollist ::= sclp expr as", + /* 144 */ "selcollist ::= sclp STAR", + /* 145 */ "as ::= AS ids", + /* 146 */ "as ::= ids", + /* 147 */ "as ::=", + /* 148 */ "from ::= FROM tablelist", + /* 149 */ "tablelist ::= ids cpxName", + /* 150 */ "tablelist ::= ids cpxName ids", + /* 151 */ "tablelist ::= tablelist COMMA ids cpxName", + /* 152 */ "tablelist ::= tablelist COMMA ids cpxName ids", + /* 153 */ "tmvar ::= VARIABLE", + /* 154 */ "interval_opt ::= INTERVAL LP tmvar RP", + /* 155 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", + /* 156 */ "interval_opt ::=", + /* 157 */ "fill_opt ::=", + /* 158 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 159 */ "fill_opt ::= FILL LP ID RP", + /* 160 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 161 */ "sliding_opt ::=", + /* 162 */ "orderby_opt ::=", + /* 163 */ "orderby_opt ::= ORDER BY sortlist", + /* 164 */ "sortlist ::= sortlist COMMA item sortorder", + /* 165 */ "sortlist ::= item sortorder", + /* 166 */ "item ::= ids cpxName", + /* 167 */ "sortorder ::= ASC", + /* 168 */ "sortorder ::= DESC", + /* 169 */ "sortorder ::=", + /* 170 */ "groupby_opt ::=", + /* 171 */ "groupby_opt ::= GROUP BY grouplist", + /* 172 */ "grouplist ::= grouplist COMMA item", + /* 173 */ "grouplist ::= item", + /* 174 */ "having_opt ::=", + /* 175 */ "having_opt ::= HAVING expr", + /* 176 */ "limit_opt ::=", + /* 177 */ "limit_opt ::= LIMIT signed", + /* 178 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 179 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 180 */ "slimit_opt ::=", + /* 181 */ "slimit_opt ::= SLIMIT signed", + /* 182 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 183 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 184 */ "where_opt ::=", + /* 185 */ "where_opt ::= WHERE expr", + /* 186 */ "expr ::= LP expr RP", + /* 187 */ "expr ::= ID", + /* 188 */ "expr ::= ID DOT ID", + /* 189 */ "expr ::= ID DOT STAR", + /* 190 */ "expr ::= INTEGER", + /* 191 */ "expr ::= MINUS INTEGER", + /* 192 */ "expr ::= PLUS INTEGER", + /* 193 */ "expr ::= FLOAT", + /* 194 */ "expr ::= MINUS FLOAT", + /* 195 */ "expr ::= PLUS FLOAT", + /* 196 */ "expr ::= STRING", + /* 197 */ "expr ::= NOW", + /* 198 */ "expr ::= VARIABLE", + /* 199 */ "expr ::= BOOL", + /* 200 */ "expr ::= ID LP exprlist RP", + /* 201 */ "expr ::= ID LP STAR RP", + /* 202 */ "expr ::= expr IS NULL", + /* 203 */ "expr ::= expr IS NOT NULL", + /* 204 */ "expr ::= expr LT expr", + /* 205 */ "expr ::= expr GT expr", + /* 206 */ "expr ::= expr LE expr", + /* 207 */ "expr ::= expr GE expr", + /* 208 */ "expr ::= expr NE expr", + /* 209 */ "expr ::= expr EQ expr", + /* 210 */ "expr ::= expr AND expr", + /* 211 */ "expr ::= expr OR expr", + /* 212 */ "expr ::= expr PLUS expr", + /* 213 */ "expr ::= expr MINUS expr", + /* 214 */ "expr ::= expr STAR expr", + /* 215 */ "expr ::= expr SLASH expr", + /* 216 */ "expr ::= expr REM expr", + /* 217 */ "expr ::= expr LIKE expr", + /* 218 */ "expr ::= expr IN LP exprlist RP", + /* 219 */ "exprlist ::= exprlist COMMA expritem", + /* 220 */ "exprlist ::= expritem", + /* 221 */ "expritem ::= expr", + /* 222 */ "expritem ::=", + /* 223 */ "cmd ::= RESET QUERY CACHE", + /* 224 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 225 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 226 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 227 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 228 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 229 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 230 */ "cmd ::= KILL CONNECTION INTEGER", + /* 231 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 232 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1359,50 +1366,50 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 226: /* keep */ - case 227: /* tagitemlist */ - case 251: /* fill_opt */ - case 253: /* groupby_opt */ - case 254: /* orderby_opt */ - case 264: /* sortlist */ - case 268: /* grouplist */ + case 227: /* keep */ + case 228: /* tagitemlist */ + case 253: /* fill_opt */ + case 255: /* groupby_opt */ + case 256: /* orderby_opt */ + case 266: /* sortlist */ + case 270: /* grouplist */ { -tVariantListDestroy((yypminor->yy494)); +tVariantListDestroy((yypminor->yy498)); } break; - case 243: /* columnlist */ + case 245: /* columnlist */ { -tFieldListDestroy((yypminor->yy449)); +tFieldListDestroy((yypminor->yy523)); } break; - case 244: /* select */ + case 246: /* select */ { -doDestroyQuerySql((yypminor->yy150)); +doDestroyQuerySql((yypminor->yy414)); } break; - case 247: /* selcollist */ - case 259: /* sclp */ - case 269: /* exprlist */ + case 249: /* selcollist */ + case 261: /* sclp */ + case 271: /* exprlist */ { -tSQLExprListDestroy((yypminor->yy224)); +tSQLExprListDestroy((yypminor->yy290)); } break; - case 249: /* where_opt */ - case 255: /* having_opt */ - case 260: /* expr */ - case 270: /* expritem */ + case 251: /* where_opt */ + case 257: /* having_opt */ + case 262: /* expr */ + case 272: /* expritem */ { -tSQLExprDestroy((yypminor->yy66)); +tSQLExprDestroy((yypminor->yy64)); } break; - case 258: /* union */ + case 260: /* union */ { -destroyAllSelectClause((yypminor->yy25)); +destroyAllSelectClause((yypminor->yy231)); } break; - case 265: /* sortitem */ + case 267: /* sortitem */ { -tVariantDestroy(&(yypminor->yy312)); +tVariantDestroy(&(yypminor->yy134)); } break; /********* End destructor definitions *****************************************/ @@ -1696,236 +1703,239 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 207, -1 }, /* (0) program ::= cmd */ - { 208, -2 }, /* (1) cmd ::= SHOW DATABASES */ - { 208, -2 }, /* (2) cmd ::= SHOW MNODES */ - { 208, -2 }, /* (3) cmd ::= SHOW DNODES */ - { 208, -2 }, /* (4) cmd ::= SHOW ACCOUNTS */ - { 208, -2 }, /* (5) cmd ::= SHOW USERS */ - { 208, -2 }, /* (6) cmd ::= SHOW MODULES */ - { 208, -2 }, /* (7) cmd ::= SHOW QUERIES */ - { 208, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ - { 208, -2 }, /* (9) cmd ::= SHOW STREAMS */ - { 208, -2 }, /* (10) cmd ::= SHOW VARIABLES */ - { 208, -2 }, /* (11) cmd ::= SHOW SCORES */ - { 208, -2 }, /* (12) cmd ::= SHOW GRANTS */ - { 208, -2 }, /* (13) cmd ::= SHOW VNODES */ - { 208, -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */ - { 209, 0 }, /* (15) dbPrefix ::= */ - { 209, -2 }, /* (16) dbPrefix ::= ids DOT */ - { 211, 0 }, /* (17) cpxName ::= */ - { 211, -2 }, /* (18) cpxName ::= DOT ids */ - { 208, -5 }, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */ - { 208, -4 }, /* (20) cmd ::= SHOW CREATE DATABASE ids */ - { 208, -3 }, /* (21) cmd ::= SHOW dbPrefix TABLES */ - { 208, -5 }, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - { 208, -3 }, /* (23) cmd ::= SHOW dbPrefix STABLES */ - { 208, -5 }, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - { 208, -3 }, /* (25) cmd ::= SHOW dbPrefix VGROUPS */ - { 208, -4 }, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */ - { 208, -5 }, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */ - { 208, -4 }, /* (28) cmd ::= DROP DATABASE ifexists ids */ - { 208, -3 }, /* (29) cmd ::= DROP DNODE ids */ - { 208, -3 }, /* (30) cmd ::= DROP USER ids */ - { 208, -3 }, /* (31) cmd ::= DROP ACCOUNT ids */ - { 208, -2 }, /* (32) cmd ::= USE ids */ - { 208, -3 }, /* (33) cmd ::= DESCRIBE ids cpxName */ - { 208, -5 }, /* (34) cmd ::= ALTER USER ids PASS ids */ - { 208, -5 }, /* (35) cmd ::= ALTER USER ids PRIVILEGE ids */ - { 208, -4 }, /* (36) cmd ::= ALTER DNODE ids ids */ - { 208, -5 }, /* (37) cmd ::= ALTER DNODE ids ids ids */ - { 208, -3 }, /* (38) cmd ::= ALTER LOCAL ids */ - { 208, -4 }, /* (39) cmd ::= ALTER LOCAL ids ids */ - { 208, -4 }, /* (40) cmd ::= ALTER DATABASE ids alter_db_optr */ - { 208, -4 }, /* (41) cmd ::= ALTER ACCOUNT ids acct_optr */ - { 208, -6 }, /* (42) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - { 210, -1 }, /* (43) ids ::= ID */ - { 210, -1 }, /* (44) ids ::= STRING */ - { 212, -2 }, /* (45) ifexists ::= IF EXISTS */ - { 212, 0 }, /* (46) ifexists ::= */ - { 215, -3 }, /* (47) ifnotexists ::= IF NOT EXISTS */ - { 215, 0 }, /* (48) ifnotexists ::= */ - { 208, -3 }, /* (49) cmd ::= CREATE DNODE ids */ - { 208, -6 }, /* (50) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - { 208, -5 }, /* (51) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - { 208, -5 }, /* (52) cmd ::= CREATE USER ids PASS ids */ - { 217, 0 }, /* (53) pps ::= */ - { 217, -2 }, /* (54) pps ::= PPS INTEGER */ - { 218, 0 }, /* (55) tseries ::= */ - { 218, -2 }, /* (56) tseries ::= TSERIES INTEGER */ - { 219, 0 }, /* (57) dbs ::= */ - { 219, -2 }, /* (58) dbs ::= DBS INTEGER */ - { 220, 0 }, /* (59) streams ::= */ - { 220, -2 }, /* (60) streams ::= STREAMS INTEGER */ - { 221, 0 }, /* (61) storage ::= */ - { 221, -2 }, /* (62) storage ::= STORAGE INTEGER */ - { 222, 0 }, /* (63) qtime ::= */ - { 222, -2 }, /* (64) qtime ::= QTIME INTEGER */ - { 223, 0 }, /* (65) users ::= */ - { 223, -2 }, /* (66) users ::= USERS INTEGER */ - { 224, 0 }, /* (67) conns ::= */ - { 224, -2 }, /* (68) conns ::= CONNS INTEGER */ - { 225, 0 }, /* (69) state ::= */ - { 225, -2 }, /* (70) state ::= STATE ids */ - { 214, -9 }, /* (71) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - { 226, -2 }, /* (72) keep ::= KEEP tagitemlist */ - { 228, -2 }, /* (73) cache ::= CACHE INTEGER */ - { 229, -2 }, /* (74) replica ::= REPLICA INTEGER */ - { 230, -2 }, /* (75) quorum ::= QUORUM INTEGER */ - { 231, -2 }, /* (76) days ::= DAYS INTEGER */ - { 232, -2 }, /* (77) minrows ::= MINROWS INTEGER */ - { 233, -2 }, /* (78) maxrows ::= MAXROWS INTEGER */ - { 234, -2 }, /* (79) blocks ::= BLOCKS INTEGER */ - { 235, -2 }, /* (80) ctime ::= CTIME INTEGER */ - { 236, -2 }, /* (81) wal ::= WAL INTEGER */ - { 237, -2 }, /* (82) fsync ::= FSYNC INTEGER */ - { 238, -2 }, /* (83) comp ::= COMP INTEGER */ - { 239, -2 }, /* (84) prec ::= PRECISION STRING */ - { 216, 0 }, /* (85) db_optr ::= */ - { 216, -2 }, /* (86) db_optr ::= db_optr cache */ - { 216, -2 }, /* (87) db_optr ::= db_optr replica */ - { 216, -2 }, /* (88) db_optr ::= db_optr quorum */ - { 216, -2 }, /* (89) db_optr ::= db_optr days */ - { 216, -2 }, /* (90) db_optr ::= db_optr minrows */ - { 216, -2 }, /* (91) db_optr ::= db_optr maxrows */ - { 216, -2 }, /* (92) db_optr ::= db_optr blocks */ - { 216, -2 }, /* (93) db_optr ::= db_optr ctime */ - { 216, -2 }, /* (94) db_optr ::= db_optr wal */ - { 216, -2 }, /* (95) db_optr ::= db_optr fsync */ - { 216, -2 }, /* (96) db_optr ::= db_optr comp */ - { 216, -2 }, /* (97) db_optr ::= db_optr prec */ - { 216, -2 }, /* (98) db_optr ::= db_optr keep */ - { 213, 0 }, /* (99) alter_db_optr ::= */ - { 213, -2 }, /* (100) alter_db_optr ::= alter_db_optr replica */ - { 213, -2 }, /* (101) alter_db_optr ::= alter_db_optr quorum */ - { 213, -2 }, /* (102) alter_db_optr ::= alter_db_optr keep */ - { 213, -2 }, /* (103) alter_db_optr ::= alter_db_optr blocks */ - { 213, -2 }, /* (104) alter_db_optr ::= alter_db_optr comp */ - { 213, -2 }, /* (105) alter_db_optr ::= alter_db_optr wal */ - { 213, -2 }, /* (106) alter_db_optr ::= alter_db_optr fsync */ - { 240, -1 }, /* (107) typename ::= ids */ - { 240, -4 }, /* (108) typename ::= ids LP signed RP */ - { 241, -1 }, /* (109) signed ::= INTEGER */ - { 241, -2 }, /* (110) signed ::= PLUS INTEGER */ - { 241, -2 }, /* (111) signed ::= MINUS INTEGER */ - { 208, -6 }, /* (112) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ - { 242, -3 }, /* (113) create_table_args ::= LP columnlist RP */ - { 242, -7 }, /* (114) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ - { 242, -7 }, /* (115) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ - { 242, -2 }, /* (116) create_table_args ::= AS select */ - { 243, -3 }, /* (117) columnlist ::= columnlist COMMA column */ - { 243, -1 }, /* (118) columnlist ::= column */ - { 245, -2 }, /* (119) column ::= ids typename */ - { 227, -3 }, /* (120) tagitemlist ::= tagitemlist COMMA tagitem */ - { 227, -1 }, /* (121) tagitemlist ::= tagitem */ - { 246, -1 }, /* (122) tagitem ::= INTEGER */ - { 246, -1 }, /* (123) tagitem ::= FLOAT */ - { 246, -1 }, /* (124) tagitem ::= STRING */ - { 246, -1 }, /* (125) tagitem ::= BOOL */ - { 246, -1 }, /* (126) tagitem ::= NULL */ - { 246, -2 }, /* (127) tagitem ::= MINUS INTEGER */ - { 246, -2 }, /* (128) tagitem ::= MINUS FLOAT */ - { 246, -2 }, /* (129) tagitem ::= PLUS INTEGER */ - { 246, -2 }, /* (130) tagitem ::= PLUS FLOAT */ - { 244, -12 }, /* (131) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ - { 258, -1 }, /* (132) union ::= select */ - { 258, -3 }, /* (133) union ::= LP union RP */ - { 258, -4 }, /* (134) union ::= union UNION ALL select */ - { 258, -6 }, /* (135) union ::= union UNION ALL LP select RP */ - { 208, -1 }, /* (136) cmd ::= union */ - { 244, -2 }, /* (137) select ::= SELECT selcollist */ - { 259, -2 }, /* (138) sclp ::= selcollist COMMA */ - { 259, 0 }, /* (139) sclp ::= */ - { 247, -3 }, /* (140) selcollist ::= sclp expr as */ - { 247, -2 }, /* (141) selcollist ::= sclp STAR */ - { 261, -2 }, /* (142) as ::= AS ids */ - { 261, -1 }, /* (143) as ::= ids */ - { 261, 0 }, /* (144) as ::= */ - { 248, -2 }, /* (145) from ::= FROM tablelist */ - { 262, -2 }, /* (146) tablelist ::= ids cpxName */ - { 262, -3 }, /* (147) tablelist ::= ids cpxName ids */ - { 262, -4 }, /* (148) tablelist ::= tablelist COMMA ids cpxName */ - { 262, -5 }, /* (149) tablelist ::= tablelist COMMA ids cpxName ids */ - { 263, -1 }, /* (150) tmvar ::= VARIABLE */ - { 250, -4 }, /* (151) interval_opt ::= INTERVAL LP tmvar RP */ - { 250, -6 }, /* (152) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ - { 250, 0 }, /* (153) interval_opt ::= */ - { 251, 0 }, /* (154) fill_opt ::= */ - { 251, -6 }, /* (155) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - { 251, -4 }, /* (156) fill_opt ::= FILL LP ID RP */ - { 252, -4 }, /* (157) sliding_opt ::= SLIDING LP tmvar RP */ - { 252, 0 }, /* (158) sliding_opt ::= */ - { 254, 0 }, /* (159) orderby_opt ::= */ - { 254, -3 }, /* (160) orderby_opt ::= ORDER BY sortlist */ - { 264, -4 }, /* (161) sortlist ::= sortlist COMMA item sortorder */ - { 264, -2 }, /* (162) sortlist ::= item sortorder */ - { 266, -2 }, /* (163) item ::= ids cpxName */ - { 267, -1 }, /* (164) sortorder ::= ASC */ - { 267, -1 }, /* (165) sortorder ::= DESC */ - { 267, 0 }, /* (166) sortorder ::= */ - { 253, 0 }, /* (167) groupby_opt ::= */ - { 253, -3 }, /* (168) groupby_opt ::= GROUP BY grouplist */ - { 268, -3 }, /* (169) grouplist ::= grouplist COMMA item */ - { 268, -1 }, /* (170) grouplist ::= item */ - { 255, 0 }, /* (171) having_opt ::= */ - { 255, -2 }, /* (172) having_opt ::= HAVING expr */ - { 257, 0 }, /* (173) limit_opt ::= */ - { 257, -2 }, /* (174) limit_opt ::= LIMIT signed */ - { 257, -4 }, /* (175) limit_opt ::= LIMIT signed OFFSET signed */ - { 257, -4 }, /* (176) limit_opt ::= LIMIT signed COMMA signed */ - { 256, 0 }, /* (177) slimit_opt ::= */ - { 256, -2 }, /* (178) slimit_opt ::= SLIMIT signed */ - { 256, -4 }, /* (179) slimit_opt ::= SLIMIT signed SOFFSET signed */ - { 256, -4 }, /* (180) slimit_opt ::= SLIMIT signed COMMA signed */ - { 249, 0 }, /* (181) where_opt ::= */ - { 249, -2 }, /* (182) where_opt ::= WHERE expr */ - { 260, -3 }, /* (183) expr ::= LP expr RP */ - { 260, -1 }, /* (184) expr ::= ID */ - { 260, -3 }, /* (185) expr ::= ID DOT ID */ - { 260, -3 }, /* (186) expr ::= ID DOT STAR */ - { 260, -1 }, /* (187) expr ::= INTEGER */ - { 260, -2 }, /* (188) expr ::= MINUS INTEGER */ - { 260, -2 }, /* (189) expr ::= PLUS INTEGER */ - { 260, -1 }, /* (190) expr ::= FLOAT */ - { 260, -2 }, /* (191) expr ::= MINUS FLOAT */ - { 260, -2 }, /* (192) expr ::= PLUS FLOAT */ - { 260, -1 }, /* (193) expr ::= STRING */ - { 260, -1 }, /* (194) expr ::= NOW */ - { 260, -1 }, /* (195) expr ::= VARIABLE */ - { 260, -1 }, /* (196) expr ::= BOOL */ - { 260, -4 }, /* (197) expr ::= ID LP exprlist RP */ - { 260, -4 }, /* (198) expr ::= ID LP STAR RP */ - { 260, -3 }, /* (199) expr ::= expr IS NULL */ - { 260, -4 }, /* (200) expr ::= expr IS NOT NULL */ - { 260, -3 }, /* (201) expr ::= expr LT expr */ - { 260, -3 }, /* (202) expr ::= expr GT expr */ - { 260, -3 }, /* (203) expr ::= expr LE expr */ - { 260, -3 }, /* (204) expr ::= expr GE expr */ - { 260, -3 }, /* (205) expr ::= expr NE expr */ - { 260, -3 }, /* (206) expr ::= expr EQ expr */ - { 260, -3 }, /* (207) expr ::= expr AND expr */ - { 260, -3 }, /* (208) expr ::= expr OR expr */ - { 260, -3 }, /* (209) expr ::= expr PLUS expr */ - { 260, -3 }, /* (210) expr ::= expr MINUS expr */ - { 260, -3 }, /* (211) expr ::= expr STAR expr */ - { 260, -3 }, /* (212) expr ::= expr SLASH expr */ - { 260, -3 }, /* (213) expr ::= expr REM expr */ - { 260, -3 }, /* (214) expr ::= expr LIKE expr */ - { 260, -5 }, /* (215) expr ::= expr IN LP exprlist RP */ - { 269, -3 }, /* (216) exprlist ::= exprlist COMMA expritem */ - { 269, -1 }, /* (217) exprlist ::= expritem */ - { 270, -1 }, /* (218) expritem ::= expr */ - { 270, 0 }, /* (219) expritem ::= */ - { 208, -3 }, /* (220) cmd ::= RESET QUERY CACHE */ - { 208, -7 }, /* (221) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 208, -7 }, /* (222) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 208, -7 }, /* (223) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 208, -7 }, /* (224) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 208, -8 }, /* (225) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 208, -9 }, /* (226) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 208, -3 }, /* (227) cmd ::= KILL CONNECTION INTEGER */ - { 208, -5 }, /* (228) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 208, -5 }, /* (229) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + { 208, -1 }, /* (0) program ::= cmd */ + { 209, -2 }, /* (1) cmd ::= SHOW DATABASES */ + { 209, -2 }, /* (2) cmd ::= SHOW MNODES */ + { 209, -2 }, /* (3) cmd ::= SHOW DNODES */ + { 209, -2 }, /* (4) cmd ::= SHOW ACCOUNTS */ + { 209, -2 }, /* (5) cmd ::= SHOW USERS */ + { 209, -2 }, /* (6) cmd ::= SHOW MODULES */ + { 209, -2 }, /* (7) cmd ::= SHOW QUERIES */ + { 209, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ + { 209, -2 }, /* (9) cmd ::= SHOW STREAMS */ + { 209, -2 }, /* (10) cmd ::= SHOW VARIABLES */ + { 209, -2 }, /* (11) cmd ::= SHOW SCORES */ + { 209, -2 }, /* (12) cmd ::= SHOW GRANTS */ + { 209, -2 }, /* (13) cmd ::= SHOW VNODES */ + { 209, -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */ + { 210, 0 }, /* (15) dbPrefix ::= */ + { 210, -2 }, /* (16) dbPrefix ::= ids DOT */ + { 212, 0 }, /* (17) cpxName ::= */ + { 212, -2 }, /* (18) cpxName ::= DOT ids */ + { 209, -5 }, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */ + { 209, -4 }, /* (20) cmd ::= SHOW CREATE DATABASE ids */ + { 209, -3 }, /* (21) cmd ::= SHOW dbPrefix TABLES */ + { 209, -5 }, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */ + { 209, -3 }, /* (23) cmd ::= SHOW dbPrefix STABLES */ + { 209, -5 }, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */ + { 209, -3 }, /* (25) cmd ::= SHOW dbPrefix VGROUPS */ + { 209, -4 }, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */ + { 209, -5 }, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */ + { 209, -4 }, /* (28) cmd ::= DROP DATABASE ifexists ids */ + { 209, -3 }, /* (29) cmd ::= DROP DNODE ids */ + { 209, -3 }, /* (30) cmd ::= DROP USER ids */ + { 209, -3 }, /* (31) cmd ::= DROP ACCOUNT ids */ + { 209, -2 }, /* (32) cmd ::= USE ids */ + { 209, -3 }, /* (33) cmd ::= DESCRIBE ids cpxName */ + { 209, -5 }, /* (34) cmd ::= ALTER USER ids PASS ids */ + { 209, -5 }, /* (35) cmd ::= ALTER USER ids PRIVILEGE ids */ + { 209, -4 }, /* (36) cmd ::= ALTER DNODE ids ids */ + { 209, -5 }, /* (37) cmd ::= ALTER DNODE ids ids ids */ + { 209, -3 }, /* (38) cmd ::= ALTER LOCAL ids */ + { 209, -4 }, /* (39) cmd ::= ALTER LOCAL ids ids */ + { 209, -4 }, /* (40) cmd ::= ALTER DATABASE ids alter_db_optr */ + { 209, -4 }, /* (41) cmd ::= ALTER ACCOUNT ids acct_optr */ + { 209, -6 }, /* (42) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ + { 211, -1 }, /* (43) ids ::= ID */ + { 211, -1 }, /* (44) ids ::= STRING */ + { 213, -2 }, /* (45) ifexists ::= IF EXISTS */ + { 213, 0 }, /* (46) ifexists ::= */ + { 216, -3 }, /* (47) ifnotexists ::= IF NOT EXISTS */ + { 216, 0 }, /* (48) ifnotexists ::= */ + { 209, -3 }, /* (49) cmd ::= CREATE DNODE ids */ + { 209, -6 }, /* (50) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ + { 209, -5 }, /* (51) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + { 209, -5 }, /* (52) cmd ::= CREATE USER ids PASS ids */ + { 218, 0 }, /* (53) pps ::= */ + { 218, -2 }, /* (54) pps ::= PPS INTEGER */ + { 219, 0 }, /* (55) tseries ::= */ + { 219, -2 }, /* (56) tseries ::= TSERIES INTEGER */ + { 220, 0 }, /* (57) dbs ::= */ + { 220, -2 }, /* (58) dbs ::= DBS INTEGER */ + { 221, 0 }, /* (59) streams ::= */ + { 221, -2 }, /* (60) streams ::= STREAMS INTEGER */ + { 222, 0 }, /* (61) storage ::= */ + { 222, -2 }, /* (62) storage ::= STORAGE INTEGER */ + { 223, 0 }, /* (63) qtime ::= */ + { 223, -2 }, /* (64) qtime ::= QTIME INTEGER */ + { 224, 0 }, /* (65) users ::= */ + { 224, -2 }, /* (66) users ::= USERS INTEGER */ + { 225, 0 }, /* (67) conns ::= */ + { 225, -2 }, /* (68) conns ::= CONNS INTEGER */ + { 226, 0 }, /* (69) state ::= */ + { 226, -2 }, /* (70) state ::= STATE ids */ + { 215, -9 }, /* (71) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + { 227, -2 }, /* (72) keep ::= KEEP tagitemlist */ + { 229, -2 }, /* (73) cache ::= CACHE INTEGER */ + { 230, -2 }, /* (74) replica ::= REPLICA INTEGER */ + { 231, -2 }, /* (75) quorum ::= QUORUM INTEGER */ + { 232, -2 }, /* (76) days ::= DAYS INTEGER */ + { 233, -2 }, /* (77) minrows ::= MINROWS INTEGER */ + { 234, -2 }, /* (78) maxrows ::= MAXROWS INTEGER */ + { 235, -2 }, /* (79) blocks ::= BLOCKS INTEGER */ + { 236, -2 }, /* (80) ctime ::= CTIME INTEGER */ + { 237, -2 }, /* (81) wal ::= WAL INTEGER */ + { 238, -2 }, /* (82) fsync ::= FSYNC INTEGER */ + { 239, -2 }, /* (83) comp ::= COMP INTEGER */ + { 240, -2 }, /* (84) prec ::= PRECISION STRING */ + { 241, -2 }, /* (85) update ::= UPDATE INTEGER */ + { 217, 0 }, /* (86) db_optr ::= */ + { 217, -2 }, /* (87) db_optr ::= db_optr cache */ + { 217, -2 }, /* (88) db_optr ::= db_optr replica */ + { 217, -2 }, /* (89) db_optr ::= db_optr quorum */ + { 217, -2 }, /* (90) db_optr ::= db_optr days */ + { 217, -2 }, /* (91) db_optr ::= db_optr minrows */ + { 217, -2 }, /* (92) db_optr ::= db_optr maxrows */ + { 217, -2 }, /* (93) db_optr ::= db_optr blocks */ + { 217, -2 }, /* (94) db_optr ::= db_optr ctime */ + { 217, -2 }, /* (95) db_optr ::= db_optr wal */ + { 217, -2 }, /* (96) db_optr ::= db_optr fsync */ + { 217, -2 }, /* (97) db_optr ::= db_optr comp */ + { 217, -2 }, /* (98) db_optr ::= db_optr prec */ + { 217, -2 }, /* (99) db_optr ::= db_optr keep */ + { 217, -2 }, /* (100) db_optr ::= db_optr update */ + { 214, 0 }, /* (101) alter_db_optr ::= */ + { 214, -2 }, /* (102) alter_db_optr ::= alter_db_optr replica */ + { 214, -2 }, /* (103) alter_db_optr ::= alter_db_optr quorum */ + { 214, -2 }, /* (104) alter_db_optr ::= alter_db_optr keep */ + { 214, -2 }, /* (105) alter_db_optr ::= alter_db_optr blocks */ + { 214, -2 }, /* (106) alter_db_optr ::= alter_db_optr comp */ + { 214, -2 }, /* (107) alter_db_optr ::= alter_db_optr wal */ + { 214, -2 }, /* (108) alter_db_optr ::= alter_db_optr fsync */ + { 214, -2 }, /* (109) alter_db_optr ::= alter_db_optr update */ + { 242, -1 }, /* (110) typename ::= ids */ + { 242, -4 }, /* (111) typename ::= ids LP signed RP */ + { 243, -1 }, /* (112) signed ::= INTEGER */ + { 243, -2 }, /* (113) signed ::= PLUS INTEGER */ + { 243, -2 }, /* (114) signed ::= MINUS INTEGER */ + { 209, -6 }, /* (115) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ + { 244, -3 }, /* (116) create_table_args ::= LP columnlist RP */ + { 244, -7 }, /* (117) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ + { 244, -7 }, /* (118) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ + { 244, -2 }, /* (119) create_table_args ::= AS select */ + { 245, -3 }, /* (120) columnlist ::= columnlist COMMA column */ + { 245, -1 }, /* (121) columnlist ::= column */ + { 247, -2 }, /* (122) column ::= ids typename */ + { 228, -3 }, /* (123) tagitemlist ::= tagitemlist COMMA tagitem */ + { 228, -1 }, /* (124) tagitemlist ::= tagitem */ + { 248, -1 }, /* (125) tagitem ::= INTEGER */ + { 248, -1 }, /* (126) tagitem ::= FLOAT */ + { 248, -1 }, /* (127) tagitem ::= STRING */ + { 248, -1 }, /* (128) tagitem ::= BOOL */ + { 248, -1 }, /* (129) tagitem ::= NULL */ + { 248, -2 }, /* (130) tagitem ::= MINUS INTEGER */ + { 248, -2 }, /* (131) tagitem ::= MINUS FLOAT */ + { 248, -2 }, /* (132) tagitem ::= PLUS INTEGER */ + { 248, -2 }, /* (133) tagitem ::= PLUS FLOAT */ + { 246, -12 }, /* (134) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + { 260, -1 }, /* (135) union ::= select */ + { 260, -3 }, /* (136) union ::= LP union RP */ + { 260, -4 }, /* (137) union ::= union UNION ALL select */ + { 260, -6 }, /* (138) union ::= union UNION ALL LP select RP */ + { 209, -1 }, /* (139) cmd ::= union */ + { 246, -2 }, /* (140) select ::= SELECT selcollist */ + { 261, -2 }, /* (141) sclp ::= selcollist COMMA */ + { 261, 0 }, /* (142) sclp ::= */ + { 249, -3 }, /* (143) selcollist ::= sclp expr as */ + { 249, -2 }, /* (144) selcollist ::= sclp STAR */ + { 263, -2 }, /* (145) as ::= AS ids */ + { 263, -1 }, /* (146) as ::= ids */ + { 263, 0 }, /* (147) as ::= */ + { 250, -2 }, /* (148) from ::= FROM tablelist */ + { 264, -2 }, /* (149) tablelist ::= ids cpxName */ + { 264, -3 }, /* (150) tablelist ::= ids cpxName ids */ + { 264, -4 }, /* (151) tablelist ::= tablelist COMMA ids cpxName */ + { 264, -5 }, /* (152) tablelist ::= tablelist COMMA ids cpxName ids */ + { 265, -1 }, /* (153) tmvar ::= VARIABLE */ + { 252, -4 }, /* (154) interval_opt ::= INTERVAL LP tmvar RP */ + { 252, -6 }, /* (155) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ + { 252, 0 }, /* (156) interval_opt ::= */ + { 253, 0 }, /* (157) fill_opt ::= */ + { 253, -6 }, /* (158) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + { 253, -4 }, /* (159) fill_opt ::= FILL LP ID RP */ + { 254, -4 }, /* (160) sliding_opt ::= SLIDING LP tmvar RP */ + { 254, 0 }, /* (161) sliding_opt ::= */ + { 256, 0 }, /* (162) orderby_opt ::= */ + { 256, -3 }, /* (163) orderby_opt ::= ORDER BY sortlist */ + { 266, -4 }, /* (164) sortlist ::= sortlist COMMA item sortorder */ + { 266, -2 }, /* (165) sortlist ::= item sortorder */ + { 268, -2 }, /* (166) item ::= ids cpxName */ + { 269, -1 }, /* (167) sortorder ::= ASC */ + { 269, -1 }, /* (168) sortorder ::= DESC */ + { 269, 0 }, /* (169) sortorder ::= */ + { 255, 0 }, /* (170) groupby_opt ::= */ + { 255, -3 }, /* (171) groupby_opt ::= GROUP BY grouplist */ + { 270, -3 }, /* (172) grouplist ::= grouplist COMMA item */ + { 270, -1 }, /* (173) grouplist ::= item */ + { 257, 0 }, /* (174) having_opt ::= */ + { 257, -2 }, /* (175) having_opt ::= HAVING expr */ + { 259, 0 }, /* (176) limit_opt ::= */ + { 259, -2 }, /* (177) limit_opt ::= LIMIT signed */ + { 259, -4 }, /* (178) limit_opt ::= LIMIT signed OFFSET signed */ + { 259, -4 }, /* (179) limit_opt ::= LIMIT signed COMMA signed */ + { 258, 0 }, /* (180) slimit_opt ::= */ + { 258, -2 }, /* (181) slimit_opt ::= SLIMIT signed */ + { 258, -4 }, /* (182) slimit_opt ::= SLIMIT signed SOFFSET signed */ + { 258, -4 }, /* (183) slimit_opt ::= SLIMIT signed COMMA signed */ + { 251, 0 }, /* (184) where_opt ::= */ + { 251, -2 }, /* (185) where_opt ::= WHERE expr */ + { 262, -3 }, /* (186) expr ::= LP expr RP */ + { 262, -1 }, /* (187) expr ::= ID */ + { 262, -3 }, /* (188) expr ::= ID DOT ID */ + { 262, -3 }, /* (189) expr ::= ID DOT STAR */ + { 262, -1 }, /* (190) expr ::= INTEGER */ + { 262, -2 }, /* (191) expr ::= MINUS INTEGER */ + { 262, -2 }, /* (192) expr ::= PLUS INTEGER */ + { 262, -1 }, /* (193) expr ::= FLOAT */ + { 262, -2 }, /* (194) expr ::= MINUS FLOAT */ + { 262, -2 }, /* (195) expr ::= PLUS FLOAT */ + { 262, -1 }, /* (196) expr ::= STRING */ + { 262, -1 }, /* (197) expr ::= NOW */ + { 262, -1 }, /* (198) expr ::= VARIABLE */ + { 262, -1 }, /* (199) expr ::= BOOL */ + { 262, -4 }, /* (200) expr ::= ID LP exprlist RP */ + { 262, -4 }, /* (201) expr ::= ID LP STAR RP */ + { 262, -3 }, /* (202) expr ::= expr IS NULL */ + { 262, -4 }, /* (203) expr ::= expr IS NOT NULL */ + { 262, -3 }, /* (204) expr ::= expr LT expr */ + { 262, -3 }, /* (205) expr ::= expr GT expr */ + { 262, -3 }, /* (206) expr ::= expr LE expr */ + { 262, -3 }, /* (207) expr ::= expr GE expr */ + { 262, -3 }, /* (208) expr ::= expr NE expr */ + { 262, -3 }, /* (209) expr ::= expr EQ expr */ + { 262, -3 }, /* (210) expr ::= expr AND expr */ + { 262, -3 }, /* (211) expr ::= expr OR expr */ + { 262, -3 }, /* (212) expr ::= expr PLUS expr */ + { 262, -3 }, /* (213) expr ::= expr MINUS expr */ + { 262, -3 }, /* (214) expr ::= expr STAR expr */ + { 262, -3 }, /* (215) expr ::= expr SLASH expr */ + { 262, -3 }, /* (216) expr ::= expr REM expr */ + { 262, -3 }, /* (217) expr ::= expr LIKE expr */ + { 262, -5 }, /* (218) expr ::= expr IN LP exprlist RP */ + { 271, -3 }, /* (219) exprlist ::= exprlist COMMA expritem */ + { 271, -1 }, /* (220) exprlist ::= expritem */ + { 272, -1 }, /* (221) expritem ::= expr */ + { 272, 0 }, /* (222) expritem ::= */ + { 209, -3 }, /* (223) cmd ::= RESET QUERY CACHE */ + { 209, -7 }, /* (224) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + { 209, -7 }, /* (225) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + { 209, -7 }, /* (226) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + { 209, -7 }, /* (227) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + { 209, -8 }, /* (228) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + { 209, -9 }, /* (229) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + { 209, -3 }, /* (230) cmd ::= KILL CONNECTION INTEGER */ + { 209, -5 }, /* (231) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + { 209, -5 }, /* (232) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2156,13 +2166,13 @@ static void yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 40: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy158, &t);} +{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} break; case 41: /* cmd ::= ALTER ACCOUNT ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy73);} +{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);} break; case 42: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy73);} +{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);} break; case 43: /* ids ::= ID */ case 44: /* ids ::= STRING */ yytestcase(yyruleno==44); @@ -2183,10 +2193,10 @@ static void yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} break; case 50: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy73);} +{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);} break; case 51: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ -{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy158, &yymsp[-2].minor.yy0);} +{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &yymsp[-2].minor.yy0);} break; case 52: /* cmd ::= CREATE USER ids PASS ids */ { setCreateUserSQL(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} @@ -2215,20 +2225,20 @@ static void yy_reduce( break; case 71: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ { - yylhsminor.yy73.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; - yylhsminor.yy73.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; - yylhsminor.yy73.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; - yylhsminor.yy73.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; - yylhsminor.yy73.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; - yylhsminor.yy73.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy73.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy73.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; - yylhsminor.yy73.stat = yymsp[0].minor.yy0; -} - yymsp[-8].minor.yy73 = yylhsminor.yy73; + yylhsminor.yy149.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; + yylhsminor.yy149.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; + yylhsminor.yy149.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; + yylhsminor.yy149.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; + yylhsminor.yy149.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; + yylhsminor.yy149.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy149.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy149.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; + yylhsminor.yy149.stat = yymsp[0].minor.yy0; +} + yymsp[-8].minor.yy149 = yylhsminor.yy149; break; case 72: /* keep ::= KEEP tagitemlist */ -{ yymsp[-1].minor.yy494 = yymsp[0].minor.yy494; } +{ yymsp[-1].minor.yy498 = yymsp[0].minor.yy498; } break; case 73: /* cache ::= CACHE INTEGER */ case 74: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==74); @@ -2242,540 +2252,546 @@ static void yy_reduce( case 82: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==82); case 83: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==83); case 84: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==84); + case 85: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==85); { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 85: /* db_optr ::= */ -{setDefaultCreateDbOption(&yymsp[1].minor.yy158);} - break; - case 86: /* db_optr ::= db_optr cache */ -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; - break; - case 87: /* db_optr ::= db_optr replica */ - case 100: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==100); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; - break; - case 88: /* db_optr ::= db_optr quorum */ - case 101: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==101); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; - break; - case 89: /* db_optr ::= db_optr days */ -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; - break; - case 90: /* db_optr ::= db_optr minrows */ -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; - break; - case 91: /* db_optr ::= db_optr maxrows */ -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; - break; - case 92: /* db_optr ::= db_optr blocks */ - case 103: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==103); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; - break; - case 93: /* db_optr ::= db_optr ctime */ -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; - break; - case 94: /* db_optr ::= db_optr wal */ - case 105: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==105); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; - break; - case 95: /* db_optr ::= db_optr fsync */ - case 106: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==106); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; - break; - case 96: /* db_optr ::= db_optr comp */ - case 104: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==104); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; - break; - case 97: /* db_optr ::= db_optr prec */ -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.precision = yymsp[0].minor.yy0; } - yymsp[-1].minor.yy158 = yylhsminor.yy158; - break; - case 98: /* db_optr ::= db_optr keep */ - case 102: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==102); -{ yylhsminor.yy158 = yymsp[-1].minor.yy158; yylhsminor.yy158.keep = yymsp[0].minor.yy494; } - yymsp[-1].minor.yy158 = yylhsminor.yy158; - break; - case 99: /* alter_db_optr ::= */ -{ setDefaultCreateDbOption(&yymsp[1].minor.yy158);} - break; - case 107: /* typename ::= ids */ + case 86: /* db_optr ::= */ +{setDefaultCreateDbOption(&yymsp[1].minor.yy268);} + break; + case 87: /* db_optr ::= db_optr cache */ +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 88: /* db_optr ::= db_optr replica */ + case 102: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==102); +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 89: /* db_optr ::= db_optr quorum */ + case 103: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==103); +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 90: /* db_optr ::= db_optr days */ +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 91: /* db_optr ::= db_optr minrows */ +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 92: /* db_optr ::= db_optr maxrows */ +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 93: /* db_optr ::= db_optr blocks */ + case 105: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==105); +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 94: /* db_optr ::= db_optr ctime */ +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 95: /* db_optr ::= db_optr wal */ + case 107: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==107); +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 96: /* db_optr ::= db_optr fsync */ + case 108: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==108); +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 97: /* db_optr ::= db_optr comp */ + case 106: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==106); +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 98: /* db_optr ::= db_optr prec */ +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.precision = yymsp[0].minor.yy0; } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 99: /* db_optr ::= db_optr keep */ + case 104: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==104); +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.keep = yymsp[0].minor.yy498; } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 100: /* db_optr ::= db_optr update */ + case 109: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==109); +{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy268 = yylhsminor.yy268; + break; + case 101: /* alter_db_optr ::= */ +{ setDefaultCreateDbOption(&yymsp[1].minor.yy268);} + break; + case 110: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; - tSQLSetColumnType (&yylhsminor.yy181, &yymsp[0].minor.yy0); + tSQLSetColumnType (&yylhsminor.yy223, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy181 = yylhsminor.yy181; + yymsp[0].minor.yy223 = yylhsminor.yy223; break; - case 108: /* typename ::= ids LP signed RP */ + case 111: /* typename ::= ids LP signed RP */ { - if (yymsp[-1].minor.yy271 <= 0) { + if (yymsp[-1].minor.yy207 <= 0) { yymsp[-3].minor.yy0.type = 0; - tSQLSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0); + tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0); } else { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy271; // negative value of name length - tSQLSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0); + yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy207; // negative value of name length + tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0); } } - yymsp[-3].minor.yy181 = yylhsminor.yy181; + yymsp[-3].minor.yy223 = yylhsminor.yy223; break; - case 109: /* signed ::= INTEGER */ -{ yylhsminor.yy271 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[0].minor.yy271 = yylhsminor.yy271; + case 112: /* signed ::= INTEGER */ +{ yylhsminor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[0].minor.yy207 = yylhsminor.yy207; break; - case 110: /* signed ::= PLUS INTEGER */ -{ yymsp[-1].minor.yy271 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 113: /* signed ::= PLUS INTEGER */ +{ yymsp[-1].minor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 111: /* signed ::= MINUS INTEGER */ -{ yymsp[-1].minor.yy271 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} + case 114: /* signed ::= MINUS INTEGER */ +{ yymsp[-1].minor.yy207 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} break; - case 112: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ + case 115: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ { yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-2].minor.yy0, &yymsp[-3].minor.yy0); } break; - case 113: /* create_table_args ::= LP columnlist RP */ + case 116: /* create_table_args ::= LP columnlist RP */ { - yymsp[-2].minor.yy374 = tSetCreateSQLElems(yymsp[-1].minor.yy449, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE); - setSQLInfo(pInfo, yymsp[-2].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE); + yymsp[-2].minor.yy470 = tSetCreateSQLElems(yymsp[-1].minor.yy523, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE); + setSQLInfo(pInfo, yymsp[-2].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE); } break; - case 114: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ + case 117: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ { - yymsp[-6].minor.yy374 = tSetCreateSQLElems(yymsp[-5].minor.yy449, yymsp[-1].minor.yy449, NULL, NULL, NULL, TSQL_CREATE_STABLE); - setSQLInfo(pInfo, yymsp[-6].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE); + yymsp[-6].minor.yy470 = tSetCreateSQLElems(yymsp[-5].minor.yy523, yymsp[-1].minor.yy523, NULL, NULL, NULL, TSQL_CREATE_STABLE); + setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE); } break; - case 115: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ + case 118: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; - yymsp[-6].minor.yy374 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy494, NULL, TSQL_CREATE_TABLE_FROM_STABLE); - setSQLInfo(pInfo, yymsp[-6].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE); + yymsp[-6].minor.yy470 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy498, NULL, TSQL_CREATE_TABLE_FROM_STABLE); + setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE); } break; - case 116: /* create_table_args ::= AS select */ + case 119: /* create_table_args ::= AS select */ { - yymsp[-1].minor.yy374 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy150, TSQL_CREATE_STREAM); - setSQLInfo(pInfo, yymsp[-1].minor.yy374, NULL, TSDB_SQL_CREATE_TABLE); + yymsp[-1].minor.yy470 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy414, TSQL_CREATE_STREAM); + setSQLInfo(pInfo, yymsp[-1].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE); } break; - case 117: /* columnlist ::= columnlist COMMA column */ -{yylhsminor.yy449 = tFieldListAppend(yymsp[-2].minor.yy449, &yymsp[0].minor.yy181); } - yymsp[-2].minor.yy449 = yylhsminor.yy449; + case 120: /* columnlist ::= columnlist COMMA column */ +{yylhsminor.yy523 = tFieldListAppend(yymsp[-2].minor.yy523, &yymsp[0].minor.yy223); } + yymsp[-2].minor.yy523 = yylhsminor.yy523; break; - case 118: /* columnlist ::= column */ -{yylhsminor.yy449 = tFieldListAppend(NULL, &yymsp[0].minor.yy181);} - yymsp[0].minor.yy449 = yylhsminor.yy449; + case 121: /* columnlist ::= column */ +{yylhsminor.yy523 = tFieldListAppend(NULL, &yymsp[0].minor.yy223);} + yymsp[0].minor.yy523 = yylhsminor.yy523; break; - case 119: /* column ::= ids typename */ + case 122: /* column ::= ids typename */ { - tSQLSetColumnInfo(&yylhsminor.yy181, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy181); + tSQLSetColumnInfo(&yylhsminor.yy223, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy223); } - yymsp[-1].minor.yy181 = yylhsminor.yy181; + yymsp[-1].minor.yy223 = yylhsminor.yy223; break; - case 120: /* tagitemlist ::= tagitemlist COMMA tagitem */ -{ yylhsminor.yy494 = tVariantListAppend(yymsp[-2].minor.yy494, &yymsp[0].minor.yy312, -1); } - yymsp[-2].minor.yy494 = yylhsminor.yy494; + case 123: /* tagitemlist ::= tagitemlist COMMA tagitem */ +{ yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1); } + yymsp[-2].minor.yy498 = yylhsminor.yy498; break; - case 121: /* tagitemlist ::= tagitem */ -{ yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[0].minor.yy312, -1); } - yymsp[0].minor.yy494 = yylhsminor.yy494; + case 124: /* tagitemlist ::= tagitem */ +{ yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); } + yymsp[0].minor.yy498 = yylhsminor.yy498; break; - case 122: /* tagitem ::= INTEGER */ - case 123: /* tagitem ::= FLOAT */ yytestcase(yyruleno==123); - case 124: /* tagitem ::= STRING */ yytestcase(yyruleno==124); - case 125: /* tagitem ::= BOOL */ yytestcase(yyruleno==125); -{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy312, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy312 = yylhsminor.yy312; + case 125: /* tagitem ::= INTEGER */ + case 126: /* tagitem ::= FLOAT */ yytestcase(yyruleno==126); + case 127: /* tagitem ::= STRING */ yytestcase(yyruleno==127); + case 128: /* tagitem ::= BOOL */ yytestcase(yyruleno==128); +{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy134 = yylhsminor.yy134; break; - case 126: /* tagitem ::= NULL */ -{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy312, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy312 = yylhsminor.yy312; + case 129: /* tagitem ::= NULL */ +{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy134 = yylhsminor.yy134; break; - case 127: /* tagitem ::= MINUS INTEGER */ - case 128: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==128); - case 129: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==129); - case 130: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==130); + case 130: /* tagitem ::= MINUS INTEGER */ + case 131: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==131); + case 132: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==132); + case 133: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==133); { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; toTSDBType(yymsp[-1].minor.yy0.type); - tVariantCreate(&yylhsminor.yy312, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy312 = yylhsminor.yy312; + yymsp[-1].minor.yy134 = yylhsminor.yy134; break; - case 131: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + case 134: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ { - yylhsminor.yy150 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy224, yymsp[-9].minor.yy494, yymsp[-8].minor.yy66, yymsp[-4].minor.yy494, yymsp[-3].minor.yy494, &yymsp[-7].minor.yy314, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy494, &yymsp[0].minor.yy188, &yymsp[-1].minor.yy188); + yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy290, yymsp[-9].minor.yy498, yymsp[-8].minor.yy64, yymsp[-4].minor.yy498, yymsp[-3].minor.yy498, &yymsp[-7].minor.yy532, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy498, &yymsp[0].minor.yy216, &yymsp[-1].minor.yy216); } - yymsp[-11].minor.yy150 = yylhsminor.yy150; + yymsp[-11].minor.yy414 = yylhsminor.yy414; break; - case 132: /* union ::= select */ -{ yylhsminor.yy25 = setSubclause(NULL, yymsp[0].minor.yy150); } - yymsp[0].minor.yy25 = yylhsminor.yy25; + case 135: /* union ::= select */ +{ yylhsminor.yy231 = setSubclause(NULL, yymsp[0].minor.yy414); } + yymsp[0].minor.yy231 = yylhsminor.yy231; break; - case 133: /* union ::= LP union RP */ -{ yymsp[-2].minor.yy25 = yymsp[-1].minor.yy25; } + case 136: /* union ::= LP union RP */ +{ yymsp[-2].minor.yy231 = yymsp[-1].minor.yy231; } break; - case 134: /* union ::= union UNION ALL select */ -{ yylhsminor.yy25 = appendSelectClause(yymsp[-3].minor.yy25, yymsp[0].minor.yy150); } - yymsp[-3].minor.yy25 = yylhsminor.yy25; + case 137: /* union ::= union UNION ALL select */ +{ yylhsminor.yy231 = appendSelectClause(yymsp[-3].minor.yy231, yymsp[0].minor.yy414); } + yymsp[-3].minor.yy231 = yylhsminor.yy231; break; - case 135: /* union ::= union UNION ALL LP select RP */ -{ yylhsminor.yy25 = appendSelectClause(yymsp[-5].minor.yy25, yymsp[-1].minor.yy150); } - yymsp[-5].minor.yy25 = yylhsminor.yy25; + case 138: /* union ::= union UNION ALL LP select RP */ +{ yylhsminor.yy231 = appendSelectClause(yymsp[-5].minor.yy231, yymsp[-1].minor.yy414); } + yymsp[-5].minor.yy231 = yylhsminor.yy231; break; - case 136: /* cmd ::= union */ -{ setSQLInfo(pInfo, yymsp[0].minor.yy25, NULL, TSDB_SQL_SELECT); } + case 139: /* cmd ::= union */ +{ setSQLInfo(pInfo, yymsp[0].minor.yy231, NULL, TSDB_SQL_SELECT); } break; - case 137: /* select ::= SELECT selcollist */ + case 140: /* select ::= SELECT selcollist */ { - yylhsminor.yy150 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy224, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy290, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } - yymsp[-1].minor.yy150 = yylhsminor.yy150; + yymsp[-1].minor.yy414 = yylhsminor.yy414; break; - case 138: /* sclp ::= selcollist COMMA */ -{yylhsminor.yy224 = yymsp[-1].minor.yy224;} - yymsp[-1].minor.yy224 = yylhsminor.yy224; + case 141: /* sclp ::= selcollist COMMA */ +{yylhsminor.yy290 = yymsp[-1].minor.yy290;} + yymsp[-1].minor.yy290 = yylhsminor.yy290; break; - case 139: /* sclp ::= */ -{yymsp[1].minor.yy224 = 0;} + case 142: /* sclp ::= */ +{yymsp[1].minor.yy290 = 0;} break; - case 140: /* selcollist ::= sclp expr as */ + case 143: /* selcollist ::= sclp expr as */ { - yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224, yymsp[-1].minor.yy66, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); + yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290, yymsp[-1].minor.yy64, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } - yymsp[-2].minor.yy224 = yylhsminor.yy224; + yymsp[-2].minor.yy290 = yylhsminor.yy290; break; - case 141: /* selcollist ::= sclp STAR */ + case 144: /* selcollist ::= sclp STAR */ { tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL); - yylhsminor.yy224 = tSQLExprListAppend(yymsp[-1].minor.yy224, pNode, 0); + yylhsminor.yy290 = tSQLExprListAppend(yymsp[-1].minor.yy290, pNode, 0); } - yymsp[-1].minor.yy224 = yylhsminor.yy224; + yymsp[-1].minor.yy290 = yylhsminor.yy290; break; - case 142: /* as ::= AS ids */ + case 145: /* as ::= AS ids */ { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 143: /* as ::= ids */ + case 146: /* as ::= ids */ { yylhsminor.yy0 = yymsp[0].minor.yy0; } yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 144: /* as ::= */ + case 147: /* as ::= */ { yymsp[1].minor.yy0.n = 0; } break; - case 145: /* from ::= FROM tablelist */ -{yymsp[-1].minor.yy494 = yymsp[0].minor.yy494;} + case 148: /* from ::= FROM tablelist */ +{yymsp[-1].minor.yy498 = yymsp[0].minor.yy498;} break; - case 146: /* tablelist ::= ids cpxName */ + case 149: /* tablelist ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy494 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[-1].minor.yy0, -1); // table alias name + yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[-1].minor.yy0, -1); // table alias name } - yymsp[-1].minor.yy494 = yylhsminor.yy494; + yymsp[-1].minor.yy498 = yylhsminor.yy498; break; - case 147: /* tablelist ::= ids cpxName ids */ + case 150: /* tablelist ::= ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy494 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[0].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[0].minor.yy0, -1); } - yymsp[-2].minor.yy494 = yylhsminor.yy494; + yymsp[-2].minor.yy498 = yylhsminor.yy498; break; - case 148: /* tablelist ::= tablelist COMMA ids cpxName */ + case 151: /* tablelist ::= tablelist COMMA ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy494 = tVariantListAppendToken(yymsp[-3].minor.yy494, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[-1].minor.yy0, -1); } - yymsp[-3].minor.yy494 = yylhsminor.yy494; + yymsp[-3].minor.yy498 = yylhsminor.yy498; break; - case 149: /* tablelist ::= tablelist COMMA ids cpxName ids */ + case 152: /* tablelist ::= tablelist COMMA ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy494 = tVariantListAppendToken(yymsp[-4].minor.yy494, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy494 = tVariantListAppendToken(yylhsminor.yy494, &yymsp[0].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yymsp[-4].minor.yy498, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[0].minor.yy0, -1); } - yymsp[-4].minor.yy494 = yylhsminor.yy494; + yymsp[-4].minor.yy498 = yylhsminor.yy498; break; - case 150: /* tmvar ::= VARIABLE */ + case 153: /* tmvar ::= VARIABLE */ {yylhsminor.yy0 = yymsp[0].minor.yy0;} yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 151: /* interval_opt ::= INTERVAL LP tmvar RP */ -{yymsp[-3].minor.yy314.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy314.offset.n = 0; yymsp[-3].minor.yy314.offset.z = NULL; yymsp[-3].minor.yy314.offset.type = 0;} + case 154: /* interval_opt ::= INTERVAL LP tmvar RP */ +{yymsp[-3].minor.yy532.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy532.offset.n = 0; yymsp[-3].minor.yy532.offset.z = NULL; yymsp[-3].minor.yy532.offset.type = 0;} break; - case 152: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ -{yymsp[-5].minor.yy314.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy314.offset = yymsp[-1].minor.yy0;} + case 155: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ +{yymsp[-5].minor.yy532.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy532.offset = yymsp[-1].minor.yy0;} break; - case 153: /* interval_opt ::= */ -{memset(&yymsp[1].minor.yy314, 0, sizeof(yymsp[1].minor.yy314));} + case 156: /* interval_opt ::= */ +{memset(&yymsp[1].minor.yy532, 0, sizeof(yymsp[1].minor.yy532));} break; - case 154: /* fill_opt ::= */ -{yymsp[1].minor.yy494 = 0; } + case 157: /* fill_opt ::= */ +{yymsp[1].minor.yy498 = 0; } break; - case 155: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 158: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); tVariantCreate(&A, &yymsp[-3].minor.yy0); - tVariantListInsert(yymsp[-1].minor.yy494, &A, -1, 0); - yymsp[-5].minor.yy494 = yymsp[-1].minor.yy494; + tVariantListInsert(yymsp[-1].minor.yy498, &A, -1, 0); + yymsp[-5].minor.yy498 = yymsp[-1].minor.yy498; } break; - case 156: /* fill_opt ::= FILL LP ID RP */ + case 159: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-3].minor.yy494 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yymsp[-3].minor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 157: /* sliding_opt ::= SLIDING LP tmvar RP */ + case 160: /* sliding_opt ::= SLIDING LP tmvar RP */ {yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } break; - case 158: /* sliding_opt ::= */ + case 161: /* sliding_opt ::= */ {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } break; - case 159: /* orderby_opt ::= */ - case 167: /* groupby_opt ::= */ yytestcase(yyruleno==167); -{yymsp[1].minor.yy494 = 0;} + case 162: /* orderby_opt ::= */ + case 170: /* groupby_opt ::= */ yytestcase(yyruleno==170); +{yymsp[1].minor.yy498 = 0;} break; - case 160: /* orderby_opt ::= ORDER BY sortlist */ - case 168: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==168); -{yymsp[-2].minor.yy494 = yymsp[0].minor.yy494;} + case 163: /* orderby_opt ::= ORDER BY sortlist */ + case 171: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==171); +{yymsp[-2].minor.yy498 = yymsp[0].minor.yy498;} break; - case 161: /* sortlist ::= sortlist COMMA item sortorder */ + case 164: /* sortlist ::= sortlist COMMA item sortorder */ { - yylhsminor.yy494 = tVariantListAppend(yymsp[-3].minor.yy494, &yymsp[-1].minor.yy312, yymsp[0].minor.yy82); + yylhsminor.yy498 = tVariantListAppend(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); } - yymsp[-3].minor.yy494 = yylhsminor.yy494; + yymsp[-3].minor.yy498 = yylhsminor.yy498; break; - case 162: /* sortlist ::= item sortorder */ + case 165: /* sortlist ::= item sortorder */ { - yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[-1].minor.yy312, yymsp[0].minor.yy82); + yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); } - yymsp[-1].minor.yy494 = yylhsminor.yy494; + yymsp[-1].minor.yy498 = yylhsminor.yy498; break; - case 163: /* item ::= ids cpxName */ + case 166: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - tVariantCreate(&yylhsminor.yy312, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy312 = yylhsminor.yy312; + yymsp[-1].minor.yy134 = yylhsminor.yy134; break; - case 164: /* sortorder ::= ASC */ -{yymsp[0].minor.yy82 = TSDB_ORDER_ASC; } + case 167: /* sortorder ::= ASC */ +{yymsp[0].minor.yy46 = TSDB_ORDER_ASC; } break; - case 165: /* sortorder ::= DESC */ -{yymsp[0].minor.yy82 = TSDB_ORDER_DESC;} + case 168: /* sortorder ::= DESC */ +{yymsp[0].minor.yy46 = TSDB_ORDER_DESC;} break; - case 166: /* sortorder ::= */ -{yymsp[1].minor.yy82 = TSDB_ORDER_ASC;} + case 169: /* sortorder ::= */ +{yymsp[1].minor.yy46 = TSDB_ORDER_ASC;} break; - case 169: /* grouplist ::= grouplist COMMA item */ + case 172: /* grouplist ::= grouplist COMMA item */ { - yylhsminor.yy494 = tVariantListAppend(yymsp[-2].minor.yy494, &yymsp[0].minor.yy312, -1); + yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1); } - yymsp[-2].minor.yy494 = yylhsminor.yy494; + yymsp[-2].minor.yy498 = yylhsminor.yy498; break; - case 170: /* grouplist ::= item */ + case 173: /* grouplist ::= item */ { - yylhsminor.yy494 = tVariantListAppend(NULL, &yymsp[0].minor.yy312, -1); + yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); } - yymsp[0].minor.yy494 = yylhsminor.yy494; + yymsp[0].minor.yy498 = yylhsminor.yy498; break; - case 171: /* having_opt ::= */ - case 181: /* where_opt ::= */ yytestcase(yyruleno==181); - case 219: /* expritem ::= */ yytestcase(yyruleno==219); -{yymsp[1].minor.yy66 = 0;} + case 174: /* having_opt ::= */ + case 184: /* where_opt ::= */ yytestcase(yyruleno==184); + case 222: /* expritem ::= */ yytestcase(yyruleno==222); +{yymsp[1].minor.yy64 = 0;} break; - case 172: /* having_opt ::= HAVING expr */ - case 182: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==182); -{yymsp[-1].minor.yy66 = yymsp[0].minor.yy66;} + case 175: /* having_opt ::= HAVING expr */ + case 185: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==185); +{yymsp[-1].minor.yy64 = yymsp[0].minor.yy64;} break; - case 173: /* limit_opt ::= */ - case 177: /* slimit_opt ::= */ yytestcase(yyruleno==177); -{yymsp[1].minor.yy188.limit = -1; yymsp[1].minor.yy188.offset = 0;} + case 176: /* limit_opt ::= */ + case 180: /* slimit_opt ::= */ yytestcase(yyruleno==180); +{yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;} break; - case 174: /* limit_opt ::= LIMIT signed */ - case 178: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==178); -{yymsp[-1].minor.yy188.limit = yymsp[0].minor.yy271; yymsp[-1].minor.yy188.offset = 0;} + case 177: /* limit_opt ::= LIMIT signed */ + case 181: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==181); +{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;} break; - case 175: /* limit_opt ::= LIMIT signed OFFSET signed */ - case 179: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==179); -{yymsp[-3].minor.yy188.limit = yymsp[-2].minor.yy271; yymsp[-3].minor.yy188.offset = yymsp[0].minor.yy271;} + case 178: /* limit_opt ::= LIMIT signed OFFSET signed */ + case 182: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==182); +{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;} break; - case 176: /* limit_opt ::= LIMIT signed COMMA signed */ - case 180: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==180); -{yymsp[-3].minor.yy188.limit = yymsp[0].minor.yy271; yymsp[-3].minor.yy188.offset = yymsp[-2].minor.yy271;} + case 179: /* limit_opt ::= LIMIT signed COMMA signed */ + case 183: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==183); +{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;} break; - case 183: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy66 = yymsp[-1].minor.yy66; } + case 186: /* expr ::= LP expr RP */ +{yymsp[-2].minor.yy64 = yymsp[-1].minor.yy64; } break; - case 184: /* expr ::= ID */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 187: /* expr ::= ID */ +{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} + yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 185: /* expr ::= ID DOT ID */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 188: /* expr ::= ID DOT ID */ +{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 186: /* expr ::= ID DOT STAR */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 189: /* expr ::= ID DOT STAR */ +{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 187: /* expr ::= INTEGER */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 190: /* expr ::= INTEGER */ +{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} + yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 188: /* expr ::= MINUS INTEGER */ - case 189: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==189); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} - yymsp[-1].minor.yy66 = yylhsminor.yy66; + case 191: /* expr ::= MINUS INTEGER */ + case 192: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==192); +{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} + yymsp[-1].minor.yy64 = yylhsminor.yy64; break; - case 190: /* expr ::= FLOAT */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 193: /* expr ::= FLOAT */ +{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} + yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 191: /* expr ::= MINUS FLOAT */ - case 192: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==192); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} - yymsp[-1].minor.yy66 = yylhsminor.yy66; + case 194: /* expr ::= MINUS FLOAT */ + case 195: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==195); +{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} + yymsp[-1].minor.yy64 = yylhsminor.yy64; break; - case 193: /* expr ::= STRING */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 196: /* expr ::= STRING */ +{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} + yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 194: /* expr ::= NOW */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 197: /* expr ::= NOW */ +{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } + yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 195: /* expr ::= VARIABLE */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 198: /* expr ::= VARIABLE */ +{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} + yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 196: /* expr ::= BOOL */ -{yylhsminor.yy66 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 199: /* expr ::= BOOL */ +{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} + yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 197: /* expr ::= ID LP exprlist RP */ -{ yylhsminor.yy66 = tSQLExprCreateFunction(yymsp[-1].minor.yy224, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy66 = yylhsminor.yy66; + case 200: /* expr ::= ID LP exprlist RP */ +{ yylhsminor.yy64 = tSQLExprCreateFunction(yymsp[-1].minor.yy290, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy64 = yylhsminor.yy64; break; - case 198: /* expr ::= ID LP STAR RP */ -{ yylhsminor.yy66 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy66 = yylhsminor.yy66; + case 201: /* expr ::= ID LP STAR RP */ +{ yylhsminor.yy64 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy64 = yylhsminor.yy64; break; - case 199: /* expr ::= expr IS NULL */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, NULL, TK_ISNULL);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 202: /* expr ::= expr IS NULL */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, NULL, TK_ISNULL);} + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 200: /* expr ::= expr IS NOT NULL */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-3].minor.yy66, NULL, TK_NOTNULL);} - yymsp[-3].minor.yy66 = yylhsminor.yy66; + case 203: /* expr ::= expr IS NOT NULL */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-3].minor.yy64, NULL, TK_NOTNULL);} + yymsp[-3].minor.yy64 = yylhsminor.yy64; break; - case 201: /* expr ::= expr LT expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LT);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 204: /* expr ::= expr LT expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LT);} + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 202: /* expr ::= expr GT expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_GT);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 205: /* expr ::= expr GT expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GT);} + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 203: /* expr ::= expr LE expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LE);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 206: /* expr ::= expr LE expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LE);} + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 204: /* expr ::= expr GE expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_GE);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 207: /* expr ::= expr GE expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GE);} + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 205: /* expr ::= expr NE expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_NE);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 208: /* expr ::= expr NE expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_NE);} + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 206: /* expr ::= expr EQ expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_EQ);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 209: /* expr ::= expr EQ expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_EQ);} + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 207: /* expr ::= expr AND expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_AND);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 210: /* expr ::= expr AND expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_AND);} + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 208: /* expr ::= expr OR expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_OR); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 211: /* expr ::= expr OR expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_OR); } + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 209: /* expr ::= expr PLUS expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_PLUS); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 212: /* expr ::= expr PLUS expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_PLUS); } + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 210: /* expr ::= expr MINUS expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_MINUS); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 213: /* expr ::= expr MINUS expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_MINUS); } + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 211: /* expr ::= expr STAR expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_STAR); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 214: /* expr ::= expr STAR expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_STAR); } + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 212: /* expr ::= expr SLASH expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_DIVIDE);} - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 215: /* expr ::= expr SLASH expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_DIVIDE);} + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 213: /* expr ::= expr REM expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_REM); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 216: /* expr ::= expr REM expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_REM); } + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 214: /* expr ::= expr LIKE expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LIKE); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 217: /* expr ::= expr LIKE expr */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LIKE); } + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 215: /* expr ::= expr IN LP exprlist RP */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-4].minor.yy66, (tSQLExpr*)yymsp[-1].minor.yy224, TK_IN); } - yymsp[-4].minor.yy66 = yylhsminor.yy66; + case 218: /* expr ::= expr IN LP exprlist RP */ +{yylhsminor.yy64 = tSQLExprCreate(yymsp[-4].minor.yy64, (tSQLExpr*)yymsp[-1].minor.yy290, TK_IN); } + yymsp[-4].minor.yy64 = yylhsminor.yy64; break; - case 216: /* exprlist ::= exprlist COMMA expritem */ -{yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224,yymsp[0].minor.yy66,0);} - yymsp[-2].minor.yy224 = yylhsminor.yy224; + case 219: /* exprlist ::= exprlist COMMA expritem */ +{yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290,yymsp[0].minor.yy64,0);} + yymsp[-2].minor.yy290 = yylhsminor.yy290; break; - case 217: /* exprlist ::= expritem */ -{yylhsminor.yy224 = tSQLExprListAppend(0,yymsp[0].minor.yy66,0);} - yymsp[0].minor.yy224 = yylhsminor.yy224; + case 220: /* exprlist ::= expritem */ +{yylhsminor.yy290 = tSQLExprListAppend(0,yymsp[0].minor.yy64,0);} + yymsp[0].minor.yy290 = yylhsminor.yy290; break; - case 218: /* expritem ::= expr */ -{yylhsminor.yy66 = yymsp[0].minor.yy66;} - yymsp[0].minor.yy66 = yylhsminor.yy66; + case 221: /* expritem ::= expr */ +{yylhsminor.yy64 = yymsp[0].minor.yy64;} + yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 220: /* cmd ::= RESET QUERY CACHE */ + case 223: /* cmd ::= RESET QUERY CACHE */ { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 221: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 224: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); + SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 222: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 225: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2786,14 +2802,14 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 223: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 226: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); + SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 224: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 227: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2804,7 +2820,7 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 225: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 228: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -2818,25 +2834,25 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 226: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 229: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; toTSDBType(yymsp[-2].minor.yy0.type); tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy312, -1); + A = tVariantListAppend(A, &yymsp[0].minor.yy134, -1); SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 227: /* cmd ::= KILL CONNECTION INTEGER */ + case 230: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 228: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 231: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 229: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 232: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: @@ -3130,4 +3146,4 @@ void Parse( } #endif return; -} +} \ No newline at end of file diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index b78c5314f243874e348748a6434d224592489528..bada3194bd90e30de465dceaef07742cf6e8ac07 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -42,7 +42,7 @@ void simpleTest() { EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); - EXPECT_EQ(pTSBuf->numOfVnodes, 1); + EXPECT_EQ(pTSBuf->numOfGroups, 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -69,7 +69,7 @@ void largeTSTest() { // the data has been flush to disk, no data in cache EXPECT_EQ(pTSBuf->tsData.len, 0); EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); - EXPECT_EQ(pTSBuf->numOfVnodes, 1); + EXPECT_EQ(pTSBuf->numOfGroups, 1); EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); tsBufFlush(pTSBuf); @@ -105,7 +105,7 @@ void multiTagsTest() { EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); - EXPECT_EQ(pTSBuf->numOfVnodes, 1); + EXPECT_EQ(pTSBuf->numOfGroups, 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -139,7 +139,7 @@ void multiVnodeTagsTest() { start += step * num; } - EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + EXPECT_EQ(pTSBuf->numOfGroups, j + 1); } EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); @@ -184,7 +184,7 @@ void loadDataTest() { start += step * num; } - EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + EXPECT_EQ(pTSBuf->numOfGroups, j + 1); } EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); @@ -203,7 +203,7 @@ void loadDataTest() { // create from exists file STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false); EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder); - EXPECT_EQ(pNewBuf->numOfVnodes, numOfVnode); + EXPECT_EQ(pNewBuf->numOfGroups, numOfVnode); EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize); EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset); @@ -269,7 +269,7 @@ void TSTraverse() { start += step * num; } - EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + EXPECT_EQ(pTSBuf->numOfGroups, j + 1); } tsBufResetPos(pTSBuf); @@ -304,7 +304,7 @@ void TSTraverse() { int32_t totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -352,7 +352,7 @@ void TSTraverse() { totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -416,8 +416,8 @@ void mergeDiffVnodeBufferTest() { int64_t* list = createTsList(num, start, step); t.i64Key = i; - tsBufAppend(pTSBuf1, 0, &t, (const char*)list, num * sizeof(int64_t)); - tsBufAppend(pTSBuf2, 0, &t, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf1, 1, &t, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf2, 9, &t, (const char*)list, num * sizeof(int64_t)); free(list); @@ -426,8 +426,8 @@ void mergeDiffVnodeBufferTest() { tsBufFlush(pTSBuf2); - tsBufMerge(pTSBuf1, pTSBuf2, 9); - EXPECT_EQ(pTSBuf1->numOfVnodes, 2); + tsBufMerge(pTSBuf1, pTSBuf2); + EXPECT_EQ(pTSBuf1->numOfGroups, 2); EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); tsBufDisplay(pTSBuf1); @@ -459,8 +459,6 @@ void mergeIdenticalVnodeBufferTest() { start += step * num; } - - for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { int64_t* list = createTsList(num, start, step); @@ -473,16 +471,23 @@ void mergeIdenticalVnodeBufferTest() { tsBufFlush(pTSBuf2); - tsBufMerge(pTSBuf1, pTSBuf2, 12); - EXPECT_EQ(pTSBuf1->numOfVnodes, 1); + tsBufMerge(pTSBuf1, pTSBuf2); + EXPECT_EQ(pTSBuf1->numOfGroups, 2); EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); tsBufResetPos(pTSBuf1); + + int32_t count = 0; while (tsBufNextPos(pTSBuf1)) { STSElem elem = tsBufGetElem(pTSBuf1); - EXPECT_EQ(elem.vnode, 12); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); + if (count++ < numOfTags * num) { + EXPECT_EQ(elem.id, 12); + } else { + EXPECT_EQ(elem.id, 77); + } + + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64Key, elem.ts); } tsBufDestroy(pTSBuf1); diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index 46b0d4e3bb9428f98c1022556f2f57c6ec0e9a14..09d8f3bff1faad5596d85e931adea7f83670a48a 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -101,9 +101,9 @@ void rpcCloseConnCache(void *handle) { if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool); - taosTFree(pCache->connHashList); - taosTFree(pCache->count); - taosTFree(pCache->lockedBy); + tfree(pCache->connHashList); + tfree(pCache->count); + tfree(pCache->lockedBy); pthread_mutex_unlock(&pCache->mutex); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 6e9088d9fb82a80bb28857d563ad51f19594e6b7..f963eeb68dd4ffcecf4a945c81b7658924670b71 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -20,6 +20,7 @@ #include "ttimer.h" #include "tutil.h" #include "lz4.h" +#include "tref.h" #include "taoserror.h" #include "tsocket.h" #include "tglobal.h" @@ -72,7 +73,6 @@ typedef struct { SRpcInfo *pRpc; // associated SRpcInfo SRpcEpSet epSet; // ip list provided by app void *ahandle; // handle provided by app - void *signature; // for validation struct SRpcConn *pConn; // pConn allocated char msgType; // message type uint8_t *pCont; // content provided by app @@ -82,6 +82,7 @@ typedef struct { int8_t oldInUse; // server EP inUse passed by app int8_t redirect; // flag to indicate redirect int8_t connType; // connection type + int64_t rid; // refId returned by taosAddRef SRpcMsg *pRsp; // for synchronous API tsem_t *pSem; // for synchronous API SRpcEpSet *pSet; // for synchronous API @@ -132,6 +133,10 @@ int tsRpcMaxRetry; int tsRpcHeadSize; int tsRpcOverhead; +static int tsRpcRefId = -1; +static int32_t tsRpcNum = 0; +static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; + // server:0 client:1 tcp:2 udp:0 #define RPC_CONN_UDPS 0 #define RPC_CONN_UDPC 1 @@ -211,14 +216,30 @@ static void rpcUnlockConn(SRpcConn *pConn); static void rpcAddRef(SRpcInfo *pRpc); static void rpcDecRef(SRpcInfo *pRpc); -void *rpcOpen(const SRpcInit *pInit) { - SRpcInfo *pRpc; +static void rpcFree(void *p) { + tTrace("free mem: %p", p); + free(p); +} +void rpcInit(void) { tsProgressTimer = tsRpcTimer/2; tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer; tsRpcHeadSize = RPC_MSG_OVERHEAD; tsRpcOverhead = sizeof(SRpcReqContext); + tsRpcRefId = taosOpenRef(200, rpcFree); +} + +void rpcCleanup(void) { + taosCloseRef(tsRpcRefId); + tsRpcRefId = -1; +} + +void *rpcOpen(const SRpcInit *pInit) { + SRpcInfo *pRpc; + + pthread_once(&tsRpcInit, rpcInit); + pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) return NULL; @@ -237,6 +258,8 @@ void *rpcOpen(const SRpcInit *pInit) { pRpc->afp = pInit->afp; pRpc->refCount = 1; + atomic_add_fetch_32(&tsRpcNum, 1); + size_t size = sizeof(SRpcConn) * pRpc->sessions; pRpc->connList = (SRpcConn *)calloc(1, size); if (pRpc->connList == NULL) { @@ -323,7 +346,7 @@ void *rpcMallocCont(int contLen) { tError("failed to malloc msg, size:%d", size); return NULL; } else { - tTrace("malloc mem: %p", start); + tTrace("malloc mem:%p size:%d", start, size); } return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); @@ -356,14 +379,13 @@ void *rpcReallocCont(void *ptr, int contLen) { return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } -void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) { +int64_t rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) { SRpcInfo *pRpc = (SRpcInfo *)shandle; SRpcReqContext *pContext; int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); pContext = (SRpcReqContext *) ((char*)pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); pContext->ahandle = pMsg->ahandle; - pContext->signature = pContext; pContext->pRpc = (SRpcInfo *)shandle; pContext->epSet = *pEpSet; pContext->contLen = contLen; @@ -386,9 +408,11 @@ void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) { // set the handle to pContext, so app can cancel the request if (pMsg->handle) *((void **)pMsg->handle) = pContext; + pContext->rid = taosAddRef(tsRpcRefId, pContext); + rpcSendReqToServer(pRpc, pContext); - return; + return pContext->rid; } void rpcSendResponse(const SRpcMsg *pRsp) { @@ -533,17 +557,14 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) { return code; } -void rpcCancelRequest(void *handle) { - SRpcReqContext *pContext = handle; +void rpcCancelRequest(int64_t rid) { - // signature is used to check if pContext is freed. - // pContext may have been released just before app calls the rpcCancelRequest - if (pContext == NULL || pContext->signature != pContext) return; + SRpcReqContext *pContext = taosAcquireRef(tsRpcRefId, rid); + if (pContext == NULL) return; - if (pContext->pConn) { - tDebug("%s, app tries to cancel request", pContext->pConn->info); - rpcCloseConn(pContext->pConn); - } + rpcCloseConn(pContext->pConn); + + taosReleaseRef(tsRpcRefId, rid); } static void rpcFreeMsg(void *msg) { @@ -612,7 +633,7 @@ static void rpcReleaseConn(SRpcConn *pConn) { // if there is an outgoing message, free it if (pConn->outType && pConn->pReqMsg) { if (pConn->pContext) pConn->pContext->pConn = NULL; - rpcFreeMsg(pConn->pReqMsg); + taosRemoveRef(tsRpcRefId, pConn->pContext->rid); } } @@ -636,6 +657,7 @@ static void rpcReleaseConn(SRpcConn *pConn) { static void rpcCloseConn(void *thandle) { SRpcConn *pConn = (SRpcConn *)thandle; + if (pConn == NULL) return; rpcLockConn(pConn); @@ -1007,6 +1029,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) { if (pConn->outType) { SRpcReqContext *pContext = pConn->pContext; pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + pContext->pConn = NULL; pConn->pReqMsg = NULL; taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl); } @@ -1057,6 +1080,13 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code); } } else { // msg is passed to app only parsing is ok + + if (pHead->msgType == TSDB_MSG_TYPE_NETWORK_TEST) { + rpcSendQuickRsp(pConn, TSDB_CODE_SUCCESS); + rpcFreeMsg(pRecv->msg); + return pConn; + } + rpcProcessIncomingMsg(pConn, pHead, pContext); } } @@ -1068,7 +1098,6 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { SRpcInfo *pRpc = pContext->pRpc; - pContext->signature = NULL; pContext->pConn = NULL; if (pContext->pRsp) { // for synchronous API @@ -1085,7 +1114,7 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { } // free the request message - rpcFreeCont(pContext->pCont); + taosRemoveRef(tsRpcRefId, pContext->rid); } static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) { @@ -1110,6 +1139,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte // it's a response rpcMsg.handle = pContext; rpcMsg.ahandle = pContext->ahandle; + pContext->pConn = NULL; // for UDP, port may be changed by server, the port in epSet shall be used for cache if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) { @@ -1345,6 +1375,7 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) { tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort); if (pConn->pContext) { pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + pConn->pContext->pConn = NULL; pConn->pReqMsg = NULL; taosTmrStart(rpcProcessConnError, 0, pConn->pContext, pRpc->tmrCtrl); rpcReleaseConn(pConn); @@ -1453,7 +1484,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { pNewHead->msgLen = rpcMsgLenFromCont(origLen); rpcFreeMsg(pHead); // free the compressed message buffer pHead = pNewHead; - tTrace("decomp malloc mem: %p", temp); + tTrace("decomp malloc mem:%p", temp); } else { tError("failed to allocate memory to decompress msg, contLen:%d", contLen); } @@ -1589,10 +1620,12 @@ static void rpcDecRef(SRpcInfo *pRpc) taosTmrCleanUp(pRpc->tmrCtrl); taosIdPoolCleanUp(pRpc->idPool); - taosTFree(pRpc->connList); + tfree(pRpc->connList); pthread_mutex_destroy(&pRpc->mutex); tDebug("%s rpc resources are released", pRpc->label); - taosTFree(pRpc); + tfree(pRpc); + + atomic_sub_fetch_32(&tsRpcNum, 1); } } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index bc8d360d39509ce9a2fdbe6a9dd883c5c5c99190..7b32d3416d784b4cbc6e319a73664a28dd578d4d 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -236,8 +236,8 @@ void taosCleanUpTcpServer(void *handle) { tDebug("%s TCP server is cleaned up", pServerObj->label); - taosTFree(pServerObj->pThreadObj); - taosTFree(pServerObj); + tfree(pServerObj->pThreadObj); + tfree(pServerObj); } static void *taosAcceptTcpConnection(void *arg) { @@ -437,12 +437,13 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { } msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); - buffer = malloc(msgLen + tsRpcOverhead); + int32_t size = msgLen + tsRpcOverhead; + buffer = malloc(size); if (NULL == buffer) { tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { - tTrace("TCP malloc mem: %p", buffer); + tTrace("TCP malloc mem:%p size:%d", buffer, size); } msg = buffer + tsRpcOverhead; @@ -534,7 +535,7 @@ static void *taosProcessTcpData(void *param) { pthread_mutex_destroy(&(pThreadObj->mutex)); tDebug("%s TCP thread exits ...", pThreadObj->label); - taosTFree(pThreadObj); + tfree(pThreadObj); return NULL; } @@ -555,7 +556,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pFdObj; if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { - taosTFree(pFdObj); + tfree(pFdObj); terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -608,5 +609,5 @@ static void taosFreeFdObj(SFdObj *pFdObj) { tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); - taosTFree(pFdObj); + tfree(pFdObj); } diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 6f653046615f162c516b5eebf08995d30c6214d7..22301fcecc83fb1f4c9a29132b0311e05b2382e6 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -147,7 +147,7 @@ void taosStopUdpConnection(void *handle) { if (taosCheckPthreadValid(pConn->thread)) { pthread_join(pConn->thread, NULL); } - taosTFree(pConn->buffer); + tfree(pConn->buffer); // tTrace("%s UDP thread is closed, index:%d", pConn->label, i); } @@ -166,7 +166,7 @@ void taosCleanUpUdpConnection(void *handle) { } tDebug("%s UDP is cleaned up", pSet->label); - taosTFree(pSet); + tfree(pSet); } void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { @@ -209,12 +209,13 @@ static void *taosRecvUdpData(void *param) { continue; } - char *tmsg = malloc(dataLen + tsRpcOverhead); + int32_t size = dataLen + tsRpcOverhead; + char *tmsg = malloc(size); if (NULL == tmsg) { tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen); continue; } else { - tTrace("UDP malloc mem: %p", tmsg); + tTrace("UDP malloc mem:%p size:%d", tmsg, size); } tmsg += tsRpcOverhead; // overhead for SRpcReqContext diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index f6818106462cb9c9d694d59c7cb9012cd1a54c8b..93c6efc20a8d79a7d6436680f01cee87dadda853 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -35,6 +35,8 @@ extern "C" { #define TAOS_SMSG_SYNC_MUST 6 #define TAOS_SMSG_STATUS 7 +#define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16) + #define nodeRole pNode->peerInfo[pNode->selfIndex]->role #define nodeVersion pNode->peerInfo[pNode->selfIndex]->version #define nodeSStatus pNode->peerInfo[pNode->selfIndex]->sstatus @@ -89,11 +91,11 @@ typedef struct { #pragma pack(pop) typedef struct { - char *buffer; - int bufferSize; - char *offset; - int forwards; - int code; + char * buffer; + int32_t bufferSize; + char * offset; + int32_t forwards; + int32_t code; } SRecvBuffer; typedef struct { @@ -107,10 +109,10 @@ typedef struct { } SFwdInfo; typedef struct { - int first; - int last; - int fwds; // number of forwards - SFwdInfo fwdInfo[]; + int32_t first; + int32_t last; + int32_t fwds; // number of forwards + SFwdInfo fwdInfo[]; } SSyncFwds; typedef struct SsyncPeer { @@ -123,15 +125,15 @@ typedef struct SsyncPeer { int8_t sstatus; // sync status uint64_t version; uint64_t sversion; // track the peer version in retrieve process - int syncFd; - int peerFd; // forward FD - int numOfRetrieves; // number of retrieves tried - int fileChanged; // a flag to indicate file is changed during retrieving process + int32_t syncFd; + int32_t peerFd; // forward FD + int32_t numOfRetrieves; // number of retrieves tried + int32_t fileChanged; // a flag to indicate file is changed during retrieving process void * timer; void * pConn; - int notifyFd; - int watchNum; - int * watchFd; + int32_t notifyFd; + int32_t watchNum; + int32_t *watchFd; int8_t refCount; // reference count struct SSyncNode *pSyncNode; } SSyncPeer; @@ -141,6 +143,7 @@ typedef struct SSyncNode { int8_t replica; int8_t quorum; uint32_t vgId; + int64_t rid; void *ahandle; int8_t selfIndex; SSyncPeer *peerInfo[TAOS_SYNC_MAX_REPLICA+1]; // extra one for arbitrator @@ -160,16 +163,16 @@ typedef struct SSyncNode { } SSyncNode; // sync module global -extern int tsSyncNum; -extern char tsNodeFqdn[TSDB_FQDN_LEN]; +extern int32_t tsSyncNum; +extern char tsNodeFqdn[TSDB_FQDN_LEN]; void *syncRetrieveData(void *param); void *syncRestoreData(void *param); -int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); -void syncRestartConnection(SSyncPeer *pPeer); -void syncBroadcastStatus(SSyncNode *pNode); -void syncAddPeerRef(SSyncPeer *pPeer); -int syncDecPeerRef(SSyncPeer *pPeer); +int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); +void syncRestartConnection(SSyncPeer *pPeer); +void syncBroadcastStatus(SSyncNode *pNode); +void syncAddPeerRef(SSyncPeer *pPeer); +int32_t syncDecPeerRef(SSyncPeer *pPeer); #ifdef __cplusplus } diff --git a/src/sync/inc/taosTcpPool.h b/src/sync/inc/taosTcpPool.h index 261d190ad3b7cbe4fbf46a061a1432c217262d24..41043b0cd4c886616d5cecd2739eae684052c395 100644 --- a/src/sync/inc/taosTcpPool.h +++ b/src/sync/inc/taosTcpPool.h @@ -20,23 +20,23 @@ extern "C" { #endif -typedef void* ttpool_h; -typedef void* tthread_h; +typedef void *ttpool_h; +typedef void *tthread_h; typedef struct { - int numOfThreads; + int32_t numOfThreads; uint32_t serverIp; int16_t port; - int bufferSize; - void (*processBrokenLink)(void *ahandle); - int (*processIncomingMsg)(void *ahandle, void *buffer); - void (*processIncomingConn)(int fd, uint32_t ip); + int32_t bufferSize; + void (*processBrokenLink)(void *ahandle); + int32_t (*processIncomingMsg)(void *ahandle, void *buffer); + void (*processIncomingConn)(int32_t fd, uint32_t ip); } SPoolInfo; -ttpool_h taosOpenTcpThreadPool(SPoolInfo *pInfo); -void taosCloseTcpThreadPool(ttpool_h); -void *taosAllocateTcpConn(void *, void *ahandle, int connFd); -void taosFreeTcpConn(void *); +ttpool_h taosOpenTcpThreadPool(SPoolInfo *pInfo); +void taosCloseTcpThreadPool(ttpool_h); +void * taosAllocateTcpConn(void *, void *ahandle, int32_t connFd); +void taosFreeTcpConn(void *); #ifdef __cplusplus } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index ef635e6efc1ca5f071c64dbe00920c3987837494..5c92801cc389df6745377d81b4110ccf8fbc8441 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -20,6 +20,7 @@ #include "tlog.h" #include "tutil.h" #include "ttimer.h" +#include "tref.h" #include "tsocket.h" #include "tglobal.h" #include "taoserror.h" @@ -30,37 +31,38 @@ #include "syncInt.h" // global configurable -int tsMaxSyncNum = 2; -int tsSyncTcpThreads = 2; -int tsMaxWatchFiles = 500; -int tsMaxFwdInfo = 200; -int tsSyncTimer = 1; +int32_t tsMaxSyncNum = 2; +int32_t tsSyncTcpThreads = 2; +int32_t tsMaxWatchFiles = 500; +int32_t tsMaxFwdInfo = 200; +int32_t tsSyncTimer = 1; // module global, not configurable -int tsSyncNum; // number of sync in process in whole system -char tsNodeFqdn[TSDB_FQDN_LEN]; +int32_t tsSyncNum; // number of sync in process in whole system +char tsNodeFqdn[TSDB_FQDN_LEN]; static ttpool_h tsTcpPool; -static void * syncTmrCtrl = NULL; -static void * vgIdHash; +static void * tsSyncTmrCtrl = NULL; +static void * tsVgIdHash; +static int32_t tsSyncRefId = -1; // local functions -static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer); -static void syncRecoverFromMaster(SSyncPeer *pPeer); -static void syncCheckPeerConnection(void *param, void *tmrId); -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack); -static void syncProcessBrokenLink(void *param); -static int syncProcessPeerMsg(void *param, void *buffer); -static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp); -static void syncRemovePeer(SSyncPeer *pPeer); -static void syncAddArbitrator(SSyncNode *pNode); -static void syncAddNodeRef(SSyncNode *pNode); -static void syncDecNodeRef(SSyncNode *pNode); -static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode); -static void syncMonitorFwdInfos(void *param, void *tmrId); -static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code); -static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); -static void syncRestartPeer(SSyncPeer *pPeer); +static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer); +static void syncRecoverFromMaster(SSyncPeer *pPeer); +static void syncCheckPeerConnection(void *param, void *tmrId); +static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack); +static void syncProcessBrokenLink(void *param); +static int32_t syncProcessPeerMsg(void *param, void *buffer); +static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +static void syncRemovePeer(SSyncPeer *pPeer); +static void syncAddArbitrator(SSyncNode *pNode); +static void syncFreeNode(void *); +static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode); +static void syncMonitorFwdInfos(void *param, void *tmrId); +static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code); +static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); +static void syncRestartPeer(SSyncPeer *pPeer); +static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtyp); static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo); char* syncRole[] = { @@ -77,7 +79,7 @@ int32_t syncInit() { info.numOfThreads = tsSyncTcpThreads; info.serverIp = 0; info.port = tsSyncPort; - info.bufferSize = 640000; + info.bufferSize = SYNC_MAX_SIZE; info.processBrokenLink = syncProcessBrokenLink; info.processIncomingMsg = syncProcessPeerMsg; info.processIncomingConn = syncProcessIncommingConnection; @@ -88,21 +90,27 @@ int32_t syncInit() { return -1; } - syncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC"); - if (syncTmrCtrl == NULL) { + tsSyncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC"); + if (tsSyncTmrCtrl == NULL) { sError("failed to init tmrCtrl"); taosCloseTcpThreadPool(tsTcpPool); tsTcpPool = NULL; return -1; } - vgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); - if (vgIdHash == NULL) { - sError("failed to init vgIdHash"); - taosTmrCleanUp(syncTmrCtrl); + tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); + if (tsVgIdHash == NULL) { + sError("failed to init tsVgIdHash"); + taosTmrCleanUp(tsSyncTmrCtrl); taosCloseTcpThreadPool(tsTcpPool); tsTcpPool = NULL; - syncTmrCtrl = NULL; + tsSyncTmrCtrl = NULL; + return -1; + } + + tsSyncRefId = taosOpenRef(200, syncFreeNode); + if (tsSyncRefId < 0) { + syncCleanUp(); return -1; } @@ -118,27 +126,30 @@ void syncCleanUp() { tsTcpPool = NULL; } - if (syncTmrCtrl) { - taosTmrCleanUp(syncTmrCtrl); - syncTmrCtrl = NULL; + if (tsSyncTmrCtrl) { + taosTmrCleanUp(tsSyncTmrCtrl); + tsSyncTmrCtrl = NULL; } - if (vgIdHash) { - taosHashCleanup(vgIdHash); - vgIdHash = NULL; + if (tsVgIdHash) { + taosHashCleanup(tsVgIdHash); + tsVgIdHash = NULL; } + taosCloseRef(tsSyncRefId); + tsSyncRefId = -1; + sInfo("sync module is cleaned up"); } -void *syncStart(const SSyncInfo *pInfo) { +int64_t syncStart(const SSyncInfo *pInfo) { const SSyncCfg *pCfg = &pInfo->syncCfg; SSyncNode *pNode = (SSyncNode *)calloc(sizeof(SSyncNode), 1); if (pNode == NULL) { sError("no memory to allocate syncNode"); terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; + return -1; } tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path)); @@ -159,7 +170,13 @@ void *syncStart(const SSyncInfo *pInfo) { pNode->quorum = pCfg->quorum; if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica; - for (int i = 0; i < pCfg->replica; ++i) { + pNode->rid = taosAddRef(tsSyncRefId, pNode); + if (pNode->rid < 0) { + syncFreeNode(pNode); + return -1; + } + + for (int32_t i = 0; i < pCfg->replica; ++i) { const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i; pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo); if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) { @@ -167,13 +184,11 @@ void *syncStart(const SSyncInfo *pInfo) { } } - syncAddNodeRef(pNode); - if (pNode->selfIndex < 0) { sInfo("vgId:%d, this node is not configured", pNode->vgId); terrno = TSDB_CODE_SYN_INVALID_CONFIG; - syncStop(pNode); - return NULL; + syncStop(pNode->rid); + return -1; } nodeVersion = pInfo->version; // set the initial version @@ -185,40 +200,41 @@ void *syncStart(const SSyncInfo *pInfo) { if (pNode->pSyncFwds == NULL) { sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId); terrno = TAOS_SYSTEM_ERROR(errno); - syncStop(pNode); - return NULL; + syncStop(pNode->rid); + return -1; } - pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl); + pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl); if (pNode->pFwdTimer == NULL) { sError("vgId:%d, failed to allocate timer", pNode->vgId); - syncStop(pNode); - return NULL; + syncStop(pNode->rid); + return -1; } syncAddArbitrator(pNode); - taosHashPut(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *)); + taosHashPut(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *)); if (pNode->notifyRole) { (*pNode->notifyRole)(pNode->ahandle, nodeRole); } - return pNode; + return pNode->rid; } -void syncStop(void *param) { - SSyncNode *pNode = param; +void syncStop(int64_t rid) { SSyncPeer *pPeer; + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); if (pNode == NULL) return; + sInfo("vgId:%d, cleanup sync", pNode->vgId); pthread_mutex_lock(&(pNode->mutex)); - if (vgIdHash) taosHashRemove(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t)); + if (tsVgIdHash) taosHashRemove(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t)); if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer); - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { pPeer = pNode->peerInfo[i]; if (pPeer) syncRemovePeer(pPeer); } @@ -228,14 +244,16 @@ void syncStop(void *param) { pthread_mutex_unlock(&(pNode->mutex)); - syncDecNodeRef(pNode); + taosReleaseRef(tsSyncRefId, rid); + taosRemoveRef(tsSyncRefId, rid); } -int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) { - SSyncNode *pNode = param; - int i, j; +int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { + int32_t i, j; + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG; + sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica, pNode->replica); @@ -298,105 +316,58 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) { syncRole[nodeRole]); syncBroadcastStatus(pNode); + taosReleaseRef(tsSyncRefId, rid); + return 0; } -int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) { - SSyncNode *pNode = param; - SSyncPeer *pPeer; - SSyncHead *pSyncHead; - SWalHead * pWalHead = data; - int fwdLen; - int code = 0; - - if (pNode == NULL) return 0; - - if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) { - sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId, - pWalHead->version, nodeVersion); - for (int i = 0; i < pNode->replica; ++i) { - pPeer = pNode->peerInfo[i]; - syncRestartConnection(pPeer); - } - return TSDB_CODE_SYN_INVALID_VERSION; - } +int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype) { + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + if (pNode == NULL) return 0; - // always update version - nodeVersion = pWalHead->version; - sDebug("vgId:%d, replica:%d nodeRole:%s qtype:%d ver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole], - qtype, pWalHead->version); + int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype); - if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0; - - // only pkt from RPC or CQ can be forwarded - if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0; - - // a hacker way to improve the performance - pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead)); - pSyncHead->type = TAOS_SMSG_FORWARD; - pSyncHead->pversion = 0; - pSyncHead->len = sizeof(SWalHead) + pWalHead->len; - fwdLen = pSyncHead->len + sizeof(SSyncHead); // include the WAL and SYNC head - - pthread_mutex_lock(&(pNode->mutex)); - - for (int i = 0; i < pNode->replica; ++i) { - pPeer = pNode->peerInfo[i]; - if (pPeer == NULL || pPeer->peerFd < 0) continue; - if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue; - - if (pNode->quorum > 1 && code == 0) { - syncSaveFwdInfo(pNode, pWalHead->version, mhandle); - code = 1; - } - - int retLen = write(pPeer->peerFd, pSyncHead, fwdLen); - if (retLen == fwdLen) { - sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len); - } else { - sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen); - syncRestartConnection(pPeer); - } - } - - pthread_mutex_unlock(&(pNode->mutex)); + taosReleaseRef(tsSyncRefId, rid); return code; } -void syncConfirmForward(void *param, uint64_t version, int32_t code) { - SSyncNode *pNode = param; +void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); if (pNode == NULL) return; - if (pNode->quorum <= 1) return; SSyncPeer *pPeer = pNode->pMaster; - if (pPeer == NULL) return; - - char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0}; + if (pPeer && pNode->quorum > 1) { + char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0}; - SSyncHead *pHead = (SSyncHead *)msg; - pHead->type = TAOS_SMSG_FORWARD_RSP; - pHead->len = sizeof(SFwdRsp); + SSyncHead *pHead = (SSyncHead *)msg; + pHead->type = TAOS_SMSG_FORWARD_RSP; + pHead->len = sizeof(SFwdRsp); - SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead)); - pFwdRsp->version = version; - pFwdRsp->code = code; + SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead)); + pFwdRsp->version = version; + pFwdRsp->code = code; - int msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp); - int retLen = write(pPeer->peerFd, msg, msgLen); + int32_t msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp); + int32_t retLen = write(pPeer->peerFd, msg, msgLen); - if (retLen == msgLen) { - sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version); - } else { - sDebug("%s, failed to send forward ack, restart", pPeer->id); - syncRestartConnection(pPeer); + if (retLen == msgLen) { + sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version); + } else { + sDebug("%s, failed to send forward ack, restart", pPeer->id); + syncRestartConnection(pPeer); + } } + + taosReleaseRef(tsSyncRefId, rid); } -void syncRecover(void *param) { - SSyncNode *pNode = param; +void syncRecover(int64_t rid) { SSyncPeer *pPeer; + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + if (pNode == NULL) return; + // to do: add a few lines to check if recover is OK // if take this node to unsync state, the whole system may not work @@ -406,7 +377,7 @@ void syncRecover(void *param) { pthread_mutex_lock(&(pNode->mutex)); - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { pPeer = (SSyncPeer *)pNode->peerInfo[i]; if (pPeer->peerFd >= 0) { syncRestartConnection(pPeer); @@ -414,17 +385,22 @@ void syncRecover(void *param) { } pthread_mutex_unlock(&(pNode->mutex)); + + taosReleaseRef(tsSyncRefId, rid); } -int syncGetNodesRole(void *param, SNodesRole *pNodesRole) { - SSyncNode *pNode = param; +int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + if (pNode == NULL) return -1; pNodesRole->selfIndex = pNode->selfIndex; - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId; pNodesRole->role[i] = pNode->peerInfo[i]->role; } + taosReleaseRef(tsSyncRefId, rid); + return 0; } @@ -440,7 +416,7 @@ static void syncAddArbitrator(SSyncNode *pNode) { SNodeInfo nodeInfo; nodeInfo.nodeId = 0; - int ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort); + int32_t ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort); if (-1 == ret) { nodeInfo.nodePort = tsArbitratorPort; } @@ -457,26 +433,24 @@ static void syncAddArbitrator(SSyncNode *pNode) { pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo); } -static void syncAddNodeRef(SSyncNode *pNode) { atomic_add_fetch_8(&pNode->refCount, 1); } +static void syncFreeNode(void *param) { + SSyncNode *pNode = param; -static void syncDecNodeRef(SSyncNode *pNode) { - if (atomic_sub_fetch_8(&pNode->refCount, 1) == 0) { - pthread_mutex_destroy(&pNode->mutex); - taosTFree(pNode->pRecv); - taosTFree(pNode->pSyncFwds); - taosTFree(pNode); - } + pthread_mutex_destroy(&pNode->mutex); + tfree(pNode->pRecv); + tfree(pNode->pSyncFwds); + tfree(pNode); } void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); } -int syncDecPeerRef(SSyncPeer *pPeer) { +int32_t syncDecPeerRef(SSyncPeer *pPeer) { if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) { - syncDecNodeRef(pPeer->pSyncNode); + taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid); sDebug("%s, resource is freed", pPeer->id); - taosTFree(pPeer->watchFd); - taosTFree(pPeer); + tfree(pPeer->watchFd); + tfree(pPeer); return 0; } @@ -512,7 +486,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { pPeer->ip = ip; pPeer->port = pInfo->nodePort; pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0; - snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d peer:%s:%u", pNode->vgId, pPeer->fqdn, pPeer->port); + snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, peer:%s:%u", pNode->vgId, pPeer->fqdn, pPeer->port); pPeer->peerFd = -1; pPeer->syncFd = -1; @@ -521,22 +495,22 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { pPeer->refCount = 1; sInfo("%s, it is configured", pPeer->id); - int ret = strcmp(pPeer->fqdn, tsNodeFqdn); + int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { int32_t checkMs = 100 + (pNode->vgId * 10) % 100; if (pNode->vgId > 1) checkMs = tsStatusInterval * 2000 + checkMs; sDebug("%s, start to check peer connection after %d ms", pPeer->id, checkMs); - taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, syncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer); } - syncAddNodeRef(pNode); + taosAcquireRef(tsSyncRefId, pNode->rid); return pPeer; } void syncBroadcastStatus(SSyncNode *pNode) { SSyncPeer *pPeer; - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { if (i == pNode->selfIndex) continue; pPeer = pNode->peerInfo[i]; syncSendPeersStatusMsgToPeer(pPeer, 1); @@ -544,7 +518,7 @@ void syncBroadcastStatus(SSyncNode *pNode) { } static void syncResetFlowCtrl(SSyncNode *pNode) { - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { pNode->peerInfo[i]->numOfRetrieves = 0; } @@ -555,13 +529,13 @@ static void syncResetFlowCtrl(SSyncNode *pNode) { static void syncChooseMaster(SSyncNode *pNode) { SSyncPeer *pPeer; - int onlineNum = 0; - int index = -1; - int replica = pNode->replica; + int32_t onlineNum = 0; + int32_t index = -1; + int32_t replica = pNode->replica; sDebug("vgId:%d, choose master", pNode->vgId); - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) { onlineNum++; } @@ -570,7 +544,7 @@ static void syncChooseMaster(SSyncNode *pNode) { if (onlineNum == pNode->replica) { // if all peers are online, peer with highest version shall be master index = 0; - for (int i = 1; i < pNode->replica; ++i) { + for (int32_t i = 1; i < pNode->replica; ++i) { if (pNode->peerInfo[i]->version > pNode->peerInfo[index]->version) { index = i; } @@ -586,7 +560,7 @@ static void syncChooseMaster(SSyncNode *pNode) { if (index < 0 && onlineNum > replica / 2.0) { // over half of nodes are online - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { // slave with highest version shall be master pPeer = pNode->peerInfo[i]; if (pPeer->role == TAOS_SYNC_ROLE_SLAVE || pPeer->role == TAOS_SYNC_ROLE_MASTER) { @@ -613,11 +587,11 @@ static void syncChooseMaster(SSyncNode *pNode) { } static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { - int onlineNum = 0; - int index = -1; - int replica = pNode->replica; + int32_t onlineNum = 0; + int32_t index = -1; + int32_t replica = pNode->replica; - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) { onlineNum++; } @@ -638,7 +612,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } } else { - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { SSyncPeer *pTemp = pNode->peerInfo[i]; if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue; if (index < 0) { @@ -657,9 +631,9 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { return pMaster; } -static int syncValidateMaster(SSyncPeer *pPeer) { +static int32_t syncValidateMaster(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - int code = 0; + int32_t code = 0; if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) { sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id); @@ -667,7 +641,7 @@ static int syncValidateMaster(SSyncPeer *pPeer) { (*pNode->notifyRole)(pNode->ahandle, nodeRole); code = -1; - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { if (i == pNode->selfIndex) continue; syncRestartPeer(pNode->peerInfo[i]); } @@ -709,7 +683,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne } } else { // master not there, if all peer's state and version are consistent, choose the master - int consistent = 0; + int32_t consistent = 0; if (peersStatus) { for (i = 0; i < pNode->replica; ++i) { SSyncPeer *pTemp = pNode->peerInfo[i]; @@ -747,9 +721,9 @@ static void syncRestartPeer(SSyncPeer *pPeer) { pPeer->sstatus = TAOS_SYNC_STATUS_INIT; - int ret = strcmp(pPeer->fqdn, tsNodeFqdn); + int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) { - taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); } } @@ -783,7 +757,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { pthread_t thread; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); - int ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer); + int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer); pthread_attr_destroy(&thattr); if (ret != 0) { @@ -828,7 +802,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) { // Ensure the sync of mnode not interrupted if (pNode->vgId != 1 && tsSyncNum >= tsMaxSyncNum) { sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum); - taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, syncTmrCtrl, &pPeer->timer); + taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, tsSyncTmrCtrl, &pPeer->timer); return; } @@ -841,7 +815,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) { firstPkt.syncHead.len = sizeof(firstPkt) - sizeof(SSyncHead); tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); firstPkt.port = tsSyncPort; - taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer); + taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); if (write(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) { sError("%s, failed to send sync-req to peer", pPeer->id); @@ -862,7 +836,7 @@ static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) { if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) { // find the forwardInfo from first - for (int i = 0; i < pSyncFwds->fwds; ++i) { + for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { pFwdInfo = pSyncFwds->fwdInfo + (i + pSyncFwds->first) % tsMaxFwdInfo; if (pFwdRsp->version == pFwdInfo->version) break; } @@ -876,16 +850,16 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SWalHead * pHead = (SWalHead *)cont; - sDebug("%s, forward is received, ver:%" PRIu64, pPeer->id, pHead->version); + sDebug("%s, forward is received, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len); if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { // nodeVersion = pHead->version; - (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD); + (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL); } else { if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { syncSaveIntoBuffer(pPeer, pHead); } else { - sError("%s, forward discarded, ver:%" PRIu64, pPeer->id, pHead->version); + sError("%s, forward discarded, hver:%" PRIu64, pPeer->id, pHead->version); } } } @@ -905,10 +879,10 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { } } -static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { +static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { if (pPeer->peerFd < 0) return -1; - int hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead)); + int32_t hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead)); if (hlen != sizeof(SSyncHead)) { sDebug("%s, failed to read msg, hlen:%d", pPeer->id, hlen); return -1; @@ -916,11 +890,12 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { // head.len = htonl(head.len); if (pHead->len < 0) { - sError("%s, invalid pkt length, len:%d", pPeer->id, pHead->len); + sError("%s, invalid pkt length, hlen:%d", pPeer->id, pHead->len); return -1; } - int bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len); + assert(pHead->len <= TSDB_MAX_WAL_SIZE); + int32_t bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len); if (bytes != pHead->len) { sError("%s, failed to read, bytes:%d len:%d", pPeer->id, bytes, pHead->len); return -1; @@ -929,7 +904,7 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { return 0; } -static int syncProcessPeerMsg(void *param, void *buffer) { +static int32_t syncProcessPeerMsg(void *param, void *buffer) { SSyncPeer *pPeer = param; SSyncHead head; char * cont = buffer; @@ -937,7 +912,7 @@ static int syncProcessPeerMsg(void *param, void *buffer) { SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&(pNode->mutex)); - int code = syncReadPeerMsg(pPeer, &head, cont); + int32_t code = syncReadPeerMsg(pPeer, &head, cont); if (code == 0) { if (head.type == TAOS_SMSG_FORWARD) { @@ -974,12 +949,12 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) { pPeersStatus->role = nodeRole; pPeersStatus->ack = ack; - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { pPeersStatus->peersStatus[i].role = pNode->peerInfo[i]->role; pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version; } - int retLen = write(pPeer->peerFd, msg, statusMsgLen); + int32_t retLen = write(pPeer->peerFd, msg, statusMsgLen); if (retLen == statusMsgLen) { sDebug("%s, status msg is sent, self:%s ver:%" PRIu64 ", ack:%d", pPeer->id, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack); @@ -1001,10 +976,10 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { return; } - int connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (connFd < 0) { sDebug("%s, failed to open tcp socket(%s)", pPeer->id, strerror(errno)); - taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); return; } @@ -1025,7 +1000,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { } else { sDebug("try later"); close(connFd); - taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); } } @@ -1050,7 +1025,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); syncAddPeerRef(pPeer); - int ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer); + int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer); pthread_attr_destroy(&thattr); if (ret < 0) { @@ -1062,9 +1037,9 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { } } -static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) { - char ipstr[24]; - int i; +static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { + char ipstr[24]; + int32_t i; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); @@ -1077,7 +1052,7 @@ static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) { } int32_t vgId = firstPkt.syncHead.vgId; - SSyncNode **ppNode = (SSyncNode **)taosHashGet(vgIdHash, (const char *)&vgId, sizeof(int32_t)); + SSyncNode **ppNode = (SSyncNode **)taosHashGet(tsVgIdHash, (const char *)&vgId, sizeof(int32_t)); if (ppNode == NULL || *ppNode == NULL) { sError("vgId:%d, vgId could not be found", vgId); taosCloseSocket(connFd); @@ -1122,7 +1097,7 @@ static void syncProcessBrokenLink(void *param) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; - syncAddNodeRef(pNode); + if (taosAcquireRef(tsSyncRefId, pNode->rid) < 0) return; pthread_mutex_lock(&(pNode->mutex)); sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno)); @@ -1133,7 +1108,7 @@ static void syncProcessBrokenLink(void *param) { } pthread_mutex_unlock(&(pNode->mutex)); - syncDecNodeRef(pNode); + taosReleaseRef(tsSyncRefId, pNode->rid); } static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { @@ -1163,8 +1138,8 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; - int fwds = pSyncFwds->fwds; - for (int i = 0; i < fwds; ++i) { + int32_t fwds = pSyncFwds->fwds; + for (int32_t i = 0; i < fwds; ++i) { SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->first; if (pFwdInfo->confirmed == 0) break; @@ -1178,7 +1153,7 @@ static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { } static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) { - int confirm = 0; + int32_t confirm = 0; if (pFwdInfo->code == 0) pFwdInfo->code = code; if (code == 0) { @@ -1201,23 +1176,90 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code } static void syncMonitorFwdInfos(void *param, void *tmrId) { - SSyncNode *pNode = param; + int64_t rid = (int64_t) param; + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + if (pNode == NULL) return; + SSyncFwds *pSyncFwds = pNode->pSyncFwds; - if (pSyncFwds == NULL) return; - uint64_t time = taosGetTimestampMs(); + if (pSyncFwds) {; + uint64_t time = taosGetTimestampMs(); - if (pSyncFwds->fwds > 0) { - pthread_mutex_lock(&(pNode->mutex)); - for (int i = 0; i < pSyncFwds->fwds; ++i) { - SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo; - if (time - pFwdInfo->time < 2000) break; - syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL); + if (pSyncFwds->fwds > 0) { + pthread_mutex_lock(&(pNode->mutex)); + for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { + SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo; + if (time - pFwdInfo->time < 2000) break; + syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL); + } + + syncRemoveConfirmedFwdInfo(pNode); + pthread_mutex_unlock(&(pNode->mutex)); } - syncRemoveConfirmedFwdInfo(pNode); - pthread_mutex_unlock(&(pNode->mutex)); + pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl); } - pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl); + taosReleaseRef(tsSyncRefId, rid); } + +static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype) { + SSyncPeer *pPeer; + SSyncHead *pSyncHead; + SWalHead * pWalHead = data; + int32_t fwdLen; + int32_t code = 0; + + if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) { + sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId, + pWalHead->version, nodeVersion); + for (int32_t i = 0; i < pNode->replica; ++i) { + pPeer = pNode->peerInfo[i]; + syncRestartConnection(pPeer); + } + return TSDB_CODE_SYN_INVALID_VERSION; + } + + // always update version + nodeVersion = pWalHead->version; + sDebug("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, + syncRole[nodeRole], qtypeStr[qtype], pWalHead->version); + + if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0; + + // only pkt from RPC or CQ can be forwarded + if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0; + + // a hacker way to improve the performance + pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead)); + pSyncHead->type = TAOS_SMSG_FORWARD; + pSyncHead->pversion = 0; + pSyncHead->len = sizeof(SWalHead) + pWalHead->len; + fwdLen = pSyncHead->len + sizeof(SSyncHead); // include the WAL and SYNC head + + pthread_mutex_lock(&(pNode->mutex)); + + for (int32_t i = 0; i < pNode->replica; ++i) { + pPeer = pNode->peerInfo[i]; + if (pPeer == NULL || pPeer->peerFd < 0) continue; + if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue; + + if (pNode->quorum > 1 && code == 0) { + syncSaveFwdInfo(pNode, pWalHead->version, mhandle); + code = 1; + } + + int32_t retLen = write(pPeer->peerFd, pSyncHead, fwdLen); + if (retLen == fwdLen) { + sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len); + } else { + sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen); + syncRestartConnection(pPeer); + } + } + + pthread_mutex_unlock(&(pNode->mutex)); + + return code; +} + diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index ebb6c3a0a9edff5acfc5f2ce7da8b58f03d8ab4a..44aed220d7369f63c4e7cdb5b3ce86d75c91b73e 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -48,20 +48,21 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex } } -static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { +static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; SFileInfo minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */ // master file info SFileInfo sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */ // slave file info SFileAck fileAck; - int code = -1; + int32_t code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; uint32_t pindex = 0; // index in last restore + bool fileChanged = false; *fversion = 0; sinfo.index = 0; while (1) { // read file info - int ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo)); + int32_t ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo)); if (ret < 0) break; // if no more file from master, break; @@ -103,7 +104,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { minfo.name[sizeof(minfo.name) - 1] = 0; snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name); - int dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); if (dfd < 0) { sError("%s, failed to open file:%s", pPeer->id, name); break; @@ -114,10 +115,11 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { close(dfd); if (ret < 0) break; + fileChanged = true; sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); } - if (code == 0 && (minfo.fversion != sinfo.fversion)) { + if (code == 0 && fileChanged) { // data file is changed, code shall be set to 1 *fversion = minfo.fversion; code = 1; @@ -130,9 +132,9 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { return code; } -static int syncRestoreWal(SSyncPeer *pPeer) { +static int32_t syncRestoreWal(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - int ret, code = -1; + int32_t ret, code = -1; void *buffer = calloc(1024000, 1); // size for one record if (buffer == NULL) return -1; @@ -151,8 +153,8 @@ static int syncRestoreWal(SSyncPeer *pPeer) { ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len); if (ret < 0) break; - sDebug("%s, restore a record, ver:%" PRIu64, pPeer->id, pHead->version); - (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL); + sDebug("%s, restore a record, qtype:wal hver:%" PRIu64, pPeer->id, pHead->version); + (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL, NULL); } if (code < 0) { @@ -167,16 +169,16 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) { SSyncNode *pNode = pPeer->pSyncNode; SWalHead * pHead = (SWalHead *)offset; - (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD); + (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL); offset += pHead->len + sizeof(SWalHead); return offset; } -static int syncProcessBufferedFwd(SSyncPeer *pPeer) { +static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) { SSyncNode * pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; - int forwards = 0; + int32_t forwards = 0; sDebug("%s, number of buffered forwards:%d", pPeer->id, pRecv->forwards); @@ -201,12 +203,12 @@ static int syncProcessBufferedFwd(SSyncPeer *pPeer) { return pRecv->code; } -int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { +int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { SSyncNode * pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; if (pRecv == NULL) return -1; - int len = pHead->len + sizeof(SWalHead); + int32_t len = pHead->len + sizeof(SWalHead); if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) { memcpy(pRecv->offset, pHead, len); @@ -223,13 +225,13 @@ int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { static void syncCloseRecvBuffer(SSyncNode *pNode) { if (pNode->pRecv) { - taosTFree(pNode->pRecv->buffer); + tfree(pNode->pRecv->buffer); } - taosTFree(pNode->pRecv); + tfree(pNode->pRecv); } -static int syncOpenRecvBuffer(SSyncNode *pNode) { +static int32_t syncOpenRecvBuffer(SSyncNode *pNode) { syncCloseRecvBuffer(pNode); SRecvBuffer *pRecv = calloc(sizeof(SRecvBuffer), 1); @@ -250,13 +252,13 @@ static int syncOpenRecvBuffer(SSyncNode *pNode) { return 0; } -static int syncRestoreDataStepByStep(SSyncPeer *pPeer) { +static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; nodeSStatus = TAOS_SYNC_STATUS_FILE; uint64_t fversion = 0; sDebug("%s, start to restore file", pPeer->id); - int code = syncRestoreFile(pPeer, &fversion); + int32_t code = syncRestoreFile(pPeer, &fversion); if (code < 0) { sError("%s, failed to restore file", pPeer->id); return -1; diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 60625d75eccdbe6bbb29f97b31ecd8e9855480a7..968f5becaddc7b06c06171a4d91cc0e0ffffc0da 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -27,7 +27,7 @@ #include "tsync.h" #include "syncInt.h" -static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { +static int32_t syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { sDebug("%s, start to monitor:%s", pPeer->id, name); if (pPeer->notifyFd <= 0) { @@ -38,16 +38,16 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { return -1; } - if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles); + if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles); if (pPeer->watchFd == NULL) { sError("%s, failed to allocate watchFd", pPeer->id); return -1; } - memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles); + memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles); } - int *wd = pPeer->watchFd + pPeer->watchNum; + int32_t *wd = pPeer->watchFd + pPeer->watchNum; if (*wd >= 0) { if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) { @@ -69,17 +69,17 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { return 0; } -static int syncAreFilesModified(SSyncPeer *pPeer) { +static int32_t syncAreFilesModified(SSyncPeer *pPeer) { if (pPeer->notifyFd <= 0) return 0; - char buf[2048]; - int len = read(pPeer->notifyFd, buf, sizeof(buf)); + char buf[2048]; + int32_t len = read(pPeer->notifyFd, buf, sizeof(buf)); if (len < 0 && errno != EAGAIN) { sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); return -1; } - int code = 0; + int32_t code = 0; if (len > 0) { const struct inotify_event *event; char *ptr; @@ -97,11 +97,11 @@ static int syncAreFilesModified(SSyncPeer *pPeer) { return code; } -static int syncRetrieveFile(SSyncPeer *pPeer) { +static int32_t syncRetrieveFile(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SFileInfo fileInfo; SFileAck fileAck; - int code = -1; + int32_t code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; memset(&fileInfo, 0, sizeof(fileInfo)); @@ -146,10 +146,10 @@ static int syncRetrieveFile(SSyncPeer *pPeer) { } // send the file to peer - int sfd = open(name, O_RDONLY); + int32_t sfd = open(name, O_RDONLY); if (sfd < 0) break; - ret = taosTSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); + ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); close(sfd); if (ret < 0) break; @@ -169,8 +169,8 @@ static int syncRetrieveFile(SSyncPeer *pPeer) { /* if only a partial record is read out, set the IN_MODIFY flag in event, so upper layer will reload the file to get a complete record */ -static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) { - int ret; +static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead, uint32_t *pEvent) { + int32_t ret; ret = read(sfd, pHead, sizeof(SWalHead)); if (ret < 0) return -1; @@ -194,7 +194,7 @@ static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) { return sizeof(SWalHead) + pHead->len; } -static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) { +static int32_t syncMonitorLastWal(SSyncPeer *pPeer, char *name) { pPeer->watchNum = 0; taosClose(pPeer->notifyFd); pPeer->notifyFd = inotify_init1(IN_NONBLOCK); @@ -203,14 +203,14 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) { return -1; } - if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles); + if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles); if (pPeer->watchFd == NULL) { sError("%s, failed to allocate watchFd", pPeer->id); return -1; } - memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles); - int *wd = pPeer->watchFd; + memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles); + int32_t *wd = pPeer->watchFd; *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE); if (*wd == -1) { @@ -222,8 +222,8 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) { } static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) { - char buf[2048]; - int len = read(pPeer->notifyFd, buf, sizeof(buf)); + char buf[2048]; + int32_t len = read(pPeer->notifyFd, buf, sizeof(buf)); if (len < 0 && errno != EAGAIN) { sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); return -1; @@ -243,11 +243,11 @@ static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) { return 0; } -static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) { - SWalHead *pHead = malloc(640000); - int code = -1; +static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) { + SWalHead *pHead = malloc(SYNC_MAX_SIZE); + int32_t code = -1; int32_t bytes = 0; - int sfd; + int32_t sfd; sfd = open(name, O_RDONLY); if (sfd < 0) { @@ -256,10 +256,10 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, } (void)lseek(sfd, offset, SEEK_SET); - sDebug("%s, retrieve last wal, offset:%" PRId64 " fversion:%" PRIu64, pPeer->id, offset, fversion); + sDebug("%s, retrieve last wal, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, offset, fversion); while (1) { - int wsize = syncReadOneWalRecord(sfd, pHead, pEvent); + int32_t wsize = syncReadOneWalRecord(sfd, pHead, pEvent); if (wsize < 0) break; if (wsize == 0) { code = 0; @@ -267,7 +267,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, } sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version); - int ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); + int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); if (ret != wsize) break; pPeer->sversion = pHead->version; @@ -287,9 +287,9 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, return -1; } -static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) { +static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { SSyncNode *pNode = pPeer->pSyncNode; - int code = -1; + int32_t code = -1; char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file if (syncAreFilesModified(pPeer) != 0) return -1; @@ -325,7 +325,7 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) { // if all data up to fversion is read out, it is over if (pPeer->sversion >= fversion && fversion > 0) { code = 0; - sDebug("%s, data up to fversion:%" PRId64 " has been read out, bytes:%d", pPeer->id, fversion, bytes); + sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d", pPeer->id, fversion, bytes); break; } @@ -370,14 +370,14 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) { return code; } -static int syncRetrieveWal(SSyncPeer *pPeer) { +static int32_t syncRetrieveWal(SSyncPeer *pPeer) { SSyncNode * pNode = pPeer->pSyncNode; char fname[TSDB_FILENAME_LEN * 3]; char wname[TSDB_FILENAME_LEN * 2]; int32_t size; struct stat fstat; - int code = -1; - uint32_t index = 0; + int32_t code = -1; + int64_t index = 0; while (1) { // retrieve wal info @@ -403,10 +403,10 @@ static int syncRetrieveWal(SSyncPeer *pPeer) { size = fstat.st_size; sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); - int sfd = open(fname, O_RDONLY); + int32_t sfd = open(fname, O_RDONLY); if (sfd < 0) break; - code = taosTSendFile(pPeer->syncFd, sfd, NULL, size); + code = taosSendFile(pPeer->syncFd, sfd, NULL, size); close(sfd); if (code < 0) break; @@ -428,7 +428,7 @@ static int syncRetrieveWal(SSyncPeer *pPeer) { return code; } -static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) { +static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SFirstPkt firstPkt; diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c index 6a210a136ffe67b2e1394d26bac4cb5083452c8c..3024d7d4e31ba999972c533099710cad6650af8d 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/taosTcpPool.c @@ -13,18 +13,22 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "tulog.h" #include "tutil.h" #include "tsocket.h" #include "taoserror.h" #include "taosTcpPool.h" +#include "twal.h" +#include "tsync.h" +#include "syncInt.h" typedef struct SThreadObj { pthread_t thread; bool stop; - int pollFd; - int numOfFds; + int32_t pollFd; + int32_t numOfFds; struct SPoolObj *pPool; } SThreadObj; @@ -32,15 +36,15 @@ typedef struct SPoolObj { SPoolInfo info; SThreadObj **pThread; pthread_t thread; - int nextId; - int acceptFd; // FD for accept new connection + int32_t nextId; + int32_t acceptFd; // FD for accept new connection } SPoolObj; typedef struct { SThreadObj *pThread; - void *ahandle; - int fd; - int closedByApp; + void * ahandle; + int32_t fd; + int32_t closedByApp; } SConnObj; static void *taosAcceptPeerTcpConnection(void *argv); @@ -53,66 +57,66 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { SPoolObj *pPool = calloc(sizeof(SPoolObj), 1); if (pPool == NULL) { - uError("TCP server, no enough memory"); + sError("failed to alloc pool for TCP server since no enough memory"); return NULL; } pPool->info = *pInfo; - pPool->pThread = (SThreadObj **)calloc(sizeof(SThreadObj *), pInfo->numOfThreads); + pPool->pThread = calloc(sizeof(SThreadObj *), pInfo->numOfThreads); if (pPool->pThread == NULL) { - uError("TCP server, no enough memory"); - free(pPool); + sError("failed to alloc pool thread for TCP server since no enough memory"); + tfree(pPool); return NULL; } pPool->acceptFd = taosOpenTcpServerSocket(pInfo->serverIp, pInfo->port); if (pPool->acceptFd < 0) { - free(pPool->pThread); - free(pPool); - uError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno)); + tfree(pPool->pThread); + tfree(pPool); + sError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno)); return NULL; } pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pPool->thread), &thattr, (void *)taosAcceptPeerTcpConnection, pPool) != 0) { - uError("TCP server, failed to create accept thread, reason:%s", strerror(errno)); + sError("failed to create accept thread for TCP server since %s", strerror(errno)); close(pPool->acceptFd); - free(pPool->pThread); - free(pPool); + tfree(pPool->pThread); + tfree(pPool); return NULL; } pthread_attr_destroy(&thattr); - uDebug("%p TCP pool is created", pPool); + sDebug("%p TCP pool is created", pPool); return pPool; } void taosCloseTcpThreadPool(void *param) { - SPoolObj * pPool = (SPoolObj *)param; + SPoolObj * pPool = param; SThreadObj *pThread; shutdown(pPool->acceptFd, SHUT_RD); pthread_join(pPool->thread, NULL); - for (int i = 0; i < pPool->info.numOfThreads; ++i) { + for (int32_t i = 0; i < pPool->info.numOfThreads; ++i) { pThread = pPool->pThread[i]; if (pThread) taosStopPoolThread(pThread); } - uDebug("%p TCP pool is closed", pPool); + sDebug("%p TCP pool is closed", pPool); - taosTFree(pPool->pThread); - free(pPool); + tfree(pPool->pThread); + tfree(pPool); } -void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) { +void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { struct epoll_event event; - SPoolObj *pPool = (SPoolObj *)param; + SPoolObj *pPool = param; - SConnObj *pConn = (SConnObj *)calloc(sizeof(SConnObj), 1); + SConnObj *pConn = calloc(sizeof(SConnObj), 1); if (pConn == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -120,7 +124,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) { SThreadObj *pThread = taosGetTcpThread(pPool); if (pThread == NULL) { - free(pConn); + tfree(pConn); return NULL; } @@ -133,13 +137,13 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) { event.data.ptr = pConn; if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - uError("failed to add fd:%d(%s)", connFd, strerror(errno)); + sError("failed to add fd:%d since %s", connFd, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); - free(pConn); + tfree(pConn); pConn = NULL; } else { pThread->numOfFds++; - uDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds); + sDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds); } return pConn; @@ -149,7 +153,7 @@ void taosFreeTcpConn(void *param) { SConnObj * pConn = (SConnObj *)param; SThreadObj *pThread = pConn->pThread; - uDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd); + sDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd); pConn->closedByApp = 1; shutdown(pConn->fd, SHUT_WR); } @@ -164,9 +168,9 @@ static void taosProcessBrokenLink(SConnObj *pConn) { pThread->numOfFds--; epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); - uDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); + sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); taosClose(pConn->fd); - free(pConn); + tfree(pConn); } #define maxEvents 10 @@ -183,18 +187,18 @@ static void *taosProcessTcpData(void *param) { while (1) { if (pThread->stop) break; - int fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); + int32_t fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); if (pThread->stop) { - uDebug("%p TCP epoll thread is exiting...", pThread); + sDebug("%p TCP epoll thread is exiting...", pThread); break; } if (fdNum < 0) { - uError("epoll_wait failed (%s)", strerror(errno)); + sError("epoll_wait failed since %s", strerror(errno)); continue; } - for (int i = 0; i < fdNum; ++i) { + for (int32_t i = 0; i < fdNum; ++i) { pConn = events[i].data.ptr; assert(pConn); @@ -219,17 +223,16 @@ static void *taosProcessTcpData(void *param) { continue; } } - } if (pThread->stop) break; } - uDebug("%p TCP epoll thread exits", pThread); + sDebug("%p TCP epoll thread exits", pThread); close(pThread->pollFd); - free(pThread); - free(buffer); + tfree(pThread); + tfree(buffer); return NULL; } @@ -242,18 +245,18 @@ static void *taosAcceptPeerTcpConnection(void *argv) { while (1) { struct sockaddr_in clientAddr; socklen_t addrlen = sizeof(clientAddr); - int connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + int32_t connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); if (connFd < 0) { if (errno == EINVAL) { - uDebug("%p TCP server accept is exiting...", pPool); + sDebug("%p TCP server accept is exiting...", pPool); break; } else { - uError("TCP accept failure, reason:%s", strerror(errno)); + sError("TCP accept failure since %s", strerror(errno)); continue; } } - // uDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port); + // sDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port); taosKeepTcpAlive(connFd); (*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr); } @@ -273,23 +276,23 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) { pThread->pPool = pPool; pThread->pollFd = epoll_create(10); // size does not matter if (pThread->pollFd < 0) { - free(pThread); + tfree(pThread); return NULL; } pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread); + int32_t ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread); pthread_attr_destroy(&thattr); if (ret != 0) { close(pThread->pollFd); - free(pThread); + tfree(pThread); return NULL; } - uDebug("%p TCP epoll thread is created", pThread); + sDebug("%p TCP epoll thread is created", pThread); pPool->pThread[pPool->nextId] = pThread; pPool->nextId++; pPool->nextId = pPool->nextId % pPool->info.numOfThreads; @@ -314,12 +317,12 @@ static void taosStopPoolThread(SThreadObj *pThread) { eventfd_t fd = eventfd(1, 0); if (fd == -1) { // failed to create eventfd, call pthread_cancel instead, which may result in data corruption - uError("failed to create eventfd(%s)", strerror(errno)); + sError("failed to create eventfd since %s", strerror(errno)); pthread_cancel(pThread->thread); pThread->stop = true; } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { // failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption - uError("failed to call epoll_ctl(%s)", strerror(errno)); + sError("failed to call epoll_ctl since %s", strerror(errno)); pthread_cancel(pThread->thread); } diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 360ea93f6c3fcd41e1df03702a0c963029deaacc..b7f819a3cd8f890d3580352322d8313f7cb2828e 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -28,22 +28,22 @@ #include "syncInt.h" static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); -static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp); +static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); static void arbProcessBrokenLink(void *param); -static int arbProcessPeerMsg(void *param, void *buffer); +static int32_t arbProcessPeerMsg(void *param, void *buffer); static tsem_t tsArbSem; static ttpool_h tsArbTcpPool; typedef struct { - char id[TSDB_EP_LEN + 24]; - int nodeFd; - void *pConn; + char id[TSDB_EP_LEN + 24]; + int32_t nodeFd; + void * pConn; } SNodeConn; -int main(int argc, char *argv[]) { +int32_t main(int32_t argc, char *argv[]) { char arbLogPath[TSDB_FILENAME_LEN + 16] = {0}; - for (int i = 1; i < argc; ++i) { + for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { tsArbitratorPort = atoi(argv[++i]); } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { @@ -86,7 +86,7 @@ int main(int argc, char *argv[]) { info.numOfThreads = 1; info.serverIp = 0; info.port = tsArbitratorPort; - info.bufferSize = 640000; + info.bufferSize = SYNC_MAX_SIZE; info.processBrokenLink = arbProcessBrokenLink; info.processIncomingMsg = arbProcessPeerMsg; info.processIncomingConn = arbProcessIncommingConnection; @@ -108,7 +108,7 @@ int main(int argc, char *argv[]) { return 0; } -static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) { +static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { char ipstr[24]; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); @@ -128,10 +128,10 @@ static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) { } firstPkt.fqdn[sizeof(firstPkt.fqdn) - 1] = 0; - snprintf(pNode->id, sizeof(pNode->id), "vgId:%d peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port); + snprintf(pNode->id, sizeof(pNode->id), "vgId:%d, peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port); if (firstPkt.syncHead.vgId) { sDebug("%s, vgId in head is not zero, close the connection", pNode->id); - taosTFree(pNode); + tfree(pNode); taosCloseSocket(connFd); return; } @@ -147,16 +147,16 @@ static void arbProcessBrokenLink(void *param) { SNodeConn *pNode = param; sDebug("%s, TCP link is broken(%s), close connection", pNode->id, strerror(errno)); - taosTFree(pNode); + tfree(pNode); } -static int arbProcessPeerMsg(void *param, void *buffer) { +static int32_t arbProcessPeerMsg(void *param, void *buffer) { SNodeConn *pNode = param; SSyncHead head; - int bytes = 0; + int32_t bytes = 0; char * cont = (char *)buffer; - int hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head)); + int32_t hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head)); if (hlen != sizeof(head)) { sDebug("%s, failed to read msg, hlen:%d", pNode->id, hlen); return -1; diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index 380b971fa89bd1726e138c973a974bc995500693..9dd3feb4614105c13ba6a2d1069931345167b472 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -30,7 +30,7 @@ int dataFd = -1; void * qhandle = NULL; int walNum = 0; uint64_t tversion = 0; -void * syncHandle; +int64_t syncHandle; int role; int nodeId; char path[256]; @@ -254,7 +254,7 @@ uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex return magic; } -int getWalInfo(void *ahandle, char *name, uint32_t *index) { +int getWalInfo(void *ahandle, char *name, int64_t *index) { struct stat fstat; char aname[280]; diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 256b8189f8fc69345b27fdf702fb705d22ac3c10..0962e7c2cbb7fa0576fe1f00495ebbcbd23959d3 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -220,8 +220,7 @@ typedef struct { SMemTable* mem; SMemTable* imem; STsdbFileH* tsdbFileH; - int commit; - pthread_t commitThread; + sem_t readyToCommit; pthread_mutex_t mutex; bool repoLocked; } STsdbRepo; @@ -320,6 +319,15 @@ typedef struct { void* compBuffer; // Buffer for temperary compress/decompress purpose } SRWHelper; +typedef struct { + int rowsInserted; + int rowsUpdated; + int rowsDeleteSucceed; + int rowsDeleteFailed; + int nOperations; + TSKEY keyFirst; + TSKEY keyLast; +} SMergeInfo; // ------------------ tsdbScan.c typedef struct { SFileGroup fGroup; @@ -422,7 +430,7 @@ void tsdbCloseBufPool(STsdbRepo* pRepo); SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); // ------------------ tsdbMemTable.c -int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable); +int tsdbUpdateRowInMem(STsdbRepo* pRepo, SDataRow row, STable* pTable); int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); @@ -430,7 +438,8 @@ void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem) void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); int tsdbAsyncCommit(STsdbRepo* pRepo); int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, - TSKEY* filterKeys, int nFilterKeys); + TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo); +void* tsdbCommitData(STsdbRepo* pRepo); static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { if (pIter == NULL) return NULL; @@ -438,16 +447,23 @@ static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { SSkipListNode* node = tSkipListIterGet(pIter); if (node == NULL) return NULL; - return *(SDataRow *)SL_GET_NODE_DATA(node); + return (SDataRow)SL_GET_NODE_DATA(node); } static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { SDataRow row = tsdbNextIterRow(pIter); - if (row == NULL) return -1; + if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL; return dataRowKey(row); } +static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) { + SDataRow row = tsdbNextIterRow(pIter); + if (row == NULL) return TKEY_NULL; + + return dataRowTKey(row); +} + static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) { ASSERT(pRepo != NULL); if (pRepo->mem == NULL) return NULL; @@ -572,6 +588,9 @@ int tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx); int tsdbCloseScanFile(STsdbScanHandle* pScanHandle); void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle); +// ------------------ tsdbCommitQueue.c +int tsdbScheduleCommit(STsdbRepo *pRepo); + #ifdef __cplusplus } #endif diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index 2e097c6ff73543f2dd858f8424b1e942ebdd8c3e..7cea27658c80d689972e3cb0f5dda3269a34b720 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -110,7 +110,7 @@ void tsdbCloseBufPool(STsdbRepo *pRepo) { } } - tsdbDebug("vgId:%d buffer pool is closed", REPO_ID(pRepo)); + tsdbDebug("vgId:%d, buffer pool is closed", REPO_ID(pRepo)); } SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) { @@ -134,7 +134,7 @@ SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) { pBufBlock->offset = 0; pBufBlock->remain = pBufPool->bufBlockSize; - tsdbDebug("vgId:%d buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId); + tsdbDebug("vgId:%d, buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId); return pNode; } @@ -157,4 +157,4 @@ _err: return NULL; } -static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { taosTFree(pBufBlock); } \ No newline at end of file +static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } \ No newline at end of file diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c new file mode 100644 index 0000000000000000000000000000000000000000..3c158a2201c1641d57b8c0902b2b3a9c1c828c9e --- /dev/null +++ b/src/tsdb/src/tsdbCommitQueue.c @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "tlist.h" +#include "tsdbMain.h" + +typedef struct { + bool stop; + pthread_mutex_t lock; + pthread_cond_t queueNotEmpty; + int nthreads; + SList * queue; + pthread_t * threads; +} SCommitQueue; + +typedef struct { + STsdbRepo *pRepo; +} SCommitReq; + +static void *tsdbLoopCommit(void *arg); + +SCommitQueue tsCommitQueue = {0}; + +int tsdbInitCommitQueue(int nthreads) { + SCommitQueue *pQueue = &tsCommitQueue; + + if (nthreads < 1) nthreads = 1; + + pQueue->stop = false; + pQueue->nthreads = nthreads; + + pQueue->queue = tdListNew(0); + if (pQueue->queue == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + pQueue->threads = (pthread_t *)calloc(nthreads, sizeof(pthread_t)); + if (pQueue->threads == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tdListFree(pQueue->queue); + return -1; + } + + pthread_mutex_init(&(pQueue->lock), NULL); + pthread_cond_init(&(pQueue->queueNotEmpty), NULL); + + for (int i = 0; i < nthreads; i++) { + pthread_create(pQueue->threads + i, NULL, tsdbLoopCommit, NULL); + } + + return 0; +} + +void tsdbDestroyCommitQueue() { + SCommitQueue *pQueue = &tsCommitQueue; + + pthread_mutex_lock(&(pQueue->lock)); + + if (pQueue->stop) { + pthread_mutex_unlock(&(pQueue->lock)); + return; + } + + pQueue->stop = true; + pthread_cond_broadcast(&(pQueue->queueNotEmpty)); + + pthread_mutex_unlock(&(pQueue->lock)); + + for (size_t i = 0; i < pQueue->nthreads; i++) { + pthread_join(pQueue->threads[i], NULL); + } + + free(pQueue->threads); + tdListFree(pQueue->queue); + pthread_cond_destroy(&(pQueue->queueNotEmpty)); + pthread_mutex_destroy(&(pQueue->lock)); +} + +int tsdbScheduleCommit(STsdbRepo *pRepo) { + SCommitQueue *pQueue = &tsCommitQueue; + + SListNode *pNode = (SListNode *)calloc(1, sizeof(SListNode) + sizeof(SCommitReq)); + if (pNode == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + ((SCommitReq *)pNode->data)->pRepo = pRepo; + + pthread_mutex_lock(&(pQueue->lock)); + + ASSERT(!pQueue->stop); + + tdListAppendNode(pQueue->queue, pNode); + pthread_cond_signal(&(pQueue->queueNotEmpty)); + + pthread_mutex_unlock(&(pQueue->lock)); + return 0; +} + +static void *tsdbLoopCommit(void *arg) { + SCommitQueue *pQueue = &tsCommitQueue; + SListNode * pNode = NULL; + STsdbRepo * pRepo = NULL; + + while (true) { + pthread_mutex_lock(&(pQueue->lock)); + + while (true) { + pNode = tdListPopHead(pQueue->queue); + if (pNode == NULL) { + if (pQueue->stop) { + pthread_mutex_unlock(&(pQueue->lock)); + goto _exit; + } else { + pthread_cond_wait(&(pQueue->queueNotEmpty), &(pQueue->lock)); + } + } else { + break; + } + } + + pthread_mutex_unlock(&(pQueue->lock)); + + pRepo = ((SCommitReq *)pNode->data)->pRepo; + + tsdbCommitData(pRepo); + listNodeFree(pNode); + } + +_exit: + return NULL; +} diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 626ad77da2eab4be9e94516c4e5c7c0e5a45837e..1efde49ed0dba50cf01967265bcef63ed36839b7 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -13,10 +13,8 @@ * along with this program. If not, see . */ #define _DEFAULT_SOURCE -#include - #define TAOS_RANDOM_FILE_FAIL_TEST - +#include #include "os.h" #include "talgo.h" #include "tchecksum.h" @@ -67,7 +65,7 @@ _err: void tsdbFreeFileH(STsdbFileH *pFileH) { if (pFileH) { pthread_rwlock_destroy(&pFileH->fhlock); - taosTFree(pFileH->pFGroup); + tfree(pFileH->pFGroup); free(pFileH); } } @@ -79,7 +77,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { DIR * dir = NULL; int fid = 0; int vid = 0; - regex_t regex1, regex2; + regex_t regex1 = {0}, regex2 = {0}; int code = 0; char fname[TSDB_FILENAME_LEN] = "\0"; @@ -95,9 +93,27 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { dir = opendir(tDataDir); if (dir == NULL) { - tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + if (errno == ENOENT) { + tsdbError("vgId:%d directory %s not exist", REPO_ID(pRepo), tDataDir); + terrno = TAOS_SYSTEM_ERROR(errno); + + if (taosMkDir(tDataDir, 0755) < 0) { + tsdbError("vgId:%d failed to create directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + dir = opendir(tDataDir); + if (dir == NULL) { + tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } else { + tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } } code = regcomp(®ex1, "^v[0-9]+f[0-9]+\\.(head|data|last|stat)$", REG_EXTENDED); @@ -183,7 +199,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { regfree(®ex1); regfree(®ex2); - taosTFree(tDataDir); + tfree(tDataDir); closedir(dir); return 0; @@ -193,7 +209,7 @@ _err: regfree(®ex1); regfree(®ex2); - taosTFree(tDataDir); + tfree(tDataDir); if (dir != NULL) closedir(dir); tsdbCloseFileH(pRepo); return -1; @@ -410,7 +426,7 @@ int tsdbUpdateFileHeader(SFile *pFile) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (taosTWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + if (taosWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -475,7 +491,7 @@ int tsdbLoadFileHeader(SFile *pFile, uint32_t *version) { return -1; } - if (taosTRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + if (taosRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { tsdbError("failed to read file %s header part with %d bytes, reason:%s", pFile->fname, TSDB_FILE_HEAD_SIZE, strerror(errno)); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index a1e6376304a54ea810f3662bd30e3f97fd53765c..2ded3b668b47ac3dcf35fef287bae8f056977cb2 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -163,7 +163,7 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { if (toCommit) { tsdbAsyncCommit(pRepo); - if (pRepo->commit) pthread_join(pRepo->commitThread, NULL); + sem_wait(&(pRepo->readyToCommit)); } tsdbUnRefMemTable(pRepo, pRepo->mem); tsdbUnRefMemTable(pRepo, pRepo->imem); @@ -228,7 +228,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ int prefixLen = (int)strlen(prefix); if (name[0] == 0) { // get the file from index or after, but not larger than eindex - taosTFree(sdup); + tfree(sdup); int fid = (*index) / TSDB_FILE_TYPE_MAX; if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { @@ -260,8 +260,8 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ fname = malloc(prefixLen + strlen(name) + 2); sprintf(fname, "%s/%s", prefix, name); if (access(fname, F_OK) != 0) { - taosFree(fname); - taosFree(sdup); + tfree(fname); + tfree(sdup); return 0; } if (*index == TSDB_META_FILE_INDEX) { // get meta file @@ -269,20 +269,20 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ } else { tsdbGetFileInfoImpl(fname, &magic, size); } - taosFree(fname); - taosFree(sdup); + tfree(fname); + tfree(sdup); return magic; } if (stat(fname, &fState) < 0) { - taosTFree(fname); + tfree(fname); return 0; } *size = fState.st_size; // magic = *size; - taosTFree(fname); + tfree(fname); return magic; } @@ -512,6 +512,9 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } } + // update check + if (pCfg->update != 0) pCfg->update = 1; + return 0; _err: @@ -579,7 +582,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); - if (taosTWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + if (taosWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, TSDB_FILE_HEAD_SIZE, fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -597,7 +600,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { return 0; _err: - taosTFree(fname); + tfree(fname); if (fd >= 0) close(fd); return -1; } @@ -620,7 +623,7 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) { goto _err; } - if (taosTRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + if (taosRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { tsdbError("failed to read %d bytes from file %s since %s", TSDB_FILE_HEAD_SIZE, fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -634,13 +637,13 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) { tsdbDecodeCfg(buf, pCfg); - taosTFree(fname); + tfree(fname); close(fd); return 0; _err: - taosTFree(fname); + tfree(fname); if (fd >= 0) close(fd); return -1; } @@ -672,6 +675,12 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { goto _err; } + code = sem_init(&(pRepo->readyToCommit), 0, 1); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + goto _err; + } + pRepo->repoLocked = false; pRepo->rootDir = strdup(rootDir); @@ -715,7 +724,8 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { tsdbFreeMeta(pRepo->tsdbMeta); // tsdbFreeMemTable(pRepo->mem); // tsdbFreeMemTable(pRepo->imem); - taosTFree(pRepo->rootDir); + tfree(pRepo->rootDir); + sem_destroy(&(pRepo->readyToCommit)); pthread_mutex_destroy(&pRepo->mutex); free(pRepo); } @@ -762,7 +772,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY return -1; } - if (tsdbInsertRowToMem(pRepo, row, pTable) < 0) return -1; + if (tsdbUpdateRowInMem(pRepo, row, pTable) < 0) return -1; (*affectedrows)++; points++; @@ -923,6 +933,7 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) { tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock); tlen += taosEncodeFixedI8(buf, pCfg->precision); tlen += taosEncodeFixedI8(buf, pCfg->compression); + tlen += taosEncodeFixedI8(buf, pCfg->update); return tlen; } @@ -939,6 +950,7 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) { buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock)); buf = taosDecodeFixedI8(buf, &(pCfg->precision)); buf = taosDecodeFixedI8(buf, &(pCfg->compression)); + buf = taosDecodeFixedI8(buf, &(pCfg->update)); return buf; } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 4cf8ddd4bd8df396352ad66b8499552018d5d322..5680abcc6f4bf7874b59583cbfe3815c8a72a0a2 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -24,7 +24,6 @@ static void tsdbFreeMemTable(SMemTable *pMemTable); static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); static void tsdbFreeTableData(STableData *pTableData); static char * tsdbGetTsTupleKey(const void *data); -static void * tsdbCommitData(void *arg); static int tsdbCommitMeta(STsdbRepo *pRepo); static void tsdbEndCommit(STsdbRepo *pRepo); static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey); @@ -32,43 +31,40 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo); static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables); static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row); // ---------------- INTERNAL FUNCTIONS ---------------- -int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { +int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { STsdbCfg * pCfg = &pRepo->config; STsdbMeta * pMeta = pRepo->tsdbMeta; - int32_t level = 0; - int32_t headSize = 0; + TKEY tkey = dataRowTKey(row); TSKEY key = dataRowKey(row); SMemTable * pMemTable = pRepo->mem; STableData *pTableData = NULL; - SSkipList * pSList = NULL; + bool isRowDelete = TKEY_IS_DELETED(tkey); - if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL && - pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) { - pTableData = pMemTable->tData[TABLE_TID(pTable)]; - pSList = pTableData->pData; - } - - tSkipListNewNodeInfo(pSList, &level, &headSize); + if (isRowDelete) { + if (!pCfg->update) { + tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo)); + terrno = TSDB_CODE_TDB_INVALID_ACTION; + return -1; + } - SSkipListNode *pNode = (SSkipListNode *)malloc(headSize + sizeof(SDataRow *)); - if (pNode == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; + if (key > TABLE_LASTKEY(pTable)) { + tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64, + REPO_ID(pRepo), key, TABLE_LASTKEY(pTable)); + return 0; + } } void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row)); if (pRow == NULL) { tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s", REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno)); - free(pNode); return -1; } - pNode->level = level; dataRowCpy(pRow, row); - *(SDataRow *)SL_GET_NODE_DATA(pNode) = pRow; // Operations above may change pRepo->mem, retake those values ASSERT(pRepo->mem != NULL); @@ -77,7 +73,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { if (TABLE_TID(pTable) >= pMemTable->maxTables) { if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) { tsdbFreeBytes(pRepo, pRow, dataRowLen(row)); - free(pNode); return -1; } } @@ -97,7 +92,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { " to table %s while create new table data object since %s", REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno)); tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); - free(pNode); return -1; } @@ -106,24 +100,31 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable)); - if (tSkipListPut(pTableData->pData, pNode) == NULL) { + int64_t oldSize = SL_SIZE(pTableData->pData); + if (tSkipListPut(pTableData->pData, pRow) == NULL) { tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); - free(pNode); } else { - if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; + int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize; + if (isRowDelete) { + if (TABLE_LASTKEY(pTable) == key) { + // TODO: need to update table last key here (may from file) + } + } else { + if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; + } + if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; if (pMemTable->keyLast < key) pMemTable->keyLast = key; - pMemTable->numOfRows++; + pMemTable->numOfRows += deltaSize; if (pTableData->keyFirst > key) pTableData->keyFirst = key; if (pTableData->keyLast < key) pTableData->keyLast = key; - pTableData->numOfRows++; - - ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData)); + pTableData->numOfRows += deltaSize; } - tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), - TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), key); + tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), + isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), + key); return 0; } @@ -196,6 +197,8 @@ void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem) if (pIMem != NULL) { tsdbUnRefMemTable(pRepo, pIMem); } + + tsdbDebug("vgId:%d utake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem); } void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { @@ -258,100 +261,211 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { int tsdbAsyncCommit(STsdbRepo *pRepo) { SMemTable *pIMem = pRepo->imem; - int code = 0; - if (pIMem != NULL) { - ASSERT(pRepo->commit); - tsdbDebug("vgId:%d waiting for the commit thread", REPO_ID(pRepo)); - code = pthread_join(pRepo->commitThread, NULL); - tsdbDebug("vgId:%d commit thread is finished", REPO_ID(pRepo)); - if (code != 0) { - tsdbError("vgId:%d failed to thread join since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - pRepo->commit = 0; - } - - ASSERT(pRepo->commit == 0); if (pRepo->mem != NULL) { + sem_wait(&(pRepo->readyToCommit)); + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START); if (tsdbLockRepo(pRepo) < 0) return -1; pRepo->imem = pRepo->mem; pRepo->mem = NULL; - pRepo->commit = 1; - code = pthread_create(&pRepo->commitThread, NULL, tsdbCommitData, (void *)pRepo); - if (code != 0) { - tsdbError("vgId:%d failed to create commit thread since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - tsdbUnlockRepo(pRepo); - return -1; - } + tsdbScheduleCommit(pRepo); if (tsdbUnlockRepo(pRepo) < 0) return -1; } - if (pIMem && tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1; + if (tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1; + + return 0; +} +int tsdbSyncCommit(TSDB_REPO_T *repo) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + tsdbAsyncCommit(pRepo); + sem_wait(&(pRepo->readyToCommit)); + sem_post(&(pRepo->readyToCommit)); return 0; } +/** + * This is an important function to load data or try to load data from memory skiplist iterator. + * + * This function load memory data until: + * 1. iterator ends + * 2. data key exceeds maxKey + * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead + * 4. operations in pCols not exceeds its max capacity if pCols is given + * + * The function tries to procceed AS MUSH AS POSSIBLE. + */ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, - TSKEY *filterKeys, int nFilterKeys) { - ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); + TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { + ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0 && pMergeInfo != NULL); if (pIter == NULL) return 0; STSchema *pSchema = NULL; - int numOfRows = 0; - TSKEY keyNext = 0; + TSKEY rowKey = 0; + TSKEY fKey = 0; + bool isRowDel = false; int filterIter = 0; + SDataRow row = NULL; + + memset(pMergeInfo, 0, sizeof(*pMergeInfo)); + pMergeInfo->keyFirst = INT64_MAX; + pMergeInfo->keyLast = INT64_MIN; + if (pCols) tdResetDataCols(pCols); + + row = tsdbNextIterRow(pIter); + if (row == NULL || dataRowKey(row) > maxKey) { + rowKey = INT64_MAX; + isRowDel = false; + } else { + rowKey = dataRowKey(row); + isRowDel = dataRowDeleted(row); + } - if (nFilterKeys != 0) { // for filter purpose - ASSERT(filterKeys != NULL); - keyNext = tsdbNextIterKey(pIter); - if (keyNext < 0 || keyNext > maxKey) return numOfRows; - void *ptr = taosbsearch((void *)(&keyNext), (void *)filterKeys, nFilterKeys, sizeof(TSKEY), compTSKEY, TD_GE); - filterIter = (ptr == NULL) ? nFilterKeys : (int)((POINTER_DISTANCE(ptr, filterKeys) / sizeof(TSKEY))); - } - - do { - SDataRow row = tsdbNextIterRow(pIter); - if (row == NULL) break; - - keyNext = dataRowKey(row); - if (keyNext > maxKey) break; - - bool keyFiltered = false; - if (nFilterKeys != 0) { - while (true) { - if (filterIter >= nFilterKeys) break; - if (keyNext == filterKeys[filterIter]) { - keyFiltered = true; - filterIter++; - break; - } else if (keyNext < filterKeys[filterIter]) { - break; + if (filterIter >= nFilterKeys) { + fKey = INT64_MAX; + } else { + fKey = tdGetKey(filterKeys[filterIter]); + } + + while (true) { + if (fKey == INT64_MAX && rowKey == INT64_MAX) break; + + if (fKey < rowKey) { + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); + + filterIter++; + if (filterIter >= nFilterKeys) { + fKey = INT64_MAX; + } else { + fKey = tdGetKey(filterKeys[filterIter]); + } + } else if (fKey > rowKey) { + if (isRowDel) { + pMergeInfo->rowsDeleteFailed++; + } else { + if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsInserted++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + } + + tSkipListIterNext(pIter); + row = tsdbNextIterRow(pIter); + if (row == NULL || dataRowKey(row) > maxKey) { + rowKey = INT64_MAX; + isRowDel = false; + } else { + rowKey = dataRowKey(row); + isRowDel = dataRowDeleted(row); + } + } else { + if (isRowDel) { + ASSERT(!keepDup); + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsDeleteSucceed++; + pMergeInfo->nOperations++; + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + } else { + if (keepDup) { + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsUpdated++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); } else { - filterIter++; + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); } } + + tSkipListIterNext(pIter); + row = tsdbNextIterRow(pIter); + if (row == NULL || dataRowKey(row) > maxKey) { + rowKey = INT64_MAX; + isRowDel = false; + } else { + rowKey = dataRowKey(row); + isRowDel = dataRowDeleted(row); + } + + filterIter++; + if (filterIter >= nFilterKeys) { + fKey = INT64_MAX; + } else { + fKey = tdGetKey(filterKeys[filterIter]); + } } + } - if (!keyFiltered) { - if (numOfRows >= maxRowsToRead) break; - if (pCols) { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); - if (pSchema == NULL) { - ASSERT(0); - } - } + return 0; +} + +void *tsdbCommitData(STsdbRepo *pRepo) { + SMemTable * pMem = pRepo->imem; + STsdbCfg * pCfg = &pRepo->config; + SDataCols * pDataCols = NULL; + STsdbMeta * pMeta = pRepo->tsdbMeta; + SCommitIter *iters = NULL; + SRWHelper whelper = {0}; + ASSERT(pMem != NULL); + + tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64, REPO_ID(pRepo), + pMem->keyFirst, pMem->keyLast, pMem->numOfRows); + + // Create the iterator to read from cache + if (pMem->numOfRows > 0) { + iters = tsdbCreateCommitIters(pRepo); + if (iters == NULL) { + tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _exit; + } + + if (tsdbInitWriteHelper(&whelper, pRepo) < 0) { + tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _exit; + } + + if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s", + REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno)); + goto _exit; + } + + int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision)); + int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision)); - tdAppendDataRowToDataCol(row, pSchema, pCols); + // Loop to commit to each file + for (int fid = sfid; fid <= efid; fid++) { + if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) { + tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + goto _exit; } - numOfRows++; } - } while (tSkipListIterNext(pIter)); + } + + // Commit to update meta file + if (tsdbCommitMeta(pRepo) < 0) { + tsdbError("vgId:%d failed to commit data while committing meta data since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _exit; + } - return numOfRows; + tsdbFitRetention(pRepo); + +_exit: + tdFreeDataCols(pDataCols); + tsdbDestroyCommitIters(iters, pMem->maxTables); + tsdbDestroyHelper(&whelper); + tsdbEndCommit(pRepo); + tsdbInfo("vgId:%d commit over", pRepo->config.tsdbId); + + return NULL; } // ---------------- LOCAL FUNCTIONS ---------------- @@ -423,7 +537,7 @@ static void tsdbFreeMemTable(SMemTable* pMemTable) { tdListFree(pMemTable->extraBuffList); tdListFree(pMemTable->bufBlockList); tdListFree(pMemTable->actList); - taosTFree(pMemTable->tData); + tfree(pMemTable->tData); free(pMemTable); } } @@ -440,8 +554,9 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { pTableData->keyLast = 0; pTableData->numOfRows = 0; - pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, - TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 1, tsdbGetTsTupleKey); + pTableData->pData = + tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], + tkeyComparFn, pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -461,71 +576,8 @@ static void tsdbFreeTableData(STableData *pTableData) { } } -static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(*(SDataRow *)data); } - -static void *tsdbCommitData(void *arg) { - STsdbRepo * pRepo = (STsdbRepo *)arg; - SMemTable * pMem = pRepo->imem; - STsdbCfg * pCfg = &pRepo->config; - SDataCols * pDataCols = NULL; - STsdbMeta * pMeta = pRepo->tsdbMeta; - SCommitIter *iters = NULL; - SRWHelper whelper = {0}; - ASSERT(pRepo->commit == 1); - ASSERT(pMem != NULL); +static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); } - tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64, REPO_ID(pRepo), - pMem->keyFirst, pMem->keyLast, pMem->numOfRows); - - // Create the iterator to read from cache - if (pMem->numOfRows > 0) { - iters = tsdbCreateCommitIters(pRepo); - if (iters == NULL) { - tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _exit; - } - - if (tsdbInitWriteHelper(&whelper, pRepo) < 0) { - tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _exit; - } - - if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s", - REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno)); - goto _exit; - } - - int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision)); - int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision)); - - // Loop to commit to each file - for (int fid = sfid; fid <= efid; fid++) { - if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) { - tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); - goto _exit; - } - } - } - - // Commit to update meta file - if (tsdbCommitMeta(pRepo) < 0) { - tsdbError("vgId:%d failed to commit data while committing meta data since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _exit; - } - - tsdbFitRetention(pRepo); - -_exit: - tdFreeDataCols(pDataCols); - tsdbDestroyCommitIters(iters, pMem->maxTables); - tsdbDestroyHelper(&whelper); - tsdbEndCommit(pRepo); - tsdbInfo("vgId:%d commit over", pRepo->config.tsdbId); - - return NULL; -} static int tsdbCommitMeta(STsdbRepo *pRepo) { SMemTable *pMem = pRepo->imem; @@ -576,14 +628,14 @@ _err: } static void tsdbEndCommit(STsdbRepo *pRepo) { - ASSERT(pRepo->commit == 1); if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER); + sem_post(&(pRepo->readyToCommit)); } static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { for (int i = 0; i < nIters; i++) { TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter); - if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1; + if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1; } return 0; } @@ -680,7 +732,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe goto _err; } - taosTFree(dataDir); + tfree(dataDir); tsdbCloseHelperFile(pHelper, 0, pGroup); pthread_rwlock_wrlock(&(pFileH->fhlock)); @@ -702,7 +754,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe return 0; _err: - taosTFree(dataDir); + tfree(dataDir); tsdbCloseHelperFile(pHelper, 1, NULL); return -1; } @@ -777,7 +829,23 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { pMemTable->tData = pTableData; taosWUnLockLatch(&(pMemTable->latch)); - taosTFree(tData); + tfree(tData); + + return 0; +} + +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) { + if (pCols) { + if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) { + *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); + if (*ppSchema == NULL) { + ASSERT(false); + return -1; + } + } + + tdAppendDataRowToDataCol(row, *ppSchema, pCols); + } return 0; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index f3bd91f038cf209827b1c252160019a6b0aac27f..25c815b74e3bd5593bef9157cbd42ba869298cab 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -86,7 +86,8 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { if (pTable != NULL) { tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); - return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + goto _err; } if (pCfg->type == TSDB_CHILD_TABLE) { @@ -191,7 +192,7 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { return 0; _err: - taosTFree(tbname); + tfree(tbname); return -1; } @@ -461,7 +462,7 @@ void tsdbFreeMeta(STsdbMeta *pMeta) { if (pMeta) { taosHashCleanup(pMeta->uidMap); tdListFree(pMeta->superList); - taosTFree(pMeta->tables); + tfree(pMeta->tables); pthread_rwlock_destroy(&pMeta->rwLock); free(pMeta); } @@ -485,11 +486,11 @@ int tsdbOpenMeta(STsdbRepo *pRepo) { } tsdbDebug("vgId:%d open TSDB meta succeed", REPO_ID(pRepo)); - taosTFree(fname); + tfree(fname); return 0; _err: - taosTFree(fname); + tfree(fname); return -1; } @@ -562,12 +563,12 @@ int tsdbUnlockRepoMeta(STsdbRepo *pRepo) { void tsdbRefTable(STable *pTable) { int32_t ref = T_REF_INC(pTable); UNUSED(ref); - // tsdbDebug("ref table %"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref); + tsdbDebug("ref table %s uid %" PRIu64 " tid:%d, refCount:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), ref); } void tsdbUnRefTable(STable *pTable) { int32_t ref = T_REF_DEC(pTable); - tsdbDebug("unref table uid:%"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref); + tsdbDebug("unref table %s uid:%"PRIu64" tid:%d, refCount:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), ref); if (ref == 0) { // tsdbDebug("destory table name:%s uid:%"PRIu64", tid:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable)); @@ -643,7 +644,7 @@ static void tsdbOrgMeta(void *pHandle) { } static char *getTagIndexKey(const void *pData) { - STable *pTable = *(STable **)pData; + STable *pTable = (STable *)pData; STSchema *pSchema = tsdbGetTableTagSchema(pTable); STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN); @@ -700,7 +701,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { } pTable->tagVal = NULL; STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey); + pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, SL_ALLOW_DUP_KEY, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -745,7 +746,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { T_REF_INC(pTable); - tsdbTrace("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), + tsdbDebug("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); return pTable; @@ -760,7 +761,7 @@ static void tsdbFreeTable(STable *pTable) { if (pTable->name != NULL) tsdbTrace("table %s tid %d uid %" PRIu64 " is freed", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); - taosTFree(TABLE_NAME(pTable)); + tfree(TABLE_NAME(pTable)); if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { for (int i = 0; i < TSDB_MAX_TABLE_SCHEMAS; i++) { tdFreeSchema(pTable->schema[i]); @@ -774,7 +775,7 @@ static void tsdbFreeTable(STable *pTable) { kvRowFree(pTable->tagVal); tSkipListDestroy(pTable->pIndex); - taosTFree(pTable->sql); + tfree(pTable->sql); free(pTable); } } @@ -889,7 +890,7 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro } if (lock) tsdbUnlockRepoMeta(pRepo); - tsdbDebug("vgId:%d table %s is removed from meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable)); + tsdbDebug("vgId:%d table %s uid %" PRIu64 " is removed from meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_UID(pTable)); tsdbUnRefTable(pTable); } @@ -900,23 +901,8 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper pTable->pSuper = pSTable; - int32_t level = 0; - int32_t headSize = 0; - - tSkipListNewNodeInfo(pSTable->pIndex, &level, &headSize); - - // NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the - // actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function - SSkipListNode *pNode = calloc(1, headSize + sizeof(STable *)); - if (pNode == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - pNode->level = level; - - memcpy(SL_GET_NODE_DATA(pNode), &pTable, sizeof(STable *)); + tSkipListPut(pSTable->pIndex, (void *)pTable); - tSkipListPut(pSTable->pIndex, pNode); if (refSuper) T_REF_INC(pSTable); return 0; } @@ -940,7 +926,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { SSkipListNode *pNode = taosArrayGetP(res, i); // STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode); - if (*(STable **)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need + if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need tSkipListRemoveNode(pSTable->pIndex, pNode); } } @@ -1080,9 +1066,9 @@ void tsdbClearTableCfg(STableCfg *config) { if (config->schema) tdFreeSchema(config->schema); if (config->tagSchema) tdFreeSchema(config->tagSchema); if (config->tagValues) kvRowFree(config->tagValues); - taosTFree(config->name); - taosTFree(config->sname); - taosTFree(config->sql); + tfree(config->name); + tfree(config->sname); + tfree(config->sql); free(config); } } @@ -1170,8 +1156,8 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { buf = tdDecodeSchema(buf, &(pTable->tagSchema)); STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - pTable->pIndex = - tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey); + pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, + SL_ALLOW_DUP_KEY, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbFreeTable(pTable); @@ -1197,7 +1183,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) { tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM); } else { if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (tSkipListGetSize(pTable->pIndex) + 1)); + tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_SIZE(pTable->pIndex) + 1)); } else { tlen = sizeof(SListNode) + sizeof(SActObj); } @@ -1244,7 +1230,7 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) { } while (tSkipListIterNext(pIter)) { - STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); + STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE); pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, tTable); } @@ -1269,7 +1255,7 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) { tsdbWLockRepoMeta(pRepo); while (tSkipListIterNext(pIter)) { - STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); + STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); tsdbRemoveTableFromMeta(pRepo, tTable, false, false); } @@ -1304,7 +1290,7 @@ static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) { STable **tTables = pMeta->tables; pMeta->tables = tables; - taosTFree(tTables); + tfree(tTables); tsdbDebug("vgId:%d tsdb meta maxTables is adjusted as %d", REPO_ID(pRepo), maxTables); return 0; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 357093bd9e125567ef3d3629c00877f54778b500..98f815f78acb5f8beb77b27bc43c88e93bf1b917 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -14,9 +14,7 @@ */ #define _DEFAULT_SOURCE - #define TAOS_RANDOM_FILE_FAIL_TEST - #include "os.h" #include "talgo.h" #include "tchecksum.h" @@ -27,6 +25,7 @@ #define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM)) #define TSDB_KEY_COL_OFFSET 0 #define TSDB_GET_COMPBLOCK_IDX(h, b) (POINTER_DISTANCE(b, (h)->pCompInfo->blocks)/sizeof(SCompBlock)) +#define TSDB_IS_LAST_BLOCK(pb) ((pb)->last) static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock, @@ -34,7 +33,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pD static int compareKeyBlock(const void *arg1, const void *arg2); static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize); static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); -static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded); +static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo); static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); static void tsdbResetHelperFileImpl(SRWHelper *pHelper); static int tsdbInitHelperFile(SRWHelper *pHelper); @@ -61,8 +60,10 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pComp static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock); static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, int *blkIdx); -static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows); +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, + TSKEY maxKey, int maxRows, int8_t update); +static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps); +static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx); // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { @@ -279,7 +280,7 @@ int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols while (true) { ASSERT(blkIdx <= (int)pIdx->numOfBlocks); TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - if (keyFirst < 0 || keyFirst > maxKey) break; // iter over + if (keyFirst == TSDB_DATA_TIMESTAMP_NULL || keyFirst > maxKey) break; // iter over if (pIdx->len <= 0 || keyFirst > pIdx->maxKey) { if (tsdbProcessAppendCommit(pHelper, pCommitIter, pDataCols, maxKey) < 0) return -1; @@ -335,7 +336,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { return -1; } - if (taosTSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) { + if (taosSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) { tsdbError("vgId:%d failed to sendfile from file %s to file %s since %s", REPO_ID(pHelper->pRepo), helperLastF(pHelper)->fname, helperNewLastF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -380,7 +381,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->tid = pHelper->tableInfo.tid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - if (taosTWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) { + if (taosWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -432,7 +433,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { ASSERT(offset == pFile->info.size); - if (taosTWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < (int)pFile->info.len) { + if (taosWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < (int)pFile->info.len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -454,7 +455,7 @@ int tsdbLoadCompIdxImpl(SFile *pFile, uint32_t offset, uint32_t len, void *buffe return -1; } - if (taosTRead(pFile->fd, buffer, len) < len) { + if (taosRead(pFile->fd, buffer, len) < len) { tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, offset, len, strerror(errno)); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; @@ -551,7 +552,7 @@ int tsdbLoadCompInfoImpl(SFile *pFile, SCompIdx *pIdx, SCompInfo **ppCompInfo) { return -1; } - if (taosTRead(pFile->fd, (void *)(*ppCompInfo), pIdx->len) < (int)pIdx->len) { + if (taosRead(pFile->fd, (void *)(*ppCompInfo), pIdx->len) < (int)pIdx->len) { tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, pIdx->offset, pIdx->len, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -608,7 +609,7 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { return -1; } - if (taosTRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) { + if (taosRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) { tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -823,7 +824,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa sizeof(TSCKSUM)); // Write the whole block to file - if (taosTWrite(pFile->fd, (void *)pCompData, lsize) < lsize) { + if (taosWrite(pFile->fd, (void *)pCompData, lsize) < lsize) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -925,7 +926,7 @@ _err: return -1; } -static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) { +static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo) { ASSERT(pCompBlock->numOfSubBlocks == 0); SCompIdx *pIdx = &(pHelper->curCompIdx); @@ -958,9 +959,9 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId pSCompBlock->numOfSubBlocks++; ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS); pSCompBlock->len += sizeof(SCompBlock); - pSCompBlock->numOfRows += rowsAdded; - pSCompBlock->keyFirst = MIN(pSCompBlock->keyFirst, pCompBlock->keyFirst); - pSCompBlock->keyLast = MAX(pSCompBlock->keyLast, pCompBlock->keyLast); + pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; + pSCompBlock->keyFirst = pMergeInfo->keyFirst; + pSCompBlock->keyLast = pMergeInfo->keyLast; pIdx->len += sizeof(SCompBlock); } else { // Need to create two sub-blocks void *ptr = NULL; @@ -989,11 +990,11 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId ((SCompBlock *)ptr)[1] = *pCompBlock; pSCompBlock->numOfSubBlocks = 2; - pSCompBlock->numOfRows += rowsAdded; + pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo); pSCompBlock->len = sizeof(SCompBlock) * 2; - pSCompBlock->keyFirst = MIN(((SCompBlock *)ptr)[0].keyFirst, ((SCompBlock *)ptr)[1].keyFirst); - pSCompBlock->keyLast = MAX(((SCompBlock *)ptr)[0].keyLast, ((SCompBlock *)ptr)[1].keyLast); + pSCompBlock->keyFirst = pMergeInfo->keyFirst; + pSCompBlock->keyLast = pMergeInfo->keyLast; pIdx->len += (sizeof(SCompBlock) * 2); } @@ -1047,6 +1048,45 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int return 0; } +static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) { + SCompIdx *pCompIdx = &(pHelper->curCompIdx); + + ASSERT(pCompIdx->numOfBlocks > 0 && blkIdx < pCompIdx->numOfBlocks); + + SCompBlock *pCompBlock= blockAtIdx(pHelper, blkIdx); + SCompBlock compBlock = *pCompBlock; + ASSERT(pCompBlock->numOfSubBlocks > 0 && pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS); + + if (pCompIdx->numOfBlocks == 1) { + memset(pCompIdx, 0, sizeof(*pCompIdx)); + } else { + int tsize = 0; + + if (compBlock.numOfSubBlocks > 1) { + tsize = (int)(pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks)); + + ASSERT(tsize > 0); + memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset), + POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks), + tsize); + + pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks; + } + + tsize = (int)(pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo)); + ASSERT(tsize > 0); + memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize); + + pCompIdx->len -= sizeof(SCompBlock); + + pCompIdx->numOfBlocks--; + pCompIdx->hasLast = (uint32_t)(blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last); + pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast; + } + + return 0; +} + static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { pHelper->idxH.numOfIdx = 0; pHelper->idxH.curIdx = 0; @@ -1210,7 +1250,7 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl return -1; } - if (taosTRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) { + if (taosRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompCol->len, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -1325,7 +1365,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (taosTRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) { + if (taosRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompBlock->len, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -1439,51 +1479,62 @@ static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { } static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { - STsdbCfg * pCfg = &(pHelper->pRepo->config); - STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = &(pHelper->curCompIdx); - TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; - SCompBlock compBlock = {0}; + STsdbCfg * pCfg = &(pHelper->pRepo->config); + STable * pTable = pCommitIter->pTable; + SCompIdx * pIdx = &(pHelper->curCompIdx); + TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); + int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; + SCompBlock compBlock = {0}; + SMergeInfo mergeInfo = {0}; + SMergeInfo *pMergeInfo = &mergeInfo; ASSERT(pIdx->len <= 0 || keyFirst > pIdx->maxKey); if (pIdx->hasLast) { // append to with last block ASSERT(pIdx->len > 0); SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1); ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock); - tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, - pDataCols, NULL, 0); - ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); - if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && - pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { - if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; - if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, rowsRead) < 0) return -1; - } else { - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows); + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols, + NULL, 0, pCfg->update, pMergeInfo); - if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows); + ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); - if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1; - if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; - } + if (pDataCols->numOfRows > 0) { + ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols))); - if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && + pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { + if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, pCompBlock->keyFirst); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, pCompBlock->keyLast); + if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1; + } else { + if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; + ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows); + + if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1; + ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows); + + if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1; + if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; + } + + if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + } } else { ASSERT(!pHelper->hasOldLastBlock); - tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0); - ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo); + ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); - if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; - if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1; + if (pDataCols->numOfRows > 0) { + ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols))); + if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; + if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1; + } } #ifndef NDEBUG TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter); - ASSERT(keyNext < 0 || keyNext > pIdx->maxKey); + ASSERT(keyNext == TSDB_DATA_TIMESTAMP_NULL || keyNext > pIdx->maxKey); #endif return 0; @@ -1491,13 +1542,16 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, int *blkIdx) { - STsdbCfg * pCfg = &(pHelper->pRepo->config); - STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = &(pHelper->curCompIdx); - SCompBlock compBlock = {0}; - TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; - SDataCols *pDataCols0 = pHelper->pDataCols[0]; + STsdbCfg * pCfg = &(pHelper->pRepo->config); + STable * pTable = pCommitIter->pTable; + SCompIdx * pIdx = &(pHelper->curCompIdx); + SCompBlock compBlock = {0}; + TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); + int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; + SDataCols * pDataCols0 = pHelper->pDataCols[0]; + SMergeInfo mergeInfo = {0}; + SMergeInfo *pMergeInfo = &mergeInfo; + SCompBlock oBlock = {0}; SSkipListIterator slIter = {0}; @@ -1507,123 +1561,82 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE); ASSERT(pCompBlock != NULL); int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock)); + oBlock = *pCompBlock; + + ASSERT((!TSDB_IS_LAST_BLOCK(&oBlock)) || (tblkIdx == pIdx->numOfBlocks - 1)); - if (pCompBlock->last) { - ASSERT(pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && tblkIdx == pIdx->numOfBlocks - 1); + if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) { + while (true) { + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0, + pCfg->update, pMergeInfo); + ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); + if (pDataCols->numOfRows == 0) break; + + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; + if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; + tblkIdx++; + } + ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) == TSDB_DATA_TIMESTAMP_NULL || + tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); + } else { int16_t colId = 0; + if (tsdbLoadBlockDataCols(pHelper, &oBlock, NULL, &colId, 1) < 0) return -1; + + TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1); + slIter = *(pCommitIter->pIter); - if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1; - ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows); + tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows, + pCfg->update, pMergeInfo); - int rows1 = defaultRowsInBlock - pCompBlock->numOfRows; - int rows2 = - tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows); - if (rows2 == 0) { // all data filtered out + if (pMergeInfo->nOperations == 0) { + // Do nothing + ASSERT(pMergeInfo->rowsDeleteFailed >= 0); *(pCommitIter->pIter) = slIter; + tblkIdx++; + } else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) { + // Delete the block and do some stuff + ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN); + if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1; + *pCommitIter->pIter = slIter; + if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + } else if (tsdbCheckAddSubBlockCond(pHelper, &oBlock, pMergeInfo, pDataCols->maxPoints)) { + // Append as a sub-block of the searched block + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData, + pDataCols0->numOfRows, pCfg->update, pMergeInfo); + ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0); + if (tsdbWriteBlockToFile(pHelper, oBlock.last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols, + &compBlock, oBlock.last, false) < 0) { + return -1; + } + if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) { + return -1; + } + tblkIdx++; } else { - if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock && - pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { - tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows); - ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; - if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; - tblkIdx++; - } else { - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - int round = 0; - int dIter = 0; - while (true) { - tdResetDataCols(pDataCols); - int rowsRead = - tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, defaultRowsInBlock); - if (rowsRead == 0) break; + // load the block data, merge with the memory data + if (tsdbLoadBlockData(pHelper, &oBlock, NULL) < 0) return -1; + int round = 0; + int dIter = 0; + while (true) { + tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock, + pCfg->update); + if (pDataCols->numOfRows == 0) break; + if (tblkIdx == pIdx->numOfBlocks - 1) { if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; - if (round == 0) { - if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } else { - if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } - - tblkIdx++; - round++; + } else { + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; } - } - if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; - } - } else { - TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (pCompBlock[1].keyFirst - 1); - TSKEY blkKeyFirst = pCompBlock->keyFirst; - TSKEY blkKeyLast = pCompBlock->keyLast; - if (keyFirst < blkKeyFirst) { - while (true) { - tdResetDataCols(pDataCols); - int rowsRead = - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, NULL, 0); - if (rowsRead == 0) break; - - ASSERT(rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; - if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - tblkIdx++; - } - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); - } else { - ASSERT(keyFirst <= blkKeyLast); - int16_t colId = 0; - if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1; - - slIter = *(pCommitIter->pIter); - int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows); - int rows2 = tsdbLoadDataFromCache(pTable, &slIter, blkKeyLast, INT_MAX, NULL, pDataCols0->cols[0].pData, - pDataCols0->numOfRows); - - if (rows2 == 0) { // all filtered out - *(pCommitIter->pIter) = slIter; - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); - } else { - int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0) + rows2; - - if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) { - int rows = (rows1 >= rows3) ? rows3 : rows2; - tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows); - ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0) - return -1; - if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; - tblkIdx++; - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); + if (round == 0) { + if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; } else { - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - int round = 0; - int dIter = 0; - while (true) { - int rowsRead = - tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock); - if (rowsRead == 0) break; - - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) - return -1; - if (round == 0) { - if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } else { - if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } - - round++; - tblkIdx++; - } - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); + if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; } + + round++; + tblkIdx++; } } } @@ -1632,9 +1645,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, return 0; } -static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows) { - int numOfRows = 0; +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, + TSKEY maxKey, int maxRows, int8_t update) { TSKEY key1 = INT64_MAX; TSKEY key2 = INT64_MAX; STSchema *pSchema = NULL; @@ -1644,35 +1656,62 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte while (true) { key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); + bool isRowDel = false; SDataRow row = tsdbNextIterRow(pCommitIter->pIter); - key2 = (row == NULL || dataRowKey(row) > maxKey) ? INT64_MAX : dataRowKey(row); + if (row == NULL || dataRowKey(row) > maxKey) { + key2 = INT64_MAX; + } else { + key2 = dataRowKey(row); + isRowDel = dataRowDeleted(row); + } if (key1 == INT64_MAX && key2 == INT64_MAX) break; - if (key1 <= key2) { + if (key1 < key2) { for (int i = 0; i < pDataCols->numOfCols; i++) { dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, pTarget->maxPoints); } + pTarget->numOfRows++; (*iter)++; - if (key1 == key2) tSkipListIterNext(pCommitIter->pIter); - } else { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); - ASSERT(pSchema != NULL); + } else if (key1 > key2) { + if (!isRowDel) { + if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + ASSERT(pSchema != NULL); + } + + tdAppendDataRowToDataCol(row, pSchema, pTarget); } - tdAppendDataRowToDataCol(row, pSchema, pTarget); + tSkipListIterNext(pCommitIter->pIter); + } else { + if (update) { + if (!isRowDel) { + if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + ASSERT(pSchema != NULL); + } + + tdAppendDataRowToDataCol(row, pSchema, pTarget); + } + } else { + ASSERT(!isRowDel); + + for (int i = 0; i < pDataCols->numOfCols; i++) { + dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, + pTarget->maxPoints); + } + + pTarget->numOfRows++; + } + (*iter)++; tSkipListIterNext(pCommitIter->pIter); } - numOfRows++; - if (numOfRows >= maxRows) break; - ASSERT(numOfRows == pTarget->numOfRows && numOfRows <= pTarget->maxPoints); + if (pTarget->numOfRows >= maxRows) break; } - - return numOfRows; } static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock) { @@ -1695,3 +1734,20 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, return 0; } + +static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps) { + STsdbCfg *pCfg = &(pHelper->pRepo->config); + int mergeRows = pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; + + ASSERT(mergeRows > 0); + + if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pMergeInfo->nOperations <= maxOps) { + if (pCompBlock->last) { + if (!TSDB_NLAST_FILE_OPENED(pHelper) && mergeRows < pCfg->minRowsPerFileBlock) return true; + } else { + if (mergeRows < pCfg->maxRowsPerFileBlock) return true; + } + } + + return false; +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index ac6c2e0c5a9d5590e7dd57863dd18d8726912de5..021c10ab6a53eac66d120d8b01574598ec6689ff 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -20,7 +20,6 @@ #include "exception.h" #include "../../query/inc/qAst.h" // todo move to common module -#include "../../query/inc/qExecutor.h" // todo move to common module #include "tlosertree.h" #include "tsdb.h" #include "tsdbMain.h" @@ -72,8 +71,8 @@ typedef struct STableCheckInfo { STable* pTableObj; SCompInfo* pCompInfo; int32_t compSize; - int32_t numOfBlocks; // number of qualified data blocks not the original blocks - int32_t chosen; // indicate which iterator should move forward + int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks + int8_t chosen:2; // indicate which iterator should move forward bool initBuf; // whether to initialize the in-memory skip list iterator or not SSkipListIterator* iter; // mem buffer skip list iterator SSkipListIterator* iiter; // imem buffer skip list iterator @@ -120,8 +119,9 @@ typedef struct STsdbQueryHandle { SDataCols *pDataCols; // in order to hold current file data block int32_t allocSize; // allocated data block size - SMemTable *mem; // mem-table - SMemTable *imem; // imem-table, acquired from snapshot + SMemRef *pMemRef; +// SMemTable *mem; // mem-table +// SMemTable *imem; // imem-table, acquired from snapshot SArray *defaultLoadColumn;// default load column SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */ @@ -184,26 +184,26 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS return pLocalIdList; } -static void tsdbMayTakeMemSnapshot(TsdbQueryHandleT pHandle) { - STsdbQueryHandle* pSecQueryHandle = (STsdbQueryHandle*) pHandle; - SQInfo *pQInfo = (SQInfo *)(pSecQueryHandle->qinfo); +static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { + assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL); - if (pQInfo->memRef.ref++ == 0) { - tsdbTakeMemSnapshot(pSecQueryHandle->pTsdb, &pSecQueryHandle->mem, &pSecQueryHandle->imem); - pQInfo->memRef.mem = pSecQueryHandle->mem; - pQInfo->memRef.imem = pSecQueryHandle->imem; - } else { - pSecQueryHandle->mem = (SMemTable *)(pQInfo->memRef.mem); - pSecQueryHandle->imem = (SMemTable *)(pQInfo->memRef.imem); + SMemRef* pMemRef = pQueryHandle->pMemRef; + if (pQueryHandle->pMemRef->ref++ == 0) { + tsdbTakeMemSnapshot(pQueryHandle->pTsdb, (SMemTable**)&(pMemRef->mem), (SMemTable**)&(pMemRef->imem)); } } -static void tsdbMayUnTakeMemSnapshot(TsdbQueryHandleT pHandle) { - STsdbQueryHandle* pSecQueryHandle = (STsdbQueryHandle*) pHandle; - SQInfo *pQInfo = (SQInfo *)(pSecQueryHandle->qinfo); - if (--pQInfo->memRef.ref == 0) { - tsdbUnTakeMemSnapShot(pSecQueryHandle->pTsdb, pSecQueryHandle->mem, pSecQueryHandle->imem); +static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { + assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL); + + SMemRef* pMemRef = pQueryHandle->pMemRef; + if (--pMemRef->ref == 0) { + tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, pMemRef->mem, pMemRef->imem); + pMemRef->mem = NULL; + pMemRef->imem = NULL; } + + pQueryHandle->pMemRef = NULL; } static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta) { size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList); @@ -270,7 +270,7 @@ static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey) return pNew; } -static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo) { +static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo, SMemRef* pMemRef) { STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); if (pQueryHandle == NULL) { goto out_of_memory; @@ -288,13 +288,14 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; pQueryHandle->allocSize = 0; pQueryHandle->locateStart = false; + pQueryHandle->pMemRef = pMemRef; if (tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb) != 0) { goto out_of_memory; } tsdbMayTakeMemSnapshot(pQueryHandle); - assert(pCond != NULL && pCond->numOfCols > 0); + assert(pCond != NULL && pCond->numOfCols > 0 && pMemRef != NULL); if (ASCENDING_TRAVERSE(pCond->order)) { assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); @@ -348,8 +349,8 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* return NULL; } -TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) { - STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo); +TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo, SMemRef* pRef) { + STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo, pRef); STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); @@ -366,7 +367,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab return (TsdbQueryHandleT) pQueryHandle; } -TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { +TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) { pCond->twindow = changeTableGroupByLastrow(groupList); // no qualified table @@ -374,7 +375,7 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab return NULL; } - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pMemRef); assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey); return pQueryHandle; @@ -396,8 +397,8 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) { return res; } -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo) { - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); +TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) { + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pRef); if (pQueryHandle != NULL) { pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; changeQueryHandleForInterpQuery(pQueryHandle); @@ -417,7 +418,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh int32_t order = pHandle->order; // no data in buffer, abort - if (pHandle->mem == NULL && pHandle->imem == NULL) { + if (pHandle->pMemRef->mem == NULL && pHandle->pMemRef->imem == NULL) { return false; } @@ -426,16 +427,19 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh STableData* pMem = NULL; STableData* pIMem = NULL; - if (pHandle->mem && pCheckInfo->tableId.tid < pHandle->mem->maxTables) { - pMem = pHandle->mem->tData[pCheckInfo->tableId.tid]; + SMemTable* pMemT = pHandle->pMemRef->mem; + SMemTable* pIMemT = pHandle->pMemRef->imem; + + if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { + pMem = pMemT->tData[pCheckInfo->tableId.tid]; if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid pCheckInfo->iter = tSkipListCreateIterFromVal(pMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } - if (pHandle->imem && pCheckInfo->tableId.tid < pHandle->imem->maxTables) { - pIMem = pHandle->imem->tData[pCheckInfo->tableId.tid]; + if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { + pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid pCheckInfo->iiter = tSkipListCreateIterFromVal(pIMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); @@ -457,7 +461,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); assert(node != NULL); - SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); + SDataRow row = (SDataRow)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", @@ -479,7 +483,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); assert(node != NULL); - SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); + SDataRow row = (SDataRow)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", @@ -504,19 +508,19 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { +static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) { SDataRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { - rmem = *(SDataRow *)SL_GET_NODE_DATA(node); + rmem = (SDataRow)SL_GET_NODE_DATA(node); } } if (pCheckInfo->iiter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { - rimem = *(SDataRow *)SL_GET_NODE_DATA(node); + rimem = (SDataRow)SL_GET_NODE_DATA(node); } } @@ -538,9 +542,15 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order TSKEY r2 = dataRowKey(rimem); if (r1 == r2) { // data ts are duplicated, ignore the data in mem - tSkipListIterNext(pCheckInfo->iter); - pCheckInfo->chosen = 1; - return rimem; + if (!update) { + tSkipListIterNext(pCheckInfo->iter); + pCheckInfo->chosen = 1; + return rimem; + } else { + tSkipListIterNext(pCheckInfo->iiter); + pCheckInfo->chosen = 0; + return rmem; + } } else { if (ASCENDING_TRAVERSE(order)) { if (r1 < r2) { @@ -594,6 +604,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { } static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { + STsdbCfg *pCfg = &pHandle->pTsdb->config; size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); pHandle->cur.fid = -1; @@ -607,7 +618,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { initTableMemIterator(pHandle, pCheckInfo); } - SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update); if (row == NULL) { return false; } @@ -827,11 +838,12 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){ SQueryFilePos* cur = &pQueryHandle->cur; + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); int32_t code = TSDB_CODE_SUCCESS; /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); assert(cur->pos >= 0 && cur->pos <= binfo.rows); @@ -1263,7 +1275,6 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl int32_t end = endPos; if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(start >= end); SWAP(start, end, int32_t); } @@ -1317,6 +1328,7 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { SQueryFilePos* cur = &pQueryHandle->cur; SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; initTableMemIterator(pQueryHandle, pCheckInfo); @@ -1354,7 +1366,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); if (row == NULL) { break; } @@ -1384,7 +1396,22 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it - moveToNextRowInMem(pCheckInfo); + if (pCfg->update) { + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable); + numOfRows += 1; + if (cur->win.skey == TSKEY_INITIAL_VAL) { + cur->win.skey = key; + } + + cur->win.ekey = key; + cur->lastKey = key + step; + cur->mixBlock = true; + + moveToNextRowInMem(pCheckInfo); + pos += step; + } else { + moveToNextRowInMem(pCheckInfo); + } } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { if (cur->win.skey == TSKEY_INITIAL_VAL) { @@ -1395,7 +1422,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* assert(end != -1); if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it - moveToNextRowInMem(pCheckInfo); + if (!pCfg->update) { + moveToNextRowInMem(pCheckInfo); + } else { + end -= step; + } } int32_t qstart = 0, qend = 0; @@ -1415,8 +1446,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* * copy them all to result buffer, since it may be overlapped with file data block. */ if (node == NULL || - ((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || + ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { // no data in cache or data in cache is greater than the ekey of time window, load data from file block if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; @@ -1513,15 +1544,15 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { } static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { - taosTFree(pSupporter->numOfBlocksPerTable); - taosTFree(pSupporter->blockIndexArray); + tfree(pSupporter->numOfBlocksPerTable); + tfree(pSupporter->blockIndexArray); for (int32_t i = 0; i < numOfTables; ++i) { STableBlockInfo* pBlockInfo = pSupporter->pDataBlockInfo[i]; - taosTFree(pBlockInfo); + tfree(pBlockInfo); } - taosTFree(pSupporter->pDataBlockInfo); + tfree(pSupporter->pDataBlockInfo); } static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) { @@ -1863,13 +1894,14 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STsdbQueryHandle* pQueryHandle) { int numOfRows = 0; int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; win->skey = TSKEY_INITIAL_VAL; int64_t st = taosGetTimestampUs(); STable* pTable = pCheckInfo->pTableObj; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); if (row == NULL) { break; } @@ -1920,9 +1952,9 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) { while (tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); - STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); + STable* pTable = (STable*) SL_GET_NODE_DATA((SSkipListNode*) pNode); - STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = pTable, .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(list, &info); } @@ -1938,7 +1970,7 @@ static void destroyHelper(void* param) { tQueryInfo* pInfo = (tQueryInfo*)param; if (pInfo->optr != TSDB_RELATION_IN) { - taosTFree(pInfo->q); + tfree(pInfo->q); } free(param); @@ -2001,9 +2033,9 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); } - STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo); + STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pQueryHandle->pMemRef); - taosTFree(cond.colList); + tfree(cond.colList); pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey); if (pSecQueryHandle->pTableCheckInfo == NULL) { @@ -2122,7 +2154,16 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { } } - // clear current group + // clear current group, unref unused table + for (int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pGroup, i); + + // keyInfo.pTable may be NULL here. + if (pKeyInfo->pTable != keyInfo.pTable) { + tsdbUnRefTable(pKeyInfo->pTable); + } + } + taosArrayClear(pGroup); // more than one table in each group, only one table left for each group @@ -2303,7 +2344,11 @@ void filterPrepare(void* expr, void* param) { if (pInfo->optr == TSDB_RELATION_IN) { pInfo->q = (char*) pCond->arr; } else { - pInfo->q = calloc(1, pSchema->bytes + TSDB_NCHAR_SIZE); // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space. + uint32_t size = pCond->nLen * TSDB_NCHAR_SIZE; + if (size < (uint32_t)pSchema->bytes) { + size = pSchema->bytes; + } + pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE); // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space. tVariantDump(pCond, pInfo->q, pSchema->type, true); } } @@ -2452,7 +2497,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC static bool indexedNodeFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; - STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); + STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); char* val = NULL; @@ -2705,7 +2750,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); destroyTableMemIterator(pTableCheckInfo); - taosTFree(pTableCheckInfo->pCompInfo); + tfree(pTableCheckInfo->pCompInfo); } taosArrayDestroy(pQueryHandle->pTableCheckInfo); } @@ -2714,14 +2759,14 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { size_t cols = taosArrayGetSize(pQueryHandle->pColumns); for (int32_t i = 0; i < cols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - taosTFree(pColInfo->pData); + tfree(pColInfo->pData); } taosArrayDestroy(pQueryHandle->pColumns); } taosArrayDestroy(pQueryHandle->defaultLoadColumn); - taosTFree(pQueryHandle->pDataBlockInfo); - taosTFree(pQueryHandle->statis); + tfree(pQueryHandle->pDataBlockInfo); + tfree(pQueryHandle->statis); // todo check error tsdbMayUnTakeMemSnapshot(pQueryHandle); @@ -2735,7 +2780,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %p", pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo); - taosTFree(pQueryHandle); + tfree(pQueryHandle); } void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index 605586515b7b1626751c01a550925c6e9ac4e183..ef5ed6f04459a4213e761f94ad00363ede9ecd26 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -80,7 +80,7 @@ static int insertData(SInsertInfo *pInfo) { pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) { - taosTFree(pMsg); + tfree(pMsg); return -1; } } @@ -88,7 +88,7 @@ static int insertData(SInsertInfo *pInfo) { double etime = getCurTime(); printf("Spent %f seconds to write %d records\n", etime - stime, pInfo->totalRows); - taosTFree(pMsg); + tfree(pMsg); return 0; } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 89c8e3dc39211eca1b6c877f7789ad4313917ea2..78b9c90979fa74bdd0ce5ecfd53042db30e4e8fa 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -1,13 +1,14 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z) IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) - ADD_SUBDIRECTORY(tests) + # ADD_SUBDIRECTORY(tests) FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 2a58bec8830faab4cf9909bd417a44e2e8881570..42bc136584d618e62826c307b698746ca387ff41 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -31,14 +31,16 @@ extern "C" { typedef void (*_hash_free_fn_t)(void *param); typedef struct SHashNode { - char *key; -// struct SHashNode *prev; +// char *key; struct SHashNode *next; - uint32_t hashVal; // the hash value of key, if hashVal == HASH_VALUE_IN_TRASH, this node is moved to trash + uint32_t hashVal; // the hash value of key uint32_t keyLen; // length of the key - char *data; +// char *data; } SHashNode; +#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode)) +#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->keyLen) + typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, HASH_ENTRY_LOCK = 1, @@ -175,6 +177,8 @@ void* taosHashDestroyIter(SHashMutableIterator* iter); */ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj); +size_t taosHashGetMemSize(const SHashObj *pHashObj); + #ifdef __cplusplus } #endif diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index 0520cf29a87c9d4727ef6db48d8f5712ac845b89..33819f6a20ee64ada194d520ef09c6133d4dad96 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -53,7 +53,7 @@ enum { TAOS_CFG_UTYPE_NONE, TAOS_CFG_UTYPE_PERCENT, TAOS_CFG_UTYPE_GB, - TAOS_CFG_UTYPE_Mb, + TAOS_CFG_UTYPE_MB, TAOS_CFG_UTYPE_BYTE, TAOS_CFG_UTYPE_SECOND, TAOS_CFG_UTYPE_MS diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h new file mode 100644 index 0000000000000000000000000000000000000000..10b7c1df35caaa675c9334106785b91af3fdeb05 --- /dev/null +++ b/src/util/inc/tfile.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TFILE_H +#define TDENGINE_TFILE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +// init taos file module +int32_t tfinit(); + +// clean up taos file module +void tfcleanup(); + +// the same syntax as UNIX standard open/close/read/write +// but FD is int64_t and will never be reused +int64_t tfopen(const char *pathname, int32_t flags); +int64_t tfclose(int64_t tfd); +int64_t tfwrite(int64_t tfd, void *buf, int64_t count); +int64_t tfread(int64_t tfd, void *buf, int64_t count); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TREF_H diff --git a/src/util/inc/tnettest.h b/src/util/inc/tnettest.h index 3fe1dfa9204fbbf85f193078b17e0bb4f9643848..426df5cbb28b9c0fcada049c7242730359d2a3fc 100644 --- a/src/util/inc/tnettest.h +++ b/src/util/inc/tnettest.h @@ -20,7 +20,27 @@ extern "C" { #endif -void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole); +typedef struct CmdArguments { + char* host; + char* password; + char* user; + char* auth; + char* database; + char* timezone; + bool is_raw_time; + bool is_use_passwd; + char file[TSDB_FILENAME_LEN]; + char dir[TSDB_FILENAME_LEN]; + int threadNum; + char* commands; + int abort; + int port; + int endPort; + int pktLen; + char* netTestRole; +} CmdArguments; + +void taosNetTest(CmdArguments* args); #ifdef __cplusplus } diff --git a/src/util/inc/tqueue.h b/src/util/inc/tqueue.h index 8493a64315966aa32b58359652d0f429e8e0916a..c3051464e556860178be36f3473f5e4686f6082e 100644 --- a/src/util/inc/tqueue.h +++ b/src/util/inc/tqueue.h @@ -20,6 +20,23 @@ extern "C" { #endif +/* + +This set of API for queue is designed specially for vnode/mnode. The main purpose is to +consume all the items instead of one item from a queue by one single read. Also, it can +combine multiple queues into a queue set, a consumer thread can consume a queue set via +a single API instead of looping every queue by itself. + +Notes: +1: taosOpenQueue/taosCloseQueue, taosOpenQset/taosCloseQset is NOT multi-thread safe +2: after taosCloseQueue/taosCloseQset is called, read/write operation APIs are not safe. +3: read/write operation APIs are multi-thread safe + +To remove the limitation and make this set of queue APIs multi-thread safe, REF(tref.c) +shall be used to set up the protection. + +*/ + typedef void* taos_queue; typedef void* taos_qset; typedef void* taos_qall; diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h new file mode 100644 index 0000000000000000000000000000000000000000..cd5092f30a290c51de49e38b0226bbed637dc0e6 --- /dev/null +++ b/src/util/inc/tref.h @@ -0,0 +1,75 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TREF_H +#define TDENGINE_TREF_H + +#ifdef __cplusplus +extern "C" { +#endif + +// open a reference set, max is the mod used by hash, fp is the pointer to free resource function +// return rsetId which will be used by other APIs. On error, -1 is returned, and terrno is set appropriately +int taosOpenRef(int max, void (*fp)(void *)); + +// close the reference set, refId is the return value by taosOpenRef +// return 0 if success. On error, -1 is returned, and terrno is set appropriately +int taosCloseRef(int refId); + +// add ref, p is the pointer to resource or pointer ID +// return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately +int64_t taosAddRef(int refId, void *p); + +// remove ref, rid is the reference ID returned by taosAddRef +// return 0 if success. On error, -1 is returned, and terrno is set appropriately +int taosRemoveRef(int rsetId, int64_t rid); + +// acquire ref, rid is the reference ID returned by taosAddRef +// return the resource p. On error, NULL is returned, and terrno is set appropriately +void *taosAcquireRef(int rsetId, int64_t rid); + +// release ref, rid is the reference ID returned by taosAddRef +// return 0 if success. On error, -1 is returned, and terrno is set appropriately +int taosReleaseRef(int rsetId, int64_t rid); + +// return the first reference if rid is 0, otherwise return the next after current reference. +// if return value is NULL, it means list is over(if terrno is set, it means error happens) +void *taosIterateRef(int rsetId, int64_t rid); + +// return the number of references in system +int taosListRef(); + +/* sample code to iterate the refs + +void demoIterateRefs(int rsetId) { + + void *p = taosIterateRef(refId, 0); + while (p) { + // process P + + // get the rid from p + + p = taosIterateRef(rsetId, rid); + } +} + +*/ + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TREF_H diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index a14a8565617bc08eefc989168a93f523f2e6caff..65b6482520c445fd65433300a5fcb569506c85e2 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -27,33 +27,25 @@ extern "C" { #define MAX_SKIP_LIST_LEVEL 15 #define SKIP_LIST_RECORD_PERFORMANCE 0 +// For key property setting +#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage) +#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case) +#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update=1 case) +// For thread safety setting +#define SL_THREAD_SAFE (uint8_t)0x4 + typedef char *SSkipListKey; typedef char *(*__sl_key_fn_t)(const void *); -/** - * the skip list node is located in a consecutive memory area, - * the format of skip list node is as follows: - * +------------+-----------------------+------------------------+-----+------+ - * | node level | forward pointer array | backward pointer array | key | data | - * +------------+-----------------------+------------------------+-----+------+ - */ typedef struct SSkipListNode { - uint8_t level; + uint8_t level; + void * pData; + struct SSkipListNode *forwards[]; } SSkipListNode; -#define SL_NODE_HEADER_SIZE(_l) (sizeof(SSkipListNode) + ((_l) << 1u) * POINTER_BYTES) - -#define SL_GET_FORWARD_POINTER(n, _l) ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode)))[(_l)] -#define SL_GET_BACKWARD_POINTER(n, _l) \ - ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)] - -#define SL_GET_NODE_DATA(n) ((char *)(n) + SL_NODE_HEADER_SIZE((n)->level)) -#define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n))) - -#define SL_GET_SL_MIN_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_FORWARD_POINTER((s)->pHead, 0))) -#define SL_GET_SL_MAX_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_BACKWARD_POINTER((s)->pTail, 0))) - -#define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n)) +#define SL_GET_NODE_DATA(n) (n)->pData +#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] +#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)] /* * @version 0.3 @@ -103,34 +95,23 @@ typedef struct tSkipListState { uint64_t nTotalElapsedTimeForInsert; } tSkipListState; -typedef struct SSkipListKeyInfo { - uint8_t dupKey : 2; // if allow duplicated key in the skip list - uint8_t type : 4; // key type - uint8_t freeNode:2; // free node when destroy the skiplist - uint8_t len; // maximum key length, used in case of string key -} SSkipListKeyInfo; - typedef struct SSkipList { __compar_fn_t comparFn; __sl_key_fn_t keyFn; - uint32_t size; + pthread_rwlock_t *lock; + uint16_t len; uint8_t maxLevel; + uint8_t flags; + uint8_t type; // static info above uint8_t level; - SSkipListKeyInfo keyInfo; - pthread_rwlock_t *lock; - SSkipListNode * pHead; // point to the first element - SSkipListNode * pTail; // point to the last element + uint32_t size; + SSkipListNode * pHead; // point to the first element + SSkipListNode * pTail; // point to the last element #if SKIP_LIST_RECORD_PERFORMANCE tSkipListState state; // skiplist state #endif } SSkipList; -/* - * iterate the skiplist - * this will cause the multi-thread problem, when the skiplist is destroyed, the iterate may - * continue iterating the skiplist, so add the reference count for skiplist - * TODO add the ref for skip list when one iterator is created - */ typedef struct SSkipListIterator { SSkipList * pSkipList; SSkipListNode *cur; @@ -139,114 +120,26 @@ typedef struct SSkipListIterator { SSkipListNode *next; // next points to the true qualified node in skip list } SSkipListIterator; -/** - * - * @param nMaxLevel maximum skip list level - * @param keyType type of key - * @param dupKey allow the duplicated key in the skip list - * @return - */ -SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe, - uint8_t freeNode, __sl_key_fn_t fn); - -/** - * - * @param pSkipList - * @return NULL will always be returned - */ -void *tSkipListDestroy(SSkipList *pSkipList); - -/** - * - * @param pSkipList - * @param level - * @param headSize - */ -void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize); - -/** - * put the skip list node into the skip list. - * If failed, NULL will be returned, otherwise, the pNode will be returned. - * - * @param pSkipList - * @param pNode - * @return - */ -SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode); - -/** - * get *all* nodes which key are equivalent to pKey - * - * @param pSkipList - * @param pKey - * @return - */ -SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); - -/** - * get the size of skip list - * @param pSkipList - * @return - */ -size_t tSkipListGetSize(const SSkipList *pSkipList); - -/** - * display skip list of the given level, for debug purpose only - * @param pSkipList - * @param nlevel - */ -void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); - -/** - * create skiplist iterator - * @param pSkipList - * @return - */ +#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE) +#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1)) +#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData)) +#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0)) +#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0)) +#define SL_SIZE(s) (s)->size + +SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, + __sl_key_fn_t fn); +void tSkipListDestroy(SSkipList *pSkipList); +SSkipListNode * tSkipListPut(SSkipList *pSkipList, void *pData); +SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); +void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); - -/** - * create skip list iterator from the given node and specified the order - * @param pSkipList - * @param pNode start position, instead of the first node in skip list - * @param order traverse order of the iterator - * @return - */ -SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order); - -/** - * forward the skip list iterator - * @param iter - * @return - */ -bool tSkipListIterNext(SSkipListIterator *iter); - -/** - * get the element of skip list node - * @param iter - * @return - */ -SSkipListNode *tSkipListIterGet(SSkipListIterator *iter); - -/** - * destroy the skip list node - * @param iter - * @return - */ -void *tSkipListDestroyIter(SSkipListIterator *iter); - -/* - * remove nodes of the pKey value. - * If more than one node has the same value, all will be removed - * - * @Return - * the count of removed nodes - */ -uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); - -/* - * remove the specified node in parameters - */ -void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); +SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order); +bool tSkipListIterNext(SSkipListIterator *iter); +SSkipListNode * tSkipListIterGet(SSkipListIterator *iter); +void * tSkipListDestroyIter(SSkipListIterator *iter); +uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); +void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); #ifdef __cplusplus } diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h index f14e8dbb356e131bbd580f8c5310bea944e71033..391cc44accadd785fb46e02aa79cde7617834c70 100644 --- a/src/util/inc/tsocket.h +++ b/src/util/inc/tsocket.h @@ -20,21 +20,21 @@ extern "C" { #endif -int taosReadn(SOCKET sock, char *buffer, int len); -int taosWriteMsg(SOCKET fd, void *ptr, int nbytes); -int taosReadMsg(SOCKET fd, void *ptr, int nbytes); -int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes); -int taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len); -int taosSetNonblocking(SOCKET sock, int on); +int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); +int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); +int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); +int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes); +int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len); +int32_t taosSetNonblocking(SOCKET sock, int32_t on); -SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); -SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); -SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); -int taosKeepTcpAlive(SOCKET sockFd); +SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); +SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); +int32_t taosKeepTcpAlive(SOCKET sockFd); -int taosGetFqdn(char *); +int32_t taosGetFqdn(char *); uint32_t taosGetIpFromFqdn(const char *); -void tinet_ntoa(char *ipstr, unsigned int ip); +void tinet_ntoa(char *ipstr, uint32_t ip); uint32_t ip2uint(const char *const ip_addr); #ifdef __cplusplus diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 625d4af1ac981c5d1f5f079f5b15533a4d63ef24..03a73424971ba7e4b77d508b12f71c729889a51a 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -22,14 +22,13 @@ #define DO_FREE_HASH_NODE(_n) \ do { \ - taosTFree((_n)->data); \ - taosTFree(_n); \ + tfree(_n); \ } while (0) #define FREE_HASH_NODE(_h, _n) \ do { \ if ((_h)->freeFp) { \ - (_h)->freeFp((_n)->data); \ + (_h)->freeFp(GET_HASH_NODE_DATA(_n)); \ } \ \ DO_FREE_HASH_NODE(_n); \ @@ -77,7 +76,7 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { assert(pNode->hashVal == hashVal); break; } @@ -115,11 +114,15 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p * @param dsize size of actual data * @return hash node */ -static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode *pNode, SHashNode *pNewNode) { +static FORCE_INLINE SHashNode *doUpdateHashNode(SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); - SWAP(pNode->key, pNewNode->key, void *); - SWAP(pNode->data, pNewNode->data, void *); + if (prev != NULL) { + prev->next = pNewNode; + } else { + pe->next = pNewNode; + } + pNewNode->next = pNode->next; return pNewNode; } @@ -208,12 +211,14 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da assert(pNode == NULL); } + SHashNode* prev = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { assert(pNode->hashVal == hashVal); break; } + prev = pNode; pNode = pNode->next; } @@ -239,7 +244,10 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da } else { // not support the update operation, return error if (pHashObj->enableUpdate) { - doUpdateHashNode(pNode, pNewNode); + doUpdateHashNode(pe, prev, pNode, pNewNode); + DO_FREE_HASH_NODE(pNode); + } else { + DO_FREE_HASH_NODE(pNewNode); } if (pHashObj->type == HASH_ENTRY_LOCK) { @@ -249,7 +257,6 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da // enable resize __rd_unlock(&pHashObj->lock, pHashObj->type); - DO_FREE_HASH_NODE(pNewNode); return pHashObj->enableUpdate ? 0 : -1; } } @@ -293,13 +300,13 @@ void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*f SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal); if (pNode != NULL) { if (fp != NULL) { - fp(pNode->data); + fp(GET_HASH_NODE_DATA(pNode)); } if (d != NULL) { - memcpy(d, pNode->data, dsize); + memcpy(d, GET_HASH_NODE_DATA(pNode), dsize); } else { - data = pNode->data; + data = GET_HASH_NODE_DATA(pNode); } } @@ -357,13 +364,13 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe SHashNode *pRes = NULL; // remove it - if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { pe->num -= 1; pRes = pNode; pe->next = pNode->next; } else { while (pNode->next != NULL) { - if (((pNode->next)->keyLen == keyLen) && (memcmp((pNode->next)->key, key, keyLen) == 0)) { + if (((pNode->next)->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY((pNode->next)), key, keyLen) == 0)) { assert((pNode->next)->hashVal == hashVal); break; } @@ -392,7 +399,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe __rd_unlock(&pHashObj->lock, pHashObj->type); if (data != NULL && pRes != NULL) { - memcpy(data, pRes->data, dsize); + memcpy(data, GET_HASH_NODE_DATA(pRes), dsize); } if (pRes != NULL) { @@ -426,7 +433,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi // todo remove the first node SHashNode *pNode = NULL; while((pNode = pEntry->next) != NULL) { - if (fp && (!fp(param, pNode->data))) { + if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) { pEntry->num -= 1; atomic_sub_fetch_64(&pHashObj->size, 1); @@ -451,7 +458,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi while ((pNext = pNode->next) != NULL) { // not qualified, remove it - if (fp && (!fp(param, pNext->data))) { + if (fp && (!fp(param, GET_HASH_NODE_DATA(pNext)))) { pNode->next = pNext->next; pEntry->num -= 1; atomic_sub_fetch_64(&pHashObj->size, 1); @@ -515,7 +522,7 @@ void taosHashCleanup(SHashObj *pHashObj) { size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); for (int32_t i = 0; i < memBlock; ++i) { void *p = taosArrayGetP(pHashObj->pMemBlock, i); - taosTFree(p); + tfree(p); } taosArrayDestroy(pHashObj->pMemBlock); @@ -605,7 +612,7 @@ bool taosHashIterNext(SHashMutableIterator *pIter) { } } -void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : iter->pCur->data; } +void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : GET_HASH_NODE_DATA(iter->pCur); } void *taosHashDestroyIter(SHashMutableIterator *iter) { if (iter == NULL) { @@ -743,21 +750,19 @@ void taosHashTableResize(SHashObj *pHashObj) { } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - SHashNode *pNewNode = calloc(1, sizeof(SHashNode)); + SHashNode *pNewNode = calloc(1, sizeof(SHashNode) + keyLen + dsize); if (pNewNode == NULL) { uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } - pNewNode->data = malloc(dsize + keyLen); - memcpy(pNewNode->data, pData, dsize); - - pNewNode->key = pNewNode->data + dsize; - memcpy(pNewNode->key, key, keyLen); - pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; + + memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); + memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); + return pNewNode; } @@ -798,3 +803,11 @@ SHashNode *getNextHashNode(SHashMutableIterator *pIter) { return NULL; } + +size_t taosHashGetMemSize(const SHashObj *pHashObj) { + if (pHashObj == NULL) { + return 0; + } + + return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); +} diff --git a/src/util/src/talgo.c b/src/util/src/talgo.c index 4b96e62e91ff440684328d7e7ea8c8c6cd783caa..278683539e3247b4b6dcd43687ac281368a7d31d 100644 --- a/src/util/src/talgo.c +++ b/src/util/src/talgo.c @@ -153,7 +153,7 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, size_t size, const void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ext_compar_fn_t comparFn) { char *buf = calloc(1, size); // prepare the swap buffer tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf); - taosTFree(buf); + tfree(buf); } void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t compar, int flags) { diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 6e20c1708dfc81728c6b961b9259d50e953b4b9d..9bdaf737009ea25f3d76353e394f43b5c070b027 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -228,7 +228,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v pCacheObj->freeFp(p->data); } - taosTFree(p); + tfree(p); } else { taosAddToTrashcan(pCacheObj, p); uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p->data); @@ -335,7 +335,7 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) { } void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { - if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0) { + if (pCacheObj == NULL) { return; } @@ -343,7 +343,12 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { uError("cache:%s, NULL data to release", pCacheObj->name); return; } - + + + // The operation of removal from hash table and addition to trashcan is not an atomic operation, + // therefore the check for the empty of both the hash table and the trashcan has a race condition. + // It happens when there is only one object in the cache, and two threads which has referenced this object + // start to free the it simultaneously [TD-1569]. size_t offset = offsetof(SCacheDataNode, data); SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset); @@ -609,7 +614,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { __cache_lock_destroy(pCacheObj); - taosTFree(pCacheObj->name); + tfree(pCacheObj->name); memset(pCacheObj, 0, sizeof(SCacheObj)); free(pCacheObj); } diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index ba711ced8ff074e639a046c5dd9908865a3e9468..1090ab810121bfbd11ae3d5c26011924a2bdc77e 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -367,7 +367,14 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_BOOL: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); case TSDB_DATA_TYPE_NCHAR: { - int32_t ret = wcsncmp((wchar_t*) f1, (wchar_t*) f2, size/TSDB_NCHAR_SIZE); + tstr* t1 = (tstr*) f1; + tstr* t2 = (tstr*) f2; + + if (t1->len != t2->len) { + return t1->len > t2->len? 1:-1; + } + + int32_t ret = wcsncmp((wchar_t*) t1->data, (wchar_t*) t2->data, t2->len/TSDB_NCHAR_SIZE); if (ret == 0) { return ret; } diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 0ec55841a060a49f4aa9e29981fa426e42d29d5c..e89dea5a244acb5823c49d4b7c9aefb4c254db4c 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -288,7 +288,7 @@ void taosReadGlobalLogCfg() { option = value = NULL; olen = vlen = 0; - taosGetline(&line, &len, fp); + tgetline(&line, &len, fp); line[len - 1] = 0; paGetToken(line, &option, &olen); @@ -302,7 +302,7 @@ void taosReadGlobalLogCfg() { taosReadLogOption(option, value); } - taosTFree(line); + tfree(line); fclose(fp); } @@ -334,7 +334,7 @@ bool taosReadGlobalCfg() { option = value = NULL; olen = vlen = 0; - taosGetline(&line, &len, fp); + tgetline(&line, &len, fp); line[len - 1] = 0; paGetToken(line, &option, &olen); @@ -354,8 +354,12 @@ bool taosReadGlobalCfg() { fclose(fp); - taosTFree(line); - + tfree(line); + + if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { + taosSetAllDebugFlag(); + } + return true; } diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c new file mode 100644 index 0000000000000000000000000000000000000000..27ba30fe8179ecd5e213d4cf78862fd4982b3ba4 --- /dev/null +++ b/src/util/src/tfile.c @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taoserror.h" +#include "tulog.h" +#include "tutil.h" +#include "tref.h" + +static int32_t tsFileRsetId = -1; + +static void taosCloseFile(void *p) { + close((int32_t)(uintptr_t)p); +} + +int32_t tfinit() { + tsFileRsetId = taosOpenRef(2000, taosCloseFile); + return tsFileRsetId; +} + +void tfcleanup() { + if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); + tsFileRsetId = -1; +} + +int64_t tfopen(const char *pathname, int32_t flags) { + int32_t fd = open(pathname, flags); + + if (fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + void *p = (void *)(int64_t)fd; + int64_t rid = taosAddRef(tsFileRsetId, p); + if (rid < 0) close(fd); + + return rid; +} + +int64_t tfclose(int64_t tfd) { + return taosRemoveRef(tsFileRsetId, tfd); +} + +int64_t tfwrite(int64_t tfd, void *buf, int64_t count) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return -1; + + int32_t fd = (int32_t)(uintptr_t)p; + + int64_t ret = taosWrite(fd, buf, count); + if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); + + taosReleaseRef(tsFileRsetId, tfd); + return ret; +} + +int64_t tfread(int64_t tfd, void *buf, int64_t count) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return -1; + + int32_t fd = (int32_t)(uintptr_t)p; + + int64_t ret = taosRead(fd, buf, count); + if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); + + taosReleaseRef(tsFileRsetId, tfd); + return ret; +} diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 6ba1d87d92ecf216bfde346f9d3ff1563515d34d..101dd4a6f4e726dc2f74f96f4a56db5135af5c24 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -14,9 +14,7 @@ */ #define _DEFAULT_SOURCE - #define TAOS_RANDOM_FILE_FAIL_TEST - #include "os.h" #include "hash.h" #include "taoserror.h" @@ -188,7 +186,7 @@ int tdKVStoreStartCommit(SKVStore *pStore) { goto _err; } - if (taosTSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { + if (taosSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { uError("failed to send file %d bytes since %s", TD_KVSTORE_HEADER_SIZE, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -248,13 +246,13 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe ASSERT(tlen == POINTER_DISTANCE(pBuf, buf)); ASSERT(tlen == sizeof(SKVRecord)); - if (taosTWrite(pStore->fd, buf, tlen) < tlen) { + if (taosWrite(pStore->fd, buf, tlen) < tlen) { uError("failed to write %d bytes to file %s since %s", tlen, pStore->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (taosTWrite(pStore->fd, cont, contLen) < contLen) { + if (taosWrite(pStore->fd, cont, contLen) < contLen) { uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno)); return -1; } @@ -292,7 +290,7 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) { void *pBuf = buf; tdEncodeKVRecord(&pBuf, &rInfo); - if (taosTWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) { + if (taosWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) { uError("failed to write %" PRId64 " bytes to file %s since %s", (int64_t)(POINTER_DISTANCE(pBuf, buf)), pStore->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -339,7 +337,7 @@ void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size) { int fd = open(fname, O_RDONLY); if (fd < 0) goto _err; - if (taosTRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err; + if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err; if (!taosCheckChecksumWhole((uint8_t *)buf, TD_KVSTORE_HEADER_SIZE)) goto _err; void *pBuf = (void *)buf; @@ -368,7 +366,7 @@ static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo, uint32_t return -1; } - if (taosTRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { + if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { uError("failed to read %d bytes from file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -402,7 +400,7 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) { ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE); taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE); - if (taosTWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { + if (taosWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -477,9 +475,9 @@ _err: static void tdFreeKVStore(SKVStore *pStore) { if (pStore) { - taosTFree(pStore->fname); - taosTFree(pStore->fsnap); - taosTFree(pStore->fnew); + tfree(pStore->fname); + tfree(pStore->fsnap); + tfree(pStore->fnew); taosHashCleanup(pStore->map); free(pStore); } @@ -535,7 +533,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE); while (true) { - ssize_t tsize = taosTRead(pStore->fd, tbuf, sizeof(SKVRecord)); + int64_t tsize = taosRead(pStore->fd, tbuf, sizeof(SKVRecord)); if (tsize == 0) break; if (tsize < sizeof(SKVRecord)) { uError("failed to read %" PRIzu " bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname, @@ -598,7 +596,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { goto _err; } - if (taosTRead(pStore->fd, buf, (size_t)pRecord->size) < pRecord->size) { + if (taosRead(pStore->fd, buf, (size_t)pRecord->size) < pRecord->size) { uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname, strerror(errno), pRecord->offset); terrno = TAOS_SYSTEM_ERROR(errno); @@ -618,11 +616,11 @@ static int tdRestoreKVStore(SKVStore *pStore) { if (pStore->aFunc) (*pStore->aFunc)(pStore->appH); taosHashDestroyIter(pIter); - taosTFree(buf); + tfree(buf); return 0; _err: taosHashDestroyIter(pIter); - taosTFree(buf); + tfree(buf); return -1; } diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 09b0933fd6e32e9b65c8c7acbb81fbfe7d5c005b..4157d88c45dda82d2ecef00a3aa52f1f492314a8 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -336,11 +336,11 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { lseek(tsLogObj.logHandle->fd, 0, SEEK_END); sprintf(name, "==================================================\n"); - taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); sprintf(name, " new log file \n"); - taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); sprintf(name, "==================================================\n"); - taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); return 0; } @@ -390,7 +390,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosTWrite(tsLogObj.logHandle->fd, buffer, len); + taosWrite(tsLogObj.logHandle->fd, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -400,7 +400,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } } - if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) taosWrite(1, buffer, (uint32_t)len); } void taosDumpData(unsigned char *msg, int32_t len) { @@ -419,7 +419,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { pos += 3; if (c >= 16) { temp[pos++] = '\n'; - taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos); c = 0; pos = 0; } @@ -427,9 +427,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { temp[pos++] = '\n'; - taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos); - - return; + taosWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos); } void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { @@ -467,7 +465,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosTWrite(tsLogObj.logHandle->fd, buffer, len); + taosWrite(tsLogObj.logHandle->fd, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -477,7 +475,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . } } - if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) taosWrite(1, buffer, (uint32_t)len); } #if 0 @@ -514,8 +512,8 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { return tLogBuff; _err: - taosTFree(LOG_BUF_BUFFER(tLogBuff)); - taosTFree(tLogBuff); + tfree(LOG_BUF_BUFFER(tLogBuff)); + tfree(tLogBuff); return NULL; } @@ -524,7 +522,7 @@ static void taosLogBuffDestroy(SLogBuff *tLogBuff) { tsem_destroy(&(tLogBuff->buffNotEmpty)); pthread_mutex_destroy(&(tLogBuff->buffMutex)); free(tLogBuff->buffer); - taosTFree(tLogBuff); + tfree(tLogBuff); } #endif @@ -606,7 +604,7 @@ static void *taosAsyncOutputLog(void *param) { while (1) { log_size = taosPollLogBuffer(tLogBuff, tempBuffer, TSDB_DEFAULT_LOG_BUF_UNIT); if (log_size) { - taosTWrite(tLogBuff->fd, tempBuffer, log_size); + taosWrite(tLogBuff->fd, tempBuffer, log_size); LOG_BUF_START(tLogBuff) = (LOG_BUF_START(tLogBuff) + log_size) % LOG_BUF_SIZE(tLogBuff); } else { break; diff --git a/src/util/src/tmempool.c b/src/util/src/tmempool.c index a3d10355003c85d993b3a37190081436c0fb19ab..678c965eb1a7315977616778c0e4b39ceb4c7525 100644 --- a/src/util/src/tmempool.c +++ b/src/util/src/tmempool.c @@ -52,9 +52,9 @@ mpool_h taosMemPoolInit(int numOfBlock, int blockSize) { if (pool_p->pool == NULL || pool_p->freeList == NULL) { uError("failed to allocate memory\n"); - taosTFree(pool_p->freeList); - taosTFree(pool_p->pool); - taosTFree(pool_p); + tfree(pool_p->freeList); + tfree(pool_p->pool); + tfree(pool_p); return NULL; } diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 3793f3d3a944cc5c8d86c0dc0c0fa5bfe3cee764..6fd526598365f831addd1bacb0b7f748d9552fdb 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -15,11 +15,16 @@ #include "os.h" #include "taosdef.h" +#include "taosmsg.h" #include "taoserror.h" #include "tulog.h" #include "tconfig.h" #include "tglobal.h" #include "tsocket.h" +#include "trpc.h" +#include "rpcHead.h" +#include "tutil.h" +#include "tnettest.h" #define MAX_PKG_LEN (64*1000) #define BUFFER_SIZE (MAX_PKG_LEN + 1024) @@ -30,9 +35,15 @@ typedef struct { uint16_t pktLen; } info_s; -static char serverFqdn[TSDB_FQDN_LEN]; +extern int tsRpcMaxUdpSize; + +static char g_user[TSDB_USER_LEN+1] = {0}; +static char g_pass[TSDB_PASSWORD_LEN+1] = {0}; +static char g_serverFqdn[TSDB_FQDN_LEN] = {0}; static uint16_t g_startPort = 0; static uint16_t g_endPort = 6042; +static uint32_t g_pktLen = 0; + static void *bindUdpPort(void *sarg) { info_s *pinfo = (info_s *)sarg; @@ -321,19 +332,145 @@ static void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uin return ; } -static void taosNetTestClient(const char* serverFqdn, uint16_t startPort, uint16_t endPort, int pktLen) { - uint32_t serverIp = taosGetIpFromFqdn(serverFqdn); - if (serverIp == 0xFFFFFFFF) { - printf("Failed to resolve FQDN:%s", serverFqdn); - exit(-1); +void* tnetInitRpc(char* secretEncrypt, char spi) { + SRpcInit rpcInit; + void* pRpcConn = NULL; + + taosEncryptPass((uint8_t *)g_pass, strlen(g_pass), secretEncrypt); + + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "NET-TEST"; + rpcInit.numOfThreads = 1; // every DB connection has only one thread + rpcInit.cfp = NULL; + rpcInit.sessions = 16; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = g_user; + rpcInit.idleTime = 2000; + rpcInit.ckey = "key"; + rpcInit.spi = spi; + rpcInit.secret = secretEncrypt; + + pRpcConn = rpcOpen(&rpcInit); + return pRpcConn; +} + +static int rpcCheckPortImpl(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi) { + SRpcEpSet epSet; + SRpcMsg reqMsg; + SRpcMsg rspMsg; + void* pRpcConn; + + char secretEncrypt[32] = {0}; + + pRpcConn = tnetInitRpc(secretEncrypt, spi); + if (NULL == pRpcConn) { + return -1; } - checkPort(serverIp, startPort, endPort, pktLen); + memset(&epSet, 0, sizeof(SRpcEpSet)); + epSet.inUse = 0; + epSet.numOfEps = 1; + epSet.port[0] = port; + strcpy(epSet.fqdn[0], serverFqdn); + + reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST; + reqMsg.pCont = rpcMallocCont(pktLen); + reqMsg.contLen = pktLen; + reqMsg.code = 0; + reqMsg.handle = NULL; // rpc handle returned to app + reqMsg.ahandle = NULL; // app handle set by client + + rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg); + + // handle response + if ((rspMsg.code != 0) || (rspMsg.msgType != TSDB_MSG_TYPE_NETWORK_TEST + 1)) { + //printf("code:%d[%s]\n", rspMsg.code, tstrerror(rspMsg.code)); + return -1; + } + + rpcFreeCont(rspMsg.pCont); - return; + rpcClose(pRpcConn); + + return 0; +} + +static void rpcCheckPort(uint32_t hostIp) { + int ret; + char spi; + + for (uint16_t port = g_startPort; port <= g_endPort; port++) { + //printf("test: %s:%d\n", info.host, port); + printf("\n"); + + //================ check tcp port ================ + int32_t pktLen; + if (g_pktLen <= tsRpcMaxUdpSize) { + pktLen = tsRpcMaxUdpSize + 1000; + } else { + pktLen = g_pktLen; + } + + spi = 1; + ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); + if (ret != 0) { + spi = 0; + ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); + if (ret != 0) { + printf("TCP port:%d test fail.\t\t", port); + } else { + //printf("tcp port:%d test ok.\t\t", port); + printf("TCP port:\033[32m%d test OK\033[0m\t\t", port); + } + } else { + //printf("tcp port:%d test ok.\t\t", port); + printf("TCP port:\033[32m%d test OK\033[0m\t\t", port); + } + + //================ check udp port ================ + if (g_pktLen >= tsRpcMaxUdpSize) { + pktLen = tsRpcMaxUdpSize - 1000; + } else { + pktLen = g_pktLen; + } + + spi = 0; + ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); + if (ret != 0) { + spi = 1; + ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); + if (ret != 0) { + printf("udp port:%d test fail.\t\n", port); + } else { + //printf("udp port:%d test ok.\t\n", port); + printf("UDP port:\033[32m%d test OK\033[0m\t\n", port); + } + } else { + //printf("udp port:%d test ok.\t\n", port); + printf("UDP port:\033[32m%d test OK\033[0m\t\n", port); + } + } + + printf("\n"); + return ; } +static void taosNetTestClient(int flag) { + uint32_t serverIp = taosGetIpFromFqdn(g_serverFqdn); + if (serverIp == 0xFFFFFFFF) { + printf("Failed to resolve FQDN:%s", g_serverFqdn); + exit(-1); + } + if (0 == flag) { + checkPort(serverIp, g_startPort, g_endPort, g_pktLen); + } else { + rpcCheckPort(serverIp); + } + + return; +} static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen) { @@ -375,49 +512,66 @@ static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen) } -void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole) { - if (pktLen > MAX_PKG_LEN) { - printf("test packet len overflow: %d, max len not greater than %d bytes\n", pktLen, MAX_PKG_LEN); - exit(-1); +void taosNetTest(CmdArguments *args) { + if (0 == args->pktLen) { + g_pktLen = 1000; + } else { + g_pktLen = args->pktLen; } - if (port && endPort) { - if (port > endPort) { - printf("endPort[%d] must not lesss port[%d]\n", endPort, port); + if (args->port && args->endPort) { + if (args->port > args->endPort) { + printf("endPort[%d] must not lesss port[%d]\n", args->endPort, args->port); exit(-1); } } - if (host && host[0] != 0) { - if (strlen(host) >= TSDB_EP_LEN) { - printf("host invalid: %s\n", host); + if (args->host && args->host[0] != 0) { + if (strlen(args->host) >= TSDB_EP_LEN) { + printf("host invalid: %s\n", args->host); exit(-1); } - taosGetFqdnPortFromEp(host, serverFqdn, &g_startPort); + taosGetFqdnPortFromEp(args->host, g_serverFqdn, &g_startPort); } else { - tstrncpy(serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN); + tstrncpy(g_serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN); g_startPort = tsServerPort; } - if (port) { - g_startPort = port; + if (args->port) { + g_startPort = args->port; } - if (endPort) { - g_endPort = endPort; + if (args->endPort) { + g_endPort = args->endPort; } - if (port > endPort) { + if (g_startPort > g_endPort) { printf("endPort[%d] must not lesss port[%d]\n", g_endPort, g_startPort); exit(-1); } + + + if (args->is_use_passwd) { + if (args->password == NULL) args->password = getpass("Enter password: "); + } else { + args->password = TSDB_DEFAULT_PASS; + } + tstrncpy(g_pass, args->password, TSDB_PASSWORD_LEN); + + if (args->user == NULL) { + args->user = TSDB_DEFAULT_USER; + } + tstrncpy(g_user, args->user, TSDB_USER_LEN); - if (0 == strcmp("client", netTestRole)) { - printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", serverFqdn, g_startPort, g_endPort, pktLen); - taosNetTestClient(serverFqdn, g_startPort, g_endPort, pktLen); - } else if (0 == strcmp("server", netTestRole)) { - taosNetTestServer(g_startPort, g_endPort, pktLen); + if (0 == strcmp("client", args->netTestRole)) { + printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen); + taosNetTestClient(0); + } else if (0 == strcmp("clients", args->netTestRole)) { + printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen); + taosNetTestClient(1); + } else if (0 == strcmp("server", args->netTestRole)) { + taosNetTestServer(g_startPort, g_endPort, g_pktLen); } } diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index 4f05277a847a07831371278915011ebc4da400c6..9536f6fb70f9fe6d74c981274c849cab0c00ef1f 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -265,7 +265,7 @@ void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...) buffer[len] = 0; if (pNote->taosNoteFd >= 0) { - taosTWrite(pNote->taosNoteFd, buffer, (unsigned int)len); + taosWrite(pNote->taosNoteFd, buffer, (unsigned int)len); if (pNote->taosNoteMaxLines > 0) { pNote->taosNoteLines++; diff --git a/src/util/src/tref.c b/src/util/src/tref.c new file mode 100644 index 0000000000000000000000000000000000000000..760d1c0eb47c8af1991d6551cf063630a24ce2ab --- /dev/null +++ b/src/util/src/tref.c @@ -0,0 +1,496 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taoserror.h" +#include "tulog.h" +#include "tutil.h" + +#define TSDB_REF_OBJECTS 50 +#define TSDB_REF_STATE_EMPTY 0 +#define TSDB_REF_STATE_ACTIVE 1 +#define TSDB_REF_STATE_DELETED 2 + +typedef struct SRefNode { + struct SRefNode *prev; // previous node + struct SRefNode *next; // next node + void *p; // pointer to resource protected, + int64_t rid; // reference ID + int32_t count; // number of references + int removed; // 1: removed +} SRefNode; + +typedef struct { + SRefNode **nodeList; // array of SRefNode linked list + int state; // 0: empty, 1: active; 2: deleted + int rsetId; // refSet ID, global unique + int64_t rid; // increase by one for each new reference + int max; // mod + int32_t count; // total number of SRefNodes in this set + int64_t *lockedBy; + void (*fp)(void *); +} SRefSet; + +static SRefSet tsRefSetList[TSDB_REF_OBJECTS]; +static pthread_once_t tsRefModuleInit = PTHREAD_ONCE_INIT; +static pthread_mutex_t tsRefMutex; +static int tsRefSetNum = 0; +static int tsNextId = 0; + +static void taosInitRefModule(void); +static void taosLockList(int64_t *lockedBy); +static void taosUnlockList(int64_t *lockedBy); +static void taosIncRsetCount(SRefSet *pSet); +static void taosDecRsetCount(SRefSet *pSet); +static int taosDecRefCount(int rsetId, int64_t rid, int remove); + +int taosOpenRef(int max, void (*fp)(void *)) +{ + SRefNode **nodeList; + SRefSet *pSet; + int64_t *lockedBy; + int i, rsetId; + + pthread_once(&tsRefModuleInit, taosInitRefModule); + + nodeList = calloc(sizeof(SRefNode *), (size_t)max); + if (nodeList == NULL) { + terrno = TSDB_CODE_REF_NO_MEMORY; + return -1; + } + + lockedBy = calloc(sizeof(int64_t), (size_t)max); + if (lockedBy == NULL) { + free(nodeList); + terrno = TSDB_CODE_REF_NO_MEMORY; + return -1; + } + + pthread_mutex_lock(&tsRefMutex); + + for (i = 0; i < TSDB_REF_OBJECTS; ++i) { + tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS; + if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId + if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break; + } + + if (i < TSDB_REF_OBJECTS) { + rsetId = tsNextId; + pSet = tsRefSetList + rsetId; + pSet->max = max; + pSet->nodeList = nodeList; + pSet->lockedBy = lockedBy; + pSet->fp = fp; + pSet->rid = 1; + pSet->rsetId = rsetId; + pSet->state = TSDB_REF_STATE_ACTIVE; + taosIncRsetCount(pSet); + + tsRefSetNum++; + uTrace("rsetId:%d is opened, max:%d, fp:%p refSetNum:%d", rsetId, max, fp, tsRefSetNum); + } else { + rsetId = TSDB_CODE_REF_FULL; + free (nodeList); + free (lockedBy); + uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); + } + + pthread_mutex_unlock(&tsRefMutex); + + return rsetId; +} + +int taosCloseRef(int rsetId) +{ + SRefSet *pSet; + int deleted = 0; + + if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { + uTrace("rsetId:%d is invalid, out of range", rsetId); + terrno = TSDB_CODE_REF_INVALID_ID; + return -1; + } + + pSet = tsRefSetList + rsetId; + + pthread_mutex_lock(&tsRefMutex); + + if (pSet->state == TSDB_REF_STATE_ACTIVE) { + pSet->state = TSDB_REF_STATE_DELETED; + deleted = 1; + uTrace("rsetId:%d is closed, count:%d", rsetId, pSet->count); + } else { + uTrace("rsetId:%d is already closed, count:%d", rsetId, pSet->count); + } + + pthread_mutex_unlock(&tsRefMutex); + + if (deleted) taosDecRsetCount(pSet); + + return 0; +} + +int64_t taosAddRef(int rsetId, void *p) +{ + int hash; + SRefNode *pNode; + SRefSet *pSet; + int64_t rid = 0; + + if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { + uTrace("rsetId:%d p:%p failed to add, rsetId not valid", rsetId, p); + terrno = TSDB_CODE_REF_INVALID_ID; + return -1; + } + + pSet = tsRefSetList + rsetId; + taosIncRsetCount(pSet); + if (pSet->state != TSDB_REF_STATE_ACTIVE) { + taosDecRsetCount(pSet); + uTrace("rsetId:%d p:%p failed to add, not active", rsetId, p); + terrno = TSDB_CODE_REF_ID_REMOVED; + return -1; + } + + pNode = calloc(sizeof(SRefNode), 1); + if (pNode == NULL) { + terrno = TSDB_CODE_REF_NO_MEMORY; + return -1; + } + + rid = atomic_add_fetch_64(&pSet->rid, 1); + hash = rid % pSet->max; + taosLockList(pSet->lockedBy+hash); + + pNode->p = p; + pNode->rid = rid; + pNode->count = 1; + + pNode->prev = NULL; + pNode->next = pSet->nodeList[hash]; + if (pSet->nodeList[hash]) pSet->nodeList[hash]->prev = pNode; + pSet->nodeList[hash] = pNode; + + uTrace("rsetId:%d p:%p rid:%" PRId64 " is added, count:%d", rsetId, p, rid, pSet->count); + + taosUnlockList(pSet->lockedBy+hash); + + return rid; +} + +int taosRemoveRef(int rsetId, int64_t rid) +{ + return taosDecRefCount(rsetId, rid, 1); +} + +// if rid is 0, return the first p in hash list, otherwise, return the next after current rid +void *taosAcquireRef(int rsetId, int64_t rid) +{ + int hash; + SRefNode *pNode; + SRefSet *pSet; + void *p = NULL; + + if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { + uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid); + terrno = TSDB_CODE_REF_INVALID_ID; + return NULL; + } + + if (rid <= 0) { + uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rid not valid", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; + return NULL; + } + + pSet = tsRefSetList + rsetId; + taosIncRsetCount(pSet); + if (pSet->state != TSDB_REF_STATE_ACTIVE) { + uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, not active", rsetId, rid); + taosDecRsetCount(pSet); + terrno = TSDB_CODE_REF_ID_REMOVED; + return NULL; + } + + hash = rid % pSet->max; + taosLockList(pSet->lockedBy+hash); + + pNode = pSet->nodeList[hash]; + + while (pNode) { + if (pNode->rid == rid) { + break; + } + + pNode = pNode->next; + } + + if (pNode) { + if (pNode->removed == 0) { + pNode->count++; + p = pNode->p; + uTrace("rsetId:%d p:%p rid:%" PRId64 " is acquired", rsetId, pNode->p, rid); + } else { + terrno = TSDB_CODE_REF_NOT_EXIST; + uTrace("rsetId:%d p:%p rid:%" PRId64 " is already removed, failed to acquire", rsetId, pNode->p, rid); + } + } else { + terrno = TSDB_CODE_REF_NOT_EXIST; + uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to acquire", rsetId, rid); + } + + taosUnlockList(pSet->lockedBy+hash); + + taosDecRsetCount(pSet); + + return p; +} + +int taosReleaseRef(int rsetId, int64_t rid) +{ + return taosDecRefCount(rsetId, rid, 0); +} + +// if rid is 0, return the first p in hash list, otherwise, return the next after current rid +void *taosIterateRef(int rsetId, int64_t rid) { + SRefNode *pNode = NULL; + SRefSet *pSet; + + if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { + uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rsetId not valid", rsetId, rid); + terrno = TSDB_CODE_REF_INVALID_ID; + return NULL; + } + + if (rid <= 0) { + uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rid not valid", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; + return NULL; + } + + pSet = tsRefSetList + rsetId; + taosIncRsetCount(pSet); + if (pSet->state != TSDB_REF_STATE_ACTIVE) { + uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rset not active", rsetId, rid); + terrno = TSDB_CODE_REF_ID_REMOVED; + taosDecRsetCount(pSet); + return NULL; + } + + int hash = 0; + if (rid > 0) { + hash = rid % pSet->max; + taosLockList(pSet->lockedBy+hash); + + pNode = pSet->nodeList[hash]; + while (pNode) { + if (pNode->rid == rid) break; + pNode = pNode->next; + } + + if (pNode == NULL) { + uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; + taosUnlockList(pSet->lockedBy+hash); + return NULL; + } + + // rid is there + pNode = pNode->next; + if (pNode == NULL) { + taosUnlockList(pSet->lockedBy+hash); + hash++; + } + } + + if (pNode == NULL) { + for (; hash < pSet->max; ++hash) { + taosLockList(pSet->lockedBy+hash); + pNode = pSet->nodeList[hash]; + if (pNode) break; + taosUnlockList(pSet->lockedBy+hash); + } + } + + void *newP = NULL; + if (pNode) { + pNode->count++; // acquire it + newP = pNode->p; + taosUnlockList(pSet->lockedBy+hash); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); + } else { + uTrace("rsetId:%d the list is over", rsetId); + } + + if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one + + taosDecRsetCount(pSet); + + return newP; +} + +int taosListRef() { + SRefSet *pSet; + SRefNode *pNode; + int num = 0; + + pthread_mutex_lock(&tsRefMutex); + + for (int i = 0; i < TSDB_REF_OBJECTS; ++i) { + pSet = tsRefSetList + i; + + if (pSet->state == TSDB_REF_STATE_EMPTY) + continue; + + uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count); + + for (int j=0; j < pSet->max; ++j) { + pNode = pSet->nodeList[j]; + + while (pNode) { + uInfo("rsetId:%d p:%p rid:%" PRId64 "count:%d", i, pNode->p, pNode->rid, pNode->count); + pNode = pNode->next; + num++; + } + } + } + + pthread_mutex_unlock(&tsRefMutex); + + return num; +} + +static int taosDecRefCount(int rsetId, int64_t rid, int remove) { + int hash; + SRefSet *pSet; + SRefNode *pNode; + int released = 0; + int code = 0; + + if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { + uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rsetId not valid", rsetId, rid); + terrno = TSDB_CODE_REF_INVALID_ID; + return -1; + } + + if (rid <= 0) { + uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rid not valid", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; + return -1; + } + + pSet = tsRefSetList + rsetId; + if (pSet->state == TSDB_REF_STATE_EMPTY) { + uTrace("rsetId:%d rid:%" PRId64 " failed to remove, cleaned", rsetId, rid); + terrno = TSDB_CODE_REF_ID_REMOVED; + return -1; + } + + hash = rid % pSet->max; + taosLockList(pSet->lockedBy+hash); + + pNode = pSet->nodeList[hash]; + while (pNode) { + if (pNode->rid == rid) + break; + + pNode = pNode->next; + } + + if (pNode) { + pNode->count--; + if (remove) pNode->removed = 1; + + if (pNode->count <= 0) { + if (pNode->prev) { + pNode->prev->next = pNode->next; + } else { + pSet->nodeList[hash] = pNode->next; + } + + if (pNode->next) { + pNode->next->prev = pNode->prev; + } + + (*pSet->fp)(pNode->p); + + uTrace("rsetId:%d p:%p rid:%" PRId64 "is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); + free(pNode); + released = 1; + } else { + uTrace("rsetId:%d p:%p rid:%" PRId64 "is released, count:%d", rsetId, pNode->p, rid, pNode->count); + } + } else { + uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; + code = -1; + } + + taosUnlockList(pSet->lockedBy+hash); + + if (released) taosDecRsetCount(pSet); + + return code; +} + +static void taosLockList(int64_t *lockedBy) { + int64_t tid = taosGetPthreadId(); + int i = 0; + while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { + if (++i % 100 == 0) { + sched_yield(); + } + } +} + +static void taosUnlockList(int64_t *lockedBy) { + int64_t tid = taosGetPthreadId(); + if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { + assert(false); + } +} + +static void taosInitRefModule(void) { + pthread_mutex_init(&tsRefMutex, NULL); +} + +static void taosIncRsetCount(SRefSet *pSet) { + atomic_add_fetch_32(&pSet->count, 1); + // uTrace("rsetId:%d inc count:%d", pSet->rsetId, count); +} + +static void taosDecRsetCount(SRefSet *pSet) { + int32_t count = atomic_sub_fetch_32(&pSet->count, 1); + // uTrace("rsetId:%d dec count:%d", pSet->rsetId, count); + + if (count > 0) return; + + pthread_mutex_lock(&tsRefMutex); + + if (pSet->state != TSDB_REF_STATE_EMPTY) { + pSet->state = TSDB_REF_STATE_EMPTY; + pSet->max = 0; + pSet->fp = NULL; + + tfree(pSet->nodeList); + tfree(pSet->lockedBy); + + tsRefSetNum--; + uTrace("rsetId:%d is cleaned, refSetNum:%d count:%d", pSet->rsetId, tsRefSetNum, pSet->count); + } + + pthread_mutex_unlock(&tsRefMutex); +} + diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index bacdaef6c8da1e42218331aaa34d25aa40ab5dc4..5ecc8ce119a925df47832684f58af34b79e8d773 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -13,13 +13,447 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "tskiplist.h" #include "os.h" +#include "tcompare.h" #include "tulog.h" -#include "tskiplist.h" #include "tutil.h" -#include "tcompare.h" -UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level) { // record link count in each level +static int initForwardBackwardPtr(SSkipList *pSkipList); +static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); +static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); +static void tSkipListCorrectLevel(SSkipList *pSkipList); +static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **backward, SSkipListNode *pNode); +static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData); +static SSkipListNode * tSkipListNewNode(uint8_t level); +#define tSkipListFreeNode(n) tfree((n)) + +static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList); +static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList); +static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList); +static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList); + +SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, + __sl_key_fn_t fn) { + SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); + if (pSkipList == NULL) return NULL; + + if (maxLevel > MAX_SKIP_LIST_LEVEL) { + maxLevel = MAX_SKIP_LIST_LEVEL; + } + + pSkipList->maxLevel = maxLevel; + pSkipList->type = keyType; + pSkipList->len = keyLen; + pSkipList->flags = flags; + pSkipList->keyFn = fn; + if (comparFn == NULL) { + pSkipList->comparFn = getKeyComparFunc(keyType); + } else { + pSkipList->comparFn = comparFn; + } + + if (initForwardBackwardPtr(pSkipList) < 0) { + tSkipListDestroy(pSkipList); + return NULL; + } + + if (SL_IS_THREAD_SAFE(pSkipList)) { + pSkipList->lock = (pthread_rwlock_t *)calloc(1, sizeof(pthread_rwlock_t)); + if (pSkipList->lock == NULL) { + tSkipListDestroy(pSkipList); + return NULL; + } + + if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { + tSkipListDestroy(pSkipList); + return NULL; + } + } + + srand((uint32_t)time(NULL)); + +#if SKIP_LIST_RECORD_PERFORMANCE + pSkipList->state.nTotalMemSize += sizeof(SSkipList); +#endif + + return pSkipList; +} + +void tSkipListDestroy(SSkipList *pSkipList) { + if (pSkipList == NULL) return; + + tSkipListWLock(pSkipList); + + SSkipListNode *pNode = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, 0); + + while (pNode != pSkipList->pTail) { + SSkipListNode *pTemp = pNode; + pNode = SL_NODE_GET_FORWARD_POINTER(pNode, 0); + tSkipListFreeNode(pTemp); + } + + tSkipListUnlock(pSkipList); + if (pSkipList->lock != NULL) { + pthread_rwlock_destroy(pSkipList->lock); + tfree(pSkipList->lock); + } + + tSkipListFreeNode(pSkipList->pHead); + tSkipListFreeNode(pSkipList->pTail); + tfree(pSkipList); +} + +SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { + if (pSkipList == NULL || pData == NULL) return NULL; + + SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0}; + uint8_t dupMode = SL_DUP_MODE(pSkipList); + SSkipListNode *pNode = NULL; + + tSkipListWLock(pSkipList); + + bool hasDup = tSkipListGetPosToPut(pSkipList, backward, pData); + + if (hasDup && (dupMode == SL_DISCARD_DUP_KEY || dupMode == SL_UPDATE_DUP_KEY)) { + if (dupMode == SL_UPDATE_DUP_KEY) { + pNode = SL_NODE_GET_BACKWARD_POINTER(backward[0], 0); + atomic_store_ptr(&(pNode->pData), pData); + } + } else { + pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList)); + if (pNode != NULL) { + pNode->pData = pData; + + tSkipListDoInsert(pSkipList, backward, pNode); + } + } + + tSkipListUnlock(pSkipList); + + return pNode; +} + +uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { + uint32_t count = 0; + + tSkipListWLock(pSkipList); + + SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL); + while (1) { + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0); + if (p == pSkipList->pTail) { + break; + } + if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { + break; + } + + tSkipListRemoveNodeImpl(pSkipList, p); + + ++count; + } + + tSkipListCorrectLevel(pSkipList); + + tSkipListUnlock(pSkipList); + + return count; +} + +SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { + SArray *sa = taosArrayInit(1, POINTER_BYTES); + + tSkipListRLock(pSkipList); + + SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL); + while (1) { + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0); + if (p == pSkipList->pTail) { + break; + } + if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { + break; + } + taosArrayPush(sa, &p); + pNode = p; + } + + tSkipListUnlock(pSkipList); + + return sa; +} + +void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) { + tSkipListWLock(pSkipList); + tSkipListRemoveNodeImpl(pSkipList, pNode); + tSkipListCorrectLevel(pSkipList); + tSkipListUnlock(pSkipList); +} + +SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList) { + if (pSkipList == NULL) return NULL; + + return doCreateSkipListIterator(pSkipList, TSDB_ORDER_ASC); +} + +SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order) { + ASSERT(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + ASSERT(pSkipList != NULL); + + SSkipListIterator *iter = doCreateSkipListIterator(pSkipList, order); + if (val == NULL) { + return iter; + } + + tSkipListRLock(pSkipList); + + iter->cur = getPriorNode(pSkipList, val, order, &(iter->next)); + + tSkipListUnlock(pSkipList); + + return iter; +} + +bool tSkipListIterNext(SSkipListIterator *iter) { + if (iter->pSkipList == NULL) return false; + + SSkipList *pSkipList = iter->pSkipList; + + tSkipListRLock(pSkipList); + + if (iter->order == TSDB_ORDER_ASC) { + if (iter->cur == pSkipList->pTail) return false; + iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); + + // a new node is inserted into between iter->cur and iter->next, ignore it + if (iter->cur != iter->next && (iter->next != NULL)) { + iter->cur = iter->next; + } + + iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); + iter->step++; + } else { + if (iter->cur == pSkipList->pHead) return false; + iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); + + // a new node is inserted into between iter->cur and iter->next, ignore it + if (iter->cur != iter->next && (iter->next != NULL)) { + iter->cur = iter->next; + } + + iter->next = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); + iter->step++; + } + + tSkipListUnlock(pSkipList); + + return (iter->order == TSDB_ORDER_ASC) ? (iter->cur != pSkipList->pTail) : (iter->cur != pSkipList->pHead); +} + +SSkipListNode *tSkipListIterGet(SSkipListIterator *iter) { + if (iter == NULL || iter->cur == iter->pSkipList->pTail || iter->cur == iter->pSkipList->pHead) { + return NULL; + } else { + return iter->cur; + } +} + +void *tSkipListDestroyIter(SSkipListIterator *iter) { + if (iter == NULL) { + return NULL; + } + + tfree(iter); + return NULL; +} + +void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { + if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) { + return; + } + + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); + + int32_t id = 1; + char * prev = NULL; + + while (p != pSkipList->pTail) { + char *key = SL_GET_NODE_KEY(pSkipList, p); + if (prev != NULL) { + ASSERT(pSkipList->comparFn(prev, key) < 0); + } + + switch (pSkipList->type) { + case TSDB_DATA_TYPE_INT: + fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key); + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BIGINT: + fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key); + break; + case TSDB_DATA_TYPE_BINARY: + fprintf(stdout, "%d: %s \n", id++, key); + break; + case TSDB_DATA_TYPE_DOUBLE: + fprintf(stdout, "%d: %lf \n", id++, *(double *)key); + break; + default: + fprintf(stdout, "\n"); + } + + prev = SL_GET_NODE_KEY(pSkipList, p); + + p = SL_NODE_GET_FORWARD_POINTER(p, nlevel - 1); + } +} + +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **backward, SSkipListNode *pNode) { + for (int32_t i = 0; i < pNode->level; ++i) { + if (i >= pSkipList->level) { + SL_NODE_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail; + SL_NODE_GET_BACKWARD_POINTER(pNode, i) = pSkipList->pHead; + SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode; + SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode; + } else { + SSkipListNode *x = backward[i]; + SL_NODE_GET_FORWARD_POINTER(pNode, i) = x; + + SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(x, i); + SL_NODE_GET_FORWARD_POINTER(prev, i) = pNode; + + SL_NODE_GET_BACKWARD_POINTER(x, i) = pNode; + SL_NODE_GET_BACKWARD_POINTER(pNode, i) = prev; + } + } + + if (pSkipList->level < pNode->level) pSkipList->level = pNode->level; + + pSkipList->size += 1; +} + +static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { + SSkipListIterator *iter = calloc(1, sizeof(SSkipListIterator)); + + iter->pSkipList = pSkipList; + iter->order = order; + if (order == TSDB_ORDER_ASC) { + iter->cur = pSkipList->pHead; + iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); + } else { + iter->cur = pSkipList->pTail; + iter->next = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); + } + + return iter; +} + +static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList) { + if (pSkipList->lock) { + return pthread_rwlock_wrlock(pSkipList->lock); + } + return 0; +} + +static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList) { + if (pSkipList->lock) { + return pthread_rwlock_rdlock(pSkipList->lock); + } + return 0; +} + +static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) { + if (pSkipList->lock) { + return pthread_rwlock_unlock(pSkipList->lock); + } + return 0; +} + +static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData) { + int compare = 0; + bool hasDupKey = false; + char * pDataKey = pSkipList->keyFn(pData); + + if (pSkipList->size == 0) { + for (int i = 0; i < pSkipList->level; i++) { + backward[i] = pSkipList->pTail; + } + } else { + char *pKey = NULL; + + // Compare max key + pKey = SL_GET_MAX_KEY(pSkipList); + compare = pSkipList->comparFn(pDataKey, pKey); + if (compare >= 0) { + for (int i = 0; i < pSkipList->level; i++) { + backward[i] = pSkipList->pTail; + } + + return (compare == 0); + } + + // Compare min key + pKey = SL_GET_MIN_KEY(pSkipList); + compare = pSkipList->comparFn(pDataKey, pKey); + if (compare < 0) { + for (int i = 0; i < pSkipList->level; i++) { + backward[i] = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i); + } + + return (compare == 0); + } + + SSkipListNode *px = pSkipList->pTail; + for (int i = pSkipList->level - 1; i >= 0; --i) { + SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(px, i); + while (p != pSkipList->pHead) { + pKey = SL_GET_NODE_KEY(pSkipList, p); + + compare = pSkipList->comparFn(pKey, pDataKey); + if (compare <= 0) { + if (compare == 0 && !hasDupKey) hasDupKey = true; + break; + } else { + px = p; + p = SL_NODE_GET_BACKWARD_POINTER(px, i); + } + } + + backward[i] = px; + } + } + + return hasDupKey; +} + +static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) { + int32_t level = pNode->level; + uint8_t dupMode = SL_DUP_MODE(pSkipList); + ASSERT(dupMode != SL_DISCARD_DUP_KEY && dupMode != SL_UPDATE_DUP_KEY); + + for (int32_t j = level - 1; j >= 0; --j) { + SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(pNode, j); + SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(pNode, j); + + SL_NODE_GET_FORWARD_POINTER(prev, j) = next; + SL_NODE_GET_BACKWARD_POINTER(next, j) = prev; + } + + tSkipListFreeNode(pNode); + pSkipList->size--; +} + +// Function must be called after calling tSkipListRemoveNodeImpl() function +static void tSkipListCorrectLevel(SSkipList *pSkipList) { + while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) { + pSkipList->level -= 1; + } +} + +UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, + int32_t level) { // record link count in each level #if SKIP_LIST_RECORD_PERFORMANCE for (int32_t i = 0; i < level; ++i) { pSkipList->state.nLevelNodeCnt[i]++; @@ -47,40 +481,28 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) { } static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) { - int32_t level = getSkipListNodeRandomHeight(pSkipList); + int32_t level = 0; if (pSkipList->size == 0) { level = 1; - pSkipList->level = 1; } else { + level = getSkipListNodeRandomHeight(pSkipList); if (level > pSkipList->level) { if (pSkipList->level < pSkipList->maxLevel) { - level = (++pSkipList->level); + level = pSkipList->level + 1; } else { level = pSkipList->level; } } } - - assert(level <= pSkipList->maxLevel); + + ASSERT(level <= pSkipList->maxLevel); return level; } -#define DO_MEMSET_PTR_AREA(n) do {\ -int32_t _l = (n)->level;\ -memset(pNode, 0, SL_NODE_HEADER_SIZE(_l));\ -(n)->level = _l;\ -} while(0) - -static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode); -static SSkipListNode* tSkipListPushBack(SSkipList *pSkipList, SSkipListNode *pNode); -static SSkipListNode* tSkipListPushFront(SSkipList* pSkipList, SSkipListNode *pNode); -static SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); - - // when order is TSDB_ORDER_ASC, return the last node with key less than val // when order is TSDB_ORDER_DESC, return the first node with key large than val -static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_t order, SSkipListNode** pCur) { - __compar_fn_t comparFn = pSkipList->comparFn; +static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur) { + __compar_fn_t comparFn = pSkipList->comparFn; SSkipListNode *pNode = NULL; if (pCur != NULL) { *pCur = NULL; @@ -89,12 +511,12 @@ static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_ if (order == TSDB_ORDER_ASC) { pNode = pSkipList->pHead; for (int32_t i = pSkipList->level - 1; i >= 0; --i) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, i); + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, i); while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); if (comparFn(key, val) < 0) { pNode = p; - p = SL_GET_FORWARD_POINTER(p, i); + p = SL_NODE_GET_FORWARD_POINTER(p, i); } else { if (pCur != NULL) { *pCur = p; @@ -106,12 +528,12 @@ static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_ } else { pNode = pSkipList->pTail; for (int32_t i = pSkipList->level - 1; i >= 0; --i) { - SSkipListNode *p = SL_GET_BACKWARD_POINTER(pNode, i); + SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(pNode, i); while (p != pSkipList->pHead) { char *key = SL_GET_NODE_KEY(pSkipList, p); if (comparFn(key, val) > 0) { pNode = p; - p = SL_GET_BACKWARD_POINTER(p, i); + p = SL_NODE_GET_BACKWARD_POINTER(p, i); } else { if (pCur != NULL) { *pCur = p; @@ -125,215 +547,38 @@ static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_ return pNode; } - -static bool initForwardBackwardPtr(SSkipList* pSkipList) { +static int initForwardBackwardPtr(SSkipList *pSkipList) { uint32_t maxLevel = pSkipList->maxLevel; - - // head info - pSkipList->pHead = (SSkipListNode *)calloc(1, SL_NODE_HEADER_SIZE(maxLevel) * 2); - if (pSkipList->pHead == NULL) { - return false; - } - - pSkipList->pHead->level = pSkipList->maxLevel; - - // tail info - pSkipList->pTail = (SSkipListNode*) ((char*) pSkipList->pHead + SL_NODE_HEADER_SIZE(maxLevel)); - pSkipList->pTail->level = pSkipList->maxLevel; - - for (uint32_t i = 0; i < maxLevel; ++i) { - SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail; - SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead; - } - - return true; -} - -SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t lock, - uint8_t freeNode, __sl_key_fn_t fn) { - SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); - if (pSkipList == NULL) { - return NULL; - } - - if (maxLevel > MAX_SKIP_LIST_LEVEL) { - maxLevel = MAX_SKIP_LIST_LEVEL; - } - - pSkipList->keyInfo.type = keyType; - pSkipList->keyInfo.len = keyLen; - pSkipList->keyInfo.dupKey = dupKey; - pSkipList->keyInfo.freeNode = freeNode; - - pSkipList->keyFn = fn; - pSkipList->comparFn = getKeyComparFunc(keyType); - pSkipList->maxLevel = maxLevel; - pSkipList->level = 1; - - if (!initForwardBackwardPtr(pSkipList)) { - taosTFree(pSkipList); - return NULL; - } - - if (lock) { - pSkipList->lock = calloc(1, sizeof(pthread_rwlock_t)); - - if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { - taosTFree(pSkipList->pHead); - taosTFree(pSkipList); - - return NULL; - } - } - - srand((uint32_t)time(NULL)); - -#if SKIP_LIST_RECORD_PERFORMANCE - pSkipList->state.nTotalMemSize += sizeof(SSkipList); -#endif - - return pSkipList; -} - -void *tSkipListDestroy(SSkipList *pSkipList) { - if (pSkipList == NULL) { - return NULL; - } - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - if (pSkipList->keyInfo.freeNode) { - SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); - while (pNode != pSkipList->pTail) { - SSkipListNode *pTemp = pNode; - pNode = SL_GET_FORWARD_POINTER(pNode, 0); - taosTFree(pTemp); - } - } - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - pthread_rwlock_destroy(pSkipList->lock); + // head info + pSkipList->pHead = tSkipListNewNode(maxLevel); + if (pSkipList->pHead == NULL) return -1; - taosTFree(pSkipList->lock); + // tail info + pSkipList->pTail = tSkipListNewNode(maxLevel); + if (pSkipList->pTail == NULL) { + tSkipListFreeNode(pSkipList->pHead); + return -1; } - taosTFree(pSkipList->pHead); - taosTFree(pSkipList); - return NULL; -} - -void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize) { - if (pSkipList == NULL) { - *level = 1; - *headSize = SL_NODE_HEADER_SIZE(*level); - return; + for (uint32_t i = 0; i < maxLevel; ++i) { + SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail; + SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead; } - *level = getSkipListRandLevel(pSkipList); - *headSize = SL_NODE_HEADER_SIZE(*level); + return 0; } -SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { - if (pSkipList == NULL || pNode == NULL) { - return NULL; - } - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - // if the new key is greater than the maximum key of skip list, push back this node at the end of skip list - char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode); - if (pSkipList->size == 0 || pSkipList->comparFn(SL_GET_SL_MAX_KEY(pSkipList), newDatakey) < 0) { - return tSkipListPushBack(pSkipList, pNode); - } - - // if the new key is less than the minimum key of skip list, push front this node at the front of skip list - assert(pSkipList->size > 0); - char* minKey = SL_GET_SL_MIN_KEY(pSkipList); - if (pSkipList->comparFn(newDatakey, minKey) < 0) { - return tSkipListPushFront(pSkipList, pNode); - } - - // find the appropriated position to insert data - SSkipListNode *px = pSkipList->pHead; - SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; - - int32_t ret = -1; - for (int32_t i = pSkipList->level - 1; i >= 0; --i) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); - while (p != pSkipList->pTail) { - char *key = SL_GET_NODE_KEY(pSkipList, p); - - // if the forward element is less than the specified key, forward one step - ret = pSkipList->comparFn(key, newDatakey); - if (ret < 0) { - px = p; - p = SL_GET_FORWARD_POINTER(px, i); - } else { - break; - } - } - - forward[i] = px; - } +static SSkipListNode *tSkipListNewNode(uint8_t level) { + int32_t tsize = sizeof(SSkipListNode) + sizeof(SSkipListNode *) * level * 2; - // if the skip list does not allowed identical key inserted, the new data will be discarded. - if (pSkipList->keyInfo.dupKey == 0 && ret == 0) { - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } + SSkipListNode *pNode = (SSkipListNode *)calloc(1, tsize); + if (pNode == NULL) return NULL; - return NULL; - } - - tSkipListDoInsert(pSkipList, forward, pNode); + pNode->level = level; return pNode; } - - -SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { - SArray* sa = taosArrayInit(1, POINTER_BYTES); - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL); - while (1) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); - if (p == pSkipList->pTail) { - break; - } - if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { - break; - } - taosArrayPush(sa, &p); - pNode = p; - } - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return sa; -} - - - -size_t tSkipListGetSize(const SSkipList* pSkipList) { - if (pSkipList == NULL) { - return 0; - } - - return pSkipList->size; -} - // static int32_t tSkipListEndParQuery(SSkipList *pSkipList, SSkipListNode *pStartNode, SSkipListKey *pEndKey, // int32_t cond, SSkipListNode ***pRes) { // pthread_rwlock_rdlock(&pSkipList->lock); @@ -447,178 +692,13 @@ size_t tSkipListGetSize(const SSkipList* pSkipList) { // } // // // compress the minimum level of skip list -// while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) { +// while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) { // pSkipList->level -= 1; // } // // return true; //} - -uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { - uint32_t count = 0; - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL); - while (1) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); - if (p == pSkipList->pTail) { - break; - } - if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { - break; - } - - for (int32_t j = p->level - 1; j >= 0; --j) { - SSkipListNode* prev = SL_GET_BACKWARD_POINTER(p, j); - SSkipListNode* next = SL_GET_FORWARD_POINTER(p, j); - SL_GET_FORWARD_POINTER(prev, j) = next; - SL_GET_BACKWARD_POINTER(next, j) = prev; - } - - if (pSkipList->keyInfo.freeNode) { - taosTFree(p); - } - - ++count; - } - - // compress the minimum level of skip list - while (pSkipList->level > 0) { - if (SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) != NULL) { - break; - } - pSkipList->level--; - } - - pSkipList->size -= count; - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return count; -} - -void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) { - int32_t level = pNode->level; - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - for (int32_t j = level - 1; j >= 0; --j) { - SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pNode, j); - SSkipListNode* next = SL_GET_FORWARD_POINTER(pNode, j); - - SL_GET_FORWARD_POINTER(prev, j) = next; - SL_GET_BACKWARD_POINTER(next, j) = prev; - } - - if (pSkipList->keyInfo.freeNode) { - taosTFree(pNode); - } - - atomic_sub_fetch_32(&pSkipList->size, 1); - - // compress the minimum level of skip list - while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) { - pSkipList->level -= 1; - } - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } -} - -SSkipListIterator* tSkipListCreateIter(SSkipList *pSkipList) { - if (pSkipList == NULL) { - return NULL; - } - - return doCreateSkipListIterator(pSkipList, TSDB_ORDER_ASC); -} - -SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order) { - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - assert(pSkipList != NULL); - - SSkipListIterator* iter = doCreateSkipListIterator(pSkipList, order); - if (val == NULL) { - return iter; - } - - if (pSkipList->lock) { - pthread_rwlock_rdlock(pSkipList->lock); - } - - iter->cur = getPriorNode(pSkipList, val, order, &iter->next); - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return iter; -} - -bool tSkipListIterNext(SSkipListIterator *iter) { - if (iter->pSkipList == NULL) { - return false; - } - - SSkipList *pSkipList = iter->pSkipList; - - if (pSkipList->lock) { - pthread_rwlock_rdlock(pSkipList->lock); - } - - if (iter->order == TSDB_ORDER_ASC) { // ascending order iterate - iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); - - // a new node is inserted into between iter->cur and iter->next, ignore it - if (iter->cur != iter->next && (iter->next != NULL)) { - iter->cur = iter->next; - } - - iter->next = SL_GET_FORWARD_POINTER(iter->cur, 0); - } else { // descending order iterate - iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); - - // a new node is inserted into between iter->cur and iter->next, ignore it - if (iter->cur != iter->next && (iter->next != NULL)) { - iter->cur = iter->next; - } - - iter->next = SL_GET_BACKWARD_POINTER(iter->cur, 0); - } - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - iter->step += 1; - return (iter->order == TSDB_ORDER_ASC)? (iter->cur != pSkipList->pTail) : (iter->cur != pSkipList->pHead); -} - -SSkipListNode *tSkipListIterGet(SSkipListIterator *iter) { - if (iter == NULL || iter->cur == iter->pSkipList->pTail || iter->cur == iter->pSkipList->pHead) { - return NULL; - } else { - return iter->cur; - } -} - -void* tSkipListDestroyIter(SSkipListIterator* iter) { - if (iter == NULL) { - return NULL; - } - - taosTFree(iter); - return NULL; -} - +// // bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey) { // SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; // __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); @@ -638,111 +718,3 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // // return ret; //} - -void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { - if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) { - return; - } - - SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); - - int32_t id = 1; - char* prev = NULL; - - while (p != pSkipList->pTail) { - char *key = SL_GET_NODE_KEY(pSkipList, p); - if (prev != NULL) { - assert(pSkipList->comparFn(prev, key) < 0); - } - - switch (pSkipList->keyInfo.type) { - case TSDB_DATA_TYPE_INT: - fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key); - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BIGINT: - fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key); - break; - case TSDB_DATA_TYPE_BINARY: - fprintf(stdout, "%d: %s \n", id++, key); - break; - case TSDB_DATA_TYPE_DOUBLE: - fprintf(stdout, "%d: %lf \n", id++, *(double *)key); - break; - default: - fprintf(stdout, "\n"); - } - - prev = SL_GET_NODE_KEY(pSkipList, p); - - p = SL_GET_FORWARD_POINTER(p, nlevel - 1); - } -} - -void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { - DO_MEMSET_PTR_AREA(pNode); - - for (int32_t i = 0; i < pNode->level; ++i) { - SSkipListNode *x = forward[i]; - SL_GET_BACKWARD_POINTER(pNode, i) = x; - - SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); - SL_GET_BACKWARD_POINTER(next, i) = pNode; - - SL_GET_FORWARD_POINTER(pNode, i) = next; - SL_GET_FORWARD_POINTER(x, i) = pNode; - } - - atomic_add_fetch_32(&pSkipList->size, 1); - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } -} - -SSkipListNode* tSkipListPushFront(SSkipList* pSkipList, SSkipListNode *pNode) { - SSkipListNode* forward[MAX_SKIP_LIST_LEVEL] = {0}; - for(int32_t i = 0; i < pSkipList->level; ++i) { - forward[i] = pSkipList->pHead; - } - - tSkipListDoInsert(pSkipList, forward, pNode); - return pNode; -} - -SSkipListNode* tSkipListPushBack(SSkipList *pSkipList, SSkipListNode *pNode) { - // do clear pointer area - DO_MEMSET_PTR_AREA(pNode); - - for(int32_t i = 0; i < pNode->level; ++i) { - SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pSkipList->pTail, i); - SL_GET_FORWARD_POINTER(prev, i) = pNode; - SL_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail; - - SL_GET_BACKWARD_POINTER(pNode, i) = prev; - SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode; - } - - atomic_add_fetch_32(&pSkipList->size, 1); - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return pNode; -} - -SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { - SSkipListIterator* iter = calloc(1, sizeof(SSkipListIterator)); - - iter->pSkipList = pSkipList; - iter->order = order; - if(order == TSDB_ORDER_ASC) { - iter->cur = pSkipList->pHead; - iter->next = SL_GET_FORWARD_POINTER(iter->cur, 0); - } else { - iter->cur = pSkipList->pTail; - iter->next = SL_GET_BACKWARD_POINTER(iter->cur, 0); - } - - return iter; -} \ No newline at end of file diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 4cf73e6dff8aa534e87145a3e1cb04e48d759c65..1003bc6178754e2eb6a3f8dbc61d0a878022e8f5 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -18,7 +18,7 @@ #include "tsocket.h" #include "taoserror.h" -int taosGetFqdn(char *fqdn) { +int32_t taosGetFqdn(char *fqdn) { char hostname[1024]; hostname[1023] = '\0'; if (gethostname(hostname, 1023) == -1) { @@ -26,10 +26,10 @@ int taosGetFqdn(char *fqdn) { return -1; } - struct addrinfo hints = {0}; + struct addrinfo hints = {0}; struct addrinfo *result = NULL; hints.ai_flags = AI_CANONNAME; - int ret = getaddrinfo(hostname, NULL, &hints, &result); + int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); return -1; @@ -49,10 +49,10 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) { int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result); if (result) { - struct sockaddr *sa = result->ai_addr; - struct sockaddr_in *si = (struct sockaddr_in*)sa; - struct in_addr ia = si->sin_addr; - uint32_t ip = ia.s_addr; + struct sockaddr * sa = result->ai_addr; + struct sockaddr_in *si = (struct sockaddr_in *)sa; + struct in_addr ia = si->sin_addr; + uint32_t ip = ia.s_addr; freeaddrinfo(result); return ip; } else { @@ -70,7 +70,7 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) { } } -// Function converting an IP address string to an unsigned int. +// Function converting an IP address string to an uint32_t. uint32_t ip2uint(const char *const ip_addr) { char ip_addr_cpy[20]; char ip[5]; @@ -81,7 +81,7 @@ uint32_t ip2uint(const char *const ip_addr) { s_start = ip_addr_cpy; s_end = ip_addr_cpy; - int k; + int32_t k; for (k = 0; *s_start != '\0'; s_start = s_end) { for (s_end = s_start; *s_end != '.' && *s_end != '\0'; s_end++) { @@ -95,17 +95,17 @@ uint32_t ip2uint(const char *const ip_addr) { ip[k] = '\0'; - return *((unsigned int *)ip); + return *((uint32_t *)ip); } -int taosWriteMsg(SOCKET fd, void *buf, int nbytes) { - int nleft, nwritten; - char *ptr = (char *)buf; +int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { + int32_t nleft, nwritten; + char * ptr = (char *)buf; nleft = nbytes; while (nleft > 0) { - nwritten = (int)taosWriteSocket(fd, (char *)ptr, (size_t)nleft); + nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft); if (nwritten <= 0) { if (errno == EINTR) continue; @@ -120,16 +120,16 @@ int taosWriteMsg(SOCKET fd, void *buf, int nbytes) { return (nbytes - nleft); } -int taosReadMsg(SOCKET fd, void *buf, int nbytes) { - int nleft, nread; - char *ptr = (char *)buf; +int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { + int32_t nleft, nread; + char * ptr = (char *)buf; nleft = nbytes; if (fd < 0) return -1; while (nleft > 0) { - nread = (int)taosReadSocket(fd, ptr, (size_t)nleft); + nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft); if (nread == 0) { break; } else if (nread < 0) { @@ -147,11 +147,11 @@ int taosReadMsg(SOCKET fd, void *buf, int nbytes) { return (nbytes - nleft); } -int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) { +int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { taosSetNonblocking(fd, 1); - int nleft, nwritten, nready; - fd_set fset; + int32_t nleft, nwritten, nready; + fd_set fset; struct timeval tv; nleft = nbytes; @@ -160,7 +160,7 @@ int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) { tv.tv_usec = 0; FD_ZERO(&fset); FD_SET(fd, &fset); - if ((nready = select((int)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { + if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { errno = ETIMEDOUT; uError("fd %d timeout, no enough space to write", fd); break; @@ -172,7 +172,7 @@ int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) { return -1; } - nwritten = (int)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL); + nwritten = (int32_t)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL); if (nwritten <= 0) { if (errno == EAGAIN || errno == EINTR) continue; @@ -189,10 +189,10 @@ int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) { return (nbytes - nleft); } -int taosReadn(SOCKET fd, char *ptr, int nbytes) { - int nread, nready, nleft = nbytes; +int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { + int32_t nread, nready, nleft = nbytes; - fd_set fset; + fd_set fset; struct timeval tv; while (nleft > 0) { @@ -200,7 +200,7 @@ int taosReadn(SOCKET fd, char *ptr, int nbytes) { tv.tv_usec = 0; FD_ZERO(&fset); FD_SET(fd, &fset); - if ((nready = select((int)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { + if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { errno = ETIMEDOUT; uError("fd %d timeout\n", fd); break; @@ -210,7 +210,7 @@ int taosReadn(SOCKET fd, char *ptr, int nbytes) { return -1; } - if ((nread = (int)taosReadSocket(fd, ptr, (size_t)nleft)) < 0) { + if ((nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft)) < 0) { if (errno == EINTR) continue; uError("read error, %d (%s)", errno, strerror(errno)); return -1; @@ -229,8 +229,8 @@ int taosReadn(SOCKET fd, char *ptr, int nbytes) { SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { struct sockaddr_in localAddr; - SOCKET sockFd; - int bufSize = 1024000; + SOCKET sockFd; + int32_t bufSize = 1024000; uDebug("open udp socket:0x%x:%hu", ip, port); @@ -239,7 +239,7 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { localAddr.sin_addr.s_addr = ip; localAddr.sin_port = (uint16_t)htons(port); - if ((sockFd = (int)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { + if ((sockFd = (int32_t)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { uError("failed to open udp socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); return -1; @@ -268,9 +268,9 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { } SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { - SOCKET sockFd = 0; + SOCKET sockFd = 0; + int32_t ret; struct sockaddr_in serverAddr, clientAddr; - int ret; sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); @@ -281,7 +281,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie } /* set REUSEADDR option, so the portnumber can be re-used */ - int reuse = 1; + int32_t reuse = 1; if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); taosCloseSocket(sockFd); @@ -296,8 +296,8 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie /* bind socket to client address */ if (bind(sockFd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) { - uError("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", - clientIp, destIp, destPort, strerror(errno)); + uError("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort, + strerror(errno)); taosCloseSocket(sockFd); return -1; } @@ -311,7 +311,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); if (ret != 0) { - //uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); + // uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); taosCloseSocket(sockFd); sockFd = -1; } else { @@ -321,36 +321,36 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie return sockFd; } -int taosKeepTcpAlive(SOCKET sockFd) { - int alive = 1; +int32_t taosKeepTcpAlive(SOCKET sockFd) { + int32_t alive = 1; if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { uError("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); taosCloseSocket(sockFd); return -1; } - int probes = 3; + int32_t probes = 3; if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { uError("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); taosCloseSocket(sockFd); return -1; } - int alivetime = 10; + int32_t alivetime = 10; if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) { uError("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno)); taosCloseSocket(sockFd); return -1; } - int interval = 3; + int32_t interval = 3; if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) { uError("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno)); taosCloseSocket(sockFd); return -1; } - int nodelay = 1; + int32_t nodelay = 1; if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { uError("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno)); taosCloseSocket(sockFd); @@ -371,8 +371,8 @@ int taosKeepTcpAlive(SOCKET sockFd) { SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; - SOCKET sockFd; - int reuse; + SOCKET sockFd; + int32_t reuse; uDebug("open tcp server socket:0x%x:%hu", ip, port); @@ -381,7 +381,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { serverAdd.sin_addr.s_addr = ip; serverAdd.sin_port = (uint16_t)htons(port); - if ((sockFd = (int)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + if ((sockFd = (int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { uError("failed to open TCP socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); return -1; @@ -417,38 +417,38 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { return sockFd; } -void tinet_ntoa(char *ipstr, unsigned int ip) { +void tinet_ntoa(char *ipstr, uint32_t ip) { sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24); } #define COPY_SIZE 32768 // sendfile shall be used -int taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len) { +int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len) { int64_t leftLen; - int readLen, writeLen; + int32_t readLen, writeLen; char temp[COPY_SIZE]; leftLen = len; while (leftLen > 0) { if (leftLen < COPY_SIZE) - readLen = (int)leftLen; + readLen = (int32_t)leftLen; else readLen = COPY_SIZE; // 4K - int retLen = taosReadMsg(sfd, temp, (int)readLen); + int32_t retLen = taosReadMsg(sfd, temp, (int32_t)readLen); if (readLen != retLen) { - uError("read error, readLen:%d retLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, retLen, len, leftLen, - strerror(errno)); + uError("read error, readLen:%d retLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, retLen, len, + leftLen, strerror(errno)); return -1; } writeLen = taosWriteMsg(dfd, temp, readLen); if (readLen != writeLen) { - uError("copy error, readLen:%d writeLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, writeLen, len, leftLen, - strerror(errno)); + uError("copy error, readLen:%d writeLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, writeLen, + len, leftLen, strerror(errno)); return -1; } diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 6c4af437b27b6623aeebb404e0dad1c70241cf47..451976f563740b4ac933766d5c0c8f4075ad42d8 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -326,6 +326,7 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) { return 0; } +// TODO move to comm module bool taosGetVersionNumber(char *versionStr, int *versionNubmer) { if (versionStr == NULL || versionNubmer == NULL) { return false; @@ -376,7 +377,8 @@ int taosCheckVersion(char *input_client_version, char *input_server_version, int for(int32_t i = 0; i < comparedSegments; ++i) { if (clientVersionNumber[i] != serverVersionNumber[i]) { - uError("the %d-th number of server version:%s not matched with client version:%s", i, server_version, version); + uError("the %d-th number of server version:%s not matched with client version:%s", i, server_version, + client_version); return TSDB_CODE_TSC_INVALID_VERSION; } } diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt index 8687a8005ddeda7320c60c9ef90dd221f56b971f..0c96ed2a2f3dfb7f03268c9f8fbb1b0afa2397b9 100644 --- a/src/util/tests/CMakeLists.txt +++ b/src/util/tests/CMakeLists.txt @@ -9,7 +9,22 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - + + LIST(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c) ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES(utilTest tutil common osdetail gtest pthread gcov) + + LIST(APPEND BIN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c) + ADD_EXECUTABLE(trefTest ${BIN_SRC}) + TARGET_LINK_LIBRARIES(trefTest common tutil) + ENDIF() + +#IF (TD_LINUX) +# ADD_EXECUTABLE(trefTest ./trefTest.c) +# TARGET_LINK_LIBRARIES(trefTest tutil common) +#ENDIF () + +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) + + diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp index 77174f69fda1a49975c44717d8993996274ca4c9..2203ae8e4fc75c2aa0267055f79884f05e3a65e9 100644 --- a/src/util/tests/skiplistTest.cpp +++ b/src/util/tests/skiplistTest.cpp @@ -59,7 +59,7 @@ void doubleSkipListTest() { } if (size > 0) { - taosTFree(pNodes); + tfree(pNodes); } } @@ -196,7 +196,7 @@ void stringKeySkiplistTest() { tSkipListRemoveNode(pSkipList, pres[0]); if (num > 0) { - taosTFree(pres); + tfree(pres); } } @@ -247,7 +247,7 @@ void skiplistPerformanceTest() { printf("total:%" PRIu64 " ms, avg:%f\n", e - s, (e - s) / (double)size); printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level); - assert(tSkipListGetSize(pSkipList) == size); + assert(SL_GET_SIZE(pSkipList) == size); // printf("the level of skiplist is:\n"); // @@ -273,10 +273,10 @@ void skiplistPerformanceTest() { int64_t et = taosGetTimestampMs(); printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st); - assert(tSkipListGetSize(pSkipList) == size); + assert(SL_GET_SIZE(pSkipList) == size); tSkipListDestroy(pSkipList); - taosTFree(total); + tfree(total); } // todo not support duplicated key yet @@ -357,7 +357,7 @@ TEST(testCase, skiplist_test) { printf("-----%lf\n", pNodes[i]->key.dKey); } printf("the range query result size is: %d\n", size); - taosTFree(pNodes); + tfree(pNodes); SSkipListKey *pKeys = malloc(sizeof(SSkipListKey) * 20); for (int32_t i = 0; i < 8; i += 2) { @@ -371,7 +371,7 @@ TEST(testCase, skiplist_test) { for (int32_t i = 0; i < r; ++i) { // printf("%lf ", pNodes[i]->key.dKey); } - taosTFree(pNodes); + tfree(pNodes); free(pKeys);*/ } diff --git a/src/util/tests/trefTest.c b/src/util/tests/trefTest.c new file mode 100644 index 0000000000000000000000000000000000000000..6887b24abdae9f5d9949fee0073a35eb717eb4f7 --- /dev/null +++ b/src/util/tests/trefTest.c @@ -0,0 +1,195 @@ +#include +#include +#include +#include +#include +#include "os.h" +#include "tref.h" +#include "tlog.h" +#include "tglobal.h" +#include "taoserror.h" +#include "tulog.h" + +typedef struct { + int refNum; + int steps; + int rsetId; + int64_t rid; + void **p; +} SRefSpace; + +void iterateRefs(int rsetId) { + int count = 0; + + void *p = taosIterateRef(rsetId, NULL); + while (p) { + // process P + count++; + p = taosIterateRef(rsetId, p); + } + + printf(" %d ", count); +} + +void *addRef(void *param) { + SRefSpace *pSpace = (SRefSpace *)param; + int id; + int64_t rid; + + for (int i=0; i < pSpace->steps; ++i) { + printf("a"); + id = random() % pSpace->refNum; + if (pSpace->rid[id] <= 0) { + pSpace->p[id] = malloc(128); + pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]); + } + usleep(100); + } + + return NULL; +} + +void *removeRef(void *param) { + SRefSpace *pSpace = (SRefSpace *)param; + int id; + int64_t rid; + + for (int i=0; i < pSpace->steps; ++i) { + printf("d"); + id = random() % pSpace->refNum; + if (pSpace->rid[id] > 0) { + code = taosRemoveRef(pSpace->rsetId, pSpace->rid[id]); + if (code == 0) pSpace->rid[id] = 0; + } + + usleep(100); + } + + return NULL; +} + +void *acquireRelease(void *param) { + SRefSpace *pSpace = (SRefSpace *)param; + int id; + int64_t rid; + + for (int i=0; i < pSpace->steps; ++i) { + printf("a"); + + id = random() % pSpace->refNum; + code = taosAcquireRef(pSpace->rsetId, pSpace->p[id]); + if (code >= 0) { + usleep(id % 5 + 1); + taosReleaseRef(pSpace->rsetId, pSpace->p[id]); + } + } + + return NULL; +} + +void myfree(void *p) { + free(p); +} + +void *openRefSpace(void *param) { + SRefSpace *pSpace = (SRefSpace *)param; + + printf("c"); + pSpace->rsetId = taosOpenRef(50, myfree); + + if (pSpace->rsetId < 0) { + printf("failed to open ref, reson:%s\n", tstrerror(pSpace->rsetId)); + return NULL; + } + + pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + + pthread_t thread1, thread2, thread3; + pthread_create(&(thread1), &thattr, addRef, (void *)(pSpace)); + pthread_create(&(thread2), &thattr, removeRef, (void *)(pSpace)); + pthread_create(&(thread3), &thattr, acquireRelease, (void *)(pSpace)); + + pthread_join(thread1, NULL); + pthread_join(thread2, NULL); + pthread_join(thread3, NULL); + + for (int i=0; irefNum; ++i) { + taosRemoveRef(pSpace->rsetId, pSpace->rid[i]); + } + + taosCloseRef(pSpace->rsetId); + + uInfo("rsetId:%d main thread exit", pSpace->rsetId); + free(pSpace->p); + pSpace->p = NULL; + + return NULL; +} + +int main(int argc, char *argv[]) { + int refNum = 100; + int threads = 10; + int steps = 10000; + int loops = 1; + + uDebugFlag = 143; + + for (int i=1; i + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_VNODE_CFG_H +#define TDENGINE_VNODE_CFG_H + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t vnodeReadCfg(SVnodeObj *pVnode); +int32_t vnodeWriteCfg(SCreateVnodeMsg *pVnodeCfg); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index 169334c6119a35f53fd0a69dd1f7cb6952fbb727..317d4904cb6a632aad1ee83811194e7fd245ef21 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -41,13 +41,13 @@ typedef struct { int8_t status; int8_t role; int8_t accessState; - int64_t version; // current version - int64_t fversion; // version on saved data file + uint64_t version; // current version + uint64_t fversion; // version on saved data file void *wqueue; void *rqueue; void *wal; void *tsdb; - void *sync; + int64_t sync; void *events; void *cq; // continuous query int32_t cfgVersion; @@ -61,8 +61,6 @@ typedef struct { char db[TSDB_DB_NAME_LEN]; } SVnodeObj; -int vnodeWriteToQueue(void *param, void *pHead, int type); -int vnodeWriteCqMsgToQueue(void *param, void *pHead, int type); void vnodeInitWriteFp(void); void vnodeInitReadFp(void); diff --git a/src/vnode/inc/vnodeVersion.h b/src/vnode/inc/vnodeVersion.h new file mode 100644 index 0000000000000000000000000000000000000000..1d086cb21fdab0247038c7f3d32d89b38c19d871 --- /dev/null +++ b/src/vnode/inc/vnodeVersion.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_VNODE_VERSION_H +#define TDENGINE_VNODE_VERSION_H + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t vnodeReadVersion(SVnodeObj *pVnode); +int32_t vnodeSaveVersion(SVnodeObj *pVnode); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c new file mode 100644 index 0000000000000000000000000000000000000000..e2e57a756640339febbbe11cdc231c5a216ff2b0 --- /dev/null +++ b/src/vnode/src/vnodeCfg.c @@ -0,0 +1,324 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosmsg.h" +#include "taoserror.h" +#include "cJSON.h" +#include "tglobal.h" +#include "tsdb.h" +#include "dnode.h" +#include "vnodeInt.h" +#include "vnodeCfg.h" + +static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { + strcpy(pVnode->db, vnodeMsg->db); + pVnode->cfgVersion = vnodeMsg->cfg.cfgVersion; + pVnode->tsdbCfg.cacheBlockSize = vnodeMsg->cfg.cacheBlockSize; + pVnode->tsdbCfg.totalBlocks = vnodeMsg->cfg.totalBlocks; + pVnode->tsdbCfg.daysPerFile = vnodeMsg->cfg.daysPerFile; + pVnode->tsdbCfg.keep = vnodeMsg->cfg.daysToKeep; + pVnode->tsdbCfg.keep1 = vnodeMsg->cfg.daysToKeep1; + pVnode->tsdbCfg.keep2 = vnodeMsg->cfg.daysToKeep2; + pVnode->tsdbCfg.minRowsPerFileBlock = vnodeMsg->cfg.minRowsPerFileBlock; + pVnode->tsdbCfg.maxRowsPerFileBlock = vnodeMsg->cfg.maxRowsPerFileBlock; + pVnode->tsdbCfg.precision = vnodeMsg->cfg.precision; + pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression; + pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel; + pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod; + pVnode->walCfg.keep = TAOS_WAL_NOT_KEEP; + pVnode->syncCfg.replica = vnodeMsg->cfg.replications; + pVnode->syncCfg.quorum = vnodeMsg->cfg.quorum; + + for (int i = 0; i < pVnode->syncCfg.replica; ++i) { + SVnodeDesc *node = &vnodeMsg->nodes[i]; + pVnode->syncCfg.nodeInfo[i].nodeId = node->nodeId; + taosGetFqdnPortFromEp(node->nodeEp, pVnode->syncCfg.nodeInfo[i].nodeFqdn, &pVnode->syncCfg.nodeInfo[i].nodePort); + pVnode->syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC; + } + + vInfo("vgId:%d, load vnode cfg successfully, replcia:%d", pVnode->vgId, pVnode->syncCfg.replica); + for (int32_t i = 0; i < pVnode->syncCfg.replica; i++) { + SNodeInfo *node = &pVnode->syncCfg.nodeInfo[i]; + vInfo("vgId:%d, dnode:%d, %s:%u", pVnode->vgId, node->nodeId, node->nodeFqdn, node->nodePort); + } +} + +int32_t vnodeReadCfg(SVnodeObj *pVnode) { + int32_t ret = TSDB_CODE_VND_APP_ERROR; + int32_t len = 0; + int maxLen = 1000; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + bool nodeChanged = false; + SCreateVnodeMsg vnodeMsg; + + char file[TSDB_FILENAME_LEN + 30] = {0}; + sprintf(file, "%s/vnode%d/config.json", tsVnodeDir, pVnode->vgId); + + vnodeMsg.cfg.vgId = pVnode->vgId; + + fp = fopen(file, "r"); + if (!fp) { + vError("vgId:%d, failed to open vnode cfg file:%s to read, error:%s", pVnode->vgId, file, strerror(errno)); + ret = TAOS_SYSTEM_ERROR(errno); + goto PARSE_VCFG_ERROR; + } + + len = fread(content, 1, maxLen, fp); + if (len <= 0) { + vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + vError("vgId:%d, failed to read %s, invalid json format", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + + cJSON *db = cJSON_GetObjectItem(root, "db"); + if (!db || db->type != cJSON_String || db->valuestring == NULL) { + vError("vgId:%d, failed to read %s, db not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + strcpy(vnodeMsg.db, db->valuestring); + + cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion"); + if (!cfgVersion || cfgVersion->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, cfgVersion not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.cfgVersion = cfgVersion->valueint; + + cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); + if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, cacheBlockSize not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.cacheBlockSize = cacheBlockSize->valueint; + + cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks"); + if (!totalBlocks || totalBlocks->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, totalBlocks not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.totalBlocks = totalBlocks->valueint; + + cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); + if (!daysPerFile || daysPerFile->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, daysPerFile not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.daysPerFile = daysPerFile->valueint; + + cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); + if (!daysToKeep || daysToKeep->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, daysToKeep not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.daysToKeep = daysToKeep->valueint; + + cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1"); + if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, daysToKeep1 not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.daysToKeep1 = daysToKeep1->valueint; + + cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2"); + if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, daysToKeep2 not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.daysToKeep2 = daysToKeep2->valueint; + + cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock"); + if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, minRowsPerFileBlock not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.minRowsPerFileBlock = minRowsPerFileBlock->valueint; + + cJSON *maxRowsPerFileBlock = cJSON_GetObjectItem(root, "maxRowsPerFileBlock"); + if (!maxRowsPerFileBlock || maxRowsPerFileBlock->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, maxRowsPerFileBlock not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; + + cJSON *precision = cJSON_GetObjectItem(root, "precision"); + if (!precision || precision->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, precision not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.precision = (int8_t)precision->valueint; + + cJSON *compression = cJSON_GetObjectItem(root, "compression"); + if (!compression || compression->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, compression not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.compression = (int8_t)compression->valueint; + + cJSON *walLevel = cJSON_GetObjectItem(root, "walLevel"); + if (!walLevel || walLevel->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, walLevel not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.walLevel = (int8_t)walLevel->valueint; + + cJSON *fsyncPeriod = cJSON_GetObjectItem(root, "fsync"); + if (!walLevel || walLevel->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, fsyncPeriod not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.fsyncPeriod = fsyncPeriod->valueint; + + cJSON *wals = cJSON_GetObjectItem(root, "wals"); + if (!wals || wals->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, wals not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.wals = (int8_t)wals->valueint; + + cJSON *replica = cJSON_GetObjectItem(root, "replica"); + if (!replica || replica->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, replica not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.replications = (int8_t)replica->valueint; + + cJSON *quorum = cJSON_GetObjectItem(root, "quorum"); + if (!quorum || quorum->type != cJSON_Number) { + vError("vgId: %d, failed to read %s, quorum not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.quorum = (int8_t)quorum->valueint; + + cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); + if (!nodeInfos || nodeInfos->type != cJSON_Array) { + vError("vgId:%d, failed to read %s, nodeInfos not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + + int size = cJSON_GetArraySize(nodeInfos); + if (size != vnodeMsg.cfg.replications) { + vError("vgId:%d, failed to read %s, nodeInfos size not matched", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + + for (int i = 0; i < size; ++i) { + cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); + if (nodeInfo == NULL) continue; + SVnodeDesc *node = &vnodeMsg.nodes[i]; + + cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); + if (!nodeId || nodeId->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, nodeId not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + node->nodeId = nodeId->valueint; + + cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); + if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { + vError("vgId:%d, failed to read %s, nodeFqdn not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + tstrncpy(node->nodeEp, nodeEp->valuestring, TSDB_EP_LEN); + + if (!nodeChanged) { + nodeChanged = dnodeCheckEpChanged(node->nodeId, node->nodeEp); + } + } + + ret = TSDB_CODE_SUCCESS; + +PARSE_VCFG_ERROR: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + + if (nodeChanged) { + vnodeWriteCfg(&vnodeMsg); + } + + if (ret == TSDB_CODE_SUCCESS) { + vnodeLoadCfg(pVnode, &vnodeMsg); + } + + terrno = 0; + return ret; +} + +int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { + char file[TSDB_FILENAME_LEN + 30] = {0}; + sprintf(file, "%s/vnode%d/config.json", tsVnodeDir, pMsg->cfg.vgId); + + FILE *fp = fopen(file, "w"); + if (!fp) { + vError("vgId:%d, failed to write %s error:%s", pMsg->cfg.vgId, file, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } + + int32_t len = 0; + int32_t maxLen = 1000; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"db\": \"%s\",\n", pMsg->db); + len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pMsg->cfg.cfgVersion); + len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pMsg->cfg.cacheBlockSize); + len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pMsg->cfg.totalBlocks); + len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pMsg->cfg.daysPerFile); + len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pMsg->cfg.daysToKeep); + len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pMsg->cfg.daysToKeep1); + len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pMsg->cfg.daysToKeep2); + len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pMsg->cfg.minRowsPerFileBlock); + len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pMsg->cfg.maxRowsPerFileBlock); + len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pMsg->cfg.precision); + len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pMsg->cfg.compression); + len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pMsg->cfg.walLevel); + len += snprintf(content + len, maxLen - len, " \"fsync\": %d,\n", pMsg->cfg.fsyncPeriod); + len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pMsg->cfg.replications); + len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pMsg->cfg.wals); + len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pMsg->cfg.quorum); + len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); + for (int32_t i = 0; i < pMsg->cfg.replications; i++) { + SVnodeDesc *node = &pMsg->nodes[i]; + dnodeUpdateEp(node->nodeId, node->nodeEp, NULL, NULL); + len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", node->nodeId); + len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", node->nodeEp); + if (i < pMsg->cfg.replications - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + fflush(fp); + fclose(fp); + free(content); + terrno = 0; + + vInfo("vgId:%d, successed to write %s", pMsg->cfg.vgId, file); + return TSDB_CODE_SUCCESS; +} diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index e529f27f55abe99e0a842fb839f2916a0f303bfc..199619e8514faac9e663914ddf59a3ea0462afde 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -15,45 +15,35 @@ #define _DEFAULT_SOURCE #include "os.h" - -#include "tcache.h" -#include "cJSON.h" -#include "dnode.h" -#include "hash.h" #include "taoserror.h" #include "taosmsg.h" #include "tglobal.h" #include "trpc.h" #include "tsdb.h" -#include "ttimer.h" #include "tutil.h" #include "vnode.h" #include "vnodeInt.h" #include "query.h" #include "dnode.h" +#include "vnodeCfg.h" +#include "vnodeVersion.h" -#define TSDB_VNODE_VERSION_CONTENT_LEN 31 - -static SHashObj*tsDnodeVnodesHash; +static SHashObj*tsVnodesHash; static void vnodeCleanUp(SVnodeObj *pVnode); -static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg); -static int32_t vnodeReadCfg(SVnodeObj *pVnode); -static int32_t vnodeSaveVersion(SVnodeObj *pVnode); -static int32_t vnodeReadVersion(SVnodeObj *pVnode); static int vnodeProcessTsdbStatus(void *arg, int status); static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); -static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); +static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId); static void vnodeNotifyRole(void *ahandle, int8_t role); static void vnodeCtrlFlow(void *handle, int32_t mseconds); static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion); #ifndef _SYNC -tsync_h syncStart(const SSyncInfo *info) { return NULL; } -int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype) { return 0; } -void syncStop(tsync_h shandle) {} -int32_t syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; } -int syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; } -void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {} +int64_t syncStart(const SSyncInfo *info) { return NULL; } +int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype) { return 0; } +void syncStop(int64_t rid) {} +int32_t syncReconfig(int64_t rid, const SSyncCfg * cfg) { return 0; } +int syncGetNodesRole(int64_t rid, SNodesRole * cfg) { return 0; } +void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {} #endif char* vnodeStatus[] = { @@ -71,26 +61,33 @@ int32_t vnodeInitResources() { vnodeInitWriteFp(); vnodeInitReadFp(); - tsDnodeVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); - if (tsDnodeVnodesHash == NULL) { + tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); + if (tsVnodesHash == NULL) { vError("failed to init vnode list"); return TSDB_CODE_VND_OUT_OF_MEMORY; } + if (tsdbInitCommitQueue(tsNumOfCommitThreads) < 0) { + vError("failed to init vnode commit queue"); + return terrno; + } + return TSDB_CODE_SUCCESS; } void vnodeCleanupResources() { - if (tsDnodeVnodesHash != NULL) { + tsdbDestroyCommitQueue(); + + if (tsVnodesHash != NULL) { vDebug("vnode list is cleanup"); - taosHashCleanup(tsDnodeVnodesHash); - tsDnodeVnodesHash = NULL; + taosHashCleanup(tsVnodesHash); + tsVnodesHash = NULL; } syncCleanUp(); } -int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { +int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { int32_t code; SVnodeObj *pVnode = vnodeAcquire(pVnodeCfg->cfg.vgId); @@ -128,7 +125,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { } } - code = vnodeSaveCfg(pVnodeCfg); + code = vnodeWriteCfg(pVnodeCfg); if (code != TSDB_CODE_SUCCESS) { vError("vgId:%d, failed to save vnode cfg, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(code)); return code; @@ -138,13 +135,13 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId; tsdbCfg.cacheBlockSize = pVnodeCfg->cfg.cacheBlockSize; tsdbCfg.totalBlocks = pVnodeCfg->cfg.totalBlocks; - // tsdbCfg.maxTables = pVnodeCfg->cfg.maxTables; tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile; tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep; tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; tsdbCfg.precision = pVnodeCfg->cfg.precision; tsdbCfg.compression = pVnodeCfg->cfg.compression; + tsdbCfg.update = pVnodeCfg->cfg.update; char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); @@ -176,7 +173,7 @@ int32_t vnodeDrop(int32_t vgId) { return TSDB_CODE_SUCCESS; } -int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) { +int32_t vnodeAlter(void *param, SCreateVnodeMsg *pVnodeCfg) { SVnodeObj *pVnode = param; // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS @@ -186,7 +183,7 @@ int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) { return TSDB_CODE_SUCCESS; } - int32_t code = vnodeSaveCfg(pVnodeCfg); + int32_t code = vnodeWriteCfg(pVnodeCfg); if (code != TSDB_CODE_SUCCESS) { pVnode->status = TAOS_VN_STATUS_READY; return code; @@ -237,6 +234,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->vgId = vnode; pVnode->status = TAOS_VN_STATUS_INIT; + pVnode->fversion = 0; pVnode->version = 0; pVnode->tsdbCfg.tsdbId = pVnode->vgId; pVnode->rootDir = strdup(rootDir); @@ -259,8 +257,8 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->fversion = pVnode->version; - pVnode->wqueue = dnodeAllocateVnodeWqueue(pVnode); - pVnode->rqueue = dnodeAllocateVnodeRqueue(pVnode); + pVnode->wqueue = dnodeAllocVWriteQueue(pVnode); + pVnode->rqueue = dnodeAllocVReadQueue(pVnode); if (pVnode->wqueue == NULL || pVnode->rqueue == NULL) { vnodeCleanUp(pVnode); return terrno; @@ -271,7 +269,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { strcpy(cqCfg.pass, tsInternalPass); strcpy(cqCfg.db, pVnode->db); cqCfg.vgId = vnode; - cqCfg.cqWrite = vnodeWriteCqMsgToQueue; + cqCfg.cqWrite = vnodeWriteToWQueue; pVnode->cq = cqOpen(pVnode, &cqCfg); if (pVnode->cq == NULL) { vnodeCleanUp(pVnode); @@ -291,25 +289,36 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { if (pVnode->tsdb == NULL) { vnodeCleanUp(pVnode); return terrno; - } else if (terrno != TSDB_CODE_SUCCESS && pVnode->syncCfg.replica <= 1) { + } else if (terrno != TSDB_CODE_SUCCESS) { vError("vgId:%d, failed to open tsdb, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, tstrerror(terrno)); - vnodeCleanUp(pVnode); - return terrno; + if (pVnode->syncCfg.replica <= 1) { + vnodeCleanUp(pVnode); + return terrno; + } else { + pVnode->fversion = 0; + pVnode->version = 0; + } } sprintf(temp, "%s/wal", rootDir); + pVnode->walCfg.vgId = pVnode->vgId; pVnode->wal = walOpen(temp, &pVnode->walCfg); if (pVnode->wal == NULL) { vnodeCleanUp(pVnode); return terrno; } - walRestore(pVnode->wal, pVnode, vnodeWriteToQueue); + walRestore(pVnode->wal, pVnode, vnodeProcessWrite); if (pVnode->version == 0) { + pVnode->fversion = 0; pVnode->version = walGetVersion(pVnode->wal); } + tsdbSyncCommit(pVnode->tsdb); + walRemoveAllOldFiles(pVnode->wal); + walRenew(pVnode->wal); + SSyncInfo syncInfo; syncInfo.vgId = pVnode->vgId; syncInfo.version = pVnode->version; @@ -318,8 +327,8 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { syncInfo.ahandle = pVnode; syncInfo.getWalInfo = vnodeGetWalInfo; syncInfo.getFileInfo = vnodeGetFileInfo; - syncInfo.writeToCache = vnodeWriteToQueue; - syncInfo.confirmForward = dnodeSendRpcVnodeWriteRsp; + syncInfo.writeToCache = vnodeWriteToWQueue; + syncInfo.confirmForward = dnodeSendRpcVWriteRsp; syncInfo.notifyRole = vnodeNotifyRole; syncInfo.notifyFlowCtrl = vnodeCtrlFlow; syncInfo.notifyFileSynced = vnodeNotifyFileSynced; @@ -328,7 +337,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { #ifndef _SYNC pVnode->role = TAOS_SYNC_ROLE_MASTER; #else - if (pVnode->sync == NULL) { + if (pVnode->sync <= 0) { vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, tstrerror(terrno)); vnodeCleanUp(pVnode); @@ -346,7 +355,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->status = TAOS_VN_STATUS_READY; vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); - taosHashPut(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *)); + taosHashPut(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *)); return TSDB_CODE_SUCCESS; } @@ -363,6 +372,7 @@ int32_t vnodeClose(int32_t vgId) { } void vnodeRelease(void *pVnodeRaw) { + if (pVnodeRaw == NULL) return; SVnodeObj *pVnode = pVnodeRaw; int32_t vgId = pVnode->vgId; @@ -384,6 +394,10 @@ void vnodeRelease(void *pVnodeRaw) { pVnode->qMgmt = NULL; } + if (pVnode->wal) { + walStop(pVnode->wal); + } + if (pVnode->tsdb) { tsdbCloseRepo(pVnode->tsdb, 1); pVnode->tsdb = NULL; @@ -402,16 +416,16 @@ void vnodeRelease(void *pVnodeRaw) { } if (pVnode->wqueue) { - dnodeFreeVnodeWqueue(pVnode->wqueue); + dnodeFreeVWriteQueue(pVnode->wqueue); pVnode->wqueue = NULL; } if (pVnode->rqueue) { - dnodeFreeVnodeRqueue(pVnode->rqueue); + dnodeFreeVReadQueue(pVnode->rqueue); pVnode->rqueue = NULL; } - taosTFree(pVnode->rootDir); + tfree(pVnode->rootDir); if (pVnode->dropped) { char rootDir[TSDB_FILENAME_LEN] = {0}; @@ -433,7 +447,7 @@ void vnodeRelease(void *pVnodeRaw) { tsem_destroy(&pVnode->sem); free(pVnode); - int32_t count = taosHashGetSize(tsDnodeVnodesHash); + int32_t count = taosHashGetSize(tsVnodesHash); vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count); } @@ -450,7 +464,7 @@ static void vnodeIncRef(void *ptNode) { } void *vnodeAcquire(int32_t vgId) { - SVnodeObj **ppVnode = taosHashGetCB(tsDnodeVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *)); + SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *)); if (ppVnode == NULL || *ppVnode == NULL) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; @@ -461,41 +475,11 @@ void *vnodeAcquire(int32_t vgId) { return *ppVnode; } -void *vnodeAcquireRqueue(int32_t vgId) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) return NULL; - - int32_t code = vnodeCheckRead(pVnode); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - vInfo("vgId:%d, can not provide read service, status is %s", vgId, vnodeStatus[pVnode->status]); - vnodeRelease(pVnode); - return NULL; - } - - return pVnode->rqueue; -} - -void *vnodeAcquireWqueue(int32_t vgId) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) return NULL; - - int32_t code = vnodeCheckWrite(pVnode); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - vInfo("vgId:%d, can not provide write service, status is %s", vgId, vnodeStatus[pVnode->status]); - vnodeRelease(pVnode); - return NULL; - } - - return pVnode->wqueue; -} - void *vnodeGetWal(void *pVnode) { return ((SVnodeObj *)pVnode)->wal; } -static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) { +static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) { int64_t totalStorage = 0; int64_t compStorage = 0; int64_t pointsWritten = 0; @@ -519,7 +503,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) { } int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { - SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash); + SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash); while (taosHashIterNext(pIter)) { SVnodeObj **pVnode = taosHashIterGet(pIter); if (pVnode == NULL) continue; @@ -539,8 +523,8 @@ int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { } void vnodeBuildStatusMsg(void *param) { - SDMStatusMsg *pStatus = param; - SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash); + SStatusMsg *pStatus = param; + SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash); while (taosHashIterNext(pIter)) { SVnodeObj **pVnode = taosHashIterGet(pIter); @@ -553,7 +537,7 @@ void vnodeBuildStatusMsg(void *param) { taosHashDestroyIter(pIter); } -void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes) { +void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) { for (int32_t i = 0; i < numOfVnodes; ++i) { pAccess[i].vgId = htonl(pAccess[i].vgId); SVnodeObj *pVnode = vnodeAcquire(pAccess[i].vgId); @@ -569,7 +553,7 @@ void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes) { static void vnodeCleanUp(SVnodeObj *pVnode) { // remove from hash, so new messages wont be consumed - taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t)); + taosHashRemove(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t)); if (pVnode->status != TAOS_VN_STATUS_INIT) { // it may be in updateing or reset state, then it shall wait @@ -583,9 +567,9 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { } // stop replication module - if (pVnode->sync) { - void *sync = pVnode->sync; - pVnode->sync = NULL; + if (pVnode->sync > 0) { + int64_t sync = pVnode->sync; + pVnode->sync = -1; syncStop(sync); } @@ -601,25 +585,30 @@ static int vnodeProcessTsdbStatus(void *arg, int status) { SVnodeObj *pVnode = arg; if (status == TSDB_STATUS_COMMIT_START) { - pVnode->fversion = pVnode->version; + pVnode->fversion = pVnode->version; + vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); return walRenew(pVnode->wal); } - if (status == TSDB_STATUS_COMMIT_OVER) + if (status == TSDB_STATUS_COMMIT_OVER) { + vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); + walRemoveOneOldFile(pVnode->wal); return vnodeSaveVersion(pVnode); + } - return 0; + return 0; } -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, + uint64_t *fversion) { SVnodeObj *pVnode = ahandle; *fversion = pVnode->fversion; return tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); } -static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) { +static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) { SVnodeObj *pVnode = ahandle; - return walGetWalFile(pVnode->wal, name, index); + return walGetWalFile(pVnode->wal, fileName, fileId); } static void vnodeNotifyRole(void *ahandle, int8_t role) { @@ -636,17 +625,18 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) { static void vnodeCtrlFlow(void *ahandle, int32_t mseconds) { SVnodeObj *pVnode = ahandle; - if (pVnode->delay != mseconds) + if (pVnode->delay != mseconds) { vInfo("vgId:%d, sync flow control, mseconds:%d", pVnode->vgId, mseconds); + } pVnode->delay = mseconds; } -static int vnodeResetTsdb(SVnodeObj *pVnode) -{ +static int vnodeResetTsdb(SVnodeObj *pVnode) { char rootDir[128] = "\0"; sprintf(rootDir, "%s/tsdb", pVnode->rootDir); - if (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_RESET) != TAOS_VN_STATUS_READY) { + if (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_RESET) != + TAOS_VN_STATUS_READY) { return -1; } @@ -671,375 +661,19 @@ static int vnodeResetTsdb(SVnodeObj *pVnode) pVnode->tsdb = tsdbOpenRepo(rootDir, &appH); pVnode->status = TAOS_VN_STATUS_READY; - vnodeRelease(pVnode); + vnodeRelease(pVnode); return 0; } static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion) { SVnodeObj *pVnode = ahandle; - vDebug("vgId:%d, data file is synced, fversion:%" PRId64, pVnode->vgId, fversion); pVnode->fversion = fversion; pVnode->version = fversion; vnodeSaveVersion(pVnode); + vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, + pVnode->version); return vnodeResetTsdb(pVnode); } - -static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { - char cfgFile[TSDB_FILENAME_LEN + 30] = {0}; - sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnodeCfg->cfg.vgId); - FILE *fp = fopen(cfgFile, "w"); - if (!fp) { - vError("vgId:%d, failed to open vnode cfg file for write, file:%s error:%s", pVnodeCfg->cfg.vgId, cfgFile, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - - int32_t len = 0; - int32_t maxLen = 1000; - char * content = calloc(1, maxLen + 1); - if (content == NULL) { - fclose(fp); - return TSDB_CODE_VND_OUT_OF_MEMORY; - } - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"db\": \"%s\",\n", pVnodeCfg->db); - len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnodeCfg->cfg.cfgVersion); - len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize); - len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks); - // len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables); - len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pVnodeCfg->cfg.daysPerFile); - len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep); - len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pVnodeCfg->cfg.daysToKeep1); - len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pVnodeCfg->cfg.daysToKeep2); - len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.minRowsPerFileBlock); - len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.maxRowsPerFileBlock); - // len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnodeCfg->cfg.commitTime); - len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision); - len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression); - len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pVnodeCfg->cfg.walLevel); - len += snprintf(content + len, maxLen - len, " \"fsync\": %d,\n", pVnodeCfg->cfg.fsyncPeriod); - len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnodeCfg->cfg.replications); - len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnodeCfg->cfg.wals); - len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum); - - len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); - - vInfo("vgId:%d, save vnode cfg, replica:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.replications); - for (int32_t i = 0; i < pVnodeCfg->cfg.replications; i++) { - len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", pVnodeCfg->nodes[i].nodeId); - len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", pVnodeCfg->nodes[i].nodeEp); - vInfo("vgId:%d, save vnode cfg, nodeId:%d nodeEp:%s", pVnodeCfg->cfg.vgId, pVnodeCfg->nodes[i].nodeId, - pVnodeCfg->nodes[i].nodeEp); - - if (i < pVnodeCfg->cfg.replications - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } - } - len += snprintf(content + len, maxLen - len, "}\n"); - - fwrite(content, 1, len, fp); - fflush(fp); - fclose(fp); - free(content); - - vInfo("vgId:%d, save vnode cfg successed", pVnodeCfg->cfg.vgId); - - return TSDB_CODE_SUCCESS; -} - -static int32_t vnodeReadCfg(SVnodeObj *pVnode) { - cJSON *root = NULL; - char *content = NULL; - char cfgFile[TSDB_FILENAME_LEN + 30] = {0}; - int maxLen = 1000; - - terrno = TSDB_CODE_VND_APP_ERROR; - sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnode->vgId); - FILE *fp = fopen(cfgFile, "r"); - if (!fp) { - vError("vgId:%d, failed to open vnode cfg file:%s to read, error:%s", pVnode->vgId, - cfgFile, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto PARSE_OVER; - } - - content = calloc(1, maxLen + 1); - if (content == NULL) goto PARSE_OVER; - int len = fread(content, 1, maxLen, fp); - if (len <= 0) { - vError("vgId:%d, failed to read vnode cfg, content is null", pVnode->vgId); - free(content); - fclose(fp); - return errno; - } - - root = cJSON_Parse(content); - if (root == NULL) { - vError("vgId:%d, failed to read vnode cfg, invalid json format", pVnode->vgId); - goto PARSE_OVER; - } - - cJSON *db = cJSON_GetObjectItem(root, "db"); - if (!db || db->type != cJSON_String || db->valuestring == NULL) { - vError("vgId:%d, failed to read vnode cfg, db not found", pVnode->vgId); - goto PARSE_OVER; - } - strcpy(pVnode->db, db->valuestring); - - cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion"); - if (!cfgVersion || cfgVersion->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, cfgVersion not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->cfgVersion = cfgVersion->valueint; - - cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); - if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, cacheBlockSize not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.cacheBlockSize = cacheBlockSize->valueint; - - cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks"); - if (!totalBlocks || totalBlocks->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, totalBlocks not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.totalBlocks = totalBlocks->valueint; - - // cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); - // if (!maxTables || maxTables->type != cJSON_Number) { - // vError("vgId:%d, failed to read vnode cfg, maxTables not found", pVnode->vgId); - // goto PARSE_OVER; - // } - // pVnode->tsdbCfg.maxTables = maxTables->valueint; - - cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); - if (!daysPerFile || daysPerFile->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, daysPerFile not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.daysPerFile = daysPerFile->valueint; - - cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); - if (!daysToKeep || daysToKeep->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.keep = daysToKeep->valueint; - - cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1"); - if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, daysToKeep1 not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.keep1 = daysToKeep1->valueint; - - cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2"); - if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, daysToKeep2 not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.keep2 = daysToKeep2->valueint; - - cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock"); - if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, minRowsPerFileBlock not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.minRowsPerFileBlock = minRowsPerFileBlock->valueint; - - cJSON *maxRowsPerFileBlock = cJSON_GetObjectItem(root, "maxRowsPerFileBlock"); - if (!maxRowsPerFileBlock || maxRowsPerFileBlock->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, maxRowsPerFileBlock not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; - - // cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); - // if (!commitTime || commitTime->type != cJSON_Number) { - // vError("vgId:%d, failed to read vnode cfg, commitTime not found", pVnode->vgId); - // goto PARSE_OVER; - // } - // pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; - - cJSON *precision = cJSON_GetObjectItem(root, "precision"); - if (!precision || precision->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, precision not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.precision = (int8_t)precision->valueint; - - cJSON *compression = cJSON_GetObjectItem(root, "compression"); - if (!compression || compression->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, compression not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.compression = (int8_t)compression->valueint; - - cJSON *walLevel = cJSON_GetObjectItem(root, "walLevel"); - if (!walLevel || walLevel->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, walLevel not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->walCfg.walLevel = (int8_t) walLevel->valueint; - - cJSON *fsyncPeriod = cJSON_GetObjectItem(root, "fsync"); - if (!walLevel || walLevel->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, fsyncPeriod not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->walCfg.fsyncPeriod = fsyncPeriod->valueint; - - cJSON *wals = cJSON_GetObjectItem(root, "wals"); - if (!wals || wals->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, wals not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->walCfg.wals = (int8_t)wals->valueint; - pVnode->walCfg.keep = 0; - - cJSON *replica = cJSON_GetObjectItem(root, "replica"); - if (!replica || replica->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, replica not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->syncCfg.replica = (int8_t)replica->valueint; - - cJSON *quorum = cJSON_GetObjectItem(root, "quorum"); - if (!quorum || quorum->type != cJSON_Number) { - vError("vgId: %d, failed to read vnode cfg, quorum not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->syncCfg.quorum = (int8_t)quorum->valueint; - - cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); - if (!nodeInfos || nodeInfos->type != cJSON_Array) { - vError("vgId:%d, failed to read vnode cfg, nodeInfos not found", pVnode->vgId); - goto PARSE_OVER; - } - - int size = cJSON_GetArraySize(nodeInfos); - if (size != pVnode->syncCfg.replica) { - vError("vgId:%d, failed to read vnode cfg, nodeInfos size not matched", pVnode->vgId); - goto PARSE_OVER; - } - - for (int i = 0; i < size; ++i) { - cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); - if (nodeInfo == NULL) continue; - - cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); - if (!nodeId || nodeId->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, nodeId not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->syncCfg.nodeInfo[i].nodeId = nodeId->valueint; - - cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); - if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { - vError("vgId:%d, failed to read vnode cfg, nodeFqdn not found", pVnode->vgId); - goto PARSE_OVER; - } - - taosGetFqdnPortFromEp(nodeEp->valuestring, pVnode->syncCfg.nodeInfo[i].nodeFqdn, &pVnode->syncCfg.nodeInfo[i].nodePort); - pVnode->syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC; - } - - terrno = TSDB_CODE_SUCCESS; - - vInfo("vgId:%d, read vnode cfg successfully, replcia:%d", pVnode->vgId, pVnode->syncCfg.replica); - for (int32_t i = 0; i < pVnode->syncCfg.replica; i++) { - vInfo("vgId:%d, dnode:%d, %s:%d", pVnode->vgId, pVnode->syncCfg.nodeInfo[i].nodeId, - pVnode->syncCfg.nodeInfo[i].nodeFqdn, pVnode->syncCfg.nodeInfo[i].nodePort); - } - -PARSE_OVER: - taosTFree(content); - cJSON_Delete(root); - if (fp) fclose(fp); - return terrno; -} - -static int32_t vnodeSaveVersion(SVnodeObj *pVnode) { - char versionFile[TSDB_FILENAME_LEN + 30] = {0}; - sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); - FILE *fp = fopen(versionFile, "w"); - if (!fp) { - vError("vgId:%d, failed to open vnode version file for write, file:%s error:%s", pVnode->vgId, - versionFile, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - int32_t len = 0; - int32_t maxLen = 30; - char content[TSDB_VNODE_VERSION_CONTENT_LEN] = {0}; - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"version\": %" PRId64 "\n", pVnode->fversion); - len += snprintf(content + len, maxLen - len, "}\n"); - - fwrite(content, 1, len, fp); - fflush(fp); - fclose(fp); - - vInfo("vgId:%d, save vnode version:%" PRId64 " succeed", pVnode->vgId, pVnode->fversion); - - return TSDB_CODE_SUCCESS; -} - -static int32_t vnodeReadVersion(SVnodeObj *pVnode) { - char versionFile[TSDB_FILENAME_LEN + 30] = {0}; - char *content = NULL; - cJSON *root = NULL; - int maxLen = 100; - - terrno = TSDB_CODE_VND_INVALID_VRESION_FILE; - sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); - FILE *fp = fopen(versionFile, "r"); - if (!fp) { - if (errno != ENOENT) { - vError("vgId:%d, failed to open version file:%s error:%s", pVnode->vgId, versionFile, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - } else { - terrno = TSDB_CODE_SUCCESS; - } - goto PARSE_OVER; - } - - content = calloc(1, maxLen + 1); - int len = fread(content, 1, maxLen, fp); - if (len <= 0) { - vError("vgId:%d, failed to read vnode version, content is null", pVnode->vgId); - goto PARSE_OVER; - } - - root = cJSON_Parse(content); - if (root == NULL) { - vError("vgId:%d, failed to read vnode version, invalid json format", pVnode->vgId); - goto PARSE_OVER; - } - - cJSON *ver = cJSON_GetObjectItem(root, "version"); - if (!ver || ver->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode version, version not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->version = ver->valueint; - - terrno = TSDB_CODE_SUCCESS; - vInfo("vgId:%d, read vnode version successfully, version:%" PRId64, pVnode->vgId, pVnode->version); - -PARSE_OVER: - taosTFree(content); - cJSON_Delete(root); - if (fp) fclose(fp); - return terrno; -} diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 99aed03e54ccd069e5879104f62eb01ff7bb3d05..9fa3a11c9c659b44a4edc12110969feb6779a8b7 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -15,13 +15,10 @@ #define _DEFAULT_SOURCE #define _NON_BLOCKING_RETRIEVE 0 - #include "os.h" - #include "tglobal.h" #include "taoserror.h" #include "taosmsg.h" -#include "tcache.h" #include "query.h" #include "trpc.h" #include "tsdb.h" @@ -29,9 +26,9 @@ #include "vnodeInt.h" #include "tqueue.h" -static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SReadMsg *pReadMsg); -static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg); -static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg); +static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pRead); +static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead); +static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead); static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId); void vnodeInitReadFp(void) { @@ -44,19 +41,19 @@ void vnodeInitReadFp(void) { // still required, or there will be a deadlock, so we don’t do any check here, but put the check codes before the // request enters the queue // -int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) { +int32_t vnodeProcessRead(void *param, SVReadMsg *pRead) { SVnodeObj *pVnode = (SVnodeObj *)param; - int msgType = pReadMsg->rpcMsg.msgType; + int32_t msgType = pRead->msgType; if (vnodeProcessReadMsgFp[msgType] == NULL) { vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]); return TSDB_CODE_VND_MSG_NOT_PROCESSED; } - return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); + return (*vnodeProcessReadMsgFp[msgType])(pVnode, pRead); } -int32_t vnodeCheckRead(void *param) { +static int32_t vnodeCheckRead(void *param) { SVnodeObj *pVnode = param; if (pVnode->status != TAOS_VN_STATUS_READY) { vDebug("vgId:%d, vnode status is %s, recCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status], @@ -78,24 +75,58 @@ int32_t vnodeCheckRead(void *param) { return TSDB_CODE_SUCCESS; } -static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) { - int32_t code = vnodeCheckRead(pVnode); - if (code != TSDB_CODE_SUCCESS) return code; - SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); - pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; - pRead->pCont = qhandle; - pRead->contLen = 0; - pRead->rpcMsg.ahandle = ahandle; +int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { + SVnodeObj *pVnode = vparam; - atomic_add_fetch_32(&pVnode->refCount, 1); + if (qtype == TAOS_QTYPE_RPC || qtype == TAOS_QTYPE_QUERY) { + int32_t code = vnodeCheckRead(pVnode); + if (code != TSDB_CODE_SUCCESS) return code; + } + + int32_t size = sizeof(SVReadMsg) + contLen; + SVReadMsg *pRead = taosAllocateQitem(size); + if (pRead == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + + if (rparam != NULL) { + SRpcMsg *pRpcMsg = rparam; + pRead->rpcHandle = pRpcMsg->handle; + pRead->rpcAhandle = pRpcMsg->ahandle; + pRead->msgType = pRpcMsg->msgType; + pRead->code = pRpcMsg->code; + } + + if (contLen != 0) { + pRead->contLen = contLen; + memcpy(pRead->pCont, pCont, contLen); + } else { + pRead->qhandle = pCont; + } + + pRead->qtype = qtype; - vDebug("QInfo:%p add to vread queue for exec query, msg:%p", *qhandle, pRead); - taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead); + atomic_add_fetch_32(&pVnode->refCount, 1); + vTrace("vgId:%d, get vnode rqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); + taosWriteQitem(pVnode->rqueue, qtype, pRead); return TSDB_CODE_SUCCESS; } +static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) { + SRpcMsg rpcMsg = {0}; + rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; + rpcMsg.ahandle = ahandle; + + int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg); + if (code == TSDB_CODE_SUCCESS) { + vDebug("QInfo:%p add to vread queue for exec query", *qhandle); + } + + return code; +} + /** * * @param pRet response message object @@ -146,27 +177,27 @@ static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) { pRsp->completed = true; } -static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { - void * pCont = pReadMsg->pCont; - int32_t contLen = pReadMsg->contLen; - SRspRet *pRet = &pReadMsg->rspRet; +static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { + void * pCont = pRead->pCont; + int32_t contLen = pRead->contLen; + SRspRet *pRet = &pRead->rspRet; SQueryTableMsg *pQueryTableMsg = (SQueryTableMsg *)pCont; memset(pRet, 0, sizeof(SRspRet)); // qHandle needs to be freed correctly - if (pReadMsg->rpcMsg.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont; + if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pRead->pCont; killQueryMsg->free = htons(killQueryMsg->free); killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle); - vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle); - assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1); + vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pRead->rpcHandle); + assert(pRead->contLen > 0 && killQueryMsg->free == 1); void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle); if (qhandle == NULL || *qhandle == NULL) { vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle, - pReadMsg->rpcMsg.handle); + pRead->rpcHandle); } else { assert(*qhandle == (void *)killQueryMsg->qhandle); @@ -198,7 +229,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (handle == NULL) { // failed to register qhandle pRsp->code = terrno; terrno = 0; - vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo, + vError("vgId:%d, QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo, tstrerror(pRsp->code)); qDestroyQueryInfo(pQInfo); // destroy it directly return pRsp->code; @@ -208,9 +239,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } if (handle != NULL && - vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, - pReadMsg->rpcMsg.handle); + pRead->rpcHandle); pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); return pRsp->code; @@ -221,7 +252,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (handle != NULL) { vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); - code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcMsg.ahandle); + code = vnodePutItemIntoReadQueue(pVnode, handle, pRead->rpcHandle); if (code != TSDB_CODE_SUCCESS) { pRsp->code = code; qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -230,7 +261,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } } else { assert(pCont != NULL); - void **qhandle = (void **)pCont; + void **qhandle = (void **)pRead->qhandle; vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); @@ -242,14 +273,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { // build query rsp, the retrieve request has reached here already if (buildRes) { // update the connection info according to the retrieve connection - pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*qhandle); - assert(pReadMsg->rpcMsg.handle != NULL); + pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle); + assert(pRead->rpcHandle != NULL); vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, - pReadMsg->rpcMsg.handle); + pRead->rpcHandle); // set the real rsp error code - pReadMsg->rpcMsg.code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcMsg.ahandle); + pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pRead->rpcHandle); // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client code = TSDB_CODE_QRY_HAS_RSP; @@ -274,16 +305,16 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { return code; } -static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { - void * pCont = pReadMsg->pCont; - SRspRet *pRet = &pReadMsg->rspRet; +static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { + void * pCont = pRead->pCont; + SRspRet *pRet = &pRead->rspRet; SRetrieveTableMsg *pRetrieve = pCont; pRetrieve->free = htons(pRetrieve->free); pRetrieve->qhandle = htobe64(pRetrieve->qhandle); vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle, - pRetrieve->free, pReadMsg->rpcMsg.handle); + pRetrieve->free, pRead->rpcHandle); memset(pRet, 0, sizeof(SRspRet)); @@ -298,7 +329,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } if (code != TSDB_CODE_SUCCESS) { - vDebug("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle); + vError("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle); vnodeBuildNoResultQueryRsp(pRet); return code; } @@ -314,9 +345,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } // register the qhandle to connect to quit query immediate if connection is broken - if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { - vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, - pReadMsg->rpcMsg.handle); + if (vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pRead->rpcHandle); code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -326,7 +356,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { bool freeHandle = true; bool buildRes = false; - code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle); + code = qRetrieveQueryResultInfo(*handle, &buildRes, pRead->rpcHandle); if (code != TSDB_CODE_SUCCESS) { // TODO handle malloc failure pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); @@ -337,7 +367,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { assert(buildRes == true); #if _NON_BLOCKING_RETRIEVE if (!buildRes) { - assert(pReadMsg->rpcMsg.handle != NULL); + assert(pRead->rpcHandle != NULL); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false); return TSDB_CODE_QRY_NOT_READY; @@ -345,7 +375,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { #endif // ahandle is the sqlObj pointer - code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcMsg.ahandle); + code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pRead->rpcHandle); } // If qhandle is not added into vread queue, the query should be completed already or paused with error. diff --git a/src/vnode/src/vnodeVersion.c b/src/vnode/src/vnodeVersion.c new file mode 100644 index 0000000000000000000000000000000000000000..8f6360b4f98eac8f394f5078ed2b025cea4192b0 --- /dev/null +++ b/src/vnode/src/vnodeVersion.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "cJSON.h" +#include "tglobal.h" +#include "tsdb.h" +#include "vnodeInt.h" +#include "vnodeVersion.h" + +int32_t vnodeReadVersion(SVnodeObj *pVnode) { + int32_t len = 0; + int32_t maxLen = 100; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + + terrno = TSDB_CODE_VND_INVALID_VRESION_FILE; + char file[TSDB_FILENAME_LEN + 30] = {0}; + sprintf(file, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); + + fp = fopen(file, "r"); + if (!fp) { + if (errno != ENOENT) { + vError("vgId:%d, failed to read %s, error:%s", pVnode->vgId, file, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + } else { + terrno = TSDB_CODE_SUCCESS; + } + goto PARSE_VER_ERROR; + } + + len = fread(content, 1, maxLen, fp); + if (len <= 0) { + vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file); + goto PARSE_VER_ERROR; + } + + root = cJSON_Parse(content); + if (root == NULL) { + vError("vgId:%d, failed to read %s, invalid json format", pVnode->vgId, file); + goto PARSE_VER_ERROR; + } + + cJSON *ver = cJSON_GetObjectItem(root, "version"); + if (!ver || ver->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, version not found", pVnode->vgId, file); + goto PARSE_VER_ERROR; + } + pVnode->version = (uint64_t)ver->valueint; + + terrno = TSDB_CODE_SUCCESS; + vInfo("vgId:%d, read %s successfully, fver:%" PRIu64, pVnode->vgId, file, pVnode->version); + +PARSE_VER_ERROR: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + + return terrno; +} + +int32_t vnodeSaveVersion(SVnodeObj *pVnode) { + char file[TSDB_FILENAME_LEN + 30] = {0}; + sprintf(file, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); + + FILE *fp = fopen(file, "w"); + if (!fp) { + vError("vgId:%d, failed to write %s, reason:%s", pVnode->vgId, file, strerror(errno)); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 100; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"version\": %" PRIu64 "\n", pVnode->fversion); + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + fflush(fp); + fclose(fp); + free(content); + terrno = 0; + + vInfo("vgId:%d, successed to write %s, fver:%" PRIu64, pVnode->vgId, file, pVnode->fversion); + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 855df81a1b9029d7ddd2f5e5ca6d42e8c680f7ac..e67c544fb23456d8775077e8a86f1745508bd0fc 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -19,7 +19,6 @@ #include "taoserror.h" #include "tqueue.h" #include "trpc.h" -#include "tutil.h" #include "tsdb.h" #include "twal.h" #include "tsync.h" @@ -46,26 +45,31 @@ void vnodeInitWriteFp(void) { vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessUpdateTagValMsg; } -int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { - int32_t code = 0; - SVnodeObj *pVnode = (SVnodeObj *)param1; - SWalHead * pHead = param2; +int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) { + int32_t code = 0; + SVnodeObj * pVnode = vparam; + SWalHead * pHead = wparam; + SRspRet * pRspRet = rparam; if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) { - vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[pHead->msgType]); + vError("vgId:%d, msg:%s not processed since no handle, qtype:%s hver:%" PRIu64, pVnode->vgId, + taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version); return TSDB_CODE_VND_MSG_NOT_PROCESSED; } + vTrace("vgId:%d, msg:%s will be processed in vnode, qtype:%s hver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, + taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version, pVnode->version); + if (pHead->version == 0) { // from client or CQ if (pVnode->status != TAOS_VN_STATUS_READY) { - vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[pHead->msgType], - pVnode->status); + vDebug("vgId:%d, msg:%s not processed since vstatus:%d, qtype:%s hver:%" PRIu64, pVnode->vgId, + taosMsg[pHead->msgType], pVnode->status, qtypeStr[qtype], pHead->version); return TSDB_CODE_APP_NOT_READY; // it may be in deleting or closing state } if (pVnode->role != TAOS_SYNC_ROLE_MASTER) { - vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%s", pVnode->vgId, taosMsg[pHead->msgType], - pVnode->syncCfg.replica, syncRole[pVnode->role]); + vDebug("vgId:%d, msg:%s not processed since replica:%d role:%s, qtype:%s hver:%" PRIu64, pVnode->vgId, + taosMsg[pHead->msgType], pVnode->syncCfg.replica, syncRole[pVnode->role], qtypeStr[qtype], pHead->version); return TSDB_CODE_APP_NOT_READY; } @@ -80,7 +84,7 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { // forward to peers, even it is WAL/FWD, it shall be called to update version in sync int32_t syncCode = 0; - syncCode = syncForwardToPeer(pVnode->sync, pHead, item, qtype); + syncCode = syncForwardToPeer(pVnode->sync, pHead, pRspRet, qtype); if (syncCode < 0) return syncCode; // write into WAL @@ -90,13 +94,13 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { pVnode->version = pHead->version; // write data locally - code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item); + code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, pRspRet); if (code < 0) return code; return syncCode; } -int32_t vnodeCheckWrite(void *param) { +static int32_t vnodeCheckWrite(void *param) { SVnodeObj *pVnode = param; if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) { vDebug("vgId:%d, no write auth, recCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); @@ -183,8 +187,8 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet } static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { - SMDDropSTableMsg *pTable = pCont; - int32_t code = TSDB_CODE_SUCCESS; + SDropSTableMsg *pTable = pCont; + int32_t code = TSDB_CODE_SUCCESS; vDebug("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableId); @@ -204,37 +208,37 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR return TSDB_CODE_SUCCESS; } +int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { + SVnodeObj *pVnode = vparam; + SWalHead * pHead = wparam; -int vnodeWriteCqMsgToQueue(void *param, void *data, int type) { - SVnodeObj *pVnode = param; - SWalHead * pHead = data; - - int size = sizeof(SWalHead) + pHead->len; - SSyncHead *pSync = (SSyncHead*) taosAllocateQitem(size + sizeof(SSyncHead)); - SWalHead *pWal = (SWalHead *)(pSync + 1); - memcpy(pWal, pHead, size); - - atomic_add_fetch_32(&pVnode->refCount, 1); - vTrace("CQ: vgId:%d, get vnode wqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); - - taosWriteQitem(pVnode->wqueue, type, pSync); + if (qtype == TAOS_QTYPE_RPC) { + int32_t code = vnodeCheckWrite(pVnode); + if (code != TSDB_CODE_SUCCESS) return code; + } - return 0; -} + if (pHead->len > TSDB_MAX_WAL_SIZE) { + vError("vgId:%d, wal len:%d exceeds limit, hver:%" PRIu64, pVnode->vgId, pHead->len, pHead->version); + return TSDB_CODE_WAL_SIZE_LIMIT; + } + int32_t size = sizeof(SVWriteMsg) + sizeof(SWalHead) + pHead->len; + SVWriteMsg *pWrite = taosAllocateQitem(size); + if (pWrite == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } -int vnodeWriteToQueue(void *param, void *data, int type) { - SVnodeObj *pVnode = param; - SWalHead * pHead = data; + if (rparam != NULL) { + SRpcMsg *pRpcMsg = rparam; + pWrite->rpcHandle = pRpcMsg->handle; + pWrite->rpcAhandle = pRpcMsg->ahandle; + } - int size = sizeof(SWalHead) + pHead->len; - SWalHead *pWal = (SWalHead *)taosAllocateQitem(size); - memcpy(pWal, pHead, size); + memcpy(pWrite->pHead, pHead, sizeof(SWalHead) + pHead->len); atomic_add_fetch_32(&pVnode->refCount, 1); vTrace("vgId:%d, get vnode wqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); - taosWriteQitem(pVnode->wqueue, type, pWal); - - return 0; + taosWriteQitem(pVnode->wqueue, qtype, pWrite); + return TSDB_CODE_SUCCESS; } diff --git a/src/wal/inc/walInt.h b/src/wal/inc/walInt.h new file mode 100644 index 0000000000000000000000000000000000000000..36311c8f5d92fe0ba97793224df59d4dd33b5da6 --- /dev/null +++ b/src/wal/inc/walInt.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_WAL_INT_H +#define TDENGINE_WAL_INT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tlog.h" + +extern int32_t wDebugFlag; + +#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }} +#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }} +#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }} +#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }} +#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} +#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} + +#define WAL_PREFIX "wal" +#define WAL_PREFIX_LEN 3 +#define WAL_REFRESH_MS 1000 +#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16) +#define WAL_SIGNATURE ((uint32_t)(0xFAFBFDFE)) +#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define WAL_FILE_LEN (TSDB_FILENAME_LEN + 32) +#define WAL_FILE_NUM 3 + +typedef struct { + uint64_t version; + int64_t fileId; + int64_t rid; + int32_t vgId; + int32_t fd; + int32_t keep; + int32_t level; + int32_t fsyncPeriod; + int32_t fsyncSeq; + int8_t stop; + int8_t reserved[3]; + char path[WAL_PATH_LEN]; + char name[WAL_FILE_LEN]; + pthread_mutex_t mutex; +} SWal; + +int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId); +int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId); +int32_t walGetNewFile(SWal *pWal, int64_t *newFileId); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c deleted file mode 100644 index 182600204259e703d171d6598f46a2a16cdcb27b..0000000000000000000000000000000000000000 --- a/src/wal/src/walMain.c +++ /dev/null @@ -1,588 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE - -#define TAOS_RANDOM_FILE_FAIL_TEST - -#include "os.h" -#include "tlog.h" -#include "tchecksum.h" -#include "tutil.h" -#include "ttimer.h" -#include "taoserror.h" -#include "twal.h" -#include "tqueue.h" - -#define walPrefix "wal" - -#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }} -#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }} -#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }} -#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }} -#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} -#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} - -typedef struct { - uint64_t version; - int fd; - int keep; - int level; - int32_t fsyncPeriod; - void *timer; - void *signature; - int max; // maximum number of wal files - uint32_t id; // increase continuously - int num; // number of wal files - char path[TSDB_FILENAME_LEN]; - char name[TSDB_FILENAME_LEN+16]; - pthread_mutex_t mutex; -} SWal; - -static void *walTmrCtrl = NULL; -static int tsWalNum = 0; -static pthread_once_t walModuleInit = PTHREAD_ONCE_INIT; -static uint32_t walSignature = 0xFAFBFDFE; -static int walHandleExistingFiles(const char *path); -static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp); -static int walRemoveWalFiles(const char *path); -static void walProcessFsyncTimer(void *param, void *tmrId); -static void walRelease(SWal *pWal); -static int walGetMaxOldFileId(char *odir); - -static void walModuleInitFunc() { - walTmrCtrl = taosTmrInit(1000, 100, 300000, "WAL"); - if (walTmrCtrl == NULL) - walModuleInit = PTHREAD_ONCE_INIT; - else - wDebug("WAL module is initialized"); -} - -static inline bool walNeedFsyncTimer(SWal *pWal) { - if (pWal->fsyncPeriod > 0 && pWal->level == TAOS_WAL_FSYNC) { - return true; - } - return false; -} - -void *walOpen(const char *path, const SWalCfg *pCfg) { - SWal *pWal = calloc(sizeof(SWal), 1); - if (pWal == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } - - pthread_once(&walModuleInit, walModuleInitFunc); - if (walTmrCtrl == NULL) { - free(pWal); - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } - - atomic_add_fetch_32(&tsWalNum, 1); - pWal->fd = -1; - pWal->max = pCfg->wals; - pWal->id = 0; - pWal->num = 0; - pWal->level = pCfg->walLevel; - pWal->keep = pCfg->keep; - pWal->fsyncPeriod = pCfg->fsyncPeriod; - pWal->signature = pWal; - tstrncpy(pWal->path, path, sizeof(pWal->path)); - pthread_mutex_init(&pWal->mutex, NULL); - - if (walNeedFsyncTimer(pWal)) { - pWal->timer = taosTmrStart(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, walTmrCtrl); - if (pWal->timer == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - walRelease(pWal); - return NULL; - } - } - - if (taosMkDir(path, 0755) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("wal:%s, failed to create directory(%s)", path, strerror(errno)); - walRelease(pWal); - pWal = NULL; - } - - if (pCfg->keep == 1) return pWal; - - if (walHandleExistingFiles(path) == 0) walRenew(pWal); - - if (pWal && pWal->fd < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("wal:%s, failed to open(%s)", path, strerror(errno)); - walRelease(pWal); - pWal = NULL; - } - - if (pWal) wDebug("wal:%s, it is open, level:%d fsyncPeriod:%d", path, pWal->level, pWal->fsyncPeriod); - return pWal; -} - -int walAlter(twalh wal, const SWalCfg *pCfg) { - SWal *pWal = wal; - if (pWal == NULL) { - return TSDB_CODE_WAL_APP_ERROR; - } - - if (pWal->level == pCfg->walLevel && pWal->fsyncPeriod == pCfg->fsyncPeriod) { - wDebug("wal:%s, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->name, pWal->level, - pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod); - return TSDB_CODE_SUCCESS; - } - - wInfo("wal:%s, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->name, pWal->level, pWal->fsyncPeriod, - pCfg->walLevel, pCfg->fsyncPeriod); - - pthread_mutex_lock(&pWal->mutex); - pWal->level = pCfg->walLevel; - pWal->fsyncPeriod = pCfg->fsyncPeriod; - if (walNeedFsyncTimer(pWal)) { - wInfo("wal:%s, reset fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod); - taosTmrReset(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, &pWal->timer, walTmrCtrl); - } else { - wInfo("wal:%s, stop fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod); - taosTmrStop(pWal->timer); - pWal->timer = NULL; - } - pthread_mutex_unlock(&pWal->mutex); - - return TSDB_CODE_SUCCESS; -} - -void walClose(void *handle) { - if (handle == NULL) return; - - SWal *pWal = handle; - taosClose(pWal->fd); - if (pWal->timer) taosTmrStopA(&pWal->timer); - - if (pWal->keep == 0) { - // remove all files in the directory - for (int i = 0; i < pWal->num; ++i) { - snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id - i); - if (remove(pWal->name) < 0) { - wError("wal:%s, failed to remove", pWal->name); - } else { - wDebug("wal:%s, it is removed", pWal->name); - } - } - } else { - wDebug("wal:%s, it is closed and kept", pWal->name); - } - - walRelease(pWal); -} - -int walRenew(void *handle) { - if (handle == NULL) return 0; - SWal *pWal = handle; - - terrno = 0; - - pthread_mutex_lock(&pWal->mutex); - - if (pWal->fd >= 0) { - close(pWal->fd); - pWal->id++; - wDebug("wal:%s, it is closed", pWal->name); - } - - pWal->num++; - - snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id); - pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - - if (pWal->fd < 0) { - wError("wal:%s, failed to open(%s)", pWal->name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - } else { - wDebug("wal:%s, it is created", pWal->name); - - if (pWal->num > pWal->max) { - // remove the oldest wal file - char name[TSDB_FILENAME_LEN * 3]; - snprintf(name, sizeof(name), "%s/%s%d", pWal->path, walPrefix, pWal->id - pWal->max); - if (remove(name) < 0) { - wError("wal:%s, failed to remove(%s)", name, strerror(errno)); - } else { - wDebug("wal:%s, it is removed", name); - } - - pWal->num--; - } - } - - pthread_mutex_unlock(&pWal->mutex); - - return terrno; -} - -int walWrite(void *handle, SWalHead *pHead) { - SWal *pWal = handle; - if (pWal == NULL) return -1; - - terrno = 0; - - // no wal - if (pWal->level == TAOS_WAL_NOLOG) return 0; - if (pHead->version <= pWal->version) return 0; - - pHead->signature = walSignature; - taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead)); - int contLen = pHead->len + sizeof(SWalHead); - - if (taosTWrite(pWal->fd, pHead, contLen) != contLen) { - wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - } else { - pWal->version = pHead->version; - } - - return terrno; -} - -void walFsync(void *handle) { - SWal *pWal = handle; - if (pWal == NULL || pWal->level != TAOS_WAL_FSYNC || pWal->fd < 0) return; - - if (pWal->fsyncPeriod == 0) { - if (fsync(pWal->fd) < 0) { - wError("wal:%s, fsync failed(%s)", pWal->name, strerror(errno)); - } - } -} - -int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) { - SWal *pWal = handle; - struct dirent *ent; - int count = 0; - uint32_t maxId = 0, minId = -1, index =0; - - terrno = 0; - int plen = strlen(walPrefix); - char opath[TSDB_FILENAME_LEN + 5]; - - int slen = snprintf(opath, sizeof(opath), "%s", pWal->path); - if (pWal->keep == 0) strcpy(opath + slen, "/old"); - - DIR *dir = opendir(opath); - if (dir == NULL && errno == ENOENT) return 0; - if (dir == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - - while ((ent = readdir(dir)) != NULL) { - if (strncmp(ent->d_name, walPrefix, plen) == 0) { - index = atol(ent->d_name + plen); - if (index > maxId) maxId = index; - if (index < minId) minId = index; - count++; - } - } - - closedir(dir); - - if (count == 0) { - if (pWal->keep) terrno = walRenew(pWal); - return terrno; - } - - if (count != (maxId - minId + 1)) { - wError("wal:%s, messed up, count:%d max:%d min:%d", opath, count, maxId, minId); - terrno = TSDB_CODE_WAL_APP_ERROR; - } else { - wDebug("wal:%s, %d files will be restored", opath, count); - - for (index = minId; index <= maxId; ++index) { - snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, index); - terrno = walRestoreWalFile(pWal, pVnode, writeFp); - if (terrno < 0) continue; - } - } - - if (terrno == 0) { - if (pWal->keep == 0) { - terrno = walRemoveWalFiles(opath); - if (terrno == 0) { - if (remove(opath) < 0) { - wError("wal:%s, failed to remove directory(%s)", opath, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - } - } - } else { - // open the existing WAL file in append mode - pWal->num = count; - pWal->id = maxId; - snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, maxId); - pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); - if (pWal->fd < 0) { - wError("wal:%s, failed to open file(%s)", pWal->name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - } - } - } - - return terrno; -} - -int walGetWalFile(void *handle, char *name, uint32_t *index) { - SWal * pWal = handle; - int code = 1; - int32_t first = 0; - - name[0] = 0; - if (pWal == NULL || pWal->num == 0) return 0; - - pthread_mutex_lock(&(pWal->mutex)); - - first = pWal->id + 1 - pWal->num; - if (*index == 0) *index = first; // set to first one - - if (*index < first && *index > pWal->id) { - code = -1; // index out of range - } else { - sprintf(name, "wal/%s%d", walPrefix, *index); - code = (*index == pWal->id) ? 0 : 1; - } - - pthread_mutex_unlock(&(pWal->mutex)); - - return code; -} - -static void walRelease(SWal *pWal) { - pthread_mutex_destroy(&pWal->mutex); - pWal->signature = NULL; - free(pWal); - - if (atomic_sub_fetch_32(&tsWalNum, 1) == 0) { - if (walTmrCtrl) taosTmrCleanUp(walTmrCtrl); - walTmrCtrl = NULL; - walModuleInit = PTHREAD_ONCE_INIT; - wDebug("WAL module is cleaned up"); - } -} - -static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) { - char *name = pWal->name; - int size = 1024 * 1024; // default 1M buffer size - - terrno = 0; - char *buffer = malloc(size); - if (buffer == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - - SWalHead *pHead = (SWalHead *)buffer; - - int fd = open(name, O_RDWR); - if (fd < 0) { - wError("wal:%s, failed to open for restore(%s)", name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - free(buffer); - return terrno; - } - - wDebug("wal:%s, start to restore", name); - - size_t offset = 0; - while (1) { - int ret = taosTRead(fd, pHead, sizeof(SWalHead)); - if (ret == 0) break; - - if (ret < 0) { - wError("wal:%s, failed to read wal head part since %s", name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - break; - } - - if (ret < sizeof(SWalHead)) { - wError("wal:%s, failed to read head, ret:%d, skip the rest of file", name, ret); - taosFtruncate(fd, offset); - fsync(fd); - break; - } - - if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { - wWarn("wal:%s, cksum is messed up, skip the rest of file", name); - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - // ASSERT(false); - break; - } - - if (pHead->len > size - sizeof(SWalHead)) { - size = sizeof(SWalHead) + pHead->len; - buffer = realloc(buffer, size); - if (buffer == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - break; - } - - pHead = (SWalHead *)buffer; - } - - ret = taosTRead(fd, pHead->cont, pHead->len); - if (ret < 0) { - wError("wal:%s failed to read wal body part since %s", name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - break; - } - - if (ret < pHead->len) { - wError("wal:%s, failed to read body, len:%d ret:%d, skip the rest of file", name, pHead->len, ret); - taosFtruncate(fd, offset); - fsync(fd); - break; - } - - offset = offset + sizeof(SWalHead) + pHead->len; - - if (pWal->keep) pWal->version = pHead->version; - (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL); - } - - close(fd); - free(buffer); - - return terrno; -} - -int walHandleExistingFiles(const char *path) { - char oname[TSDB_FILENAME_LEN * 3]; - char nname[TSDB_FILENAME_LEN * 3]; - char opath[TSDB_FILENAME_LEN]; - - snprintf(opath, sizeof(opath), "%s/old", path); - - struct dirent *ent; - DIR *dir = opendir(path); - int plen = strlen(walPrefix); - terrno = 0; - - int midx = walGetMaxOldFileId(opath); - int count = 0; - while ((ent = readdir(dir)) != NULL) { - if (strncmp(ent->d_name, walPrefix, plen) == 0) { - midx++; - snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name); - snprintf(nname, sizeof(nname), "%s/old/wal%d", path, midx); - if (taosMkDir(opath, 0755) != 0) { - wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - break; - } - - if (rename(oname, nname) < 0) { - wError("wal:%s, failed to move to new:%s", oname, nname); - terrno = TAOS_SYSTEM_ERROR(errno); - break; - } - - count++; - } - - wDebug("wal:%s, %d files are moved for restoration", path, count); - } - - closedir(dir); - return terrno; -} - -static int walRemoveWalFiles(const char *path) { - int plen = strlen(walPrefix); - char name[TSDB_FILENAME_LEN * 3]; - - terrno = 0; - - struct dirent *ent; - DIR *dir = opendir(path); - if (dir == NULL && errno == ENOENT) return 0; - if (dir == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - - while ((ent = readdir(dir)) != NULL) { - if (strncmp(ent->d_name, walPrefix, plen) == 0) { - snprintf(name, sizeof(name), "%s/%s", path, ent->d_name); - if (remove(name) < 0) { - wError("wal:%s, failed to remove(%s)", name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - } - } - } - - closedir(dir); - - return terrno; -} - -static void walProcessFsyncTimer(void *param, void *tmrId) { - SWal *pWal = param; - - if (pWal->signature != pWal) return; - if (pWal->fd < 0) return; - - if (fsync(pWal->fd) < 0) { - wError("wal:%s, fsync failed(%s)", pWal->name, strerror(errno)); - } - - if (walNeedFsyncTimer(pWal)) { - pWal->timer = taosTmrStart(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, walTmrCtrl); - } else { - wInfo("wal:%s, stop fsync timer for walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod); - taosTmrStop(pWal->timer); - pWal->timer = NULL; - } -} - -int64_t walGetVersion(twalh param) { - SWal *pWal = param; - if (pWal == 0) return 0; - - return pWal->version; -} - -static int walGetMaxOldFileId(char *odir) { - int midx = 0; - DIR * dir = NULL; - struct dirent *dp = NULL; - int plen = strlen(walPrefix); - - if (access(odir, F_OK) != 0) return midx; - - dir = opendir(odir); - if (dir == NULL) { - wError("failed to open directory %s since %s", odir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - while ((dp = readdir(dir)) != NULL) { - if (strncmp(dp->d_name, walPrefix, plen) == 0) { - int idx = atol(dp->d_name + plen); - if (midx < idx) midx = idx; - } - } - - closedir(dir); - return midx; -} \ No newline at end of file diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c new file mode 100644 index 0000000000000000000000000000000000000000..fb49f38217d62b9181f0eed92155434b516ab3e0 --- /dev/null +++ b/src/wal/src/walMgmt.c @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "tref.h" +#include "twal.h" +#include "walInt.h" + +typedef struct { + int32_t refId; + int32_t seq; + int8_t stop; + pthread_t thread; + pthread_mutex_t mutex; +} SWalMgmt; + +static SWalMgmt tsWal = {0}; +static int32_t walCreateThread(); +static void walStopThread(); +static int32_t walInitObj(SWal *pWal); +static void walFreeObj(void *pWal); + +int32_t walInit() { + tsWal.refId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj); + + int32_t code = walCreateThread(); + if (code != TSDB_CODE_SUCCESS) { + wError("failed to init wal module since %s", tstrerror(code)); + return code; + } + + wInfo("wal module is initialized, refId:%d", tsWal.refId); + return code; +} + +void walCleanUp() { + walStopThread(); + taosCloseRef(tsWal.refId); + wInfo("wal module is cleaned up"); +} + +void *walOpen(char *path, SWalCfg *pCfg) { + SWal *pWal = tcalloc(1, sizeof(SWal)); + if (pWal == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } + + pWal->vgId = pCfg->vgId; + pWal->fd = -1; + pWal->fileId = -1; + pWal->level = pCfg->walLevel; + pWal->keep = pCfg->keep; + pWal->fsyncPeriod = pCfg->fsyncPeriod; + tstrncpy(pWal->path, path, sizeof(pWal->path)); + pthread_mutex_init(&pWal->mutex, NULL); + + pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; + if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; + + if (walInitObj(pWal) != TSDB_CODE_SUCCESS) { + walFreeObj(pWal); + return NULL; + } + + pWal->rid = taosAddRef(tsWal.refId, pWal); + if (pWal->rid < 0) { + walFreeObj(pWal); + return NULL; + } + + wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->vgId, pWal, pWal->level, pWal->fsyncPeriod); + + return pWal; +} + +int32_t walAlter(void *handle, SWalCfg *pCfg) { + if (handle == NULL) return TSDB_CODE_WAL_APP_ERROR; + SWal *pWal = handle; + + if (pWal->level == pCfg->walLevel && pWal->fsyncPeriod == pCfg->fsyncPeriod) { + wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->vgId, pWal->level, + pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod); + return TSDB_CODE_SUCCESS; + } + + wInfo("vgId:%d, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->vgId, pWal->level, + pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod); + + pWal->level = pCfg->walLevel; + pWal->fsyncPeriod = pCfg->fsyncPeriod; + pWal->fsyncSeq = pCfg->fsyncPeriod % 1000; + if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; + + return TSDB_CODE_SUCCESS; +} + +void walStop(void *handle) { + if (handle == NULL) return; + SWal *pWal = handle; + + pthread_mutex_lock(&pWal->mutex); + pWal->stop = 1; + pthread_mutex_unlock(&pWal->mutex); + wDebug("vgId:%d, stop write wal", pWal->vgId); +} + +void walClose(void *handle) { + if (handle == NULL) return; + + SWal *pWal = handle; + pthread_mutex_lock(&pWal->mutex); + + taosClose(pWal->fd); + + if (pWal->keep != TAOS_WAL_KEEP) { + walRemoveAllOldFiles(pWal); + } else { + wDebug("vgId:%d, wal:%p file:%s, it is closed and kept", pWal->vgId, pWal, pWal->name); + } + + pthread_mutex_unlock(&pWal->mutex); + taosRemoveRef(tsWal.refId, pWal->rid); +} + +static int32_t walInitObj(SWal *pWal) { + if (taosMkDir(pWal->path, 0755) != 0) { + wError("vgId:%d, path:%s, failed to create directory since %s", pWal->vgId, pWal->path, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + wDebug("vgId:%d, object is initialized", pWal->vgId); + return TSDB_CODE_SUCCESS; +} + +static void walFreeObj(void *wal) { + SWal *pWal = wal; + wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal); + + taosClose(pWal->fd); + pthread_mutex_destroy(&pWal->mutex); + tfree(pWal); +} + +static bool walNeedFsync(SWal *pWal) { + if (pWal->fsyncPeriod <= 0 || pWal->level != TAOS_WAL_FSYNC) { + return false; + } + + if (tsWal.seq % pWal->fsyncSeq == 0) { + return true; + } + + return false; +} + +static void walUpdateSeq() { + taosMsleep(WAL_REFRESH_MS); + if (++tsWal.seq <= 0) { + tsWal.seq = 1; + } +} + +static void walFsyncAll() { + SWal *pWal = taosIterateRef(tsWal.refId, 0); + while (pWal) { + if (walNeedFsync(pWal)) { + wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq); + int32_t code = fsync(pWal->fd); + if (code != 0) { + wError("vgId:%d, file:%s, failed to fsync since %s", pWal->vgId, pWal->name, strerror(code)); + } + } + pWal = taosIterateRef(tsWal.refId, pWal->rid); + } +} + +static void *walThreadFunc(void *param) { + while (1) { + walUpdateSeq(); + walFsyncAll(); + if (tsWal.stop) break; + } + + return NULL; +} + +static int32_t walCreateThread() { + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) { + wError("failed to create wal thread since %s", strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + pthread_attr_destroy(&thAttr); + wDebug("wal thread is launched"); + + return TSDB_CODE_SUCCESS; +} + +static void walStopThread() { + tsWal.stop = 1; + if (tsWal.thread) { + pthread_join(tsWal.thread, NULL); + } + + wDebug("wal thread is stopped"); +} diff --git a/src/wal/src/walUtil.c b/src/wal/src/walUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..e4d9a555b3a60cb6be1e6584652ec4a309b1c301 --- /dev/null +++ b/src/wal/src/walUtil.c @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "walInt.h" + +int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId) { + int64_t curFileId = *nextFileId; + int64_t minFileId = INT64_MAX; + + DIR *dir = opendir(pWal->path); + if (dir == NULL) { + wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); + return -1; + } + + struct dirent *ent; + while ((ent = readdir(dir)) != NULL) { + char *name = ent->d_name; + + if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) { + int64_t id = atoll(name + WAL_PREFIX_LEN); + if (id <= curFileId) continue; + + if (id < minFileId) { + minFileId = id; + } + } + } + closedir(dir); + + if (minFileId == INT64_MAX) return -1; + + *nextFileId = minFileId; + wTrace("vgId:%d, path:%s, curFileId:%" PRId64 " nextFileId:%" PRId64, pWal->vgId, pWal->path, curFileId, *nextFileId); + + return 0; +} + +int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId) { + int64_t minFileId = INT64_MAX; + + DIR *dir = opendir(pWal->path); + if (dir == NULL) { + wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); + return -1; + } + + struct dirent *ent; + while ((ent = readdir(dir)) != NULL) { + char *name = ent->d_name; + + if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) { + int64_t id = atoll(name + WAL_PREFIX_LEN); + if (id >= curFileId) continue; + + minDiff--; + if (id < minFileId) { + minFileId = id; + } + } + } + closedir(dir); + + if (minFileId == INT64_MAX) return -1; + if (minDiff > 0) return -1; + + *oldFileId = minFileId; + wTrace("vgId:%d, path:%s, curFileId:%" PRId64 " oldFildId:%" PRId64, pWal->vgId, pWal->path, curFileId, *oldFileId); + + return 0; +} + +int32_t walGetNewFile(SWal *pWal, int64_t *newFileId) { + int64_t maxFileId = INT64_MIN; + + DIR *dir = opendir(pWal->path); + if (dir == NULL) { + wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); + return -1; + } + + struct dirent *ent; + while ((ent = readdir(dir)) != NULL) { + char *name = ent->d_name; + + if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) { + int64_t id = atoll(name + WAL_PREFIX_LEN); + if (id > maxFileId) { + maxFileId = id; + } + } + } + closedir(dir); + + if (maxFileId == INT64_MIN) { + *newFileId = 0; + } else { + *newFileId = maxFileId; + } + + wTrace("vgId:%d, path:%s, newFileId:%" PRId64, pWal->vgId, pWal->path, *newFileId); + + return 0; +} \ No newline at end of file diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c new file mode 100644 index 0000000000000000000000000000000000000000..48021eecfc3523466f1e8e878cfb6b5344c8e9c4 --- /dev/null +++ b/src/wal/src/walWrite.c @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#define TAOS_RANDOM_FILE_FAIL_TEST +#include "os.h" +#include "taoserror.h" +#include "tchecksum.h" +#include "twal.h" +#include "walInt.h" + +static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId); + +int32_t walRenew(void *handle) { + if (handle == NULL) return 0; + + SWal * pWal = handle; + int32_t code = 0; + + if (pWal->stop) { + wDebug("vgId:%d, do not create a new wal file", pWal->vgId); + return 0; + } + + pthread_mutex_lock(&pWal->mutex); + + if (pWal->fd >= 0) { + tclose(pWal->fd); + wDebug("vgId:%d, file:%s, it is closed", pWal->vgId, pWal->name); + } + + if (pWal->keep == TAOS_WAL_KEEP) { + pWal->fileId = 0; + } else { + if (walGetNewFile(pWal, &pWal->fileId) != 0) pWal->fileId = 0; + pWal->fileId++; + } + + snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId); + pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + + if (pWal->fd < 0) { + code = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno)); + } else { + wDebug("vgId:%d, file:%s, it is created", pWal->vgId, pWal->name); + } + + pthread_mutex_unlock(&pWal->mutex); + + return code; +} + +void walRemoveOneOldFile(void *handle) { + SWal *pWal = handle; + if (pWal == NULL) return; + if (pWal->keep == TAOS_WAL_KEEP) return; + + pthread_mutex_lock(&pWal->mutex); + + // remove the oldest wal file + int64_t oldFileId = -1; + if (walGetOldFile(pWal, pWal->fileId, WAL_FILE_NUM, &oldFileId) == 0) { + char walName[WAL_FILE_LEN] = {0}; + snprintf(walName, sizeof(walName), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, oldFileId); + + if (remove(walName) < 0) { + wError("vgId:%d, file:%s, failed to remove since %s", pWal->vgId, walName, strerror(errno)); + } else { + wInfo("vgId:%d, file:%s, it is removed", pWal->vgId, walName); + } + } + + pthread_mutex_unlock(&pWal->mutex); +} + +void walRemoveAllOldFiles(void *handle) { + if (handle == NULL) return; + + SWal * pWal = handle; + int64_t fileId = -1; + while (walGetNextFile(pWal, &fileId) >= 0) { + snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); + + if (remove(pWal->name) < 0) { + wError("vgId:%d, wal:%p file:%s, failed to remove", pWal->vgId, pWal, pWal->name); + } else { + wInfo("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->name); + } + } +} + +int32_t walWrite(void *handle, SWalHead *pHead) { + if (handle == NULL) return -1; + + SWal * pWal = handle; + int32_t code = 0; + + // no wal + if (pWal->fd <= 0) return 0; + if (pWal->level == TAOS_WAL_NOLOG) return 0; + if (pHead->version <= pWal->version) return 0; + + pHead->signature = WAL_SIGNATURE; + taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead)); + int32_t contLen = pHead->len + sizeof(SWalHead); + + pthread_mutex_lock(&pWal->mutex); + + if (taosWrite(pWal->fd, pHead, contLen) != contLen) { + code = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, file:%s, failed to write since %s", pWal->vgId, pWal->name, strerror(errno)); + } else { + wTrace("vgId:%d, write wal, fileId:%" PRId64 " fd:%d hver:%" PRId64 " wver:%" PRIu64 " len:%d", pWal->vgId, + pWal->fileId, pWal->fd, pHead->version, pWal->version, pHead->len); + pWal->version = pHead->version; + } + + pthread_mutex_unlock(&pWal->mutex); + + ASSERT(contLen == pHead->len + sizeof(SWalHead)); + + return code; +} + +void walFsync(void *handle, bool forceFsync) { + SWal *pWal = handle; + if (pWal == NULL || pWal->fd < 0) return; + + if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) { + wTrace("vgId:%d, file:%s, do fsync", pWal->vgId, pWal->name); + if (fsync(pWal->fd) < 0) { + wError("vgId:%d, file:%s, fsync failed since %s", pWal->vgId, pWal->name, strerror(errno)); + } + } +} + +int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) { + if (handle == NULL) return -1; + + SWal * pWal = handle; + int32_t count = 0; + int32_t code = 0; + int64_t fileId = -1; + + while ((code = walGetNextFile(pWal, &fileId)) >= 0) { + if (fileId == pWal->fileId) continue; + + char walName[WAL_FILE_LEN]; + snprintf(walName, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); + + wDebug("vgId:%d, file:%s, will be restored", pWal->vgId, walName); + int32_t code = walRestoreWalFile(pWal, pVnode, writeFp, walName, fileId); + if (code != TSDB_CODE_SUCCESS) { + wError("vgId:%d, file:%s, failed to restore since %s", pWal->vgId, walName, tstrerror(code)); + continue; + } + + wDebug("vgId:%d, file:%s, restore success", pWal->vgId, walName); + + count++; + } + + if (pWal->keep != TAOS_WAL_KEEP) return TSDB_CODE_SUCCESS; + + if (count == 0) { + wDebug("vgId:%d, wal file not exist, renew it", pWal->vgId); + return walRenew(pWal); + } else { + // open the existing WAL file in append mode + pWal->fileId = 0; + snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId); + pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); + if (pWal->fd < 0) { + wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + wDebug("vgId:%d, file:%s open success", pWal->vgId, pWal->name); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) { + if (handle == NULL) return -1; + SWal *pWal = handle; + + if (*fileId == 0) *fileId = -1; + + pthread_mutex_lock(&(pWal->mutex)); + + int32_t code = walGetNextFile(pWal, fileId); + if (code >= 0) { + sprintf(fileName, "wal/%s%" PRId64, WAL_PREFIX, *fileId); + code = (*fileId == pWal->fileId) ? 0 : 1; + } + + wTrace("vgId:%d, get wal file, code:%d curId:%" PRId64 " outId:%" PRId64, pWal->vgId, code, pWal->fileId, *fileId); + pthread_mutex_unlock(&(pWal->mutex)); + + return code; +} + +static void walFtruncate(SWal *pWal, int32_t fd, int64_t offset) { + taosFtruncate(fd, offset); + fsync(fd); +} + +static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int32_t fd, int64_t *offset) { + int64_t pos = *offset; + while (1) { + pos++; + + if (lseek(fd, pos, SEEK_SET) < 0) { + wError("vgId:%d, failed to seek from corrupted wal file since %s", pWal->vgId, strerror(errno)); + return TSDB_CODE_WAL_FILE_CORRUPTED; + } + + if (taosRead(fd, pHead, sizeof(SWalHead)) <= 0) { + wError("vgId:%d, read to end of corrupted wal file, offset:%" PRId64, pWal->vgId, pos); + return TSDB_CODE_WAL_FILE_CORRUPTED; + } + + if (pHead->signature != WAL_SIGNATURE) { + continue; + } + + if (taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { + wInfo("vgId:%d, wal head cksum check passed, offset:%" PRId64, pWal->vgId, pos); + *offset = pos; + return TSDB_CODE_SUCCESS; + } + } + + return TSDB_CODE_WAL_FILE_CORRUPTED; +} + +static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId) { + int32_t size = WAL_MAX_SIZE; + void * buffer = tmalloc(size); + if (buffer == NULL) { + wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + int32_t fd = open(name, O_RDWR); + if (fd < 0) { + wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); + tfree(buffer); + return TAOS_SYSTEM_ERROR(errno); + } + + wDebug("vgId:%d, file:%s, start to restore", pWal->vgId, name); + + int32_t code = TSDB_CODE_SUCCESS; + int64_t offset = 0; + SWalHead *pHead = buffer; + + while (1) { + int32_t ret = taosRead(fd, pHead, sizeof(SWalHead)); + if (ret == 0) break; + + if (ret < 0) { + wError("vgId:%d, file:%s, failed to read wal head since %s", pWal->vgId, name, strerror(errno)); + code = TAOS_SYSTEM_ERROR(errno); + break; + } + + if (ret < sizeof(SWalHead)) { + wError("vgId:%d, file:%s, failed to read wal head, ret is %d", pWal->vgId, name, ret); + walFtruncate(pWal, fd, offset); + break; + } + + if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { + wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, + pHead->version, pHead->len, offset); + code = walSkipCorruptedRecord(pWal, pHead, fd, &offset); + if (code != TSDB_CODE_SUCCESS) { + walFtruncate(pWal, fd, offset); + break; + } + } + + if (pHead->len > size - sizeof(SWalHead)) { + size = sizeof(SWalHead) + pHead->len; + buffer = realloc(buffer, size); + if (buffer == NULL) { + wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); + code = TAOS_SYSTEM_ERROR(errno); + break; + } + + pHead = buffer; + } + + ret = taosRead(fd, pHead->cont, pHead->len); + if (ret < 0) { + wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno)); + code = TAOS_SYSTEM_ERROR(errno); + break; + } + + if (ret < pHead->len) { + wError("vgId:%d, file:%s, failed to read wal body, ret:%d len:%d", pWal->vgId, name, ret, pHead->len); + offset += sizeof(SWalHead); + continue; + } + + offset = offset + sizeof(SWalHead) + pHead->len; + + wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId, + fileId, pHead->version, pWal->version, pHead->len); + + pWal->version = pHead->version; + (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL); + } + + tclose(fd); + tfree(buffer); + + return code; +} + +uint64_t walGetVersion(twalh param) { + SWal *pWal = param; + if (pWal == 0) return 0; + + return pWal->version; +} diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c index bbee1347b8f92aa6cfad448fdfb369de8f5a6301..7a473ed18c958afa8be3c5b94b04d2fd548a56fd 100644 --- a/src/wal/test/waltest.c +++ b/src/wal/test/waltest.c @@ -23,7 +23,7 @@ int64_t ver = 0; void *pWal = NULL; -int writeToQueue(void *pVnode, void *data, int type) { +int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { // do nothing SWalHead *pHead = data; @@ -37,7 +37,6 @@ int writeToQueue(void *pVnode, void *data, int type) { int main(int argc, char *argv[]) { char path[128] = "/home/jhtao/test/wal"; - int max = 3; int level = 2; int total = 5; int rows = 10000; @@ -47,8 +46,6 @@ int main(int argc, char *argv[]) { for (int i=1; ijava -version +java version "1.8.0_131" +Java(TM) SE Runtime Environment (build 1.8.0_131-b11) +Java HotSpot(TM) 64-Bit Server VM (build 25.131-b11, mixed mode) +``` + + +(2)安装配置maven + +官网下载maven,下载地址:http://maven.apache.org/download.cgi + +配置环境变量MAVEN_HOME,将MAVEN_HOME/bin添加到PATH + +命令行里查看maven的版本 + +```shell +>mvn --version +Apache Maven 3.5.0 (ff8f5e7444045639af65f6095c62210b5713f426; 2017-04-04T03:39:06+08:00) +Maven home: D:\apache-maven-3.5.0\bin\.. +Java version: 1.8.0_131, vendor: Oracle Corporation +Java home: C:\Program Files\Java\jdk1.8.0_131\jre +Default locale: zh_CN, platform encoding: GBK +OS name: "windows 10", version: "10.0", arch: "amd64", family: "windows" +``` + +为了加快maven下载依赖的速度,可以为maven配置mirror,修改MAVEN_HOME\config\settings.xml文件 + +```xml + + + D:\apache-maven-localRepository + + + + + alimaven + aliyun maven + http://maven.aliyun.com/nexus/content/groups/public/ + central + + + + + + + jdk-1.8 + + true + 1.8 + + + 1.8 + 1.8 + 1.8 + + + + +``` + + + +(3)在linux服务器上安装TDengine-server + +在taosdata官网下载TDengine-server,下载地址:https://www.taosdata.com/cn/all-downloads/ + +在linux服务器上安装TDengine-server + +```shell +# tar -zxvf package/TDengine-server-2.0.1.1-Linux-x64.tar.gz +# cd TDengine-server/ +# ./install.sh +``` + +启动taosd + +```shell +# systemctl start taosd +``` + +在server上用taos连接taosd + +```shell +# taos +taos> show dnodes; + id | end_point | vnodes | cores | status | role | create_time | +================================================================================================================== + 1 | td01:6030 | 2 | 4 | ready | any | 2020-08-19 18:40:25.045 | +Query OK, 1 row(s) in set (0.005765s) +``` + +如果可以正确连接到taosd实例,并打印出databases的信息,说明TDengine的server已经正确启动。这里查看server的hostname + +```shell +# hostname -f +td01 +``` + +注意,如果安装TDengine后,使用默认的taos.cfg配置文件,taosd会使用当前server的hostname创建dnode实例。之后,在client也需要使用这个hostname来连接taosd。 + + + +(4)在windows上安装TDengine-client + +在taosdata官网下载taos客户端,下载地址: +https://www.taosdata.com/cn/all-downloads/ +下载后,双击exe安装。 + +修改client的hosts文件(C:\Windows\System32\drivers\etc\hosts),将server的hostname和ip配置到client的hosts文件中 + +``` +192.168.236.136 td01 +``` + +配置完成后,在命令行内使用taos shell连接server端 + +```shell +C:\TDengine>taos +Welcome to the TDengine shell from Linux, Client Version:2.0.1.1 +Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. + +taos> show databases; + name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | precision | status | +=================================================================================================================================================================================================================================================================== + test | 2020-08-19 18:43:50.731 | 1 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready | + log | 2020-08-19 18:40:28.064 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready | +Query OK, 2 row(s) in set (0.068000s) +``` + +如果windows上的client能够正常连接,并打印database信息,说明client可以正常连接server了。 + + + +## 应用开发 + +(1)新建maven工程,在pom.xml中引入taos-jdbcdriver依赖。 + +```xml + + + 4.0.0 + + com.taosdata.demo + JdbcDemo + 1.0-SNAPSHOT + + + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.8 + + + +``` + +(2)使用jdbc查询TDengine数据库 + +下面是示例代码: + +```java +public class JdbcDemo { + + public static void main(String[] args) throws Exception { + Connection conn = getConn(); + Statement stmt = conn.createStatement(); + // create database + stmt.executeUpdate("create database if not exists db"); + // use database + stmt.executeUpdate("use db"); + // create table + stmt.executeUpdate("create table if not exists tb (ts timestamp, temperature int, humidity float)"); + // insert data + int affectedRows = stmt.executeUpdate("insert into tb values(now, 23, 10.3) (now + 1s, 20, 9.3)"); + System.out.println("insert " + affectedRows + " rows."); + // query data + ResultSet resultSet = stmt.executeQuery("select * from tb"); + Timestamp ts = null; + int temperature = 0; + float humidity = 0; + while(resultSet.next()){ + ts = resultSet.getTimestamp(1); + temperature = resultSet.getInt(2); + humidity = resultSet.getFloat("humidity"); + System.out.printf("%s, %d, %s\n", ts, temperature, humidity); + } + } + + public static Connection getConn() throws Exception{ + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String jdbcUrl = "jdbc:TAOS://td01:0/log?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + return conn; + } + +} +``` + +(3)测试jdbc访问tdengine的sever实例 + +console输出: + +``` +insert 2 rows. +2020-08-26 00:06:34.575, 23, 10.3 +2020-08-26 00:06:35.575, 20, 9.3 +``` + + + +## 指南 + +(1)如何设置主机名和hosts + +在server上查看hostname和fqdn +```shell +查看hostname +# hostname +taos-server + +查看fqdn +# hostname -f +taos-server +``` + +windows下hosts文件位于: +C:\\Windows\System32\drivers\etc\hosts +修改hosts文件,添加server的ip和hostname + +```s +192.168.56.101 node5 +``` + +(2)什么是fqdn? + + +> 什么是FQDN? +> +> FQDN(Full qualified domain name)全限定域名,fqdn由2部分组成:hostname+domainname。 +> +> 例如,一个邮件服务器的fqdn可能是:mymail.somecollege.edu,其中mymail是hostname(主机名),somcollege.edu是domainname(域名)。本例中,.edu是顶级域名,.somecollege是二级域名。 +> +> 当连接服务器时,必须指定fqdn,然后,dns服务器通过查看dns表,将hostname解析为相应的ip地址。如果只指定hostname(不指定domainname),应用程序可能服务解析主机名。因为如果你试图访问不在本地的远程服务器时,本地的dns服务器和可能没有远程服务器的hostname列表。 +> +> 参考:https://kb.iu.edu/d/aiuv diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 759e16d1de285c9490d7f1af7a94de81dbadb6e4..be60a88ad70721bd6281d9ca8f1d73263a788532 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -9,26 +9,40 @@ static void prepare_data(TAOS* taos) { - taos_query(taos, "drop database if exists test;"); + TAOS_RES *result; + result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); usleep(100000); - taos_query(taos, "create database test;"); + result = taos_query(taos, "create database test;"); + taos_free_result(result); usleep(100000); taos_select_db(taos, "test"); - taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); + result = taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); + taos_free_result(result); - taos_query(taos, "create table t0 using meters tags(0);"); - taos_query(taos, "create table t1 using meters tags(1);"); - taos_query(taos, "create table t2 using meters tags(2);"); - taos_query(taos, "create table t3 using meters tags(3);"); - taos_query(taos, "create table t4 using meters tags(4);"); - taos_query(taos, "create table t5 using meters tags(5);"); - taos_query(taos, "create table t6 using meters tags(6);"); - taos_query(taos, "create table t7 using meters tags(7);"); - taos_query(taos, "create table t8 using meters tags(8);"); - taos_query(taos, "create table t9 using meters tags(9);"); + result = taos_query(taos, "create table t0 using meters tags(0);"); + taos_free_result(result); + result = taos_query(taos, "create table t1 using meters tags(1);"); + taos_free_result(result); + result = taos_query(taos, "create table t2 using meters tags(2);"); + taos_free_result(result); + result = taos_query(taos, "create table t3 using meters tags(3);"); + taos_free_result(result); + result = taos_query(taos, "create table t4 using meters tags(4);"); + taos_free_result(result); + result = taos_query(taos, "create table t5 using meters tags(5);"); + taos_free_result(result); + result = taos_query(taos, "create table t6 using meters tags(6);"); + taos_free_result(result); + result = taos_query(taos, "create table t7 using meters tags(7);"); + taos_free_result(result); + result = taos_query(taos, "create table t8 using meters tags(8);"); + taos_free_result(result); + result = taos_query(taos, "create table t9 using meters tags(9);"); + taos_free_result(result); - TAOS_RES* res = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)" + result = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)" " ('2020-01-01 00:01:00.000', 0)" " ('2020-01-01 00:02:00.000', 0)" " t1 values('2020-01-01 00:00:00.000', 0)" @@ -46,10 +60,11 @@ static void prepare_data(TAOS* taos) { " t7 values('2020-01-01 00:01:02.000', 0)" " t8 values('2020-01-01 00:01:02.000', 0)" " t9 values('2020-01-01 00:01:02.000', 0)"); - int affected = taos_affected_rows(res); + int affected = taos_affected_rows(result); if (affected != 18) { printf("\033[31m%d rows affected by last insert statement, but it should be 18\033[0m\n", affected); } + taos_free_result(result); // super tables subscription usleep(1000000); } @@ -135,6 +150,7 @@ static void verify_query(TAOS* taos) { res = taos_query(taos, "select * from meters"); taos_stop_query(res); + taos_free_result(res); } @@ -153,23 +169,30 @@ static void verify_subscribe(TAOS* taos) { res = taos_consume(tsub); check_row_count(__LINE__, res, 0); - taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);"); - taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);"); + TAOS_RES *result; + result = taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);"); + taos_free_result(result); + result = taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);"); + taos_free_result(result); res = taos_consume(tsub); check_row_count(__LINE__, res, 2); - taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);"); - taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);"); + result = taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);"); + taos_free_result(result); + result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);"); + taos_free_result(result); res = taos_consume(tsub); check_row_count(__LINE__, res, 2); - taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);"); + result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);"); + taos_free_result(result); res = taos_consume(tsub); check_row_count(__LINE__, res, 1); // keep progress information and restart subscription taos_unsubscribe(tsub, 1); - taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);"); + result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);"); + taos_free_result(result); tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0); res = taos_consume(tsub); check_row_count(__LINE__, res, 24); @@ -196,7 +219,8 @@ static void verify_subscribe(TAOS* taos) { res = taos_consume(tsub); check_row_count(__LINE__, res, 0); - taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);"); + result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);"); + taos_free_result(result); res = taos_consume(tsub); check_row_count(__LINE__, res, 1); @@ -205,7 +229,8 @@ static void verify_subscribe(TAOS* taos) { int blockFetch = 0; tsub = taos_subscribe(taos, 1, "test", "select * from meters;", subscribe_callback, &blockFetch, 1000); usleep(2000000); - taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);"); + result = taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);"); + taos_free_result(result); usleep(2000000); taos_unsubscribe(tsub, 0); } @@ -213,8 +238,9 @@ static void verify_subscribe(TAOS* taos) { void verify_prepare(TAOS* taos) { TAOS_RES* result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); usleep(100000); - taos_query(taos, "create database test;"); + result = taos_query(taos, "create database test;"); int code = taos_errno(result); if (code != 0) { @@ -429,7 +455,8 @@ void verify_stream(TAOS* taos) { NULL); printf("waiting for stream data\n"); usleep(100000); - taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);"); + TAOS_RES* result = taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);"); + taos_free_result(result); usleep(200000000); taos_close_stream(strm); } diff --git a/tests/examples/c/asyncdemo.c b/tests/examples/c/asyncdemo.c index 1e523bd7fec9ba9fd90d1a71949f40c7be71384d..225c4f7541ec7cd7fb4784b8de94d8b47c4e3e36 100644 --- a/tests/examples/c/asyncdemo.c +++ b/tests/examples/c/asyncdemo.c @@ -46,6 +46,34 @@ void taos_insert_call_back(void *param, TAOS_RES *tres, int code); void taos_select_call_back(void *param, TAOS_RES *tres, int code); void taos_error(TAOS *taos); +static void queryDB(TAOS *taos, char *command) { + int i; + TAOS_RES *pSql = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + exit(EXIT_FAILURE); + } + + taos_free_result(pSql); +} + int main(int argc, char *argv[]) { TAOS *taos; @@ -78,16 +106,14 @@ int main(int argc, char *argv[]) printf("success to connect to server\n"); - sprintf(sql, "drop database %s", db); - taos_query(taos, sql); + sprintf(sql, "drop database if exists %s", db); + queryDB(taos, sql); sprintf(sql, "create database %s", db); - if (taos_query(taos, sql) != 0) - taos_error(taos); + queryDB(taos, sql); sprintf(sql, "use %s", db); - if (taos_query(taos, sql) != 0) - taos_error(taos); + queryDB(taos, sql); strcpy(prefix, "asytbl_"); for (i = 0; i < numOfTables; ++i) { @@ -95,8 +121,7 @@ int main(int argc, char *argv[]) tableList[i].taos = taos; sprintf(tableList[i].name, "%s%d", prefix, i); sprintf(sql, "create table %s%d (ts timestamp, volume bigint)", prefix, i); - if (taos_query(taos, sql) != 0) - taos_error(taos); + queryDB(taos, sql); } gettimeofday(&systemTime, NULL); diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 8f8a66a32593bc25d71b554808719ca42f5b32ac..74a49288e9a1aa7081db45925bf52d6516e4801a 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -22,10 +22,38 @@ #include #include // TAOS header file +static void queryDB(TAOS *taos, char *command) { + int i; + TAOS_RES *pSql = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + exit(EXIT_FAILURE); + } + + taos_free_result(pSql); +} + +void Test(char *qstr, const char *input, int i); + int main(int argc, char *argv[]) { - TAOS * taos; char qstr[1024]; - TAOS_RES *result; // connect to server if (argc < 2) { @@ -35,37 +63,26 @@ int main(int argc, char *argv[]) { // init TAOS taos_init(); - - taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); + for (int i = 0; i < 4000000; i++) { + Test(qstr, argv[1], i); + } + taos_cleanup(); +} +void Test(char *qstr, const char *input, int index) { + TAOS *taos = taos_connect(input, "root", "taosdata", NULL, 0); + printf("==================test at %d\n================================", index); + queryDB(taos, "drop database if exists demo"); + queryDB(taos, "create database demo"); + TAOS_RES *result; if (taos == NULL) { printf("failed to connect to server, reason:%s\n", "null taos"/*taos_errstr(taos)*/); exit(1); } - printf("success to connect to server\n"); - - - taos_query(taos, "drop database demo"); - - result = taos_query(taos, "create database demo"); - if (result == NULL) { - printf("failed to create database, reason:%s\n", "null result"/*taos_errstr(taos)*/); - exit(1); - } - printf("success to create database\n"); - - taos_query(taos, "use demo"); + queryDB(taos, "use demo"); - // create table - if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) { - printf("failed to create table, reason:%s\n", taos_errstr(result)); - exit(1); - } + queryDB(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))"); printf("success to create table\n"); - // sleep for one second to make sure table is created on data node - // taosMsleep(1000); - - // insert 10 records int i = 0; for (i = 0; i < 10; ++i) { sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000 + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello"); @@ -80,10 +97,11 @@ int main(int argc, char *argv[]) { printf("insert row: %i\n", i); } else { printf("failed to insert row: %i, reason:%s\n", i, "null result"/*taos_errstr(result)*/); + taos_free_result(result); exit(1); } + taos_free_result(result); - //sleep(1); } printf("success to insert rows, total %d rows\n", i); @@ -91,7 +109,8 @@ int main(int argc, char *argv[]) { sprintf(qstr, "SELECT * FROM m1"); result = taos_query(taos, qstr); if (result == NULL || taos_errno(result) != 0) { - printf("failed to select, reason:%s\n", taos_errstr(result)); + printf("failed to select, reason:%s\n", taos_errstr(result)); + taos_free_result(result); exit(1); } @@ -112,5 +131,6 @@ int main(int argc, char *argv[]) { taos_free_result(result); printf("====demo end====\n\n"); - return getchar(); + taos_close(taos); } + diff --git a/tests/examples/go/taosdemo.go b/tests/examples/go/taosdemo.go index b42e1e6d703a96bb86454f177a7207577c6d4d4c..2c3a7d09b68d84feea1ae2771b90643dbbfbc063 100644 --- a/tests/examples/go/taosdemo.go +++ b/tests/examples/go/taosdemo.go @@ -87,7 +87,7 @@ func init() { func printAllArgs() { fmt.Printf("\n============= args parse result: =============\n") - fmt.Printf("dbName: %v\n", configPara.hostName) + fmt.Printf("hostName: %v\n", configPara.hostName) fmt.Printf("serverPort: %v\n", configPara.serverPort) fmt.Printf("usr: %v\n", configPara.user) fmt.Printf("password: %v\n", configPara.password) @@ -107,7 +107,7 @@ func main() { fmt.Scanln() url = "root:taosdata@/tcp(" + configPara.hostName + ":" + strconv.Itoa(configPara.serverPort) + ")/" - //url = fmt.Sprintf("%s:%s@/tcp(%s:%d)/%s?interpolateParams=true", configPara.user, configPara.password, configPara.hostName, configPara.serverPort, configPara.dbName) + //url = fmt.Sprintf("%s:%s@/tcp(%s:%d)/%s?interpolateParams=true", configPara.user, configPara.password, configPara.hostName, configPara.serverPort, configPara.dbName) // open connect to taos server //db, err := sql.Open(taosDriverName, url) //if err != nil { @@ -115,6 +115,7 @@ func main() { // os.Exit(1) //} //defer db.Close() + rand.Seed(time.Now().Unix()) createDatabase(configPara.dbName, configPara.supTblName) fmt.Printf("======== create database success! ========\n\n") diff --git a/tests/examples/nodejs/README-win.md b/tests/examples/nodejs/README-win.md new file mode 100644 index 0000000000000000000000000000000000000000..75fec69413af2bb49498118ec7235c9947e2f89e --- /dev/null +++ b/tests/examples/nodejs/README-win.md @@ -0,0 +1,200 @@ +# 如何在windows上使用nodejs进行TDengine应用开发 + +## 环境准备 + +(1)安装nodejs-10.22.0 + +下载链接:https://nodejs.org/dist/v10.22.0/node-v10.22.0-win-x64.zip +解压安装,把node配置到环境变量里 + +cmd启动命令行,查看node的版本 + +```shell +> node.exe --version +v10.22.0 + +> npm --version +6.14.6 +``` + + + +(2)安装python2.7 + +下载链接:https://www.python.org/ftp/python/2.7.18/python-2.7.18.amd64.msi + +查看python版本 + +```shell +>python --version +Python 2.7.18 +``` + + +(3)安装TDengine-client + +下载地址:https://www.taosdata.com/cn/all-downloads/,选择一个合适的windows-client下载(client应该尽量与server端的版本保持一致) + +使用client的taos shell连接server + +```shell +>taos -h node5 + +Welcome to the TDengine shell from Linux, Client Version:2.0.6.0 +Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. + +taos> show dnodes; + id | end_point | vnodes | cores | status | role | create_time | offline reason | +============================================================================================================================================ + 1 | node5:6030 | 7 | 1 | ready | any | 2020-10-26 09:45:26.308 | | +Query OK, 1 row(s) in set (0.036000s) +``` + +注意: +* 检查能否在client的机器上ping通server的fqdn +* 如果你的dns server并没有提供到server的域名解析,可以将server的hostname配置到client的hosts文件中 + + +## 应用开发 + +(1)建立nodejs项目 + +``` +npm init +``` + +(2)安装windows-build-tools +``` +npm install --global --production windows-build-tools +``` + +(3)安装td2.0-connector驱动 + +``` tdshell +npm install td2.0-connector +``` + +(4)nodejs访问tdengine的示例程序 + +```javascript +const taos = require('td2.0-connector'); + +var host = null; +var port = 6030; +for (var i = 2; i < global.process.argv.length; i++) { + var key = global.process.argv[i].split("=")[0]; + var value = global.process.argv[i].split("=")[1]; + + if ("host" == key) { + host = value; + } + if ("port" == key) { + port = value; + } +} + +if (host == null) { + console.log("Usage: node nodejsChecker.js host= port="); + process.exit(0); +} + +// establish connection +var conn = taos.connect({host: host, user: "root", password: "taosdata", port: port}); +var cursor = conn.cursor(); +// create database +executeSql("create database if not exists testnodejs", 0); +// use db +executeSql("use testnodejs", 0); +// drop table +executeSql("drop table if exists testnodejs.weather", 0); +// create table +executeSql("create table if not exists testnodejs.weather(ts timestamp, temperature float, humidity int)", 0); +// insert +executeSql("insert into testnodejs.weather (ts, temperature, humidity) values(now, 20.5, 34)", 1); +// select +executeQuery("select * from testnodejs.weather"); +// close connection +conn.close(); + +function executeQuery(sql) { + var start = new Date().getTime(); + var promise = cursor.query(sql, true); + var end = new Date().getTime(); + promise.then(function (result) { + printSql(sql, result != null, (end - start)); + result.pretty(); + }); +} + +function executeSql(sql, affectRows) { + var start = new Date().getTime(); + var promise = cursor.execute(sql); + var end = new Date().getTime(); + printSql(sql, promise == affectRows, (end - start)); +} + +function printSql(sql, succeed, cost) { + console.log("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); +} +``` + +(5)测试nodejs程序 + +```shell +>node nodejsChecker.js +Usage: node nodejsChecker.js host= port= +# 提示指定host + +>node nodejsChecker.js host=node5 +Successfully connected to TDengine +Query OK, 0 row(s) affected (0.00997610s) +[ OK ] time cost: 14 ms, execute statement ====> create database if not exists testnodejs +Query OK, 0 row(s) affected (0.00235920s) +[ OK ] time cost: 4 ms, execute statement ====> use testnodejs +Query OK, 0 row(s) affected (0.06604280s) +[ OK ] time cost: 67 ms, execute statement ====> drop table if exists testnodejs.weather +Query OK, 0 row(s) affected (0.59403290s) +[ OK ] time cost: 595 ms, execute statement ====> create table if not exists testnodejs.weather(ts timestamp, temperature float, humidity int) +Query OK, 1 row(s) affected (0.01058950s) +[ OK ] time cost: 12 ms, execute statement ====> insert into testnodejs.weather (ts, temperature, humidity) values(now, 20.5, 34) +Query OK, 1 row(s) in set (0.00401490s) +[ OK ] time cost: 10 ms, execute statement ====> select * from testnodejs.weather +Connection is closed + + ts | temperature | humidity | +===================================================================== +2020-10-27 18:49:15.547 | 20.5 | 34 | +``` + +## 指南 + +### 如何设置主机名和hosts + +在server上查看hostname和fqdn +```shell +查看hostname +# hostname +taos-server + +查看fqdn +# hostname -f +taos-server +``` + +windows下hosts文件位于: +C:\\Windows\System32\drivers\etc\hosts +修改hosts文件,添加server的ip和hostname + +``` +192.168.56.101 node5 +``` + +> 什么是FQDN? +> +> FQDN(Full qualified domain name)全限定域名,fqdn由2部分组成:hostname+domainname。 +> +> 例如,一个邮件服务器的fqdn可能是:mymail.somecollege.edu,其中mymail是hostname(主机名),somcollege.edu是domainname(域名)。本例中,.edu是顶级域名,.somecollege是二级域名。 +> +> 当连接服务器时,必须指定fqdn,然后,dns服务器通过查看dns表,将hostname解析为相应的ip地址。如果只指定hostname(不指定domainname),应用程序可能服务解析主机名。因为如果你试图访问不在本地的远程服务器时,本地的dns服务器和可能没有远程服务器的hostname列表。 +> +> 参考:https://kb.iu.edu/d/aiuv diff --git a/tests/examples/nodejs/nodejsChecker.js b/tests/examples/nodejs/nodejsChecker.js index c77944f75243a50e6e2c738e659cb4e64f3e5574..f838d5cc8465dba70b5372a5d7720a8cff69544a 100644 --- a/tests/examples/nodejs/nodejsChecker.js +++ b/tests/examples/nodejs/nodejsChecker.js @@ -42,8 +42,8 @@ function executeQuery(sql){ var start = new Date().getTime(); var promise = cursor.query(sql, true); var end = new Date().getTime(); - printSql(sql, promise != null,(end - start)); promise.then(function(result){ + printSql(sql, result != null,(end - start)); result.pretty(); }); } diff --git a/tests/gotest/batchtest.bat b/tests/gotest/batchtest.bat old mode 100644 new mode 100755 index abe9a58f319068d5e11017abcd721a4c54d6aca9..efd8961bb0be2eb6f20e291114b92b00469b984f --- a/tests/gotest/batchtest.bat +++ b/tests/gotest/batchtest.bat @@ -7,6 +7,9 @@ set serverPort=%2 if "%severIp%"=="" (set severIp=127.0.0.1) if "%serverPort%"=="" (set serverPort=6030) +go env -w GO111MODULE=on +go env -w GOPROXY=https://goproxy.io,direct + cd case001 case001.bat %severIp% %serverPort% diff --git a/tests/gotest/batchtest.sh b/tests/gotest/batchtest.sh old mode 100644 new mode 100755 index e8ed9ecbed9f70c98e6b5db052c3e69082c1794d..0fbbf40714b3349651beea9302e66628b31a22ac --- a/tests/gotest/batchtest.sh +++ b/tests/gotest/batchtest.sh @@ -13,6 +13,9 @@ if [ ! -n "$serverPort" ]; then serverPort=6030 fi +go env -w GO111MODULE=on +go env -w GOPROXY=https://goproxy.io,direct + bash ./case001/case001.sh $severIp $serverPort #bash ./case002/case002.sh $severIp $serverPort #bash ./case003/case003.sh $severIp $serverPort diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh new file mode 100755 index 0000000000000000000000000000000000000000..bb5d9e0a9d16ccb232feeaea00a72a0972b31cc2 --- /dev/null +++ b/tests/perftest-scripts/perftest-query.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +today=`date +"%Y%m%d"` +WORK_DIR=/home/ubuntu/pxiao/ +PERFORMANCE_TEST_REPORT=$TDENGINE_DIR/tests/performance-test-report-$today.log + +# Coloured Echoes # +function red_echo { echo -e "\033[31m$@\033[0m"; } # +function green_echo { echo -e "\033[32m$@\033[0m"; } # +function yellow_echo { echo -e "\033[33m$@\033[0m"; } # +function white_echo { echo -e "\033[1;37m$@\033[0m"; } # +# Coloured Printfs # +function red_printf { printf "\033[31m$@\033[0m"; } # +function green_printf { printf "\033[32m$@\033[0m"; } # +function yellow_printf { printf "\033[33m$@\033[0m"; } # +function white_printf { printf "\033[1;37m$@\033[0m"; } # +# Debugging Outputs # +function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } # +function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } # +function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # +function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # + + +function stopTaosd { + echo "Stop taosd" + systemctl stop taosd + snap stop tdengine + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done +} + +function buildTDengine { + echoInfo "Build TDengine" + cd $WORK_DIR/TDengine + + git remote update > /dev/null + REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` + LOCAL_COMMIT=`git rev-parse --short @` + + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "repo up-to-date" + else + echo "repo need to pull" + git pull > /dev/null + + LOCAL_COMMIT=`git rev-parse --short @` + cd debug + rm -rf * + cmake .. > /dev/null + make > /dev/null + make install + fi +} + +function runQueryPerfTest { + nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taodperf/ > /dev/null 2>&1 & + echoInfo "Run Performance Test" + cd $WORK_DIR/TDengine/tests/pytest + + python3 query/queryPerformance.py | tee -a $PERFORMANCE_TEST_REPORT +} + + +function sendReport { + echo "send report" + receiver="pxiao@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + + cd $TDENGINE_DIR + + sed -i 's/\x1b\[[0-9;]*m//g' $PERFORMANCE_TEST_REPORT + BODY_CONTENT=`cat $PERFORMANCE_TEST_REPORT` + echo -e "to: ${receiver}\nsubject: Query Performace Report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode $PERFORMANCE_TEST_REPORT performance-test-report-$today.log) | \ + ssmtp "${receiver}" && echo "Report Sent!" +} + + +stopTaosd +buildTDengine +runQueryPerfTest + +echoInfo "Send Report" +sendReport +echoInfo "End of Test" diff --git a/tests/pytest/alter/db_update_options.py b/tests/pytest/alter/db_update_options.py new file mode 100644 index 0000000000000000000000000000000000000000..224e0f25b074deed55802b9ba847f7f845716a23 --- /dev/null +++ b/tests/pytest/alter/db_update_options.py @@ -0,0 +1,71 @@ + +# -*- coding: utf-8 -*- + +import random +import string +import subprocess +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + def run(self): + tdLog.debug("check database") + tdSql.prepare() + + # check default update value + sql = "create database if not exists db" + tdSql.execute(sql) + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,0) + + sql = "alter database db update 1" + + # check update value + tdSql.execute(sql) + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,1) + + + sql = "alter database db update 0" + tdSql.execute(sql) + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,0) + + sql = "alter database db update -1" + tdSql.error(sql) + + sql = "alter database db update 100" + tdSql.error(sql) + + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,0) + + tdSql.execute('drop database db') + tdSql.error('create database db update 100') + tdSql.error('create database db update -1') + + tdSql.execute('create database db update 1') + + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,1) + + tdSql.execute('drop database db') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/cluster/bananceTest.py b/tests/pytest/cluster/bananceTest.py new file mode 100644 index 0000000000000000000000000000000000000000..ef25afa7d2f7ea3b5358f8ba74d6702d28d54c85 --- /dev/null +++ b/tests/pytest/cluster/bananceTest.py @@ -0,0 +1,57 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from clusterSetup import * +from util.sql import tdSql +from util.log import tdLog +import random +import time + +class ClusterTestcase: + + ## test case 32 ## + def run(self): + + nodes = Nodes() + nodes.addConfigs("maxVgroupsPerDb", "10") + nodes.addConfigs("maxTablesPerVnode", "1000") + nodes.restartAllTaosd() + + ctest = ClusterTest(nodes.node1.hostName) + ctest.connectDB() + ctest.createSTable(1) + ctest.run() + tdSql.init(ctest.conn.cursor(), False) + + tdSql.execute("use %s" % ctest.dbName) + tdSql.query("show vgroups") + dnodes = [] + for i in range(10): + dnodes.append(int(tdSql.getData(i, 4))) + + s = set(dnodes) + if len(s) < 3: + tdLog.exit("cluster is not balanced") + + tdLog.info("cluster is balanced") + + nodes.removeConfigs("maxVgroupsPerDb", "10") + nodes.removeConfigs("maxTablesPerVnode", "1000") + nodes.restartAllTaosd() + + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +ct = ClusterTestcase() +ct.run() diff --git a/tests/pytest/cluster/basicTest.py b/tests/pytest/cluster/basicTest.py new file mode 100644 index 0000000000000000000000000000000000000000..b990d7fd982a490383939707a32635d37e546b13 --- /dev/null +++ b/tests/pytest/cluster/basicTest.py @@ -0,0 +1,47 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from clusterSetup import * +from util.sql import tdSql +from util.log import tdLog +import random + +class ClusterTestcase: + + ## test case 1, 33 ## + def run(self): + + nodes = Nodes() + ctest = ClusterTest(nodes.node1.hostName) + + ctest.connectDB() + tdSql.init(ctest.conn.cursor(), False) + + ## Test case 1 ## + tdLog.info("Test case 1 repeat %d times" % ctest.repeat) + for i in range(ctest.repeat): + tdLog.info("Start Round %d" % (i + 1)) + replica = random.randint(1,3) + ctest.createSTable(replica) + ctest.run() + tdLog.sleep(10) + tdSql.query("select count(*) from %s.%s" %(ctest.dbName, ctest.stbName)) + tdSql.checkData(0, 0, ctest.numberOfRecords * ctest.numberOfTables) + tdLog.info("Round %d completed" % (i + 1)) + + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +ct = ClusterTestcase() +ct.run() \ No newline at end of file diff --git a/tests/pytest/cluster/changeReplicaTest.py b/tests/pytest/cluster/changeReplicaTest.py new file mode 100644 index 0000000000000000000000000000000000000000..7fa68edbfee2db599076befdf9bed5f4b4be3c83 --- /dev/null +++ b/tests/pytest/cluster/changeReplicaTest.py @@ -0,0 +1,51 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from clusterSetup import * +from util.sql import tdSql +from util.log import tdLog +import random + +class ClusterTestcase: + + ## test case 7, ## + def run(self): + + nodes = Nodes() + ctest = ClusterTest(nodes.node1.hostName) + ctest.connectDB() + tdSql.init(ctest.conn.cursor(), False) + + tdSql.execute("use %s" % ctest.dbName) + tdSql.query("show vgroups") + for i in range(10): + tdSql.checkData(i, 5, "master") + + tdSql.execute("alter database %s replica 2" % ctest.dbName) + tdLog.sleep(30) + tdSql.query("show vgroups") + for i in range(10): + tdSql.checkData(i, 5, "master") + tdSql.checkData(i, 7, "slave") + + tdSql.execute("alter database %s replica 3" % ctest.dbName) + tdLog.sleep(30) + tdSql.query("show vgroups") + for i in range(10): + tdSql.checkData(i, 5, "master") + tdSql.checkData(i, 7, "slave") + tdSql.checkData(i, 9, "slave") + +ct = ClusterTestcase() +ct.run() \ No newline at end of file diff --git a/tests/pytest/cluster/clusterSetup.py b/tests/pytest/cluster/clusterSetup.py new file mode 100644 index 0000000000000000000000000000000000000000..36af8ac42e56e1b8a7ab2237305a6bf286103552 --- /dev/null +++ b/tests/pytest/cluster/clusterSetup.py @@ -0,0 +1,202 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import sys +sys.path.insert(0, os.getcwd()) +from fabric import Connection +from util.sql import * +from util.log import * +import taos +import random +import threading +import logging + +class Node: + def __init__(self, index, username, hostIP, hostName, password, homeDir): + self.index = index + self.username = username + self.hostIP = hostIP + self.hostName = hostName + self.homeDir = homeDir + self.conn = Connection("{}@{}".format(username, hostName), connect_kwargs={"password": "{}".format(password)}) + + def startTaosd(self): + try: + self.conn.run("sudo systemctl start taosd") + except Exception as e: + print("Start Taosd error for node %d " % self.index) + logging.exception(e) + + def stopTaosd(self): + try: + self.conn.run("sudo systemctl stop taosd") + except Exception as e: + print("Stop Taosd error for node %d " % self.index) + logging.exception(e) + + def restartTaosd(self): + try: + self.conn.run("sudo systemctl restart taosd") + except Exception as e: + print("Stop Taosd error for node %d " % self.index) + logging.exception(e) + + def removeTaosd(self): + try: + self.conn.run("rmtaos") + except Exception as e: + print("remove taosd error for node %d " % self.index) + logging.exception(e) + + def installTaosd(self, packagePath): + self.conn.put(packagePath, self.homeDir) + self.conn.cd(self.homeDir) + self.conn.run("tar -zxf $(basename '%s')" % packagePath) + with self.conn.cd("TDengine-enterprise-server"): + self.conn.run("yes|./install.sh") + + def configTaosd(self, taosConfigKey, taosConfigValue): + self.conn.run("sudo echo '%s %s' >> %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg")) + + def removeTaosConfig(self, taosConfigKey, taosConfigValue): + self.conn.run("sudo sed -in-place -e '/%s %s/d' %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg")) + + def configHosts(self, ip, name): + self.conn.run("echo '%s %s' >> %s" % (ip, name, '/etc/hosts')) + + def removeData(self): + try: + self.conn.run("sudo rm -rf /var/lib/taos/*") + except Exception as e: + print("remove taosd data error for node %d " % self.index) + logging.exception(e) + + def removeLog(self): + try: + self.conn.run("sudo rm -rf /var/log/taos/*") + except Exception as e: + print("remove taosd error for node %d " % self.index) + logging.exception(e) + + def removeDataForMnode(self): + try: + self.conn.run("sudo rm -rf /var/lib/taos/*") + except Exception as e: + print("remove taosd error for node %d " % self.index) + logging.exception(e) + + def removeDataForVnode(self, id): + try: + self.conn.run("sudo rm -rf /var/lib/taos/vnode%d/*.data" % id) + except Exception as e: + print("remove taosd error for node %d " % self.index) + logging.exception(e) + +class Nodes: + def __init__(self): + self.node1 = Node(1, 'ubuntu', '192.168.1.52', 'node1', 'tbase125!', '/home/ubuntu') + self.node2 = Node(2, 'ubuntu', '192.168.1.53', 'node2', 'tbase125!', '/home/ubuntu') + self.node3 = Node(3, 'ubuntu', '192.168.1.54', 'node3', 'tbase125!', '/home/ubuntu') + + def stopAllTaosd(self): + self.node1.stopTaosd() + self.node2.stopTaosd() + self.node3.stopTaosd() + + def startAllTaosd(self): + self.node1.startTaosd() + self.node2.startTaosd() + self.node3.startTaosd() + + def restartAllTaosd(self): + self.node1.restartTaosd() + self.node2.restartTaosd() + self.node3.restartTaosd() + + def addConfigs(self, configKey, configValue): + self.node1.configTaosd(configKey, configValue) + self.node2.configTaosd(configKey, configValue) + self.node3.configTaosd(configKey, configValue) + + def removeConfigs(self, configKey, configValue): + self.node1.removeTaosConfig(configKey, configValue) + self.node2.removeTaosConfig(configKey, configValue) + self.node3.removeTaosConfig(configKey, configValue) + + def removeAllDataFiles(self): + self.node1.removeData() + self.node2.removeData() + self.node3.removeData() + +class ClusterTest: + def __init__(self, hostName): + self.host = hostName + self.user = "root" + self.password = "taosdata" + self.config = "/etc/taos" + self.dbName = "mytest" + self.stbName = "meters" + self.numberOfThreads = 20 + self.numberOfTables = 10000 + self.numberOfRecords = 1000 + self.tbPrefix = "t" + self.ts = 1538548685000 + self.repeat = 1 + + def connectDB(self): + self.conn = taos.connect( + host=self.host, + user=self.user, + password=self.password, + config=self.config) + + def createSTable(self, replica): + cursor = self.conn.cursor() + tdLog.info("drop database if exists %s" % self.dbName) + cursor.execute("drop database if exists %s" % self.dbName) + tdLog.info("create database %s replica %d" % (self.dbName, replica)) + cursor.execute("create database %s replica %d" % (self.dbName, replica)) + tdLog.info("use %s" % self.dbName) + cursor.execute("use %s" % self.dbName) + tdLog.info("drop table if exists %s" % self.stbName) + cursor.execute("drop table if exists %s" % self.stbName) + tdLog.info("create table %s(ts timestamp, current float, voltage int, phase int) tags(id int)" % self.stbName) + cursor.execute("create table %s(ts timestamp, current float, voltage int, phase int) tags(id int)" % self.stbName) + cursor.close() + + def insertData(self, threadID): + print("Thread %d: starting" % threadID) + cursor = self.conn.cursor() + tablesPerThread = int(self.numberOfTables / self.numberOfThreads) + baseTableID = tablesPerThread * threadID + for i in range (tablesPerThread): + cursor.execute("create table %s%d using %s tags(%d)" % (self.tbPrefix, baseTableID + i, self.stbName, baseTableID + i)) + query = "insert into %s%d values" % (self.tbPrefix, baseTableID + i) + base = self.numberOfRecords * i + for j in range(self.numberOfRecords): + query += "(%d, %f, %d, %d)" % (self.ts + base + j, random.random(), random.randint(210, 230), random.randint(0, 10)) + cursor.execute(query) + cursor.close() + print("Thread %d: finishing" % threadID) + + def run(self): + threads = [] + tdLog.info("Inserting data") + for i in range(self.numberOfThreads): + thread = threading.Thread(target=self.insertData, args=(i,)) + threads.append(thread) + thread.start() + + for i in range(self.numberOfThreads): + threads[i].join() \ No newline at end of file diff --git a/tests/pytest/cluster/dataFileRecoveryTest.py b/tests/pytest/cluster/dataFileRecoveryTest.py new file mode 100644 index 0000000000000000000000000000000000000000..089d3fffc1499a8d9cafc87a8d94252111fcd604 --- /dev/null +++ b/tests/pytest/cluster/dataFileRecoveryTest.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from clusterSetup import * +from util.sql import tdSql +from util.log import tdLog +import random + +class ClusterTestcase: + + ## test case 20, 21, 22 ## + def run(self): + + nodes = Nodes() + ctest = ClusterTest(nodes.node1.hostName) + ctest.connectDB() + ctest.createSTable(3) + ctest.run() + tdSql.init(ctest.conn.cursor(), False) + + nodes.node2.stopTaosd() + tdSql.execute("use %s" % ctest.dbName) + tdSql.query("show vgroups") + vnodeID = tdSql.getData(0, 0) + nodes.node2.removeDataForVnode(vnodeID) + nodes.node2.startTaosd() + + # Wait for vnode file to recover + for i in range(10): + tdSql.query("select count(*) from t0") + + tdLog.sleep(10) + + for i in range(10): + tdSql.query("select count(*) from t0") + tdSql.checkData(0, 0, 1000) + + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +ct = ClusterTestcase() +ct.run() diff --git a/tests/pytest/cluster/fullDnodesTest.py b/tests/pytest/cluster/fullDnodesTest.py new file mode 100644 index 0000000000000000000000000000000000000000..3c4b10d97a24dfbb156122aa0afdbb5d22ce3941 --- /dev/null +++ b/tests/pytest/cluster/fullDnodesTest.py @@ -0,0 +1,47 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from clusterSetup import * +from util.sql import tdSql +from util.log import tdLog +import random + +class ClusterTestcase: + + ##Cover test case 5 ## + def run(self): + # cluster environment set up + nodes = Nodes() + nodes.addConfigs("maxVgroupsPerDb", "10") + nodes.addConfigs("maxTablesPerVnode", "1000") + nodes.restartAllTaosd() + + ctest = ClusterTest(nodes.node1.hostName) + ctest.connectDB() + ctest.createSTable(1) + ctest.run() + + tdSql.init(ctest.conn.cursor(), False) + tdSql.execute("use %s" % ctest.dbName) + tdSql.error("create table tt1 using %s tags(1)" % ctest.stbName) + + nodes.removeConfigs("maxVgroupsPerDb", "10") + nodes.removeConfigs("maxTablesPerVnode", "1000") + nodes.restartAllTaosd() + + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +ct = ClusterTestcase() +ct.run() \ No newline at end of file diff --git a/tests/pytest/cluster/killAndRestartDnodesTest.py b/tests/pytest/cluster/killAndRestartDnodesTest.py new file mode 100644 index 0000000000000000000000000000000000000000..be927e862f616c7fbe490e733a18984b6971ef1f --- /dev/null +++ b/tests/pytest/cluster/killAndRestartDnodesTest.py @@ -0,0 +1,75 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from clusterSetup import * +from util.sql import tdSql +from util.log import tdLog +import random + +class ClusterTestcase: + + ## test case 7, 10 ## + def run(self): + # cluster environment set up + tdLog.info("Test case 7, 10") + + nodes = Nodes() + ctest = ClusterTest(nodes.node1.hostName) + ctest.connectDB() + tdSql.init(ctest.conn.cursor(), False) + + nodes.node1.stopTaosd() + tdSql.query("show dnodes") + tdSql.checkRows(3) + tdSql.checkData(0, 4, "offline") + tdSql.checkData(1, 4, "ready") + tdSql.checkData(2, 4, "ready") + + nodes.node1.startTaosd() + tdSql.checkRows(3) + tdSql.checkData(0, 4, "ready") + tdSql.checkData(1, 4, "ready") + tdSql.checkData(2, 4, "ready") + + nodes.node2.stopTaosd() + tdSql.query("show dnodes") + tdSql.checkRows(3) + tdSql.checkData(0, 4, "ready") + tdSql.checkData(1, 4, "offline") + tdSql.checkData(2, 4, "ready") + + nodes.node2.startTaosd() + tdSql.checkRows(3) + tdSql.checkData(0, 4, "ready") + tdSql.checkData(1, 4, "ready") + tdSql.checkData(2, 4, "ready") + + nodes.node3.stopTaosd() + tdSql.query("show dnodes") + tdSql.checkRows(3) + tdSql.checkData(0, 4, "ready") + tdSql.checkData(1, 4, "ready") + tdSql.checkData(2, 4, "offline") + + nodes.node3.startTaosd() + tdSql.checkRows(3) + tdSql.checkData(0, 4, "ready") + tdSql.checkData(1, 4, "ready") + tdSql.checkData(2, 4, "ready") + + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +ct = ClusterTestcase() +ct.run() \ No newline at end of file diff --git a/tests/pytest/cluster/offlineThresholdTest.py b/tests/pytest/cluster/offlineThresholdTest.py new file mode 100644 index 0000000000000000000000000000000000000000..8373424f93c8217250907e09620c8523d63071ad --- /dev/null +++ b/tests/pytest/cluster/offlineThresholdTest.py @@ -0,0 +1,54 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from clusterSetup import * +from util.sql import tdSql +from util.log import tdLog +import random + +class ClusterTestcase: + + ## cover test case 6, 8, 9, 11 ## + def run(self): + # cluster environment set up + nodes = Nodes() + ctest = ClusterTest(nodes.node1.hostName) + ctest.connectDB() + tdSql.init(ctest.conn.cursor(), False) + + nodes.addConfigs("offlineThreshold", "10") + nodes.removeAllDataFiles() + nodes.restartAllTaosd() + nodes.node3.stopTaosd() + + tdLog.sleep(10) + tdSql.query("show dnodes") + tdSql.checkRows(3) + tdSql.checkData(2, 4, "offline") + + tdLog.sleep(60) + tdSql.checkRows(3) + tdSql.checkData(2, 4, "dropping") + + tdLog.sleep(300) + tdSql.checkRows(2) + + nodes.removeConfigs("offlineThreshold", "10") + nodes.restartAllTaosd() + + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +ct = ClusterTestcase() +ct.run() \ No newline at end of file diff --git a/tests/pytest/cluster/oneReplicaOfflineTest.py b/tests/pytest/cluster/oneReplicaOfflineTest.py new file mode 100644 index 0000000000000000000000000000000000000000..0223dfe01add9faca7987d7767f5c41a58b8edd2 --- /dev/null +++ b/tests/pytest/cluster/oneReplicaOfflineTest.py @@ -0,0 +1,65 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from clusterSetup import * +from util.sql import tdSql +from util.log import tdLog +import random + +class ClusterTestcase: + + ## test case 28, 29, 30, 31 ## + def run(self): + + nodes = Nodes() + ctest = ClusterTest(nodes.node1.hostName) + ctest.connectDB() + ctest.createSTable(3) + ctest.run() + tdSql.init(ctest.conn.cursor(), False) + + tdSql.execute("use %s" % ctest.dbName) + + nodes.node2.stopTaosd() + for i in range(100): + tdSql.execute("drop table t%d" % i) + + nodes.node2.startTaosd() + tdSql.query("show tables") + tdSql.checkRows(9900) + + nodes.node2.stopTaosd() + for i in range(10): + tdSql.execute("create table a%d using meters tags(2)" % i) + + nodes.node2.startTaosd() + tdSql.query("show tables") + tdSql.checkRows(9910) + + nodes.node2.stopTaosd() + tdSql.execute("alter table meters add col col6 int") + nodes.node2.startTaosd() + + nodes.node2.stopTaosd() + tdSql.execute("drop database %s" % ctest.dbName) + + nodes.node2.startTaosd() + tdSql.query("show databases") + tdSql.checkRows(0) + + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +ct = ClusterTestcase() +ct.run() diff --git a/tests/pytest/cluster/queryTimeTest.py b/tests/pytest/cluster/queryTimeTest.py new file mode 100644 index 0000000000000000000000000000000000000000..74a9081ccf4fd8abc175e2e0c82b0c6feedcbb26 --- /dev/null +++ b/tests/pytest/cluster/queryTimeTest.py @@ -0,0 +1,54 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from clusterSetup import * +from util.sql import tdSql +from util.log import tdLog +import random +import time + +class ClusterTestcase: + + ## test case 32 ## + def run(self): + + nodes = Nodes() + ctest = ClusterTest(nodes.node1.hostName) + ctest.connectDB() + ctest.createSTable(1) + ctest.run() + tdSql.init(ctest.conn.cursor(), False) + + tdSql.execute("use %s" % ctest.dbName) + totalTime = 0 + for i in range(10): + startTime = time.time() + tdSql.query("select * from %s" % ctest.stbName) + totalTime += time.time() - startTime + print("replica 1: avarage query time for %d records: %f seconds" % (ctest.numberOfTables * ctest.numberOfRecords,totalTime / 10)) + + tdSql.execute("alter database %s replica 3" % ctest.dbName) + tdLog.sleep(60) + totalTime = 0 + for i in range(10): + startTime = time.time() + tdSql.query("select * from %s" % ctest.stbName) + totalTime += time.time() - startTime + print("replica 3: avarage query time for %d records: %f seconds" % (ctest.numberOfTables * ctest.numberOfRecords,totalTime / 10)) + + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +ct = ClusterTestcase() +ct.run() diff --git a/tests/pytest/cluster/stopAllDnodesTest.py b/tests/pytest/cluster/stopAllDnodesTest.py new file mode 100644 index 0000000000000000000000000000000000000000..a71ae52e3d7a640bb589f3bafe16b2e4d45c7b93 --- /dev/null +++ b/tests/pytest/cluster/stopAllDnodesTest.py @@ -0,0 +1,45 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from clusterSetup import * +from util.sql import tdSql +from util.log import tdLog +import random + +class ClusterTestcase: + + ## test case 19 ## + def run(self): + + nodes = Nodes() + ctest = ClusterTest(nodes.node1.hostName) + tdSql.init(ctest.conn.cursor(), False) + + tdSql.query("show databases") + count = tdSql.queryRows; + + nodes.stopAllTaosd() + nodes.node1.startTaosd() + tdSql.error("show databases") + + nodes.node2.startTaosd() + tdSql.error("show databases") + + nodes.node3.startTaosd() + tdLog.sleep(10) + tdSql.query("show databases") + tdSql.checkRows(count) + +ct = ClusterTestcase() +ct.run() diff --git a/tests/pytest/cluster/stopTwoDnodesTest.py b/tests/pytest/cluster/stopTwoDnodesTest.py new file mode 100644 index 0000000000000000000000000000000000000000..9e9958e2d32018b6a89a3e0d08da2c1597151ff2 --- /dev/null +++ b/tests/pytest/cluster/stopTwoDnodesTest.py @@ -0,0 +1,48 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from clusterSetup import * +from util.sql import tdSql +from util.log import tdLog +import random + +class ClusterTestcase: + + ## test case 17, 18 ## + def run(self): + + nodes = Nodes() + ctest = ClusterTest(nodes.node1.hostName) + ctest.connectDB() + ctest.createSTable(1) + ctest.run() + tdSql.init(ctest.conn.cursor(), False) + + tdSql.query("show databases") + count = tdSql.queryRows; + tdSql.execute("use %s" % ctest.dbName) + tdSql.execute("alter database %s replica 3" % ctest.dbName) + nodes.node2.stopTaosd() + nodes.node3.stopTaosd() + tdSql.error("show databases") + + nodes.node2.startTaosd() + tdSql.error("show databases") + + nodes.node3.startTaosd() + tdSql.query("show databases") + tdSql.checkRows(count) + +ct = ClusterTestcase() +ct.run() diff --git a/tests/pytest/cluster/syncingTest.py b/tests/pytest/cluster/syncingTest.py new file mode 100644 index 0000000000000000000000000000000000000000..96be048d231e35f67e40fc4785d2e19337ed408b --- /dev/null +++ b/tests/pytest/cluster/syncingTest.py @@ -0,0 +1,50 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from clusterSetup import * +from util.sql import tdSql +from util.log import tdLog +import random + +class ClusterTestcase: + + ## test case 24, 25, 26, 27 ## + def run(self): + + nodes = Nodes() + ctest = ClusterTest(nodes.node1.hostName) + ctest.connectDB() + ctest.createSTable(1) + ctest.run() + tdSql.init(ctest.conn.cursor(), False) + + + tdSql.execute("use %s" % ctest.dbName) + tdSql.execute("alter database %s replica 3" % ctest.dbName) + + for i in range(100): + tdSql.execute("drop table t%d" % i) + + for i in range(100): + tdSql.execute("create table a%d using meters tags(1)" % i) + + tdSql.execute("alter table meters add col col5 int") + tdSql.execute("alter table meters drop col col5 int") + tdSql.execute("drop database %s" % ctest.dbName) + + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +ct = ClusterTestcase() +ct.run() diff --git a/tests/pytest/cluster/testcluster.sh b/tests/pytest/cluster/testcluster.sh new file mode 100644 index 0000000000000000000000000000000000000000..6e15a498c0a73db450699fe66d63d07c3b18dbe5 --- /dev/null +++ b/tests/pytest/cluster/testcluster.sh @@ -0,0 +1,12 @@ +python3 basicTest.py +python3 bananceTest.py +python3 changeReplicaTest.py +python3 dataFileRecoveryTest.py +python3 fullDnodesTest.py +python3 killAndRestartDnodesTest.py +python3 offlineThresholdTest.py +python3 oneReplicaOfflineTest.py +python3 queryTimeTest.py +python3 stopAllDnodesTest.py +python3 stopTwoDnodesTest.py +python3 syncingTest.py \ No newline at end of file diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index 4ffe35fc3c94edbdd194e03171696a1d681387c1..0af09634df5a5c418797ae4bd352c319fcbc74fa 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -54,6 +54,7 @@ export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3:$(pwd) export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR # Now we are all let, and let's see if we can find a crash. Note we pass all params +CRASH_GEN_EXEC=crash_gen_bootstrap.py if [[ $1 == '--valgrind' ]]; then shift export PYTHONMALLOC=malloc @@ -66,14 +67,16 @@ if [[ $1 == '--valgrind' ]]; then --leak-check=yes \ --suppressions=crash_gen/valgrind_taos.supp \ $PYTHON_EXEC \ - ./crash_gen/crash_gen.py $@ > $VALGRIND_OUT 2> $VALGRIND_ERR + $CRASH_GEN_EXEC $@ > $VALGRIND_OUT 2> $VALGRIND_ERR elif [[ $1 == '--helgrind' ]]; then shift + HELGRIND_OUT=helgrind.out + HELGRIND_ERR=helgrind.err valgrind \ --tool=helgrind \ $PYTHON_EXEC \ - ./crash_gen/crash_gen.py $@ + $CRASH_GEN_EXEC $@ > $HELGRIND_OUT 2> $HELGRIND_ERR else - $PYTHON_EXEC ./crash_gen/crash_gen.py $@ + $PYTHON_EXEC $CRASH_GEN_EXEC $@ fi diff --git a/tests/pytest/crash_gen/README.md b/tests/pytest/crash_gen/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6788ab1a63d0a7c515558695605d1ec8ac5fb7f9 --- /dev/null +++ b/tests/pytest/crash_gen/README.md @@ -0,0 +1,130 @@ +

User's Guide to the Crash_Gen Tool

+ +# Introduction + +To effectively test and debug our TDengine product, we have developed a simple tool to +exercise various functions of the system in a randomized fashion, hoping to expose +maximum number of problems, hopefully without a pre-determined scenario. + +# Preparation + +To run this tool, please ensure the followed preparation work is done first. + +1. Fetch a copy of the TDengine source code, and build it successfully in the `build/` + directory +1. Ensure that the system has Python3.8 or above properly installed. We use + Ubuntu 20.04LTS as our own development environment, and suggest you also use such + an environment if possible. + +# Simple Execution + +To run the tool with the simplest method, follow the steps below: + +1. Open a terminal window, start the `taosd` service in the `build/` directory + (or however you prefer to start the `taosd` service) +1. Open another terminal window, go into the `tests/pytest/` directory, and + run `./crash_gen.sh -p -t 3 -s 10` (change the two parameters here as you wish) +1. Watch the output to the end and see if you get a `SUCCESS` or `FAILURE` + +That's it! + +# Running Clusters + +This tool also makes it easy to test/verify the clustering capabilities of TDengine. You +can start a cluster quite easily with the following command: + +``` +$ cd tests/pytest/ +$ ./crash_gen.sh -e -o 3 +``` + +The `-e` option above tells the tool to start the service, and do not run any tests, while +the `-o 3` option tells the tool to start 3 DNodes and join them together in a cluster. +Obviously you can adjust the the number here. + +## Behind the Scenes + +When the tool runs a cluster, it users a number of directories, each holding the information +for a single DNode, see: + +``` +$ ls build/cluster* +build/cluster_dnode_0: +cfg data log + +build/cluster_dnode_1: +cfg data log + +build/cluster_dnode_2: +cfg data log +``` + +Therefore, when something goes wrong and you want to reset everything with the cluster, simple +erase all the files: + +``` +$ rm -rf build/cluster_dnode_* +``` + +## Addresses and Ports + +The DNodes in the cluster all binds the the `127.0.0.1` IP address (for now anyway), and +uses port 6030 for the first DNode, and 6130 for the 2nd one, and so on. + +## Testing Against a Cluster + +In a separate terminal window, you can invoke the tool in client mode and test against +a cluster, such as: + +``` +$ ./crash_gen.sh -p -t 10 -s 100 -i 3 +``` + +Here the `-i` option tells the tool to always create tables with 3 replicas, and run +all tests against such tables. + +# Additional Features + +The exhaustive features of the tool is available through the `-h` option: + +``` +$ ./crash_gen.sh -h +usage: crash_gen_bootstrap.py [-h] [-a] [-b MAX_DBS] [-c CONNECTOR_TYPE] [-d] [-e] [-g IGNORE_ERRORS] [-i MAX_REPLICAS] [-l] [-n] [-o NUM_DNODES] [-p] [-r] + [-s MAX_STEPS] [-t NUM_THREADS] [-v] [-x] + +TDengine Auto Crash Generator (PLEASE NOTICE the Prerequisites Below) +--------------------------------------------------------------------- +1. You build TDengine in the top level ./build directory, as described in offical docs +2. You run the server there before this script: ./build/bin/taosd -c test/cfg + +optional arguments: + -h, --help show this help message and exit + -a, --auto-start-service + Automatically start/stop the TDengine service (default: false) + -b MAX_DBS, --max-dbs MAX_DBS + Maximum number of DBs to keep, set to disable dropping DB. (default: 0) + -c CONNECTOR_TYPE, --connector-type CONNECTOR_TYPE + Connector type to use: native, rest, or mixed (default: 10) + -d, --debug Turn on DEBUG mode for more logging (default: false) + -e, --run-tdengine Run TDengine service in foreground (default: false) + -g IGNORE_ERRORS, --ignore-errors IGNORE_ERRORS + Ignore error codes, comma separated, 0x supported (default: None) + -i MAX_REPLICAS, --max-replicas MAX_REPLICAS + Maximum number of replicas to use, when testing against clusters. (default: 1) + -l, --larger-data Write larger amount of data during write operations (default: false) + -n, --dynamic-db-table-names + Use non-fixed names for dbs/tables, useful for multi-instance executions (default: false) + -o NUM_DNODES, --num-dnodes NUM_DNODES + Number of Dnodes to initialize, used with -e option. (default: 1) + -p, --per-thread-db-connection + Use a single shared db connection (default: false) + -r, --record-ops Use a pair of always-fsynced fils to record operations performing + performed, for power-off tests (default: false) + -s MAX_STEPS, --max-steps MAX_STEPS + Maximum number of steps to run (default: 100) + -t NUM_THREADS, --num-threads NUM_THREADS + Number of threads to run (default: 10) + -v, --verify-data Verify data written in a number of places by reading back (default: false) + -x, --continue-on-exception + Continue execution after encountering unexpected/disallowed errors/exceptions (default: false) +``` + diff --git a/tests/pytest/crash_gen/crash_gen.py b/tests/pytest/crash_gen/crash_gen_main.py similarity index 56% rename from tests/pytest/crash_gen/crash_gen.py rename to tests/pytest/crash_gen/crash_gen_main.py index 48196ab383c974b5c5d3f5ebc54773cd846353e6..8d68457ec8d07c9c00f8b8fd0a11e2f25284ce4a 100755 --- a/tests/pytest/crash_gen/crash_gen.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -14,42 +14,36 @@ # For type hinting before definition, ref: # https://stackoverflow.com/questions/33533148/how-do-i-specify-that-the-return-type-of-a-method-is-the-same-as-the-class-itsel from __future__ import annotations -import taos -from util.sql import * -from util.cases import * -from util.dnodes import * -from util.log import * -from queue import Queue, Empty -from typing import IO + from typing import Set from typing import Dict from typing import List -from requests.auth import HTTPBasicAuth +from typing import Optional # Type hinting, ref: https://stackoverflow.com/questions/19202633/python-3-type-hinting-for-none + import textwrap -import datetime -import logging import time +import datetime import random +import logging import threading -import requests import copy import argparse import getopt import sys import os -import io import signal import traceback import resource from guppy import hpy import gc -try: - import psutil -except: - print("Psutil module needed, please install: sudo pip3 install psutil") - sys.exit(-1) +from crash_gen.service_manager import ServiceManager, TdeInstance +from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress +from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager + +import taos +import requests # Require Python 3 if sys.version_info[0] < 3: @@ -59,41 +53,37 @@ if sys.version_info[0] < 3: # Command-line/Environment Configurations, will set a bit later # ConfigNameSpace = argparse.Namespace -gConfig = argparse.Namespace() # Dummy value, will be replaced later -gSvcMgr = None # TODO: refactor this hack, use dep injection -logger = None # type: Logger - -def runThread(wt: WorkerThread): - wt.run() +gConfig: argparse.Namespace +gSvcMgr: ServiceManager # TODO: refactor this hack, use dep injection +# logger: logging.Logger +gContainer: Container -class CrashGenError(Exception): - def __init__(self, msg=None, errno=None): - self.msg = msg - self.errno = errno - - def __str__(self): - return self.msg +# def runThread(wt: WorkerThread): +# wt.run() class WorkerThread: - def __init__(self, pool: ThreadPool, tid, tc: ThreadCoordinator, - # te: TaskExecutor, - ): # note: main thread context! + def __init__(self, pool: ThreadPool, tid, tc: ThreadCoordinator): + """ + Note: this runs in the main thread context + """ # self._curStep = -1 self._pool = pool self._tid = tid self._tc = tc # type: ThreadCoordinator # self.threadIdent = threading.get_ident() - self._thread = threading.Thread(target=runThread, args=(self,)) + # self._thread = threading.Thread(target=runThread, args=(self,)) + self._thread = threading.Thread(target=self.run) self._stepGate = threading.Event() # Let us have a DB connection of our own if (gConfig.per_thread_db_connection): # type: ignore # print("connector_type = {}".format(gConfig.connector_type)) - if gConfig.connector_type == 'native': - self._dbConn = DbConn.createNative() + tInst = gContainer.defTdeInstance + if gConfig.connector_type == 'native': + self._dbConn = DbConn.createNative(tInst.getDbTarget()) elif gConfig.connector_type == 'rest': - self._dbConn = DbConn.createRest() + self._dbConn = DbConn.createRest(tInst.getDbTarget()) elif gConfig.connector_type == 'mixed': if Dice.throw(2) == 0: # 1/2 chance self._dbConn = DbConn.createNative() @@ -105,10 +95,10 @@ class WorkerThread: # self._dbInUse = False # if "use db" was executed already def logDebug(self, msg): - logger.debug(" TRD[{}] {}".format(self._tid, msg)) + Logging.debug(" TRD[{}] {}".format(self._tid, msg)) def logInfo(self, msg): - logger.info(" TRD[{}] {}".format(self._tid, msg)) + Logging.info(" TRD[{}] {}".format(self._tid, msg)) # def dbInUse(self): # return self._dbInUse @@ -127,10 +117,10 @@ class WorkerThread: def run(self): # initialization after thread starts, in the thread context # self.isSleeping = False - logger.info("Starting to run thread: {}".format(self._tid)) + Logging.info("Starting to run thread: {}".format(self._tid)) if (gConfig.per_thread_db_connection): # type: ignore - logger.debug("Worker thread openning database connection") + Logging.debug("Worker thread openning database connection") self._dbConn.open() self._doTaskLoop() @@ -140,7 +130,7 @@ class WorkerThread: if self._dbConn.isOpen: #sometimes it is not open self._dbConn.close() else: - logger.warning("Cleaning up worker thread, dbConn already closed") + Logging.warning("Cleaning up worker thread, dbConn already closed") def _doTaskLoop(self): # while self._curStep < self._pool.maxSteps: @@ -151,15 +141,15 @@ class WorkerThread: tc.crossStepBarrier() # shared barrier first, INCLUDING the last one except threading.BrokenBarrierError as err: # main thread timed out print("_bto", end="") - logger.debug("[TRD] Worker thread exiting due to main thread barrier time-out") + Logging.debug("[TRD] Worker thread exiting due to main thread barrier time-out") break - logger.debug("[TRD] Worker thread [{}] exited barrier...".format(self._tid)) + Logging.debug("[TRD] Worker thread [{}] exited barrier...".format(self._tid)) self.crossStepGate() # then per-thread gate, after being tapped - logger.debug("[TRD] Worker thread [{}] exited step gate...".format(self._tid)) + Logging.debug("[TRD] Worker thread [{}] exited step gate...".format(self._tid)) if not self._tc.isRunning(): print("_wts", end="") - logger.debug("[TRD] Thread Coordinator not running any more, worker thread now stopping...") + Logging.debug("[TRD] Thread Coordinator not running any more, worker thread now stopping...") break # Before we fetch the task and run it, let's ensure we properly "use" the database (not needed any more) @@ -178,15 +168,15 @@ class WorkerThread: raise # Fetch a task from the Thread Coordinator - logger.debug( "[TRD] Worker thread [{}] about to fetch task".format(self._tid)) + Logging.debug( "[TRD] Worker thread [{}] about to fetch task".format(self._tid)) task = tc.fetchTask() # Execute such a task - logger.debug("[TRD] Worker thread [{}] about to execute task: {}".format( + Logging.debug("[TRD] Worker thread [{}] about to execute task: {}".format( self._tid, task.__class__.__name__)) task.execute(self) tc.saveExecutedTask(task) - logger.debug("[TRD] Worker thread [{}] finished executing task".format(self._tid)) + Logging.debug("[TRD] Worker thread [{}] finished executing task".format(self._tid)) # self._dbInUse = False # there may be changes between steps # print("_wtd", end=None) # worker thread died @@ -209,7 +199,7 @@ class WorkerThread: self.verifyThreadSelf() # only allowed by ourselves # Wait again at the "gate", waiting to be "tapped" - logger.debug( + Logging.debug( "[TRD] Worker thread {} about to cross the step gate".format( self._tid)) self._stepGate.wait() @@ -222,7 +212,7 @@ class WorkerThread: self.verifyThreadMain() # only allowed for main thread if self._thread.is_alive(): - logger.debug("[TRD] Tapping worker thread {}".format(self._tid)) + Logging.debug("[TRD] Tapping worker thread {}".format(self._tid)) self._stepGate.set() # wake up! time.sleep(0) # let the released thread run a bit else: @@ -253,7 +243,7 @@ class WorkerThread: class ThreadCoordinator: - WORKER_THREAD_TIMEOUT = 60 # one minute + WORKER_THREAD_TIMEOUT = 120 # Normal: 120 def __init__(self, pool: ThreadPool, dbManager: DbManager): self._curStep = -1 # first step is 0 @@ -267,7 +257,7 @@ class ThreadCoordinator: self._stepBarrier = threading.Barrier( self._pool.numThreads + 1) # one barrier for all threads self._execStats = ExecutionStats() - self._runStatus = MainExec.STATUS_RUNNING + self._runStatus = Status.STATUS_RUNNING self._initDbs() def getTaskExecutor(self): @@ -280,14 +270,14 @@ class ThreadCoordinator: self._stepBarrier.wait(timeout) def requestToStop(self): - self._runStatus = MainExec.STATUS_STOPPING + self._runStatus = Status.STATUS_STOPPING self._execStats.registerFailure("User Interruption") def _runShouldEnd(self, transitionFailed, hasAbortedTask, workerTimeout): maxSteps = gConfig.max_steps # type: ignore if self._curStep >= (maxSteps - 1): # maxStep==10, last curStep should be 9 return True - if self._runStatus != MainExec.STATUS_RUNNING: + if self._runStatus != Status.STATUS_RUNNING: return True if transitionFailed: return True @@ -308,7 +298,7 @@ class ThreadCoordinator: def _releaseAllWorkerThreads(self, transitionFailed): self._curStep += 1 # we are about to get into next step. TODO: race condition here! # Now not all threads had time to go to sleep - logger.debug( + Logging.debug( "--\r\n\n--> Step {} starts with main thread waking up".format(self._curStep)) # A new TE for the new step @@ -316,7 +306,7 @@ class ThreadCoordinator: if not transitionFailed: # only if not failed self._te = TaskExecutor(self._curStep) - logger.debug("[TRD] Main thread waking up at step {}, tapping worker threads".format( + Logging.debug("[TRD] Main thread waking up at step {}, tapping worker threads".format( self._curStep)) # Now not all threads had time to go to sleep # Worker threads will wake up at this point, and each execute it's own task self.tapAllThreads() # release all worker thread from their "gates" @@ -325,10 +315,10 @@ class ThreadCoordinator: # Now main thread (that's us) is ready to enter a step # let other threads go past the pool barrier, but wait at the # thread gate - logger.debug("[TRD] Main thread about to cross the barrier") + Logging.debug("[TRD] Main thread about to cross the barrier") self.crossStepBarrier(timeout=self.WORKER_THREAD_TIMEOUT) self._stepBarrier.reset() # Other worker threads should now be at the "gate" - logger.debug("[TRD] Main thread finished crossing the barrier") + Logging.debug("[TRD] Main thread finished crossing the barrier") def _doTransition(self): transitionFailed = False @@ -336,11 +326,11 @@ class ThreadCoordinator: for x in self._dbs: db = x # type: Database sm = db.getStateMachine() - logger.debug("[STT] starting transitions for DB: {}".format(db.getName())) + Logging.debug("[STT] starting transitions for DB: {}".format(db.getName())) # at end of step, transiton the DB state tasksForDb = db.filterTasks(self._executedTasks) sm.transition(tasksForDb, self.getDbManager().getDbConn()) - logger.debug("[STT] transition ended for DB: {}".format(db.getName())) + Logging.debug("[STT] transition ended for DB: {}".format(db.getName())) # Due to limitation (or maybe not) of the TD Python library, # we cannot share connections across threads @@ -348,14 +338,14 @@ class ThreadCoordinator: # Moving below to task loop # if sm.hasDatabase(): # for t in self._pool.threadList: - # logger.debug("[DB] use db for all worker threads") + # Logging.debug("[DB] use db for all worker threads") # t.useDb() # t.execSql("use db") # main thread executing "use # db" on behalf of every worker thread except taos.error.ProgrammingError as err: if (err.msg == 'network unavailable'): # broken DB connection - logger.info("DB connection broken, execution failed") + Logging.info("DB connection broken, execution failed") traceback.print_stack() transitionFailed = True self._te = None # Not running any more @@ -368,7 +358,7 @@ class ThreadCoordinator: self.resetExecutedTasks() # clear the tasks after we are done # Get ready for next step - logger.debug("<-- Step {} finished, trasition failed = {}".format(self._curStep, transitionFailed)) + Logging.debug("<-- Step {} finished, trasition failed = {}".format(self._curStep, transitionFailed)) return transitionFailed def run(self): @@ -382,8 +372,9 @@ class ThreadCoordinator: hasAbortedTask = False workerTimeout = False while not self._runShouldEnd(transitionFailed, hasAbortedTask, workerTimeout): - if not gConfig.debug: # print this only if we are not in debug mode - print(".", end="", flush=True) + if not gConfig.debug: # print this only if we are not in debug mode + Progress.emit(Progress.STEP_BOUNDARY) + # print(".", end="", flush=True) # if (self._curStep % 2) == 0: # print memory usage once every 10 steps # memUsage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss # print("[m:{}]".format(memUsage), end="", flush=True) # print memory usage @@ -395,10 +386,11 @@ class ThreadCoordinator: try: self._syncAtBarrier() # For now just cross the barrier + Progress.emit(Progress.END_THREAD_STEP) except threading.BrokenBarrierError as err: - logger.info("Main loop aborted, caused by worker thread time-out") + Logging.info("Main loop aborted, caused by worker thread(s) time-out") self._execStats.registerFailure("Aborted due to worker thread timeout") - print("\n\nWorker Thread time-out detected, important thread info:") + print("\n\nWorker Thread time-out detected, TAOS related threads are:") ts = ThreadStacks() ts.print(filterInternal=True) workerTimeout = True @@ -409,7 +401,7 @@ class ThreadCoordinator: # threads are QUIET. hasAbortedTask = self._hasAbortedTask() # from previous step if hasAbortedTask: - logger.info("Aborted task encountered, exiting test program") + Logging.info("Aborted task encountered, exiting test program") self._execStats.registerFailure("Aborted Task Encountered") break # do transition only if tasks are error free @@ -420,29 +412,30 @@ class ThreadCoordinator: transitionFailed = True errno2 = Helper.convertErrno(err.errno) # correct error scheme errMsg = "Transition failed: errno=0x{:X}, msg: {}".format(errno2, err) - logger.info(errMsg) + Logging.info(errMsg) traceback.print_exc() self._execStats.registerFailure(errMsg) # Then we move on to the next step + Progress.emit(Progress.BEGIN_THREAD_STEP) self._releaseAllWorkerThreads(transitionFailed) if hasAbortedTask or transitionFailed : # abnormal ending, workers waiting at "gate" - logger.debug("Abnormal ending of main thraed") + Logging.debug("Abnormal ending of main thraed") elif workerTimeout: - logger.debug("Abnormal ending of main thread, due to worker timeout") + Logging.debug("Abnormal ending of main thread, due to worker timeout") else: # regular ending, workers waiting at "barrier" - logger.debug("Regular ending, main thread waiting for all worker threads to stop...") + Logging.debug("Regular ending, main thread waiting for all worker threads to stop...") self._syncAtBarrier() self._te = None # No more executor, time to end - logger.debug("Main thread tapping all threads one last time...") + Logging.debug("Main thread tapping all threads one last time...") self.tapAllThreads() # Let the threads run one last time - logger.debug("\r\n\n--> Main thread ready to finish up...") - logger.debug("Main thread joining all threads") + Logging.debug("\r\n\n--> Main thread ready to finish up...") + Logging.debug("Main thread joining all threads") self._pool.joinAll() # Get all threads to finish - logger.info("\nAll worker threads finished") + Logging.info(". . . All worker threads finished") # No CR/LF before self._execStats.endExec() def cleanup(self): # free resources @@ -474,7 +467,7 @@ class ThreadCoordinator: wakeSeq.append(i) else: wakeSeq.insert(0, i) - logger.debug( + Logging.debug( "[TRD] Main thread waking up worker threads: {}".format( str(wakeSeq))) # TODO: set dice seed to a deterministic value @@ -492,9 +485,11 @@ class ThreadCoordinator: dbc = self.getDbManager().getDbConn() if gConfig.max_dbs == 0: self._dbs.append(Database(0, dbc)) - else: + else: + baseDbNumber = int(datetime.datetime.now().timestamp( # Don't use Dice/random, as they are deterministic + )*333) % 888 if gConfig.dynamic_db_table_names else 0 for i in range(gConfig.max_dbs): - self._dbs.append(Database(i, dbc)) + self._dbs.append(Database(baseDbNumber + i, dbc)) def pickDatabase(self): idxDb = 0 @@ -512,7 +507,7 @@ class ThreadCoordinator: # pick a task type for current state db = self.pickDatabase() - taskType = db.getStateMachine().pickTaskType() # type: Task + taskType = db.getStateMachine().pickTaskType() # dynamic name of class return taskType(self._execStats, db) # create a task from it def resetExecutedTasks(self): @@ -522,13 +517,6 @@ class ThreadCoordinator: with self._lock: self._executedTasks.append(task) -# We define a class to run a number of threads in locking steps. - -class Helper: - @classmethod - def convertErrno(cls, errno): - return errno if (errno > 0) else 0x80000000 + errno - class ThreadPool: def __init__(self, numThreads, maxSteps): self.numThreads = numThreads @@ -546,7 +534,7 @@ class ThreadPool: def joinAll(self): for workerThread in self.threadList: - logger.debug("Joining thread...") + Logging.debug("Joining thread...") workerThread._thread.join() def cleanup(self): @@ -603,7 +591,7 @@ class LinearQueue(): def allocate(self, i): with self._lock: - # logger.debug("LQ allocating item {}".format(i)) + # Logging.debug("LQ allocating item {}".format(i)) if (i in self.inUse): raise RuntimeError( "Cannot re-use same index in queue: {}".format(i)) @@ -611,7 +599,7 @@ class LinearQueue(): def release(self, i): with self._lock: - # logger.debug("LQ releasing item {}".format(i)) + # Logging.debug("LQ releasing item {}".format(i)) self.inUse.remove(i) # KeyError possible, TODO: why? def size(self): @@ -633,357 +621,6 @@ class LinearQueue(): return ret -class DbConn: - TYPE_NATIVE = "native-c" - TYPE_REST = "rest-api" - TYPE_INVALID = "invalid" - - @classmethod - def create(cls, connType): - if connType == cls.TYPE_NATIVE: - return DbConnNative() - elif connType == cls.TYPE_REST: - return DbConnRest() - else: - raise RuntimeError( - "Unexpected connection type: {}".format(connType)) - - @classmethod - def createNative(cls): - return cls.create(cls.TYPE_NATIVE) - - @classmethod - def createRest(cls): - return cls.create(cls.TYPE_REST) - - def __init__(self): - self.isOpen = False - self._type = self.TYPE_INVALID - self._lastSql = None - - def getLastSql(self): - return self._lastSql - - def open(self): - if (self.isOpen): - raise RuntimeError("Cannot re-open an existing DB connection") - - # below implemented by child classes - self.openByType() - - logger.debug("[DB] data connection opened, type = {}".format(self._type)) - self.isOpen = True - - def queryScalar(self, sql) -> int: - return self._queryAny(sql) - - def queryString(self, sql) -> str: - return self._queryAny(sql) - - def _queryAny(self, sql): # actual query result as an int - if (not self.isOpen): - raise RuntimeError("Cannot query database until connection is open") - nRows = self.query(sql) - if nRows != 1: - raise taos.error.ProgrammingError( - "Unexpected result for query: {}, rows = {}".format(sql, nRows), - (0x991 if nRows==0 else 0x992) - ) - if self.getResultRows() != 1 or self.getResultCols() != 1: - raise RuntimeError("Unexpected result set for query: {}".format(sql)) - return self.getQueryResult()[0][0] - - def use(self, dbName): - self.execute("use {}".format(dbName)) - - def existsDatabase(self, dbName: str): - ''' Check if a certain database exists ''' - self.query("show databases") - dbs = [v[0] for v in self.getQueryResult()] # ref: https://stackoverflow.com/questions/643823/python-list-transformation - # ret2 = dbName in dbs - # print("dbs = {}, str = {}, ret2={}, type2={}".format(dbs, dbName,ret2, type(dbName))) - return dbName in dbs # TODO: super weird type mangling seen, once here - - def hasTables(self): - return self.query("show tables") > 0 - - def execute(self, sql): - ''' Return the number of rows affected''' - raise RuntimeError("Unexpected execution, should be overriden") - - def safeExecute(self, sql): - '''Safely execute any SQL query, returning True/False upon success/failure''' - try: - self.execute(sql) - return True # ignore num of results, return success - except taos.error.ProgrammingError as err: - return False # failed, for whatever TAOS reason - # Not possile to reach here, non-TAOS exception would have been thrown - - def query(self, sql) -> int: # return num rows returned - ''' Return the number of rows affected''' - raise RuntimeError("Unexpected execution, should be overriden") - - def openByType(self): - raise RuntimeError("Unexpected execution, should be overriden") - - def getQueryResult(self): - raise RuntimeError("Unexpected execution, should be overriden") - - def getResultRows(self): - raise RuntimeError("Unexpected execution, should be overriden") - - def getResultCols(self): - raise RuntimeError("Unexpected execution, should be overriden") - -# Sample: curl -u root:taosdata -d "show databases" localhost:6020/rest/sql - - -class DbConnRest(DbConn): - def __init__(self): - super().__init__() - self._type = self.TYPE_REST - self._url = "http://localhost:6041/rest/sql" # fixed for now - self._result = None - - def openByType(self): # Open connection - pass # do nothing, always open - - def close(self): - if (not self.isOpen): - raise RuntimeError("Cannot clean up database until connection is open") - # Do nothing for REST - logger.debug("[DB] REST Database connection closed") - self.isOpen = False - - def _doSql(self, sql): - self._lastSql = sql # remember this, last SQL attempted - try: - r = requests.post(self._url, - data = sql, - auth = HTTPBasicAuth('root', 'taosdata')) - except: - print("REST API Failure (TODO: more info here)") - raise - rj = r.json() - # Sanity check for the "Json Result" - if ('status' not in rj): - raise RuntimeError("No status in REST response") - - if rj['status'] == 'error': # clearly reported error - if ('code' not in rj): # error without code - raise RuntimeError("REST error return without code") - errno = rj['code'] # May need to massage this in the future - # print("Raising programming error with REST return: {}".format(rj)) - raise taos.error.ProgrammingError( - rj['desc'], errno) # todo: check existance of 'desc' - - if rj['status'] != 'succ': # better be this - raise RuntimeError( - "Unexpected REST return status: {}".format( - rj['status'])) - - nRows = rj['rows'] if ('rows' in rj) else 0 - self._result = rj - return nRows - - def execute(self, sql): - if (not self.isOpen): - raise RuntimeError( - "Cannot execute database commands until connection is open") - logger.debug("[SQL-REST] Executing SQL: {}".format(sql)) - nRows = self._doSql(sql) - logger.debug( - "[SQL-REST] Execution Result, nRows = {}, SQL = {}".format(nRows, sql)) - return nRows - - def query(self, sql): # return rows affected - return self.execute(sql) - - def getQueryResult(self): - return self._result['data'] - - def getResultRows(self): - print(self._result) - raise RuntimeError("TBD") - # return self._tdSql.queryRows - - def getResultCols(self): - print(self._result) - raise RuntimeError("TBD") - - # Duplicate code from TDMySQL, TODO: merge all this into DbConnNative - - -class MyTDSql: - # Class variables - _clsLock = threading.Lock() # class wide locking - longestQuery = None # type: str - longestQueryTime = 0.0 # seconds - lqStartTime = 0.0 - # lqEndTime = 0.0 # Not needed, as we have the two above already - - def __init__(self, hostAddr, cfgPath): - # Make the DB connection - self._conn = taos.connect(host=hostAddr, config=cfgPath) - self._cursor = self._conn.cursor() - - self.queryRows = 0 - self.queryCols = 0 - self.affectedRows = 0 - - # def init(self, cursor, log=True): - # self.cursor = cursor - # if (log): - # caller = inspect.getframeinfo(inspect.stack()[1][0]) - # self.cursor.log(caller.filename + ".sql") - - def close(self): - self._cursor.close() # can we double close? - self._conn.close() # TODO: very important, cursor close does NOT close DB connection! - self._cursor.close() - - def _execInternal(self, sql): - startTime = time.time() - ret = self._cursor.execute(sql) - # print("\nSQL success: {}".format(sql)) - queryTime = time.time() - startTime - # Record the query time - cls = self.__class__ - if queryTime > (cls.longestQueryTime + 0.01) : - with cls._clsLock: - cls.longestQuery = sql - cls.longestQueryTime = queryTime - cls.lqStartTime = startTime - return ret - - def query(self, sql): - self.sql = sql - try: - self._execInternal(sql) - self.queryResult = self._cursor.fetchall() - self.queryRows = len(self.queryResult) - self.queryCols = len(self._cursor.description) - except Exception as e: - # caller = inspect.getframeinfo(inspect.stack()[1][0]) - # args = (caller.filename, caller.lineno, sql, repr(e)) - # tdLog.exit("%s(%d) failed: sql:%s, %s" % args) - raise - return self.queryRows - - def execute(self, sql): - self.sql = sql - try: - self.affectedRows = self._execInternal(sql) - except Exception as e: - # caller = inspect.getframeinfo(inspect.stack()[1][0]) - # args = (caller.filename, caller.lineno, sql, repr(e)) - # tdLog.exit("%s(%d) failed: sql:%s, %s" % args) - raise - return self.affectedRows - - -class DbConnNative(DbConn): - # Class variables - _lock = threading.Lock() - _connInfoDisplayed = False - totalConnections = 0 # Not private - - def __init__(self): - super().__init__() - self._type = self.TYPE_NATIVE - self._conn = None - # self._cursor = None - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("communit")] - else: - projPath = selfPath[:selfPath.find("tests")] - - buildPath = None - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root) - len("/build/bin")] - break - if buildPath == None: - raise RuntimeError("Failed to determine buildPath, selfPath={}, projPath={}" - .format(selfPath, projPath)) - return buildPath - - - def openByType(self): # Open connection - cfgPath = self.getBuildPath() + "/test/cfg" - hostAddr = "127.0.0.1" - - cls = self.__class__ # Get the class, to access class variables - with cls._lock: # force single threading for opening DB connections. # TODO: whaaat??!!! - if not cls._connInfoDisplayed: - cls._connInfoDisplayed = True # updating CLASS variable - logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath)) - # Make the connection - # self._conn = taos.connect(host=hostAddr, config=cfgPath) # TODO: make configurable - # self._cursor = self._conn.cursor() - # Record the count in the class - self._tdSql = MyTDSql(hostAddr, cfgPath) # making DB connection - cls.totalConnections += 1 - - self._tdSql.execute('reset query cache') - # self._cursor.execute('use db') # do this at the beginning of every - - # Open connection - # self._tdSql = MyTDSql() - # self._tdSql.init(self._cursor) - - def close(self): - if (not self.isOpen): - raise RuntimeError("Cannot clean up database until connection is open") - self._tdSql.close() - # Decrement the class wide counter - cls = self.__class__ # Get the class, to access class variables - with cls._lock: - cls.totalConnections -= 1 - - logger.debug("[DB] Database connection closed") - self.isOpen = False - - def execute(self, sql): - if (not self.isOpen): - raise RuntimeError("Cannot execute database commands until connection is open") - logger.debug("[SQL] Executing SQL: {}".format(sql)) - self._lastSql = sql - nRows = self._tdSql.execute(sql) - logger.debug( - "[SQL] Execution Result, nRows = {}, SQL = {}".format( - nRows, sql)) - return nRows - - def query(self, sql): # return rows affected - if (not self.isOpen): - raise RuntimeError( - "Cannot query database until connection is open") - logger.debug("[SQL] Executing SQL: {}".format(sql)) - self._lastSql = sql - nRows = self._tdSql.query(sql) - logger.debug( - "[SQL] Query Result, nRows = {}, SQL = {}".format( - nRows, sql)) - return nRows - # results are in: return self._tdSql.queryResult - - def getQueryResult(self): - return self._tdSql.queryResult - - def getResultRows(self): - return self._tdSql.queryRows - - def getResultCols(self): - return self._tdSql.queryCols - - class AnyState: STATE_INVALID = -1 STATE_EMPTY = 0 # nothing there, no even a DB @@ -1232,7 +869,7 @@ class StateMechine: def init(self, dbc: DbConn): # late initailization, don't save the dbConn self._curState = self._findCurrentState(dbc) # starting state - logger.debug("Found Starting State: {}".format(self._curState)) + Logging.debug("Found Starting State: {}".format(self._curState)) # TODO: seems no lnoger used, remove? def getCurrentState(self): @@ -1270,7 +907,7 @@ class StateMechine: raise RuntimeError( "No suitable task types found for state: {}".format( self._curState)) - logger.debug( + Logging.debug( "[OPS] Tasks found for state {}: {}".format( self._curState, typesToStrings(taskTypes))) @@ -1280,27 +917,27 @@ class StateMechine: ts = time.time() # we use this to debug how fast/slow it is to do the various queries to find the current DB state dbName =self._db.getName() if not dbc.existsDatabase(dbName): # dbc.hasDatabases(): # no database?! - logger.debug( "[STT] empty database found, between {} and {}".format(ts, time.time())) + Logging.debug( "[STT] empty database found, between {} and {}".format(ts, time.time())) return StateEmpty() # did not do this when openning connection, and this is NOT the worker # thread, which does this on their own dbc.use(dbName) if not dbc.hasTables(): # no tables - logger.debug("[STT] DB_ONLY found, between {} and {}".format(ts, time.time())) + Logging.debug("[STT] DB_ONLY found, between {} and {}".format(ts, time.time())) return StateDbOnly() sTable = self._db.getFixedSuperTable() if sTable.hasRegTables(dbc, dbName): # no regular tables - logger.debug("[STT] SUPER_TABLE_ONLY found, between {} and {}".format(ts, time.time())) + Logging.debug("[STT] SUPER_TABLE_ONLY found, between {} and {}".format(ts, time.time())) return StateSuperTableOnly() else: # has actual tables - logger.debug("[STT] HAS_DATA found, between {} and {}".format(ts, time.time())) + Logging.debug("[STT] HAS_DATA found, between {} and {}".format(ts, time.time())) return StateHasData() # We transition the system to a new state by examining the current state itself def transition(self, tasks, dbc: DbConn): if (len(tasks) == 0): # before 1st step, or otherwise empty - logger.debug("[STT] Starting State: {}".format(self._curState)) + Logging.debug("[STT] Starting State: {}".format(self._curState)) return # do nothing # this should show up in the server log, separating steps @@ -1336,7 +973,7 @@ class StateMechine: # Nothing for sure newState = self._findCurrentState(dbc) - logger.debug("[STT] New DB state determined: {}".format(newState)) + Logging.debug("[STT] New DB state determined: {}".format(newState)) # can old state move to new state through the tasks? self._curState.verifyTasksToState(tasks, newState) self._curState = newState @@ -1354,7 +991,7 @@ class StateMechine: # read data task, default to 10: TODO: change to a constant weights.append(10) i = self._weighted_choice_sub(weights) - # logger.debug(" (weighted random:{}/{}) ".format(i, len(taskTypes))) + # Logging.debug(" (weighted random:{}/{}) ".format(i, len(taskTypes))) return taskTypes[i] # ref: @@ -1372,6 +1009,8 @@ class Database: possibly in a cluster environment. For now we use it to manage state transitions in that database + + TODO: consider moving, but keep in mind it contains "StateMachine" ''' _clsLock = threading.Lock() # class wide lock _lastInt = 101 # next one is initial integer @@ -1433,17 +1072,18 @@ class Database: t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years t4 = datetime.datetime.fromtimestamp( t3.timestamp() + elSec2) # see explanation above - logger.info("Setting up TICKS to start from: {}".format(t4)) + Logging.debug("Setting up TICKS to start from: {}".format(t4)) return t4 @classmethod def getNextTick(cls): with cls._clsLock: # prevent duplicate tick - if cls._lastLaggingTick==0: + if cls._lastLaggingTick==0 or cls._lastTick==0 : # not initialized # 10k at 1/20 chance, should be enough to avoid overlaps - cls._lastLaggingTick = cls.setupLastTick() + datetime.timedelta(0, -10000) - if cls._lastTick==0: # should be quite a bit into the future - cls._lastTick = cls.setupLastTick() + tick = cls.setupLastTick() + cls._lastTick = tick + cls._lastLaggingTick = tick + datetime.timedelta(0, -10000) + # if : # should be quite a bit into the future if Dice.throw(20) == 0: # 1 in 20 chance, return lagging tick cls._lastLaggingTick += datetime.timedelta(0, 1) # Go back in time 100 seconds @@ -1468,64 +1108,6 @@ class Database: return ret -class DbManager(): - ''' This is a wrapper around DbConn(), to make it easier to use. - - TODO: rename this to DbConnManager - ''' - def __init__(self): - self.tableNumQueue = LinearQueue() # TODO: delete? - # self.openDbServerConnection() - self._dbConn = DbConn.createNative() if ( - gConfig.connector_type == 'native') else DbConn.createRest() - try: - self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected - except taos.error.ProgrammingError as err: - # print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err)) - if (err.msg == 'client disconnected'): # cannot open DB connection - print( - "Cannot establish DB connection, please re-run script without parameter, and follow the instructions.") - sys.exit(2) - else: - print("Failed to connect to DB, errno = {}, msg: {}" - .format(Helper.convertErrno(err.errno), err.msg)) - raise - except BaseException: - print("[=] Unexpected exception") - raise - - # Do this after dbConn is in proper shape - # Moved to Database() - # self._stateMachine = StateMechine(self._dbConn) - - def getDbConn(self): - return self._dbConn - - # TODO: not used any more, to delete - def pickAndAllocateTable(self): # pick any table, and "use" it - return self.tableNumQueue.pickAndAllocate() - - # TODO: Not used any more, to delete - def addTable(self): - with self._lock: - tIndex = self.tableNumQueue.push() - return tIndex - - # Not used any more, to delete - def releaseTable(self, i): # return the table back, so others can use it - self.tableNumQueue.release(i) - - # TODO: not used any more, delete - def getTableNameToDelete(self): - tblNum = self.tableNumQueue.pop() # TODO: race condition! - if (not tblNum): # maybe false - return False - - return "table_{}".format(tblNum) - - def cleanUp(self): - self._dbConn.close() - class TaskExecutor(): class BoundedList: def __init__(self, size=10): @@ -1584,10 +1166,10 @@ class TaskExecutor(): self._boundedList.add(n) # def logInfo(self, msg): - # logger.info(" T[{}.x]: ".format(self._curStep) + msg) + # Logging.info(" T[{}.x]: ".format(self._curStep) + msg) # def logDebug(self, msg): - # logger.debug(" T[{}.x]: ".format(self._curStep) + msg) + # Logging.debug(" T[{}.x]: ".format(self._curStep) + msg) class Task(): @@ -1596,27 +1178,31 @@ class Task(): instead. But a task is always associated with a DB ''' taskSn = 100 + _lock = threading.Lock() + _tableLocks: Dict[str, threading.Lock] = {} @classmethod def allocTaskNum(cls): Task.taskSn += 1 # IMPORTANT: cannot use cls.taskSn, since each sub class will have a copy - # logger.debug("Allocating taskSN: {}".format(Task.taskSn)) + # Logging.debug("Allocating taskSN: {}".format(Task.taskSn)) return Task.taskSn def __init__(self, execStats: ExecutionStats, db: Database): self._workerThread = None - self._err = None # type: Exception + self._err: Optional[Exception] = None self._aborted = False self._curStep = None self._numRows = None # Number of rows affected # Assign an incremental task serial number self._taskNum = self.allocTaskNum() - # logger.debug("Creating new task {}...".format(self._taskNum)) + # Logging.debug("Creating new task {}...".format(self._taskNum)) self._execStats = execStats self._db = db # A task is always associated/for a specific DB + + def isSuccess(self): return self._err is None @@ -1645,15 +1231,23 @@ class Task(): "To be implemeted by child classes, class name: {}".format( self.__class__.__name__)) + def _isServiceStable(self): + if not gSvcMgr: + return True # we don't run service, so let's assume it's stable + return gSvcMgr.isStable() # otherwise let's examine the service + def _isErrAcceptable(self, errno, msg): if errno in [ 0x05, # TSDB_CODE_RPC_NOT_READY 0x0B, # Unable to establish connection, more details in TD-1648 - # 0x200, # invalid SQL, TODO: re-examine with TD-934 + 0x200, # invalid SQL, TODO: re-examine with TD-934 + 0x20F, # query terminated, possibly due to vnoding being dropped, see TD-1776 + 0x213, # "Disconnected from service", result of "kill connection ???" 0x217, # "db not selected", client side defined error code - 0x218, # "Table does not exist" client side defined error code - 0x360, 0x362, - 0x369, # tag already exists + # 0x218, # "Table does not exist" client side defined error code + 0x360, # Table already exists + 0x362, + # 0x369, # tag already exists 0x36A, 0x36B, 0x36D, 0x381, 0x380, # "db not selected" @@ -1662,12 +1256,17 @@ class Task(): 0x503, 0x510, # vnode not in ready state 0x14, # db not ready, errno changed - 0x600, + 0x600, # Invalid table ID, why? 1000 # REST catch-all error ]: return True # These are the ALWAYS-ACCEPTABLE ones - elif (errno in [ 0x0B ]) and gConfig.auto_start_service: - return True # We may get "network unavilable" when restarting service + # This case handled below already. + # elif (errno in [ 0x0B ]) and gConfig.auto_start_service: + # return True # We may get "network unavilable" when restarting service + elif gConfig.ignore_errors: # something is specified on command line + moreErrnos = [int(v, 0) for v in gConfig.ignore_errors.split(',')] + if errno in moreErrnos: + return True elif errno == 0x200 : # invalid SQL, we need to div in a bit more if msg.find("invalid column name") != -1: return True @@ -1675,8 +1274,8 @@ class Task(): return True elif msg.find("duplicated column names") != -1: # also alter table tag issues return True - elif (gSvcMgr!=None) and gSvcMgr.isRestarting(): - logger.info("Ignoring error when service is restarting: errno = {}, msg = {}".format(errno, msg)) + elif not self._isServiceStable(): # We are managing service, and ... + Logging.info("Ignoring error when service starting/stopping: errno = {}, msg = {}".format(errno, msg)) return True return False # Not an acceptable error @@ -1725,7 +1324,7 @@ class Task(): self._err = err self._aborted = True except Exception as e: - self.logInfo("Non-TAOS exception encountered") + Logging.info("Non-TAOS exception encountered with: {}".format(self.__class__.__name__)) self._err = e self._aborted = True traceback.print_exc() @@ -1735,10 +1334,11 @@ class Task(): self._aborted = True traceback.print_exc() except BaseException: # TODO: what is this again??!! - self.logDebug( - "[=] Unexpected exception, SQL: {}".format( - wt.getDbConn().getLastSql())) - raise + raise RuntimeError("Punt") + # self.logDebug( + # "[=] Unexpected exception, SQL: {}".format( + # wt.getDbConn().getLastSql())) + # raise self._execStats.endTaskType(self.__class__.__name__, self.isSuccess()) self.logDebug("[X] task execution completed, {}, status: {}".format( @@ -1757,6 +1357,24 @@ class Task(): def getQueryResult(self, wt: WorkerThread): # execute an SQL on the worker thread return wt.getQueryResult() + def lockTable(self, ftName): # full table name + # print(" <<" + ftName + '_', end="", flush=True) + with Task._lock: + if not ftName in Task._tableLocks: + Task._tableLocks[ftName] = threading.Lock() + + Task._tableLocks[ftName].acquire() + + def unlockTable(self, ftName): + # print('_' + ftName + ">> ", end="", flush=True) + with Task._lock: + if not ftName in self._tableLocks: + raise RuntimeError("Corrupt state, no such lock") + lock = Task._tableLocks[ftName] + if not lock.locked(): + raise RuntimeError("Corrupte state, already unlocked") + lock.release() + class ExecutionStats: def __init__(self): @@ -1817,14 +1435,14 @@ class ExecutionStats: self._failureReason = reason def printStats(self): - logger.info( + Logging.info( "----------------------------------------------------------------------") - logger.info( + Logging.info( "| Crash_Gen test {}, with the following stats:". format( "FAILED (reason: {})".format( self._failureReason) if self._failed else "SUCCEEDED")) - logger.info("| Task Execution Times (success/total):") - execTimesAny = 0 + Logging.info("| Task Execution Times (success/total):") + execTimesAny = 0.0 for k, n in self._execTimes.items(): execTimesAny += n[0] errStr = None @@ -1834,28 +1452,28 @@ class ExecutionStats: errStrs = ["0x{:X}:{}".format(eno, n) for (eno, n) in errors.items()] # print("error strings = {}".format(errStrs)) errStr = ", ".join(errStrs) - logger.info("| {0:<24}: {1}/{2} (Errors: {3})".format(k, n[1], n[0], errStr)) + Logging.info("| {0:<24}: {1}/{2} (Errors: {3})".format(k, n[1], n[0], errStr)) - logger.info( + Logging.info( "| Total Tasks Executed (success or not): {} ".format(execTimesAny)) - logger.info( + Logging.info( "| Total Tasks In Progress at End: {}".format( self._tasksInProgress)) - logger.info( + Logging.info( "| Total Task Busy Time (elapsed time when any task is in progress): {:.3f} seconds".format( self._accRunTime)) - logger.info( + Logging.info( "| Average Per-Task Execution Time: {:.3f} seconds".format(self._accRunTime / execTimesAny)) - logger.info( + Logging.info( "| Total Elapsed Time (from wall clock): {:.3f} seconds".format( self._elapsedTime)) - logger.info("| Top numbers written: {}".format(TaskExecutor.getBoundedList())) - logger.info("| Active DB Native Connections (now): {}".format(DbConnNative.totalConnections)) - logger.info("| Longest native query time: {:.3f} seconds, started: {}". + Logging.info("| Top numbers written: {}".format(TaskExecutor.getBoundedList())) + Logging.info("| Active DB Native Connections (now): {}".format(DbConnNative.totalConnections)) + Logging.info("| Longest native query time: {:.3f} seconds, started: {}". format(MyTDSql.longestQueryTime, time.strftime("%x %X", time.localtime(MyTDSql.lqStartTime))) ) - logger.info("| Longest native query: {}".format(MyTDSql.longestQuery)) - logger.info( + Logging.info("| Longest native query: {}".format(MyTDSql.longestQuery)) + Logging.info( "----------------------------------------------------------------------") @@ -1865,11 +1483,14 @@ class StateTransitionTask(Task): LARGE_NUMBER_OF_RECORDS = 50 SMALL_NUMBER_OF_RECORDS = 3 + _baseTableNumber = None + + _endState = None # TODO: no longter used? + @classmethod def getInfo(cls): # each sub class should supply their own information raise RuntimeError("Overriding method expected") - - _endState = None + @classmethod def getEndState(cls): # TODO: optimize by calling it fewer times raise RuntimeError("Overriding method expected") @@ -1889,7 +1510,10 @@ class StateTransitionTask(Task): @classmethod def getRegTableName(cls, i): - return "reg_table_{}".format(i) + if ( StateTransitionTask._baseTableNumber is None): # Set it one time + StateTransitionTask._baseTableNumber = Dice.throw( + 999) if gConfig.dynamic_db_table_names else 0 + return "reg_table_{}".format(StateTransitionTask._baseTableNumber + i) def execute(self, wt: WorkerThread): super().execute(wt) @@ -1909,7 +1533,8 @@ class TaskCreateDb(StateTransitionTask): # was: self.execWtSql(wt, "create database db") repStr = "" if gConfig.max_replicas != 1: - numReplica = Dice.throw(gConfig.max_replicas) + 1 # 1,2 ... N + # numReplica = Dice.throw(gConfig.max_replicas) + 1 # 1,2 ... N + numReplica = gConfig.max_replicas # fixed, always repStr = "replica {}".format(numReplica) self.execWtSql(wt, "create database {} {}" .format(self._db.getName(), repStr) ) @@ -1925,7 +1550,7 @@ class TaskDropDb(StateTransitionTask): def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): self.execWtSql(wt, "drop database {}".format(self._db.getName())) - logger.debug("[OPS] database dropped at {}".format(time.time())) + Logging.debug("[OPS] database dropped at {}".format(time.time())) class TaskCreateSuperTable(StateTransitionTask): @classmethod @@ -1938,13 +1563,16 @@ class TaskCreateSuperTable(StateTransitionTask): def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): if not self._db.exists(wt.getDbConn()): - logger.debug("Skipping task, no DB yet") + Logging.debug("Skipping task, no DB yet") return sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place + sTable.create(wt.getDbConn(), self._db.getName(), - {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'}) + {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'}, + dropIfExists = True + ) # self.execWtSql(wt,"create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName)) # No need to create the regular tables, INSERT will do that # automatically @@ -1957,14 +1585,41 @@ class TdSuperTable: def getName(self): return self._stName + def drop(self, dbc, dbName, skipCheck = False): + if self.exists(dbc, dbName) : # if myself exists + fullTableName = dbName + '.' + self._stName + dbc.execute("DROP TABLE {}".format(fullTableName)) + else: + if not skipCheck: + raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName)) + + def exists(self, dbc, dbName): + dbc.execute("USE " + dbName) + return dbc.existsSuperTable(self._stName) + # TODO: odd semantic, create() method is usually static? - def create(self, dbc, dbName, cols: dict, tags: dict): + def create(self, dbc, dbName, cols: dict, tags: dict, + dropIfExists = False + ): + '''Creating a super table''' - sql = "CREATE TABLE {}.{} ({}) TAGS ({})".format( - dbName, - self._stName, - ",".join(['%s %s'%(k,v) for (k,v) in cols.items()]), - ",".join(['%s %s'%(k,v) for (k,v) in tags.items()]) + dbc.execute("USE " + dbName) + fullTableName = dbName + '.' + self._stName + if dbc.existsSuperTable(self._stName): + if dropIfExists: + dbc.execute("DROP TABLE {}".format(fullTableName)) + else: # error + raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName)) + + # Now let's create + sql = "CREATE TABLE {} ({})".format( + fullTableName, + ",".join(['%s %s'%(k,v) for (k,v) in cols.items()])) + if tags is None : + sql += " TAGS (dummy int) " + else: + sql += " TAGS ({})".format( + ",".join(['%s %s'%(k,v) for (k,v) in tags.items()]) ) dbc.execute(sql) @@ -1973,7 +1628,7 @@ class TdSuperTable: dbc.query("select TBNAME from {}.{}".format(dbName, self._stName)) # TODO: analyze result set later except taos.error.ProgrammingError as err: errno2 = Helper.convertErrno(err.errno) - logger.debug("[=] Failed to get tables from super table: errno=0x{:X}, msg: {}".format(errno2, err)) + Logging.debug("[=] Failed to get tables from super table: errno=0x{:X}, msg: {}".format(errno2, err)) raise qr = dbc.getQueryResult() @@ -1982,14 +1637,25 @@ class TdSuperTable: def hasRegTables(self, dbc: DbConn, dbName: str): return dbc.query("SELECT * FROM {}.{}".format(dbName, self._stName)) > 0 - def ensureTable(self, dbc: DbConn, dbName: str, regTableName: str): + def ensureTable(self, task: Task, dbc: DbConn, dbName: str, regTableName: str): sql = "select tbname from {}.{} where tbname in ('{}')".format(dbName, self._stName, regTableName) if dbc.query(sql) >= 1 : # reg table exists already return - sql = "CREATE TABLE {}.{} USING {}.{} tags ({})".format( - dbName, regTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName) - ) - dbc.execute(sql) + + # acquire a lock first, so as to be able to *verify*. More details in TD-1471 + fullTableName = dbName + '.' + regTableName + if task is not None: # optional lock + task.lockTable(fullTableName) + Progress.emit(Progress.CREATE_TABLE_ATTEMPT) # ATTEMPT to create a new table + # print("(" + fullTableName[-3:] + ")", end="", flush=True) + try: + sql = "CREATE TABLE {} USING {}.{} tags ({})".format( + fullTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName) + ) + dbc.execute(sql) + finally: + if task is not None: + task.unlockTable(fullTableName) # no matter what def _getTagStrForSql(self, dbc, dbName: str) : tags = self._getTags(dbc, dbName) @@ -2045,15 +1711,39 @@ class TaskReadData(StateTransitionTask): def canBeginFrom(cls, state: AnyState): return state.canReadData() + # def _canRestartService(self): + # if not gSvcMgr: + # return True # always + # return gSvcMgr.isActive() # only if it's running TODO: race condition here + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): sTable = self._db.getFixedSuperTable() - # 1 in 5 chance, simulate a broken connection. - if random.randrange(5) == 0: # TODO: break connection in all situations - wt.getDbConn().close() - wt.getDbConn().open() - print("_r", end="", flush=True) - + # 1 in 5 chance, simulate a broken connection, only if service stable (not restarting) + if random.randrange(20)==0: # and self._canRestartService(): # TODO: break connection in all situations + # Logging.info("Attempting to reconnect to server") # TODO: change to DEBUG + Progress.emit(Progress.SERVICE_RECONNECT_START) + try: + wt.getDbConn().close() + wt.getDbConn().open() + except ConnectionError as err: # may fail + if not gSvcMgr: + Logging.error("Failed to reconnect in client-only mode") + raise # Not OK if we are running in client-only mode + if gSvcMgr.isRunning(): # may have race conditon, but low prob, due to + Logging.error("Failed to reconnect when managed server is running") + raise # Not OK if we are running normally + + Progress.emit(Progress.SERVICE_RECONNECT_FAILURE) + # Logging.info("Ignoring DB reconnect error") + + # print("_r", end="", flush=True) + Progress.emit(Progress.SERVICE_RECONNECT_SUCCESS) + # The above might have taken a lot of time, service might be running + # by now, causing error below to be incorrectly handled due to timing issue + return # TODO: fix server restart status race condtion + + dbc = wt.getDbConn() dbName = self._db.getName() for rTbName in sTable.getRegTables(dbc, dbName): # regular tables @@ -2088,7 +1778,7 @@ class TaskReadData(StateTransitionTask): dbc.execute("select {} from {}.{}".format(aggExpr, dbName, sTable.getName())) except taos.error.ProgrammingError as err: errno2 = Helper.convertErrno(err.errno) - logger.debug("[=] Read Failure: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, dbc.getLastSql())) + Logging.debug("[=] Read Failure: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, dbc.getLastSql())) raise class TaskDropSuperTable(StateTransitionTask): @@ -2119,7 +1809,7 @@ class TaskDropSuperTable(StateTransitionTask): errno2 = Helper.convertErrno(err.errno) if (errno2 in [0x362]): # mnode invalid table name isSuccess = False - logger.debug("[DB] Acceptable error when dropping a table") + Logging.debug("[DB] Acceptable error when dropping a table") continue # try to delete next regular table if (not tickOutput): @@ -2184,7 +1874,7 @@ class TaskRestartService(StateTransitionTask): with self._classLock: if self._isRunning: - print("Skipping restart task, another running already") + Logging.info("Skipping restart task, another running already") return self._isRunning = True @@ -2199,20 +1889,19 @@ class TaskAddData(StateTransitionTask): # Track which table is being actively worked on activeTable: Set[int] = set() - # We use these two files to record operations to DB, useful for power-off - # tests - fAddLogReady = None - fAddLogDone = None + # We use these two files to record operations to DB, useful for power-off tests + fAddLogReady = None # type: TextIOWrapper + fAddLogDone = None # type: TextIOWrapper @classmethod def prepToRecordOps(cls): if gConfig.record_ops: if (cls.fAddLogReady is None): - logger.info( + Logging.info( "Recording in a file operations to be performed...") cls.fAddLogReady = open("add_log_ready.txt", "w") if (cls.fAddLogDone is None): - logger.info("Recording in a file operations completed...") + Logging.info("Recording in a file operations completed...") cls.fAddLogDone = open("add_log_done.txt", "w") @classmethod @@ -2223,13 +1912,88 @@ class TaskAddData(StateTransitionTask): def canBeginFrom(cls, state: AnyState): return state.canAddData() + def _addDataInBatch(self, db, dbc, regTableName, te: TaskExecutor): + numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS + fullTableName = db.getName() + '.' + regTableName + + sql = "insert into {} values ".format(fullTableName) + for j in range(numRecords): # number of records per table + nextInt = db.getNextInt() + nextTick = db.getNextTick() + sql += "('{}', {});".format(nextTick, nextInt) + dbc.execute(sql) + + def _addData(self, db, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches + numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS + + for j in range(numRecords): # number of records per table + nextInt = db.getNextInt() + nextTick = db.getNextTick() + if gConfig.record_ops: + self.prepToRecordOps() + self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) + self.fAddLogReady.flush() + os.fsync(self.fAddLogReady) + + # TODO: too ugly trying to lock the table reliably, refactor... + fullTableName = db.getName() + '.' + regTableName + if gConfig.verify_data: + self.lockTable(fullTableName) + # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written + + try: + sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {}) + fullTableName, + # ds.getFixedSuperTableName(), + # ds.getNextBinary(), ds.getNextFloat(), + nextTick, nextInt) + dbc.execute(sql) + except: # Any exception at all + if gConfig.verify_data: + self.unlockTable(fullTableName) + raise + + # Now read it back and verify, we might encounter an error if table is dropped + if gConfig.verify_data: # only if command line asks for it + try: + readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts='{}'". + format(db.getName(), regTableName, nextTick)) + if readBack != nextInt : + raise taos.error.ProgrammingError( + "Failed to read back same data, wrote: {}, read: {}" + .format(nextInt, readBack), 0x999) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x991, 0x992] : # not a single result + raise taos.error.ProgrammingError( + "Failed to read back same data for tick: {}, wrote: {}, read: {}" + .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"), + errno) + elif errno in [0x218, 0x362]: # table doesn't exist + # do nothing + dummy = 0 + else: + # Re-throw otherwise + raise + finally: + self.unlockTable(fullTableName) # Unlock the table no matter what + + # Successfully wrote the data into the DB, let's record it somehow + te.recordDataMark(nextInt) + + if gConfig.record_ops: + self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName)) + self.fAddLogDone.flush() + os.fsync(self.fAddLogDone) + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): # ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access db = self._db dbc = wt.getDbConn() - tblSeq = list(range( - self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES)) - random.shuffle(tblSeq) + numTables = self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES + numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS + tblSeq = list(range(numTables )) + random.shuffle(tblSeq) # now we have random sequence for i in tblSeq: if (i in self.activeTable): # wow already active print("x", end="", flush=True) # concurrent insertion @@ -2237,541 +2001,19 @@ class TaskAddData(StateTransitionTask): self.activeTable.add(i) # marking it active sTable = db.getFixedSuperTable() - regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) - sTable.ensureTable(wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists + regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) + fullTableName = db.getName() + '.' + regTableName + # self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked" + sTable.ensureTable(self, wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists + # self._unlockTable(fullTableName) - for j in range(self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS): # number of records per table - nextInt = db.getNextInt() - nextTick = db.getNextTick() - if gConfig.record_ops: - self.prepToRecordOps() - self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) - self.fAddLogReady.flush() - os.fsync(self.fAddLogReady) - sql = "insert into {}.{} values ('{}', {});".format( # removed: tags ('{}', {}) - db.getName(), - regTableName, - # ds.getFixedSuperTableName(), - # ds.getNextBinary(), ds.getNextFloat(), - nextTick, nextInt) - dbc.execute(sql) - # Successfully wrote the data into the DB, let's record it - # somehow - te.recordDataMark(nextInt) - if gConfig.record_ops: - self.fAddLogDone.write( - "Wrote {} to {}\n".format( - nextInt, regTableName)) - self.fAddLogDone.flush() - os.fsync(self.fAddLogDone) - - # Now read it back and verify, we might encounter an error if table is dropped - if gConfig.verify_data: # only if command line asks for it - try: - readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts= '{}'". - format(db.getName(), regTableName, nextTick)) - if readBack != nextInt : - raise taos.error.ProgrammingError( - "Failed to read back same data, wrote: {}, read: {}" - .format(nextInt, readBack), 0x999) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x991, 0x992] : # not a single result - raise taos.error.ProgrammingError( - "Failed to read back same data for tick: {}, wrote: {}, read: {}" - .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"), - errno) - # Re-throw no matter what - raise - - - self.activeTable.discard(i) # not raising an error, unlike remove - - -# Deterministic random number generator -class Dice(): - seeded = False # static, uninitialized - - @classmethod - def seed(cls, s): # static - if (cls.seeded): - raise RuntimeError( - "Cannot seed the random generator more than once") - cls.verifyRNG() - random.seed(s) - cls.seeded = True # TODO: protect against multi-threading - - @classmethod - def verifyRNG(cls): # Verify that the RNG is determinstic - random.seed(0) - x1 = random.randrange(0, 1000) - x2 = random.randrange(0, 1000) - x3 = random.randrange(0, 1000) - if (x1 != 864 or x2 != 394 or x3 != 776): - raise RuntimeError("System RNG is not deterministic") - - @classmethod - def throw(cls, stop): # get 0 to stop-1 - return cls.throwRange(0, stop) - - @classmethod - def throwRange(cls, start, stop): # up to stop-1 - if (not cls.seeded): - raise RuntimeError("Cannot throw dice before seeding it") - return random.randrange(start, stop) - - @classmethod - def choice(cls, cList): - return random.choice(cList) - - -class LoggingFilter(logging.Filter): - def filter(self, record: logging.LogRecord): - if (record.levelno >= logging.INFO): - return True # info or above always log - - # Commenting out below to adjust... - - # if msg.startswith("[TRD]"): - # return False - return True - - -class MyLoggingAdapter(logging.LoggerAdapter): - def process(self, msg, kwargs): - return "[{}]{}".format(threading.get_ident() % 10000, msg), kwargs - # return '[%s] %s' % (self.extra['connid'], msg), kwargs - - -class SvcManager: - def __init__(self): - print("Starting TDengine Service Manager") - # signal.signal(signal.SIGTERM, self.sigIntHandler) # Moved to MainExec - # signal.signal(signal.SIGINT, self.sigIntHandler) - # signal.signal(signal.SIGUSR1, self.sigUsrHandler) # different handler! - - self.inSigHandler = False - # self._status = MainExec.STATUS_RUNNING # set inside - # _startTaosService() - self.svcMgrThread = None # type: ServiceManagerThread - self._lock = threading.Lock() - self._isRestarting = False - - def _doMenu(self): - choice = "" - while True: - print("\nInterrupting Service Program, Choose an Action: ") - print("1: Resume") - print("2: Terminate") - print("3: Restart") - # Remember to update the if range below - # print("Enter Choice: ", end="", flush=True) - while choice == "": - choice = input("Enter Choice: ") - if choice != "": - break # done with reading repeated input - if choice in ["1", "2", "3"]: - break # we are done with whole method - print("Invalid choice, please try again.") - choice = "" # reset - return choice - - def sigUsrHandler(self, signalNumber, frame): - print("Interrupting main thread execution upon SIGUSR1") - if self.inSigHandler: # already - print("Ignoring repeated SIG...") - return # do nothing if it's already not running - self.inSigHandler = True - - choice = self._doMenu() - if choice == "1": - # TODO: can the sub-process be blocked due to us not reading from - # queue? - self.sigHandlerResume() - elif choice == "2": - self.stopTaosService() - elif choice == "3": # Restart - self.restart() - else: - raise RuntimeError("Invalid menu choice: {}".format(choice)) - - self.inSigHandler = False - - def sigIntHandler(self, signalNumber, frame): - print("SvcManager: INT Signal Handler starting...") - if self.inSigHandler: - print("Ignoring repeated SIG_INT...") - return - self.inSigHandler = True - - self.stopTaosService() - print("SvcManager: INT Signal Handler returning...") - self.inSigHandler = False - - def sigHandlerResume(self): - print("Resuming TDengine service manager thread (main thread)...\n\n") - - def _checkServiceManagerThread(self): - if self.svcMgrThread: # valid svc mgr thread - if self.svcMgrThread.isStopped(): # done? - self.svcMgrThread.procIpcBatch() # one last time. TODO: appropriate? - self.svcMgrThread = None # no more - - def _procIpcAll(self): - while self.isRunning() or self.isRestarting() : # for as long as the svc mgr thread is still here - if self.isRunning(): - self.svcMgrThread.procIpcBatch() # regular processing, - self._checkServiceManagerThread() - elif self.isRetarting(): - print("Service restarting...") - time.sleep(0.5) # pause, before next round - print( - "Service Manager Thread (with subprocess) has ended, main thread now exiting...") - - def startTaosService(self): - with self._lock: - if self.svcMgrThread: - raise RuntimeError("Cannot start TAOS service when one may already be running") - - # Find if there's already a taosd service, and then kill it - for proc in psutil.process_iter(): - if proc.name() == 'taosd': - print("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupe") - time.sleep(2.0) - proc.kill() - # print("Process: {}".format(proc.name())) - - - self.svcMgrThread = ServiceManagerThread() # create the object - print("Attempting to start TAOS service started, printing out output...") - self.svcMgrThread.start() - self.svcMgrThread.procIpcBatch(trimToTarget=10, forceOutput=True) # for printing 10 lines - print("TAOS service started") - - def stopTaosService(self, outputLines=20): - with self._lock: - if not self.isRunning(): - logger.warning("Cannot stop TAOS service, not running") - return - - print("Terminating Service Manager Thread (SMT) execution...") - self.svcMgrThread.stop() - if self.svcMgrThread.isStopped(): - self.svcMgrThread.procIpcBatch(outputLines) # one last time - self.svcMgrThread = None - print("End of TDengine Service Output") - print("----- TDengine Service (managed by SMT) is now terminated -----\n") + if Dice.throw(1) == 0: # 1 in 2 chance + self._addData(db, dbc, regTableName, te) else: - print("WARNING: SMT did not terminate as expected") - - def run(self): - self.startTaosService() - self._procIpcAll() # pump/process all the messages, may encounter SIG + restart - if self.isRunning(): # if sig handler hasn't destroyed it by now - self.stopTaosService() # should have started already - - def restart(self): - if self._isRestarting: - logger.warning("Cannot restart service when it's already restarting") - return - - self._isRestarting = True - if self.isRunning(): - self.stopTaosService() - else: - logger.warning("Service not running when restart requested") + self._addDataInBatch(db, dbc, regTableName, te) - self.startTaosService() - self._isRestarting = False - - def isRunning(self): - return self.svcMgrThread != None - - def isRestarting(self): - return self._isRestarting - -class ServiceManagerThread: - MAX_QUEUE_SIZE = 10000 - - def __init__(self): - self._tdeSubProcess = None # type: TdeSubProcess - self._thread = None - self._status = None - - def getStatus(self): - return self._status - - def isRunning(self): - # return self._thread and self._thread.is_alive() - return self._status == MainExec.STATUS_RUNNING - - def isStopping(self): - return self._status == MainExec.STATUS_STOPPING - - def isStopped(self): - return self._status == MainExec.STATUS_STOPPED - - # Start the thread (with sub process), and wait for the sub service - # to become fully operational - def start(self): - if self._thread: - raise RuntimeError("Unexpected _thread") - if self._tdeSubProcess: - raise RuntimeError("TDengine sub process already created/running") - - self._status = MainExec.STATUS_STARTING - - self._tdeSubProcess = TdeSubProcess() - self._tdeSubProcess.start() - - self._ipcQueue = Queue() - self._thread = threading.Thread( # First thread captures server OUTPUT - target=self.svcOutputReader, - args=(self._tdeSubProcess.getStdOut(), self._ipcQueue)) - self._thread.daemon = True # thread dies with the program - self._thread.start() - - self._thread2 = threading.Thread( # 2nd thread captures server ERRORs - target=self.svcErrorReader, - args=(self._tdeSubProcess.getStdErr(), self._ipcQueue)) - self._thread2.daemon = True # thread dies with the program - self._thread2.start() - - # wait for service to start - for i in range(0, 100): - time.sleep(1.0) - # self.procIpcBatch() # don't pump message during start up - print("_zz_", end="", flush=True) - if self._status == MainExec.STATUS_RUNNING: - logger.info("[] TDengine service READY to process requests") - return # now we've started - # TODO: handle this better? - self.procIpcBatch(100, True) # display output before cronking out, trim to last 20 msgs, force output - raise RuntimeError("TDengine service did not start successfully") - - def stop(self): - # can be called from both main thread or signal handler - print("Terminating TDengine service running as the sub process...") - if self.isStopped(): - print("Service already stopped") - return - if self.isStopping(): - print("Service is already being stopped") - return - # Linux will send Control-C generated SIGINT to the TDengine process - # already, ref: - # https://unix.stackexchange.com/questions/176235/fork-and-how-signals-are-delivered-to-processes - if not self._tdeSubProcess: - raise RuntimeError("sub process object missing") - - self._status = MainExec.STATUS_STOPPING - retCode = self._tdeSubProcess.stop() - print("Attempted to stop sub process, got return code: {}".format(retCode)) - if (retCode==-11): # SGV - logger.error("[[--ERROR--]]: TDengine service SEGV fault (check core file!)") - - if self._tdeSubProcess.isRunning(): # still running - print("FAILED to stop sub process, it is still running... pid = {}".format( - self._tdeSubProcess.getPid())) - else: - self._tdeSubProcess = None # not running any more - self.join() # stop the thread, change the status, etc. - - def join(self): - # TODO: sanity check - if not self.isStopping(): - raise RuntimeError( - "Unexpected status when ending svc mgr thread: {}".format( - self._status)) - - if self._thread: - self._thread.join() - self._thread = None - self._status = MainExec.STATUS_STOPPED - # STD ERR thread - self._thread2.join() - self._thread2 = None - else: - print("Joining empty thread, doing nothing") - - def _trimQueue(self, targetSize): - if targetSize <= 0: - return # do nothing - q = self._ipcQueue - if (q.qsize() <= targetSize): # no need to trim - return - - logger.debug("Triming IPC queue to target size: {}".format(targetSize)) - itemsToTrim = q.qsize() - targetSize - for i in range(0, itemsToTrim): - try: - q.get_nowait() - except Empty: - break # break out of for loop, no more trimming - - TD_READY_MSG = "TDengine is initialized successfully" - - def procIpcBatch(self, trimToTarget=0, forceOutput=False): - self._trimQueue(trimToTarget) # trim if necessary - # Process all the output generated by the underlying sub process, - # managed by IO thread - print("<", end="", flush=True) - while True: - try: - line = self._ipcQueue.get_nowait() # getting output at fast speed - self._printProgress("_o") - except Empty: - # time.sleep(2.3) # wait only if there's no output - # no more output - print(".>", end="", flush=True) - return # we are done with THIS BATCH - else: # got line, printing out - if forceOutput: - logger.info(line) - else: - logger.debug(line) - print(">", end="", flush=True) - - _ProgressBars = ["--", "//", "||", "\\\\"] - - def _printProgress(self, msg): # TODO: assuming 2 chars - print(msg, end="", flush=True) - pBar = self._ProgressBars[Dice.throw(4)] - print(pBar, end="", flush=True) - print('\b\b\b\b', end="", flush=True) - - def svcOutputReader(self, out: IO, queue): - # Important Reference: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python - # print("This is the svcOutput Reader...") - # for line in out : - for line in iter(out.readline, b''): - # print("Finished reading a line: {}".format(line)) - # print("Adding item to queue...") - try: - line = line.decode("utf-8").rstrip() - except UnicodeError: - print("\nNon-UTF8 server output: {}\n".format(line)) - - # This might block, and then causing "out" buffer to block - queue.put(line) - self._printProgress("_i") - - if self._status == MainExec.STATUS_STARTING: # we are starting, let's see if we have started - if line.find(self.TD_READY_MSG) != -1: # found - logger.info("Waiting for the service to become FULLY READY") - time.sleep(1.0) # wait for the server to truly start. TODO: remove this - logger.info("Service is now FULLY READY") - self._status = MainExec.STATUS_RUNNING - - # Trim the queue if necessary: TODO: try this 1 out of 10 times - self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10) # trim to 90% size - - if self.isStopping(): # TODO: use thread status instead - # WAITING for stopping sub process to finish its outptu - print("_w", end="", flush=True) - - # queue.put(line) - # meaning sub process must have died - print("\nNo more output from IO thread managing TDengine service") - out.close() - - def svcErrorReader(self, err: IO, queue): - for line in iter(err.readline, b''): - print("\nTDengine Service (taosd) ERROR (from stderr): {}".format(line)) - - -class TdeSubProcess: - def __init__(self): - self.subProcess = None - - def getStdOut(self): - return self.subProcess.stdout - - def getStdErr(self): - return self.subProcess.stderr - - def isRunning(self): - return self.subProcess is not None - - def getPid(self): - return self.subProcess.pid - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("communit")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root) - len("/build/bin")] - break - return buildPath + self.activeTable.discard(i) # not raising an error, unlike remove - def start(self): - ON_POSIX = 'posix' in sys.builtin_module_names - - taosdPath = self.getBuildPath() + "/build/bin/taosd" - cfgPath = self.getBuildPath() + "/test/cfg" - - # Delete the log files - logPath = self.getBuildPath() + "/test/log" - # ref: https://stackoverflow.com/questions/1995373/deleting-all-files-in-a-directory-with-python/1995397 - # filelist = [ f for f in os.listdir(logPath) ] # if f.endswith(".bak") ] - # for f in filelist: - # filePath = os.path.join(logPath, f) - # print("Removing log file: {}".format(filePath)) - # os.remove(filePath) - if os.path.exists(logPath): - logPathSaved = logPath + "_" + time.strftime('%Y-%m-%d-%H-%M-%S') - logger.info("Saving old log files to: {}".format(logPathSaved)) - os.rename(logPath, logPathSaved) - # os.mkdir(logPath) # recreate, no need actually, TDengine will auto-create with proper perms - - svcCmd = [taosdPath, '-c', cfgPath] - # svcCmdSingle = "{} -c {}".format(taosdPath, cfgPath) - # svcCmd = ['vmstat', '1'] - if self.subProcess: # already there - raise RuntimeError("Corrupt process state") - - # print("Starting service: {}".format(svcCmd)) - self.subProcess = subprocess.Popen( - svcCmd, shell=False, - # svcCmdSingle, shell=True, # capture core dump? - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - # bufsize=1, # not supported in binary mode - close_fds=ON_POSIX - ) # had text=True, which interferred with reading EOF - - def stop(self): - if not self.subProcess: - print("Sub process already stopped") - return -1 - - retCode = self.subProcess.poll() # contains real sub process return code - if retCode: # valid return code, process ended - self.subProcess = None - else: # process still alive, let's interrupt it - print( - "Sub process is running, sending SIG_INT and waiting for it to terminate...") - # sub process should end, then IPC queue should end, causing IO - # thread to end - self.subProcess.send_signal(signal.SIGINT) - try: - self.subProcess.wait(10) - retCode = self.subProcess.returncode - except subprocess.TimeoutExpired as err: - print("Time out waiting for TDengine service process to exit") - retCode = -3 - else: - print("TDengine service process terminated successfully from SIG_INT") - retCode = -4 - self.subProcess = None - return retCode class ThreadStacks: # stack info for all threads def __init__(self): @@ -2794,31 +2036,33 @@ class ThreadStacks: # stack info for all threads '__init__']: # the thread that extracted the stack continue # ignore # Now print - print("\n<----- Thread Info for ID: {}".format(thNid)) + print("\n<----- Thread Info for LWP/ID: {} (Execution stopped at Bottom Frame) <-----".format(thNid)) + stackFrame = 0 for frame in stack: # print(frame) - print("File {filename}, line {lineno}, in {name}".format( - filename=frame.filename, lineno=frame.lineno, name=frame.name)) + print("[{sf}] File {filename}, line {lineno}, in {name}".format( + sf=stackFrame, filename=frame.filename, lineno=frame.lineno, name=frame.name)) print(" {}".format(frame.line)) - print("-----> End of Thread Info\n") + stackFrame += 1 + print("-----> End of Thread Info ----->\n") class ClientManager: def __init__(self): - print("Starting service manager") + Logging.info("Starting service manager") # signal.signal(signal.SIGTERM, self.sigIntHandler) # signal.signal(signal.SIGINT, self.sigIntHandler) - self._status = MainExec.STATUS_RUNNING + self._status = Status.STATUS_RUNNING self.tc = None self.inSigHandler = False def sigIntHandler(self, signalNumber, frame): - if self._status != MainExec.STATUS_RUNNING: + if self._status != Status.STATUS_RUNNING: print("Repeated SIGINT received, forced exit...") # return # do nothing if it's already not running sys.exit(-1) - self._status = MainExec.STATUS_STOPPING # immediately set our status + self._status = Status.STATUS_STOPPING # immediately set our status print("ClientManager: Terminating program...") self.tc.requestToStop() @@ -2898,15 +2142,20 @@ class ClientManager: # self._printLastNumbers() global gConfig - dbManager = DbManager() # Regular function + # Prepare Tde Instance + global gContainer + tInst = gContainer.defTdeInstance = TdeInstance() # "subdir to hold the instance" + + dbManager = DbManager(gConfig.connector_type, tInst.getDbTarget()) # Regular function thPool = ThreadPool(gConfig.num_threads, gConfig.max_steps) self.tc = ThreadCoordinator(thPool, dbManager) + Logging.info("Starting client instance: {}".format(tInst)) self.tc.run() # print("exec stats: {}".format(self.tc.getExecStats())) # print("TC failed = {}".format(self.tc.isFailed())) if svcMgr: # gConfig.auto_start_service: - svcMgr.stopTaosService() + svcMgr.stopTaosServices() svcMgr = None # Print exec status, etc., AFTER showing messages from the server self.conclude() @@ -2936,18 +2185,10 @@ class ClientManager: # self.tc.getDbManager().cleanUp() # clean up first, so we can show ZERO db connections self.tc.printStats() - - - class MainExec: - STATUS_STARTING = 1 - STATUS_RUNNING = 2 - STATUS_STOPPING = 3 - STATUS_STOPPED = 4 - def __init__(self): self._clientMgr = None - self._svcMgr = None + self._svcMgr = None # type: ServiceManager signal.signal(signal.SIGTERM, self.sigIntHandler) signal.signal(signal.SIGINT, self.sigIntHandler) @@ -2960,219 +2201,185 @@ class MainExec: self._svcMgr.sigUsrHandler(signalNumber, frame) def sigIntHandler(self, signalNumber, frame): - if self._svcMgr: + if self._svcMgr: self._svcMgr.sigIntHandler(signalNumber, frame) - if self._clientMgr: + if self._clientMgr: self._clientMgr.sigIntHandler(signalNumber, frame) def runClient(self): global gSvcMgr if gConfig.auto_start_service: - self._svcMgr = SvcManager() - gSvcMgr = self._svcMgr # hack alert - self._svcMgr.startTaosService() # we start, don't run + gSvcMgr = self._svcMgr = ServiceManager(1) # hack alert + gSvcMgr.startTaosServices() # we start, don't run self._clientMgr = ClientManager() ret = None try: ret = self._clientMgr.run(self._svcMgr) # stop TAOS service inside except requests.exceptions.ConnectionError as err: - logger.warning("Failed to open REST connection to DB: {}".format(err.getMessage())) + Logging.warning("Failed to open REST connection to DB: {}".format(err.getMessage())) # don't raise return ret def runService(self): global gSvcMgr - self._svcMgr = SvcManager() - gSvcMgr = self._svcMgr # save it in a global variable TODO: hack alert - - self._svcMgr.run() # run to some end state - self._svcMgr = None - gSvcMgr = None - - def runTemp(self): # for debugging purposes - # # Hack to exercise reading from disk, imcreasing coverage. TODO: fix - # dbc = dbState.getDbConn() - # sTbName = dbState.getFixedSuperTableName() - # dbc.execute("create database if not exists db") - # if not dbState.getState().equals(StateEmpty()): - # dbc.execute("use db") - - # rTables = None - # try: # the super table may not exist - # sql = "select TBNAME from db.{}".format(sTbName) - # logger.info("Finding out tables in super table: {}".format(sql)) - # dbc.query(sql) # TODO: analyze result set later - # logger.info("Fetching result") - # rTables = dbc.getQueryResult() - # logger.info("Result: {}".format(rTables)) - # except taos.error.ProgrammingError as err: - # logger.info("Initial Super table OPS error: {}".format(err)) - - # # sys.exit() - # if ( not rTables == None): - # # print("rTables[0] = {}, type = {}".format(rTables[0], type(rTables[0]))) - # try: - # for rTbName in rTables : # regular tables - # ds = dbState - # logger.info("Inserting into table: {}".format(rTbName[0])) - # sql = "insert into db.{} values ('{}', {});".format( - # rTbName[0], - # ds.getNextTick(), ds.getNextInt()) - # dbc.execute(sql) - # for rTbName in rTables : # regular tables - # dbc.query("select * from db.{}".format(rTbName[0])) # TODO: check success failure - # logger.info("Initial READING operation is successful") - # except taos.error.ProgrammingError as err: - # logger.info("Initial WRITE/READ error: {}".format(err)) - - # Sandbox testing code - # dbc = dbState.getDbConn() - # while True: - # rows = dbc.query("show databases") - # print("Rows: {}, time={}".format(rows, time.time())) - return - - -def main(): - # Super cool Python argument library: - # https://docs.python.org/3/library/argparse.html - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent('''\ - TDengine Auto Crash Generator (PLEASE NOTICE the Prerequisites Below) - --------------------------------------------------------------------- - 1. You build TDengine in the top level ./build directory, as described in offical docs - 2. You run the server there before this script: ./build/bin/taosd -c test/cfg - - ''')) - - # parser.add_argument('-a', '--auto-start-service', action='store_true', - # help='Automatically start/stop the TDengine service (default: false)') - # parser.add_argument('-c', '--connector-type', action='store', default='native', type=str, - # help='Connector type to use: native, rest, or mixed (default: 10)') - # parser.add_argument('-d', '--debug', action='store_true', - # help='Turn on DEBUG mode for more logging (default: false)') - # parser.add_argument('-e', '--run-tdengine', action='store_true', - # help='Run TDengine service in foreground (default: false)') - # parser.add_argument('-l', '--larger-data', action='store_true', - # help='Write larger amount of data during write operations (default: false)') - # parser.add_argument('-p', '--per-thread-db-connection', action='store_true', - # help='Use a single shared db connection (default: false)') - # parser.add_argument('-r', '--record-ops', action='store_true', - # help='Use a pair of always-fsynced fils to record operations performing + performed, for power-off tests (default: false)') - # parser.add_argument('-s', '--max-steps', action='store', default=1000, type=int, - # help='Maximum number of steps to run (default: 100)') - # parser.add_argument('-t', '--num-threads', action='store', default=5, type=int, - # help='Number of threads to run (default: 10)') - # parser.add_argument('-x', '--continue-on-exception', action='store_true', - # help='Continue execution after encountering unexpected/disallowed errors/exceptions (default: false)') - - parser.add_argument( - '-a', - '--auto-start-service', - action='store_true', - help='Automatically start/stop the TDengine service (default: false)') - parser.add_argument( - '-b', - '--max-dbs', - action='store', - default=0, - type=int, - help='Maximum number of DBs to keep, set to disable dropping DB. (default: 0)') - parser.add_argument( - '-c', - '--connector-type', - action='store', - default='native', - type=str, - help='Connector type to use: native, rest, or mixed (default: 10)') - parser.add_argument( - '-d', - '--debug', - action='store_true', - help='Turn on DEBUG mode for more logging (default: false)') - parser.add_argument( - '-e', - '--run-tdengine', - action='store_true', - help='Run TDengine service in foreground (default: false)') - parser.add_argument( - '-i', - '--max-replicas', - action='store', - default=1, - type=int, - help='Maximum number of replicas to use, when testing against clusters. (default: 1)') - parser.add_argument( - '-l', - '--larger-data', - action='store_true', - help='Write larger amount of data during write operations (default: false)') - parser.add_argument( - '-p', - '--per-thread-db-connection', - action='store_true', - help='Use a single shared db connection (default: false)') - parser.add_argument( - '-r', - '--record-ops', - action='store_true', - help='Use a pair of always-fsynced fils to record operations performing + performed, for power-off tests (default: false)') - parser.add_argument( - '-s', - '--max-steps', - action='store', - default=1000, - type=int, - help='Maximum number of steps to run (default: 100)') - parser.add_argument( - '-t', - '--num-threads', - action='store', - default=5, - type=int, - help='Number of threads to run (default: 10)') - parser.add_argument( - '-v', - '--verify-data', - action='store_true', - help='Verify data written in a number of places by reading back (default: false)') - parser.add_argument( - '-x', - '--continue-on-exception', - action='store_true', - help='Continue execution after encountering unexpected/disallowed errors/exceptions (default: false)') - - global gConfig - gConfig = parser.parse_args() - - # Logging Stuff - global logger - _logger = logging.getLogger('CrashGen') # real logger - _logger.addFilter(LoggingFilter()) - ch = logging.StreamHandler() - _logger.addHandler(ch) - - # Logging adapter, to be used as a logger - logger = MyLoggingAdapter(_logger, []) - - if (gConfig.debug): - logger.setLevel(logging.DEBUG) # default seems to be INFO - else: - logger.setLevel(logging.INFO) - - Dice.seed(0) # initial seeding of dice - - # Run server or client - mExec = MainExec() - if gConfig.run_tdengine: # run server - mExec.runService() - else: - return mExec.runClient() - - -if __name__ == "__main__": - exitCode = main() - # print("Exiting with code: {}".format(exitCode)) - sys.exit(exitCode) + gSvcMgr = self._svcMgr = ServiceManager(gConfig.num_dnodes) # save it in a global variable TODO: hack alert + + gSvcMgr.run() # run to some end state + gSvcMgr = self._svcMgr = None + + def init(self): # TODO: refactor + global gContainer + gContainer = Container() # micky-mouse DI + + global gSvcMgr # TODO: refactor away + gSvcMgr = None + + # Super cool Python argument library: + # https://docs.python.org/3/library/argparse.html + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=textwrap.dedent('''\ + TDengine Auto Crash Generator (PLEASE NOTICE the Prerequisites Below) + --------------------------------------------------------------------- + 1. You build TDengine in the top level ./build directory, as described in offical docs + 2. You run the server there before this script: ./build/bin/taosd -c test/cfg + + ''')) + + parser.add_argument( + '-a', + '--auto-start-service', + action='store_true', + help='Automatically start/stop the TDengine service (default: false)') + parser.add_argument( + '-b', + '--max-dbs', + action='store', + default=0, + type=int, + help='Maximum number of DBs to keep, set to disable dropping DB. (default: 0)') + parser.add_argument( + '-c', + '--connector-type', + action='store', + default='native', + type=str, + help='Connector type to use: native, rest, or mixed (default: 10)') + parser.add_argument( + '-d', + '--debug', + action='store_true', + help='Turn on DEBUG mode for more logging (default: false)') + parser.add_argument( + '-e', + '--run-tdengine', + action='store_true', + help='Run TDengine service in foreground (default: false)') + parser.add_argument( + '-g', + '--ignore-errors', + action='store', + default=None, + type=str, + help='Ignore error codes, comma separated, 0x supported (default: None)') + parser.add_argument( + '-i', + '--max-replicas', + action='store', + default=1, + type=int, + help='Maximum number of replicas to use, when testing against clusters. (default: 1)') + parser.add_argument( + '-l', + '--larger-data', + action='store_true', + help='Write larger amount of data during write operations (default: false)') + parser.add_argument( + '-n', + '--dynamic-db-table-names', + action='store_true', + help='Use non-fixed names for dbs/tables, useful for multi-instance executions (default: false)') + parser.add_argument( + '-o', + '--num-dnodes', + action='store', + default=1, + type=int, + help='Number of Dnodes to initialize, used with -e option. (default: 1)') + parser.add_argument( + '-p', + '--per-thread-db-connection', + action='store_true', + help='Use a single shared db connection (default: false)') + parser.add_argument( + '-r', + '--record-ops', + action='store_true', + help='Use a pair of always-fsynced fils to record operations performing + performed, for power-off tests (default: false)') + parser.add_argument( + '-s', + '--max-steps', + action='store', + default=1000, + type=int, + help='Maximum number of steps to run (default: 100)') + parser.add_argument( + '-t', + '--num-threads', + action='store', + default=5, + type=int, + help='Number of threads to run (default: 10)') + parser.add_argument( + '-v', + '--verify-data', + action='store_true', + help='Verify data written in a number of places by reading back (default: false)') + parser.add_argument( + '-x', + '--continue-on-exception', + action='store_true', + help='Continue execution after encountering unexpected/disallowed errors/exceptions (default: false)') + + global gConfig + gConfig = parser.parse_args() + + Logging.clsInit(gConfig) + + Dice.seed(0) # initial seeding of dice + + def run(self): + if gConfig.run_tdengine: # run server + try: + self.runService() + return 0 # success + except ConnectionError as err: + Logging.error("Failed to make DB connection, please check DB instance manually") + return -1 # failure + else: + return self.runClient() + + +class Container(): + _propertyList = {'defTdeInstance'} + + def __init__(self): + self._cargo = {} # No cargo at the beginning + + def _verifyValidProperty(self, name): + if not name in self._propertyList: + raise CrashGenError("Invalid container property: {}".format(name)) + + # Called for an attribute, when other mechanisms fail (compare to __getattribute__) + def __getattr__(self, name): + self._verifyValidProperty(name) + return self._cargo[name] # just a simple lookup + + def __setattr__(self, name, value): + if name == '_cargo' : # reserved vars + super().__setattr__(name, value) + return + self._verifyValidProperty(name) + self._cargo[name] = value + diff --git a/tests/pytest/crash_gen/db.py b/tests/pytest/crash_gen/db.py new file mode 100644 index 0000000000000000000000000000000000000000..2a4b362f82c4516195becf78ef9771b7c62c3c41 --- /dev/null +++ b/tests/pytest/crash_gen/db.py @@ -0,0 +1,441 @@ +from __future__ import annotations + +import sys +import time +import threading +import requests +from requests.auth import HTTPBasicAuth + +import taos +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.log import * + +from .misc import Logging, CrashGenError, Helper, Dice +import os +import datetime +# from .service_manager import TdeInstance + +class DbConn: + TYPE_NATIVE = "native-c" + TYPE_REST = "rest-api" + TYPE_INVALID = "invalid" + + @classmethod + def create(cls, connType, dbTarget): + if connType == cls.TYPE_NATIVE: + return DbConnNative(dbTarget) + elif connType == cls.TYPE_REST: + return DbConnRest(dbTarget) + else: + raise RuntimeError( + "Unexpected connection type: {}".format(connType)) + + @classmethod + def createNative(cls, dbTarget) -> DbConn: + return cls.create(cls.TYPE_NATIVE, dbTarget) + + @classmethod + def createRest(cls, dbTarget) -> DbConn: + return cls.create(cls.TYPE_REST, dbTarget) + + def __init__(self, dbTarget): + self.isOpen = False + self._type = self.TYPE_INVALID + self._lastSql = None + self._dbTarget = dbTarget + + def __repr__(self): + return "[DbConn: type={}, target={}]".format(self._type, self._dbTarget) + + def getLastSql(self): + return self._lastSql + + def open(self): + if (self.isOpen): + raise RuntimeError("Cannot re-open an existing DB connection") + + # below implemented by child classes + self.openByType() + + Logging.debug("[DB] data connection opened: {}".format(self)) + self.isOpen = True + + def close(self): + raise RuntimeError("Unexpected execution, should be overriden") + + def queryScalar(self, sql) -> int: + return self._queryAny(sql) + + def queryString(self, sql) -> str: + return self._queryAny(sql) + + def _queryAny(self, sql): # actual query result as an int + if (not self.isOpen): + raise RuntimeError("Cannot query database until connection is open") + nRows = self.query(sql) + if nRows != 1: + raise taos.error.ProgrammingError( + "Unexpected result for query: {}, rows = {}".format(sql, nRows), + (0x991 if nRows==0 else 0x992) + ) + if self.getResultRows() != 1 or self.getResultCols() != 1: + raise RuntimeError("Unexpected result set for query: {}".format(sql)) + return self.getQueryResult()[0][0] + + def use(self, dbName): + self.execute("use {}".format(dbName)) + + def existsDatabase(self, dbName: str): + ''' Check if a certain database exists ''' + self.query("show databases") + dbs = [v[0] for v in self.getQueryResult()] # ref: https://stackoverflow.com/questions/643823/python-list-transformation + # ret2 = dbName in dbs + # print("dbs = {}, str = {}, ret2={}, type2={}".format(dbs, dbName,ret2, type(dbName))) + return dbName in dbs # TODO: super weird type mangling seen, once here + + def existsSuperTable(self, stName): + self.query("show stables") + sts = [v[0] for v in self.getQueryResult()] + return stName in sts + + def hasTables(self): + return self.query("show tables") > 0 + + def execute(self, sql): + ''' Return the number of rows affected''' + raise RuntimeError("Unexpected execution, should be overriden") + + def safeExecute(self, sql): + '''Safely execute any SQL query, returning True/False upon success/failure''' + try: + self.execute(sql) + return True # ignore num of results, return success + except taos.error.ProgrammingError as err: + return False # failed, for whatever TAOS reason + # Not possile to reach here, non-TAOS exception would have been thrown + + def query(self, sql) -> int: # return num rows returned + ''' Return the number of rows affected''' + raise RuntimeError("Unexpected execution, should be overriden") + + def openByType(self): + raise RuntimeError("Unexpected execution, should be overriden") + + def getQueryResult(self): + raise RuntimeError("Unexpected execution, should be overriden") + + def getResultRows(self): + raise RuntimeError("Unexpected execution, should be overriden") + + def getResultCols(self): + raise RuntimeError("Unexpected execution, should be overriden") + +# Sample: curl -u root:taosdata -d "show databases" localhost:6020/rest/sql + + +class DbConnRest(DbConn): + REST_PORT_INCREMENT = 11 + + def __init__(self, dbTarget: DbTarget): + super().__init__(dbTarget) + self._type = self.TYPE_REST + restPort = dbTarget.port + 11 + self._url = "http://{}:{}/rest/sql".format( + dbTarget.hostAddr, dbTarget.port + self.REST_PORT_INCREMENT) + self._result = None + + def openByType(self): # Open connection + pass # do nothing, always open + + def close(self): + if (not self.isOpen): + raise RuntimeError("Cannot clean up database until connection is open") + # Do nothing for REST + Logging.debug("[DB] REST Database connection closed") + self.isOpen = False + + def _doSql(self, sql): + self._lastSql = sql # remember this, last SQL attempted + try: + r = requests.post(self._url, + data = sql, + auth = HTTPBasicAuth('root', 'taosdata')) + except: + print("REST API Failure (TODO: more info here)") + raise + rj = r.json() + # Sanity check for the "Json Result" + if ('status' not in rj): + raise RuntimeError("No status in REST response") + + if rj['status'] == 'error': # clearly reported error + if ('code' not in rj): # error without code + raise RuntimeError("REST error return without code") + errno = rj['code'] # May need to massage this in the future + # print("Raising programming error with REST return: {}".format(rj)) + raise taos.error.ProgrammingError( + rj['desc'], errno) # todo: check existance of 'desc' + + if rj['status'] != 'succ': # better be this + raise RuntimeError( + "Unexpected REST return status: {}".format( + rj['status'])) + + nRows = rj['rows'] if ('rows' in rj) else 0 + self._result = rj + return nRows + + def execute(self, sql): + if (not self.isOpen): + raise RuntimeError( + "Cannot execute database commands until connection is open") + Logging.debug("[SQL-REST] Executing SQL: {}".format(sql)) + nRows = self._doSql(sql) + Logging.debug( + "[SQL-REST] Execution Result, nRows = {}, SQL = {}".format(nRows, sql)) + return nRows + + def query(self, sql): # return rows affected + return self.execute(sql) + + def getQueryResult(self): + return self._result['data'] + + def getResultRows(self): + print(self._result) + raise RuntimeError("TBD") # TODO: finish here to support -v under -c rest + # return self._tdSql.queryRows + + def getResultCols(self): + print(self._result) + raise RuntimeError("TBD") + + # Duplicate code from TDMySQL, TODO: merge all this into DbConnNative + + +class MyTDSql: + # Class variables + _clsLock = threading.Lock() # class wide locking + longestQuery = None # type: str + longestQueryTime = 0.0 # seconds + lqStartTime = 0.0 + # lqEndTime = 0.0 # Not needed, as we have the two above already + + def __init__(self, hostAddr, cfgPath): + # Make the DB connection + self._conn = taos.connect(host=hostAddr, config=cfgPath) + self._cursor = self._conn.cursor() + + self.queryRows = 0 + self.queryCols = 0 + self.affectedRows = 0 + + # def init(self, cursor, log=True): + # self.cursor = cursor + # if (log): + # caller = inspect.getframeinfo(inspect.stack()[1][0]) + # self.cursor.log(caller.filename + ".sql") + + def close(self): + self._cursor.close() # can we double close? + self._conn.close() # TODO: very important, cursor close does NOT close DB connection! + self._cursor.close() + + def _execInternal(self, sql): + startTime = time.time() + # Logging.debug("Executing SQL: " + sql) + ret = self._cursor.execute(sql) + # print("\nSQL success: {}".format(sql)) + queryTime = time.time() - startTime + # Record the query time + cls = self.__class__ + if queryTime > (cls.longestQueryTime + 0.01) : + with cls._clsLock: + cls.longestQuery = sql + cls.longestQueryTime = queryTime + cls.lqStartTime = startTime + return ret + + def query(self, sql): + self.sql = sql + try: + self._execInternal(sql) + self.queryResult = self._cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(self._cursor.description) + except Exception as e: + # caller = inspect.getframeinfo(inspect.stack()[1][0]) + # args = (caller.filename, caller.lineno, sql, repr(e)) + # tdLog.exit("%s(%d) failed: sql:%s, %s" % args) + raise + return self.queryRows + + def execute(self, sql): + self.sql = sql + try: + self.affectedRows = self._execInternal(sql) + except Exception as e: + # caller = inspect.getframeinfo(inspect.stack()[1][0]) + # args = (caller.filename, caller.lineno, sql, repr(e)) + # tdLog.exit("%s(%d) failed: sql:%s, %s" % args) + raise + return self.affectedRows + +class DbTarget: + def __init__(self, cfgPath, hostAddr, port): + self.cfgPath = cfgPath + self.hostAddr = hostAddr + self.port = port + + def __repr__(self): + return "[DbTarget: cfgPath={}, host={}:{}]".format( + Helper.getFriendlyPath(self.cfgPath), self.hostAddr, self.port) + + def getEp(self): + return "{}:{}".format(self.hostAddr, self.port) + +class DbConnNative(DbConn): + # Class variables + _lock = threading.Lock() + # _connInfoDisplayed = False # TODO: find another way to display this + totalConnections = 0 # Not private + + def __init__(self, dbTarget): + super().__init__(dbTarget) + self._type = self.TYPE_NATIVE + self._conn = None + # self._cursor = None + + def openByType(self): # Open connection + # global gContainer + # tInst = tInst or gContainer.defTdeInstance # set up in ClientManager, type: TdeInstance + # cfgPath = self.getBuildPath() + "/test/cfg" + # cfgPath = tInst.getCfgDir() + # hostAddr = tInst.getHostAddr() + + cls = self.__class__ # Get the class, to access class variables + with cls._lock: # force single threading for opening DB connections. # TODO: whaaat??!!! + dbTarget = self._dbTarget + # if not cls._connInfoDisplayed: + # cls._connInfoDisplayed = True # updating CLASS variable + Logging.debug("Initiating TAOS native connection to {}".format(dbTarget)) + # Make the connection + # self._conn = taos.connect(host=hostAddr, config=cfgPath) # TODO: make configurable + # self._cursor = self._conn.cursor() + # Record the count in the class + self._tdSql = MyTDSql(dbTarget.hostAddr, dbTarget.cfgPath) # making DB connection + cls.totalConnections += 1 + + self._tdSql.execute('reset query cache') + # self._cursor.execute('use db') # do this at the beginning of every + + # Open connection + # self._tdSql = MyTDSql() + # self._tdSql.init(self._cursor) + + def close(self): + if (not self.isOpen): + raise RuntimeError("Cannot clean up database until connection is open") + self._tdSql.close() + # Decrement the class wide counter + cls = self.__class__ # Get the class, to access class variables + with cls._lock: + cls.totalConnections -= 1 + + Logging.debug("[DB] Database connection closed") + self.isOpen = False + + def execute(self, sql): + if (not self.isOpen): + raise RuntimeError("Cannot execute database commands until connection is open") + Logging.debug("[SQL] Executing SQL: {}".format(sql)) + self._lastSql = sql + nRows = self._tdSql.execute(sql) + Logging.debug( + "[SQL] Execution Result, nRows = {}, SQL = {}".format( + nRows, sql)) + return nRows + + def query(self, sql): # return rows affected + if (not self.isOpen): + raise RuntimeError( + "Cannot query database until connection is open") + Logging.debug("[SQL] Executing SQL: {}".format(sql)) + self._lastSql = sql + nRows = self._tdSql.query(sql) + Logging.debug( + "[SQL] Query Result, nRows = {}, SQL = {}".format( + nRows, sql)) + return nRows + # results are in: return self._tdSql.queryResult + + def getQueryResult(self): + return self._tdSql.queryResult + + def getResultRows(self): + return self._tdSql.queryRows + + def getResultCols(self): + return self._tdSql.queryCols + + +class DbManager(): + ''' This is a wrapper around DbConn(), to make it easier to use. + + TODO: rename this to DbConnManager + ''' + def __init__(self, cType, dbTarget): + # self.tableNumQueue = LinearQueue() # TODO: delete? + # self.openDbServerConnection() + self._dbConn = DbConn.createNative(dbTarget) if ( + cType == 'native') else DbConn.createRest(dbTarget) + try: + self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected + except taos.error.ProgrammingError as err: + # print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err)) + if (err.msg == 'client disconnected'): # cannot open DB connection + print( + "Cannot establish DB connection, please re-run script without parameter, and follow the instructions.") + sys.exit(2) + else: + print("Failed to connect to DB, errno = {}, msg: {}" + .format(Helper.convertErrno(err.errno), err.msg)) + raise + except BaseException: + print("[=] Unexpected exception") + raise + + # Do this after dbConn is in proper shape + # Moved to Database() + # self._stateMachine = StateMechine(self._dbConn) + + def getDbConn(self): + return self._dbConn + + # TODO: not used any more, to delete + def pickAndAllocateTable(self): # pick any table, and "use" it + return self.tableNumQueue.pickAndAllocate() + + # TODO: Not used any more, to delete + def addTable(self): + with self._lock: + tIndex = self.tableNumQueue.push() + return tIndex + + # Not used any more, to delete + def releaseTable(self, i): # return the table back, so others can use it + self.tableNumQueue.release(i) + + # TODO: not used any more, delete + def getTableNameToDelete(self): + tblNum = self.tableNumQueue.pop() # TODO: race condition! + if (not tblNum): # maybe false + return False + + return "table_{}".format(tblNum) + + def cleanUp(self): + self._dbConn.close() + diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..2d2ce99d95582809a8940a8d777bc166b633747c --- /dev/null +++ b/tests/pytest/crash_gen/misc.py @@ -0,0 +1,186 @@ +import threading +import random +import logging +import os + + +class CrashGenError(Exception): + def __init__(self, msg=None, errno=None): + self.msg = msg + self.errno = errno + + def __str__(self): + return self.msg + + +class LoggingFilter(logging.Filter): + def filter(self, record: logging.LogRecord): + if (record.levelno >= logging.INFO): + return True # info or above always log + + # Commenting out below to adjust... + + # if msg.startswith("[TRD]"): + # return False + return True + + +class MyLoggingAdapter(logging.LoggerAdapter): + def process(self, msg, kwargs): + return "[{:04d}] {}".format(threading.get_ident() % 10000, msg), kwargs + # return '[%s] %s' % (self.extra['connid'], msg), kwargs + + +class Logging: + logger = None + + @classmethod + def getLogger(cls): + return logger + + @classmethod + def clsInit(cls, gConfig): # TODO: refactor away gConfig + if cls.logger: + return + + # Logging Stuff + # global misc.logger + _logger = logging.getLogger('CrashGen') # real logger + _logger.addFilter(LoggingFilter()) + ch = logging.StreamHandler() + _logger.addHandler(ch) + + # Logging adapter, to be used as a logger + # print("setting logger variable") + # global logger + cls.logger = MyLoggingAdapter(_logger, []) + + if (gConfig.debug): + cls.logger.setLevel(logging.DEBUG) # default seems to be INFO + else: + cls.logger.setLevel(logging.INFO) + + @classmethod + def info(cls, msg): + cls.logger.info(msg) + + @classmethod + def debug(cls, msg): + cls.logger.debug(msg) + + @classmethod + def warning(cls, msg): + cls.logger.warning(msg) + + @classmethod + def error(cls, msg): + cls.logger.error(msg) + +class Status: + STATUS_STARTING = 1 + STATUS_RUNNING = 2 + STATUS_STOPPING = 3 + STATUS_STOPPED = 4 + + def __init__(self, status): + self.set(status) + + def __repr__(self): + return "[Status: v={}]".format(self._status) + + def set(self, status): + self._status = status + + def get(self): + return self._status + + def isStarting(self): + return self._status == Status.STATUS_STARTING + + def isRunning(self): + # return self._thread and self._thread.is_alive() + return self._status == Status.STATUS_RUNNING + + def isStopping(self): + return self._status == Status.STATUS_STOPPING + + def isStopped(self): + return self._status == Status.STATUS_STOPPED + + def isStable(self): + return self.isRunning() or self.isStopped() + +# Deterministic random number generator +class Dice(): + seeded = False # static, uninitialized + + @classmethod + def seed(cls, s): # static + if (cls.seeded): + raise RuntimeError( + "Cannot seed the random generator more than once") + cls.verifyRNG() + random.seed(s) + cls.seeded = True # TODO: protect against multi-threading + + @classmethod + def verifyRNG(cls): # Verify that the RNG is determinstic + random.seed(0) + x1 = random.randrange(0, 1000) + x2 = random.randrange(0, 1000) + x3 = random.randrange(0, 1000) + if (x1 != 864 or x2 != 394 or x3 != 776): + raise RuntimeError("System RNG is not deterministic") + + @classmethod + def throw(cls, stop): # get 0 to stop-1 + return cls.throwRange(0, stop) + + @classmethod + def throwRange(cls, start, stop): # up to stop-1 + if (not cls.seeded): + raise RuntimeError("Cannot throw dice before seeding it") + return random.randrange(start, stop) + + @classmethod + def choice(cls, cList): + return random.choice(cList) + +class Helper: + @classmethod + def convertErrno(cls, errno): + return errno if (errno > 0) else 0x80000000 + errno + + @classmethod + def getFriendlyPath(cls, path): # returns .../xxx/yyy + ht1 = os.path.split(path) + ht2 = os.path.split(ht1[0]) + return ".../" + ht2[1] + '/' + ht1[1] + + +class Progress: + STEP_BOUNDARY = 0 + BEGIN_THREAD_STEP = 1 + END_THREAD_STEP = 2 + SERVICE_HEART_BEAT= 3 + SERVICE_RECONNECT_START = 4 + SERVICE_RECONNECT_SUCCESS = 5 + SERVICE_RECONNECT_FAILURE = 6 + SERVICE_START_NAP = 7 + CREATE_TABLE_ATTEMPT = 8 + + tokens = { + STEP_BOUNDARY: '.', + BEGIN_THREAD_STEP: '[', + END_THREAD_STEP: '] ', + SERVICE_HEART_BEAT: '.Y.', + SERVICE_RECONNECT_START: '', + SERVICE_RECONNECT_FAILURE: '.xr>', + SERVICE_START_NAP: '_zz', + CREATE_TABLE_ATTEMPT: '_c', + } + + @classmethod + def emit(cls, token): + print(cls.tokens[token], end="", flush=True) diff --git a/tests/pytest/crash_gen/service_manager.py b/tests/pytest/crash_gen/service_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..d249abc4396462b6dbacbfcbb1f6619e48161c3b --- /dev/null +++ b/tests/pytest/crash_gen/service_manager.py @@ -0,0 +1,773 @@ +import os +import io +import sys +import threading +import signal +import logging +import time +import subprocess + +from typing import IO, List + +try: + import psutil +except: + print("Psutil module needed, please install: sudo pip3 install psutil") + sys.exit(-1) + +from queue import Queue, Empty + +from .misc import Logging, Status, CrashGenError, Dice, Helper, Progress +from .db import DbConn, DbTarget + +class TdeInstance(): + """ + A class to capture the *static* information of a TDengine instance, + including the location of the various files/directories, and basica + configuration. + """ + + @classmethod + def _getBuildPath(cls): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("communit")] + else: + projPath = selfPath[:selfPath.find("tests")] + + buildPath = None + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + if buildPath == None: + raise RuntimeError("Failed to determine buildPath, selfPath={}, projPath={}" + .format(selfPath, projPath)) + return buildPath + + @classmethod + def prepareGcovEnv(cls, env): + # Ref: https://gcc.gnu.org/onlinedocs/gcc/Cross-profiling.html + bPath = cls._getBuildPath() # build PATH + numSegments = len(bPath.split('/')) - 1 # "/x/TDengine/build" should yield 3 + numSegments = numSegments - 1 # DEBUG only + env['GCOV_PREFIX'] = bPath + '/svc_gcov' + env['GCOV_PREFIX_STRIP'] = str(numSegments) # Strip every element, plus, ENV needs strings + Logging.info("Preparing GCOV environement to strip {} elements and use path: {}".format( + numSegments, env['GCOV_PREFIX'] )) + + def __init__(self, subdir='test', tInstNum=0, port=6030, fepPort=6030): + self._buildDir = self._getBuildPath() + self._subdir = '/' + subdir # TODO: tolerate "/" + self._port = port # TODO: support different IP address too + self._fepPort = fepPort + + self._tInstNum = tInstNum + self._smThread = ServiceManagerThread() + + def getDbTarget(self): + return DbTarget(self.getCfgDir(), self.getHostAddr(), self._port) + + def getPort(self): + return self._port + + def __repr__(self): + return "[TdeInstance: {}, subdir={}]".format( + self._buildDir, Helper.getFriendlyPath(self._subdir)) + + def generateCfgFile(self): + # print("Logger = {}".format(logger)) + # buildPath = self.getBuildPath() + # taosdPath = self._buildPath + "/build/bin/taosd" + + cfgDir = self.getCfgDir() + cfgFile = cfgDir + "/taos.cfg" # TODO: inquire if this is fixed + if os.path.exists(cfgFile): + if os.path.isfile(cfgFile): + Logging.warning("Config file exists already, skip creation: {}".format(cfgFile)) + return # cfg file already exists, nothing to do + else: + raise CrashGenError("Invalid config file: {}".format(cfgFile)) + # Now that the cfg file doesn't exist + if os.path.exists(cfgDir): + if not os.path.isdir(cfgDir): + raise CrashGenError("Invalid config dir: {}".format(cfgDir)) + # else: good path + else: + os.makedirs(cfgDir, exist_ok=True) # like "mkdir -p" + # Now we have a good cfg dir + cfgValues = { + 'runDir': self.getRunDir(), + 'ip': '127.0.0.1', # TODO: change to a network addressable ip + 'port': self._port, + 'fepPort': self._fepPort, + } + cfgTemplate = """ +dataDir {runDir}/data +logDir {runDir}/log + +charset UTF-8 + +firstEp {ip}:{fepPort} +fqdn {ip} +serverPort {port} + +# was all 135 below +dDebugFlag 135 +cDebugFlag 135 +rpcDebugFlag 135 +qDebugFlag 135 +# httpDebugFlag 143 +# asyncLog 0 +# tables 10 +maxtablesPerVnode 10 +rpcMaxTime 101 +# cache 2 +keep 36500 +# walLevel 2 +walLevel 1 +# +# maxConnections 100 +""" + cfgContent = cfgTemplate.format_map(cfgValues) + f = open(cfgFile, "w") + f.write(cfgContent) + f.close() + + def rotateLogs(self): + logPath = self.getLogDir() + # ref: https://stackoverflow.com/questions/1995373/deleting-all-files-in-a-directory-with-python/1995397 + if os.path.exists(logPath): + logPathSaved = logPath + "_" + time.strftime('%Y-%m-%d-%H-%M-%S') + Logging.info("Saving old log files to: {}".format(logPathSaved)) + os.rename(logPath, logPathSaved) + # os.mkdir(logPath) # recreate, no need actually, TDengine will auto-create with proper perms + + + def getExecFile(self): # .../taosd + return self._buildDir + "/build/bin/taosd" + + def getRunDir(self): # TODO: rename to "root dir" ?! + return self._buildDir + self._subdir + + def getCfgDir(self): # path, not file + return self.getRunDir() + "/cfg" + + def getLogDir(self): + return self.getRunDir() + "/log" + + def getHostAddr(self): + return "127.0.0.1" + + def getServiceCmdLine(self): # to start the instance + return [self.getExecFile(), '-c', self.getCfgDir()] # used in subproce.Popen() + + def _getDnodes(self, dbc): + dbc.query("show dnodes") + cols = dbc.getQueryResult() # id,end_point,vnodes,cores,status,role,create_time,offline reason + return {c[1]:c[4] for c in cols} # {'xxx:6030':'ready', 'xxx:6130':'ready'} + + def createDnode(self, dbt: DbTarget): + """ + With a connection to the "first" EP, let's create a dnode for someone else who + wants to join. + """ + dbc = DbConn.createNative(self.getDbTarget()) + dbc.open() + + if dbt.getEp() in self._getDnodes(dbc): + Logging.info("Skipping DNode creation for: {}".format(dbt)) + dbc.close() + return + + sql = "CREATE DNODE \"{}\"".format(dbt.getEp()) + dbc.execute(sql) + dbc.close() + + def getStatus(self): + return self._smThread.getStatus() + + def getSmThread(self): + return self._smThread + + def start(self): + if not self.getStatus().isStopped(): + raise CrashGenError("Cannot start instance from status: {}".format(self.getStatus())) + + Logging.info("Starting TDengine instance: {}".format(self)) + self.generateCfgFile() # service side generates config file, client does not + self.rotateLogs() + + self._smThread.start(self.getServiceCmdLine()) + + def stop(self): + self._smThread.stop() + + def isFirst(self): + return self._tInstNum == 0 + + +class TdeSubProcess: + """ + A class to to represent the actual sub process that is the run-time + of a TDengine instance. + + It takes a TdeInstance object as its parameter, with the rationale being + "a sub process runs an instance". + """ + + # RET_ALREADY_STOPPED = -1 + # RET_TIME_OUT = -3 + # RET_SUCCESS = -4 + + def __init__(self): + self.subProcess = None + # if tInst is None: + # raise CrashGenError("Empty instance not allowed in TdeSubProcess") + # self._tInst = tInst # Default create at ServiceManagerThread + + def __repr__(self): + if self.subProcess is None: + return '[TdeSubProc: Empty]' + return '[TdeSubProc: pid = {}]'.format(self.getPid()) + + def getStdOut(self): + return self.subProcess.stdout + + def getStdErr(self): + return self.subProcess.stderr + + def isRunning(self): + return self.subProcess is not None + + def getPid(self): + return self.subProcess.pid + + def start(self, cmdLine): + ON_POSIX = 'posix' in sys.builtin_module_names + + # Sanity check + if self.subProcess: # already there + raise RuntimeError("Corrupt process state") + + # Prepare environment variables for coverage information + # Ref: https://stackoverflow.com/questions/2231227/python-subprocess-popen-with-a-modified-environment + myEnv = os.environ.copy() + TdeInstance.prepareGcovEnv(myEnv) + + # print(myEnv) + # print(myEnv.items()) + # print("Starting TDengine via Shell: {}".format(cmdLineStr)) + + useShell = True + self.subProcess = subprocess.Popen( + ' '.join(cmdLine) if useShell else cmdLine, + shell=useShell, + # svcCmdSingle, shell=True, # capture core dump? + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + # bufsize=1, # not supported in binary mode + close_fds=ON_POSIX, + env=myEnv + ) # had text=True, which interferred with reading EOF + + STOP_SIGNAL = signal.SIGKILL # What signal to use (in kill) to stop a taosd process? + + def stop(self): + """ + Stop a sub process, and try to return a meaningful return code. + + Common POSIX signal values (from man -7 signal): + SIGHUP 1 + SIGINT 2 + SIGQUIT 3 + SIGILL 4 + SIGTRAP 5 + SIGABRT 6 + SIGIOT 6 + SIGBUS 7 + SIGEMT - + SIGFPE 8 + SIGKILL 9 + SIGUSR1 10 + SIGSEGV 11 + SIGUSR2 12 + """ + if not self.subProcess: + Logging.error("Sub process already stopped") + return # -1 + + retCode = self.subProcess.poll() # ret -N means killed with signal N, otherwise it's from exit(N) + if retCode: # valid return code, process ended + retCode = -retCode # only if valid + Logging.warning("TSP.stop(): process ended itself") + self.subProcess = None + return retCode + + # process still alive, let's interrupt it + Logging.info("Terminate running process, send SIG_{} and wait...".format(self.STOP_SIGNAL)) + # sub process should end, then IPC queue should end, causing IO thread to end + topSubProc = psutil.Process(self.subProcess.pid) + for child in topSubProc.children(recursive=True): # or parent.children() for recursive=False + child.send_signal(self.STOP_SIGNAL) + time.sleep(0.2) # 200 ms + # topSubProc.send_signal(sig) # now kill the main sub process (likely the Shell) + + self.subProcess.send_signal(self.STOP_SIGNAL) # main sub process (likely the Shell) + self.subProcess.wait(20) + retCode = self.subProcess.returncode # should always be there + # May throw subprocess.TimeoutExpired exception above, therefore + # The process is guranteed to have ended by now + self.subProcess = None + if retCode != 0: # != (- signal.SIGINT): + Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format( + self.STOP_SIGNAL, retCode)) + else: + Logging.info("TSP.stop(): sub proc successfully terminated with SIG {}".format(self.STOP_SIGNAL)) + return - retCode + +class ServiceManager: + PAUSE_BETWEEN_IPC_CHECK = 1.2 # seconds between checks on STDOUT of sub process + + def __init__(self, numDnodes): # >1 when we run a cluster + Logging.info("TDengine Service Manager (TSM) created") + self._numDnodes = numDnodes # >1 means we have a cluster + self._lock = threading.Lock() + # signal.signal(signal.SIGTERM, self.sigIntHandler) # Moved to MainExec + # signal.signal(signal.SIGINT, self.sigIntHandler) + # signal.signal(signal.SIGUSR1, self.sigUsrHandler) # different handler! + + self.inSigHandler = False + # self._status = MainExec.STATUS_RUNNING # set inside + # _startTaosService() + self._runCluster = (numDnodes > 1) + self._tInsts : List[TdeInstance] = [] + for i in range(0, numDnodes): + ti = self._createTdeInstance(i) # construct tInst + self._tInsts.append(ti) + + # self.svcMgrThreads : List[ServiceManagerThread] = [] + # for i in range(0, numDnodes): + # thread = self._createThread(i) # construct tInst + # self.svcMgrThreads.append(thread) + + def _createTdeInstance(self, dnIndex): + if not self._runCluster: # single instance + subdir = 'test' + else: # Create all threads in a cluster + subdir = 'cluster_dnode_{}'.format(dnIndex) + fepPort= 6030 # firstEP Port + port = fepPort + dnIndex * 100 + return TdeInstance(subdir, dnIndex, port, fepPort) + # return ServiceManagerThread(dnIndex, ti) + + def _doMenu(self): + choice = "" + while True: + print("\nInterrupting Service Program, Choose an Action: ") + print("1: Resume") + print("2: Terminate") + print("3: Restart") + # Remember to update the if range below + # print("Enter Choice: ", end="", flush=True) + while choice == "": + choice = input("Enter Choice: ") + if choice != "": + break # done with reading repeated input + if choice in ["1", "2", "3"]: + break # we are done with whole method + print("Invalid choice, please try again.") + choice = "" # reset + return choice + + def sigUsrHandler(self, signalNumber, frame): + print("Interrupting main thread execution upon SIGUSR1") + if self.inSigHandler: # already + print("Ignoring repeated SIG...") + return # do nothing if it's already not running + self.inSigHandler = True + + choice = self._doMenu() + if choice == "1": + self.sigHandlerResume() # TODO: can the sub-process be blocked due to us not reading from queue? + elif choice == "2": + self.stopTaosServices() + elif choice == "3": # Restart + self.restart() + else: + raise RuntimeError("Invalid menu choice: {}".format(choice)) + + self.inSigHandler = False + + def sigIntHandler(self, signalNumber, frame): + print("ServiceManager: INT Signal Handler starting...") + if self.inSigHandler: + print("Ignoring repeated SIG_INT...") + return + self.inSigHandler = True + + self.stopTaosServices() + print("ServiceManager: INT Signal Handler returning...") + self.inSigHandler = False + + def sigHandlerResume(self): + print("Resuming TDengine service manager (main thread)...\n\n") + + # def _updateThreadStatus(self): + # if self.svcMgrThread: # valid svc mgr thread + # if self.svcMgrThread.isStopped(): # done? + # self.svcMgrThread.procIpcBatch() # one last time. TODO: appropriate? + # self.svcMgrThread = None # no more + + def isActive(self): + """ + Determine if the service/cluster is active at all, i.e. at least + one thread is not "stopped". + """ + for ti in self._tInsts: + if not ti.getStatus().isStopped(): + return True + return False + + def isRunning(self): + for ti in self._tInsts: + if not ti.getStatus().isRunning(): + return False + return True + + + # def isRestarting(self): + # """ + # Determine if the service/cluster is being "restarted", i.e., at least + # one thread is in "restarting" status + # """ + # for thread in self.svcMgrThreads: + # if thread.isRestarting(): + # return True + # return False + + def isStable(self): + """ + Determine if the service/cluster is "stable", i.e. all of the + threads are in "stable" status. + """ + for ti in self._tInsts: + if not ti.getStatus().isStable(): + return False + return True + + def _procIpcAll(self): + while self.isActive(): + Progress.emit(Progress.SERVICE_HEART_BEAT) + for ti in self._tInsts: # all thread objects should always be valid + # while self.isRunning() or self.isRestarting() : # for as long as the svc mgr thread is still here + status = ti.getStatus() + if status.isRunning(): + th = ti.getSmThread() + th.procIpcBatch() # regular processing, + if status.isStopped(): + th.procIpcBatch() # one last time? + # self._updateThreadStatus() + + time.sleep(self.PAUSE_BETWEEN_IPC_CHECK) # pause, before next round + # raise CrashGenError("dummy") + Logging.info("Service Manager Thread (with subprocess) ended, main thread exiting...") + + def _getFirstInstance(self): + return self._tInsts[0] + + def startTaosServices(self): + with self._lock: + if self.isActive(): + raise RuntimeError("Cannot start TAOS service(s) when one/some may already be running") + + # Find if there's already a taosd service, and then kill it + for proc in psutil.process_iter(): + if proc.name() == 'taosd': + Logging.info("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupt") + time.sleep(2.0) + proc.kill() + # print("Process: {}".format(proc.name())) + + # self.svcMgrThread = ServiceManagerThread() # create the object + + for ti in self._tInsts: + ti.start() + if not ti.isFirst(): + tFirst = self._getFirstInstance() + tFirst.createDnode(ti.getDbTarget()) + ti.getSmThread().procIpcBatch(trimToTarget=10, forceOutput=True) # for printing 10 lines + + def stopTaosServices(self): + with self._lock: + if not self.isActive(): + Logging.warning("Cannot stop TAOS service(s), already not active") + return + + for ti in self._tInsts: + ti.stop() + + def run(self): + self.startTaosServices() + self._procIpcAll() # pump/process all the messages, may encounter SIG + restart + if self.isActive(): # if sig handler hasn't destroyed it by now + self.stopTaosServices() # should have started already + + def restart(self): + if not self.isStable(): + Logging.warning("Cannot restart service/cluster, when not stable") + return + + # self._isRestarting = True + if self.isActive(): + self.stopTaosServices() + else: + Logging.warning("Service not active when restart requested") + + self.startTaosServices() + # self._isRestarting = False + + # def isRunning(self): + # return self.svcMgrThread != None + + # def isRestarting(self): + # return self._isRestarting + +class ServiceManagerThread: + """ + A class representing a dedicated thread which manages the "sub process" + of the TDengine service, interacting with its STDOUT/ERR. + + It takes a TdeInstance parameter at creation time, or create a default + """ + MAX_QUEUE_SIZE = 10000 + + def __init__(self): + # Set the sub process + self._tdeSubProcess = None # type: TdeSubProcess + + # Arrange the TDengine instance + # self._tInstNum = tInstNum # instance serial number in cluster, ZERO based + # self._tInst = tInst or TdeInstance() # Need an instance + + self._thread = None # The actual thread, # type: threading.Thread + self._status = Status(Status.STATUS_STOPPED) # The status of the underlying service, actually. + + def __repr__(self): + return "[SvcMgrThread: status={}, subProc={}]".format( + self.getStatus(), self._tdeSubProcess) + + def getStatus(self): + return self._status + + # Start the thread (with sub process), and wait for the sub service + # to become fully operational + def start(self, cmdLine): + if self._thread: + raise RuntimeError("Unexpected _thread") + if self._tdeSubProcess: + raise RuntimeError("TDengine sub process already created/running") + + Logging.info("Attempting to start TAOS service: {}".format(self)) + + self._status.set(Status.STATUS_STARTING) + self._tdeSubProcess = TdeSubProcess() + self._tdeSubProcess.start(cmdLine) + + self._ipcQueue = Queue() + self._thread = threading.Thread( # First thread captures server OUTPUT + target=self.svcOutputReader, + args=(self._tdeSubProcess.getStdOut(), self._ipcQueue)) + self._thread.daemon = True # thread dies with the program + self._thread.start() + + self._thread2 = threading.Thread( # 2nd thread captures server ERRORs + target=self.svcErrorReader, + args=(self._tdeSubProcess.getStdErr(), self._ipcQueue)) + self._thread2.daemon = True # thread dies with the program + self._thread2.start() + + # wait for service to start + for i in range(0, 100): + time.sleep(1.0) + # self.procIpcBatch() # don't pump message during start up + Progress.emit(Progress.SERVICE_START_NAP) + # print("_zz_", end="", flush=True) + if self._status.isRunning(): + Logging.info("[] TDengine service READY to process requests") + Logging.info("[] TAOS service started: {}".format(self)) + # self._verifyDnode(self._tInst) # query and ensure dnode is ready + # Logging.debug("[] TAOS Dnode verified: {}".format(self)) + return # now we've started + # TODO: handle failure-to-start better? + self.procIpcBatch(100, True) # display output before cronking out, trim to last 20 msgs, force output + raise RuntimeError("TDengine service did not start successfully: {}".format(self)) + + def _verifyDnode(self, tInst: TdeInstance): + dbc = DbConn.createNative(tInst.getDbTarget()) + dbc.open() + dbc.query("show dnodes") + # dbc.query("DESCRIBE {}.{}".format(dbName, self._stName)) + cols = dbc.getQueryResult() # id,end_point,vnodes,cores,status,role,create_time,offline reason + # ret = {row[0]:row[1] for row in stCols if row[3]=='TAG'} # name:type + isValid = False + for col in cols: + # print("col = {}".format(col)) + ep = col[1].split(':') # 10.1.30.2:6030 + print("Found ep={}".format(ep)) + if tInst.getPort() == int(ep[1]): # That's us + # print("Valid Dnode matched!") + isValid = True # now we are valid + break + if not isValid: + print("Failed to start dnode, sleep for a while") + time.sleep(600) + raise RuntimeError("Failed to start Dnode, expected port not found: {}". + format(tInst.getPort())) + dbc.close() + + def stop(self): + # can be called from both main thread or signal handler + Logging.info("Terminating TDengine service running as the sub process...") + if self.getStatus().isStopped(): + Logging.info("Service already stopped") + return + if self.getStatus().isStopping(): + Logging.info("Service is already being stopped") + return + # Linux will send Control-C generated SIGINT to the TDengine process + # already, ref: + # https://unix.stackexchange.com/questions/176235/fork-and-how-signals-are-delivered-to-processes + if not self._tdeSubProcess: + raise RuntimeError("sub process object missing") + + self._status.set(Status.STATUS_STOPPING) + # retCode = self._tdeSubProcess.stop() + try: + retCode = self._tdeSubProcess.stop() + # print("Attempted to stop sub process, got return code: {}".format(retCode)) + if retCode == signal.SIGSEGV : # SGV + Logging.error("[[--ERROR--]]: TDengine service SEGV fault (check core file!)") + except subprocess.TimeoutExpired as err: + Logging.info("Time out waiting for TDengine service process to exit") + else: + if self._tdeSubProcess.isRunning(): # still running, should now never happen + Logging.error("FAILED to stop sub process, it is still running... pid = {}".format( + self._tdeSubProcess.getPid())) + else: + self._tdeSubProcess = None # not running any more + self.join() # stop the thread, change the status, etc. + + # Check if it's really stopped + outputLines = 10 # for last output + if self.getStatus().isStopped(): + self.procIpcBatch(outputLines) # one last time + Logging.debug("End of TDengine Service Output: {}".format(self)) + Logging.info("----- TDengine Service (managed by SMT) is now terminated -----\n") + else: + print("WARNING: SMT did not terminate as expected: {}".format(self)) + + def join(self): + # TODO: sanity check + if not self.getStatus().isStopping(): + raise RuntimeError( + "SMT.Join(): Unexpected status: {}".format(self._status)) + + if self._thread: + self._thread.join() + self._thread = None + self._status.set(Status.STATUS_STOPPED) + # STD ERR thread + self._thread2.join() + self._thread2 = None + else: + print("Joining empty thread, doing nothing") + + def _trimQueue(self, targetSize): + if targetSize <= 0: + return # do nothing + q = self._ipcQueue + if (q.qsize() <= targetSize): # no need to trim + return + + Logging.debug("Triming IPC queue to target size: {}".format(targetSize)) + itemsToTrim = q.qsize() - targetSize + for i in range(0, itemsToTrim): + try: + q.get_nowait() + except Empty: + break # break out of for loop, no more trimming + + TD_READY_MSG = "TDengine is initialized successfully" + + def procIpcBatch(self, trimToTarget=0, forceOutput=False): + self._trimQueue(trimToTarget) # trim if necessary + # Process all the output generated by the underlying sub process, + # managed by IO thread + print("<", end="", flush=True) + while True: + try: + line = self._ipcQueue.get_nowait() # getting output at fast speed + self._printProgress("_o") + except Empty: + # time.sleep(2.3) # wait only if there's no output + # no more output + print(".>", end="", flush=True) + return # we are done with THIS BATCH + else: # got line, printing out + if forceOutput: + Logging.info('[TAOSD] ' + line) + else: + Logging.debug('[TAOSD] ' + line) + print(">", end="", flush=True) + + _ProgressBars = ["--", "//", "||", "\\\\"] + + def _printProgress(self, msg): # TODO: assuming 2 chars + print(msg, end="", flush=True) + pBar = self._ProgressBars[Dice.throw(4)] + print(pBar, end="", flush=True) + print('\b\b\b\b', end="", flush=True) + + def svcOutputReader(self, out: IO, queue): + # Important Reference: https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python + # print("This is the svcOutput Reader...") + # for line in out : + for line in iter(out.readline, b''): + # print("Finished reading a line: {}".format(line)) + # print("Adding item to queue...") + try: + line = line.decode("utf-8").rstrip() + except UnicodeError: + print("\nNon-UTF8 server output: {}\n".format(line)) + + # This might block, and then causing "out" buffer to block + queue.put(line) + self._printProgress("_i") + + if self._status.isStarting(): # we are starting, let's see if we have started + if line.find(self.TD_READY_MSG) != -1: # found + Logging.info("Waiting for the service to become FULLY READY") + time.sleep(1.0) # wait for the server to truly start. TODO: remove this + Logging.info("Service is now FULLY READY") # TODO: more ID info here? + self._status.set(Status.STATUS_RUNNING) + + # Trim the queue if necessary: TODO: try this 1 out of 10 times + self._trimQueue(self.MAX_QUEUE_SIZE * 9 // 10) # trim to 90% size + + if self._status.isStopping(): # TODO: use thread status instead + # WAITING for stopping sub process to finish its outptu + print("_w", end="", flush=True) + + # queue.put(line) + # meaning sub process must have died + Logging.info("EOF for TDengine STDOUT: {}".format(self)) + out.close() + + def svcErrorReader(self, err: IO, queue): + for line in iter(err.readline, b''): + Logging.info("TDengine STDERR: {}".format(line)) + Logging.info("EOF for TDengine STDERR: {}".format(self)) + err.close() \ No newline at end of file diff --git a/tests/pytest/crash_gen_bootstrap.py b/tests/pytest/crash_gen_bootstrap.py new file mode 100644 index 0000000000000000000000000000000000000000..fd12284b9d7782ac7df89c37fcb653ca3bebe82b --- /dev/null +++ b/tests/pytest/crash_gen_bootstrap.py @@ -0,0 +1,23 @@ +# -----!/usr/bin/python3.7 +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +import sys +from crash_gen.crash_gen_main import MainExec + +if __name__ == "__main__": + + mExec = MainExec() + mExec.init() + exitCode = mExec.run() + + print("Exiting with code: {}".format(exitCode)) + sys.exit(exitCode) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 39d0fa3d94d36aab88366cc9f428a08f6fd3d1dc..a48dbdc48024ea7348f11b70129ebbf1a486e53d 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -24,6 +24,7 @@ python3 ./test.py -f table/alter_wal0.py python3 ./test.py -f table/column_name.py python3 ./test.py -f table/column_num.py python3 ./test.py -f table/db_table.py +python3 ./test.py -f table/create_sensitive.py #python3 ./test.py -f table/tablename-boundary.py # tag @@ -150,15 +151,17 @@ python3 ./test.py -f query/select_last_crash.py python3 ./test.py -f query/queryNullValueTest.py python3 ./test.py -f query/queryInsertValue.py python3 ./test.py -f query/queryConnection.py +python3 ./test.py -f query/queryCountCSVData.py python3 ./test.py -f query/natualInterval.py python3 ./test.py -f query/bug1471.py +python3 ./test.py -f query/dataLossTest.py #stream python3 ./test.py -f stream/metric_1.py python3 ./test.py -f stream/new.py python3 ./test.py -f stream/stream1.py python3 ./test.py -f stream/stream2.py -python3 ./test.py -f stream/parser.py +#python3 ./test.py -f stream/parser.py python3 ./test.py -f stream/history.py #alter table @@ -186,7 +189,7 @@ python3 ./test.py -f functions/function_leastsquares.py -r 1 python3 ./test.py -f functions/function_max.py -r 1 python3 ./test.py -f functions/function_min.py -r 1 python3 ./test.py -f functions/function_operations.py -r 1 -python3 ./test.py -f functions/function_percentile.py +python3 ./test.py -f functions/function_percentile.py -r 1 python3 ./test.py -f functions/function_spread.py -r 1 python3 ./test.py -f functions/function_stddev.py -r 1 python3 ./test.py -f functions/function_sum.py -r 1 @@ -204,3 +207,20 @@ python3 test.py -f tools/taosdemo.py python3 test.py -f subscribe/singlemeter.py #python3 test.py -f subscribe/stability.py python3 test.py -f subscribe/supertable.py + + +# update +python3 ./test.py -f update/allow_update.py +python3 ./test.py -f update/allow_update-0.py +python3 ./test.py -f update/append_commit_data.py +python3 ./test.py -f update/append_commit_last-0.py +python3 ./test.py -f update/append_commit_last.py +python3 ./test.py -f update/merge_commit_data.py +python3 ./test.py -f update/merge_commit_data-0.py +python3 ./test.py -f update/merge_commit_data2.py +python3 ./test.py -f update/merge_commit_data2_update0.py +python3 ./test.py -f update/merge_commit_last-0.py +python3 ./test.py -f update/merge_commit_last.py + +# wal +python3 ./test.py -f wal/addOldWalTest.py \ No newline at end of file diff --git a/tests/pytest/functions/function_percentile.py b/tests/pytest/functions/function_percentile.py index aaeb94372e44282da0d83849c18e40857463b42c..e63d65f2e6a429015e2b4d7dcbe5e8c9884eea5e 100644 --- a/tests/pytest/functions/function_percentile.py +++ b/tests/pytest/functions/function_percentile.py @@ -130,8 +130,19 @@ class TDTestCase: tdSql.query("select percentile(col6, 100) from test") tdSql.checkData(0, 0, np.percentile(floatData, 100)) tdSql.query("select apercentile(col6, 100) from test") - print("apercentile result: %s" % tdSql.getData(0, 0)) + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.execute("create table meters (ts timestamp, voltage int) tags(loc nchar(20))") + tdSql.execute("create table t0 using meters tags('beijing')") + tdSql.execute("create table t1 using meters tags('shanghai')") + for i in range(self.rowNum): + tdSql.execute("insert into t0 values(%d, %d)" % (self.ts + i, i + 1)) + tdSql.execute("insert into t1 values(%d, %d)" % (self.ts + i, i + 1)) + + tdSql.error("select percentile(voltage, 20) from meters") + tdSql.query("select apercentile(voltage, 20) from meters") + print("apercentile result: %s" % tdSql.getData(0, 0)) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh new file mode 100755 index 0000000000000000000000000000000000000000..2d48de65c9a16fdb6f5f906167baf772694e9a05 --- /dev/null +++ b/tests/pytest/handle_crash_gen_val_log.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' +nohup /root/TDinternal/debug/build/bin/taosd -c /root/TDinternal/community/sim/dnode1/cfg >/dev/null & +./crash_gen.sh --valgrind -p -t 10 -s 100 -b 4 +pidof taosd|xargs kill +grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log + +for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'` +do +if [ -n "$memError" ]; then + if [ "$memError" -gt 12 ]; then + echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ + More than our threshold! ## ${NC}" + fi +fi +done + +grep 'start to execute\|definitely lost:' valgrind.err|grep -v 'grep'|uniq|tee crash_gen-definitely-lost-out.log +for defiMemError in `grep 'definitely lost:' crash_gen-definitely-lost-out.log | awk '{print $7}'` +do + +if [ -n "$defiMemError" ]; then + if [ "$defiMemError" -gt 3 ]; then + echo -e "${RED} ## Memory errors number valgrind reports \ + Definitely lost is $defiMemError. More than our threshold! ## ${NC}" + exit 8 + fi +fi +done \ No newline at end of file diff --git a/tests/pytest/insert/ningsiInsert.py b/tests/pytest/insert/ningsiInsert.py new file mode 100644 index 0000000000000000000000000000000000000000..bcad2b03ed52bc01cae8b8b800edcd7c213ca375 --- /dev/null +++ b/tests/pytest/insert/ningsiInsert.py @@ -0,0 +1,88 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import requests, json +import threading +import string +import random +import time + +class RestfulInsert: + def init(self): + self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='} + self.url = "http://ningsi60:6041/rest/sql" + self.ts = 1104508800000 + self.numOfThreads = 10 + self.numOfTables = 3000 + self.dbName = 'netmonitortaos' + self.stbName = 'devinfomt' + self.prefix = 'dev' + + def get_random_string(self, length): + letters = string.ascii_lowercase + result_str = ''.join(random.choice(letters) for i in range(length)) + return result_str + + def createTables(self, threadID): + print("create table: thread %d started" % threadID) + tablesPerThread = int (self.numOfTables / self.numOfThreads) + for i in range(tablesPerThread): + data = "create table '%s'.dev_%d using '%s'.'%s' tags('%s', '%s')" % (self.dbName, i + threadID * tablesPerThread, self.dbName, self.stbName, self.get_random_string(25), self.get_random_string(25)) + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) + + def insertData(self, threadID): + print("insert data: thread %d started" % threadID) + tablesPerThread = int (self.numOfTables / self.numOfThreads) + base_ts = self.ts + while True: + i = 0 + for i in range(tablesPerThread): + data = "insert into %s.dev_%d values(%d, '%s', '%s', %d, %d, %d)" % (self.dbName, i + threadID * tablesPerThread, base_ts, self.get_random_string(25), self.get_random_string(30), random.randint(1, 10000), random.randint(1, 10000), random.randint(1, 10000)) + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) + + time.sleep(30) + base_ts = base_ts + 1 + + def run(self): + data = "create database if not exists %s keep 7300" % self.dbName + requests.post(self.url, data, headers = self.header) + + data = "create table '%s'.'%s' (timeid timestamp, devdesc binary(50), devname binary(50), cpu bigint, temp bigint, ram bigint) tags(devid binary(50), modelid binary(50))" % (self.dbName, self.stbName) + requests.post(self.url, data, headers = self.header) + + threads = [] + for i in range(self.numOfThreads): + thread = threading.Thread(target=self.createTables, args=(i,)) + thread.start() + threads.append(thread) + + for i in range(self.numOfThreads): + threads[i].join() + + threads = [] + for i in range(self.numOfThreads): + thread = threading.Thread(target=self.insertData, args=(i,)) + thread.start() + threads.append(thread) + + for i in range(self.numOfThreads): + threads[i].join() + +ri = RestfulInsert() +ri.init() +ri.run() diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py new file mode 100644 index 0000000000000000000000000000000000000000..a6c9b074e12c571e7ef715a520e20295c936362c --- /dev/null +++ b/tests/pytest/insert/restfulInsert.py @@ -0,0 +1,83 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import requests +import threading +import random +import time + +class RestfulInsert: + def init(self): + self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='} + self.url = "http://127.0.0.1:6041/rest/sql" + self.ts = 1500000000000 + self.numOfThreads = 20 + self.numOfTables = 10000 + self.recordsPerTable = 10000 + self.batchSize = 1000 + self.tableNamePerfix = 't' + + def createTable(self, threadID): + tablesPerThread = int (self.numOfTables / self.numOfThreads) + print("create table %d to %d" % (tablesPerThread * threadID, tablesPerThread * (threadID + 1) - 1)) + for i in range(tablesPerThread): + tableID = threadID * tablesPerThread + name = 'beijing' if tableID % 2 == 0 else 'shanghai' + data = "create table test.%s%d using test.meters tags(%d, '%s')" % (self.tableNamePerfix, tableID + i, tableID + i, name) + requests.post(self.url, data, headers = self.header) + + def insertData(self, threadID): + print("thread %d started" % threadID) + tablesPerThread = int (self.numOfTables / self.numOfThreads) + for i in range(tablesPerThread): + tableID = i + threadID * tablesPerThread + start = self.ts + for j in range(int(self.recordsPerTable / self.batchSize)): + data = "insert into test.%s%d values" % (self.tableNamePerfix, tableID) + for k in range(self.batchSize): + data += "(%d, %d, %d, %d)" % (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)) + requests.post(self.url, data, headers = self.header) + + def run(self): + data = "drop database if exists test" + requests.post(self.url, data, headers = self.header) + data = "create database test" + requests.post(self.url, data, headers = self.header) + data = "create table test.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" + requests.post(self.url, data, headers = self.header) + + threads = [] + startTime = time.time() + for i in range(self.numOfThreads): + thread = threading.Thread(target=self.createTable, args=(i,)) + thread.start() + threads.append(thread) + for i in range(self.numOfThreads): + threads[i].join() + print("createing %d tables takes %d seconds" % (self.numOfTables, (time.time() - startTime))) + + print("inserting data =======") + threads = [] + startTime = time.time() + for i in range(self.numOfThreads): + thread = threading.Thread(target=self.insertData, args=(i,)) + thread.start() + threads.append(thread) + + for i in range(self.numOfThreads): + threads[i].join() + print("inserting %d records takes %d seconds" % (self.numOfTables * self.recordsPerTable, (time.time() - startTime))) + +ri = RestfulInsert() +ri.init() +ri.run() \ No newline at end of file diff --git a/tests/pytest/query/bug1874.py b/tests/pytest/query/bug1874.py new file mode 100644 index 0000000000000000000000000000000000000000..717748487089f229cc578882b91000a7d488fcda --- /dev/null +++ b/tests/pytest/query/bug1874.py @@ -0,0 +1,59 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table join_mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))") + tdSql.execute("create table join_mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))") + stable=0 + insertRows = 1000 + tbnum = 3 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(tbnum): + tdSql.execute("create table join_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i)) + for j in range(insertRows): + ret = tdSql.execute( + "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" % + (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100))) + stable=stable+1 + for i in range(tbnum): + tdSql.execute("create table join_1_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i)) + for j in range(insertRows): + ret = tdSql.execute( + "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" % + (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100))) + print("==========step2") + print("join query ") + tdLog.info("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/bug1875.py b/tests/pytest/query/bug1875.py new file mode 100644 index 0000000000000000000000000000000000000000..12e22a7a259e1612d8da4f11647a0adcde5dae02 --- /dev/null +++ b/tests/pytest/query/bug1875.py @@ -0,0 +1,60 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table join_mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))") + tdSql.execute("create table join_mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))") + stable=0 + insertRows = 1000 + tbnum = 3 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(tbnum): + tdSql.execute("create table join_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i)) + for j in range(insertRows): + ret = tdSql.execute( + "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" % + (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100))) + stable=stable+1 + for i in range(tbnum): + tdSql.execute("create table join_1_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i)) + for j in range(insertRows): + ret = tdSql.execute( + "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" % + (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100))) + print("==========step2") + print("join query ") + tdLog.info("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc limit 3;") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc slimit 3;") + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/bug1876.py b/tests/pytest/query/bug1876.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cc4b6eb2dc3368fdb21989d3b8df27f74cc8f9 --- /dev/null +++ b/tests/pytest/query/bug1876.py @@ -0,0 +1,58 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table join_mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))") + tdSql.execute("create table join_mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))") + stable=0 + insertRows = 1000 + tbnum = 3 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(tbnum): + tdSql.execute("create table join_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i)) + for j in range(insertRows): + ret = tdSql.execute( + "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" % + (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100))) + stable=stable+1 + for i in range(tbnum): + tdSql.execute("create table join_1_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i)) + for j in range(insertRows): + ret = tdSql.execute( + "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" % + (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100))) + print("==========step2") + print("join query ") + tdLog.info("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1)-first(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;") + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/dataLossTest.py b/tests/pytest/query/dataLossTest.py new file mode 100644 index 0000000000000000000000000000000000000000..b29dc1fa9f977b1f3f68b3e8f7781c6ac5a1e646 --- /dev/null +++ b/tests/pytest/query/dataLossTest.py @@ -0,0 +1,76 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import inspect + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.numberOfTables = 240 + self.numberOfRecords = 10000 + + def run(self): + tdSql.prepare() + + os.system("yes | taosdemo -t %d -n %d" % (self.numberOfTables, self.numberOfRecords)) + print("==============step1") + + tdSql.execute("use test") + sql = "select count(*) from meters" + tdSql.query(sql) + rows = tdSql.getData(0, 0) + print ("number of records: %d" % rows) + + newRows = rows + for i in range(10000): + print("kill taosd") + time.sleep(10) + os.system("sudo kill -9 $(pgrep taosd)") + tdDnodes.startWithoutSleep(1) + while True: + try: + tdSql.query(sql) + newRows = tdSql.getData(0, 0) + print("numer of records after kill taosd %d" % newRows) + time.sleep(10) + break + except Exception as e: + pass + continue + + if newRows < rows: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, newRows, rows) + tdLog.exit("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) + break + + tdSql.query(sql) + tdSql.checkData(0, 0, rows) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/kill_query.py b/tests/pytest/query/kill_query.py new file mode 100644 index 0000000000000000000000000000000000000000..8975eea2685b93e551e0008fbf5013d5e16e934f --- /dev/null +++ b/tests/pytest/query/kill_query.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +import os +import threading +import time + + +class TDTestCase: + """ + kill query + """ + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def query(self): + conn = taos.connect(host='127.0.0.1', user='root', password='taosdata', config='/etc/config') + cursor = conn.cursor() + while True: + cursor.execute('show queries;') + print('show queries!') + temp = cursor.fetchall() + if temp: + print(temp[0][0]) + cursor.execute('kill query %s ;' % temp[0][0]) + print('kill query success') + break + time.sleep(0.5) + cursor.close() + conn.close() + + def run(self): + tdSql.prepare() + + print("==============step1") + os.system('yes | sudo taosdemo -n 100') + print('insert into test.meters 10000000 rows') + + + t1 = threading.Thread(target=self.query) + t1.setDaemon(True) + t1.start() + + print("==============step2") + tdSql.execute('use test;') + try: + print('============begin select * from 10000000 rows') + tdSql.query('select * from test.meters;') + # print(tdSql.queryResult) + except Exception as e: + if not "ProgrammingError('Query terminated'" in str(e): + raise Exception('fail') + + print('success') + print('kill query success') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + diff --git a/tests/pytest/query/queryCountCSVData.py b/tests/pytest/query/queryCountCSVData.py new file mode 100644 index 0000000000000000000000000000000000000000..6c73425faec24afeed0c8a5a168f575ec182c771 --- /dev/null +++ b/tests/pytest/query/queryCountCSVData.py @@ -0,0 +1,71 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + + +class TDTestCase: + """ + create table and insert data from disordered.csv which timestamp is disordered and + ordered.csv which timestamp is ordered. + then execute 'select count(*) from table xx;' + """ + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create database if not exists demo;"); + tdSql.execute("use demo;") + tdSql.execute("CREATE TABLE IF NOT EXISTS test1 (ts TIMESTAMP, ValueID int, " + "VariantValue float, Quality int, Flags int);") + tdSql.execute("CREATE TABLE IF NOT EXISTS test2 (ts TIMESTAMP, ValueID int, " + "VariantValue float, Quality int, Flags int);") + ordered_csv = __file__.split('query')[0] + 'test_data/ordered.csv' + disordered_csv = __file__.split('query')[0] + 'test_data/disordered.csv' + + tdSql.execute(" insert into test1 file '{file}';".format(file=ordered_csv)) + tdSql.execute(" insert into test2 file '{file}';".format(file=disordered_csv)) + print("==============insert into test1 and test2 form test file") + + + print("==============step2") + tdSql.query('select * from test1;') + with open(ordered_csv) as f1: + num1 = len(f1.readlines()) + tdSql.checkRows(num1) + + + tdSql.query('select * from test2;') + with open(disordered_csv) as f2: + num2 = len(f2.readlines()) + tdSql.checkRows(num2) + print("=============execute select count(*) from xxx") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py index 99222016048f71ae66768665fb70522d466b248b..ec3255ba7f12302f6ee00aeda391c11972f1cf26 100644 --- a/tests/pytest/query/queryInterval.py +++ b/tests/pytest/query/queryInterval.py @@ -35,10 +35,11 @@ class TDTestCase: % (self.ts, self.ts + 2000000000, self.ts + 4000000000, self.ts + 5000000000, self.ts + 7000000000)) tdSql.query("select avg(voltage) from st interval(1n)") - tdSql.checkRows(3) - tdSql.checkData(0, 1, 221.4) - tdSql.checkData(1, 1, 227.0) - tdSql.checkData(2, 1, 222.0) + tdSql.checkRows(4) + tdSql.checkData(0, 1, 220.0) + tdSql.checkData(1, 1, 222.33333333333334) + tdSql.checkData(2, 1, 227.0) + tdSql.checkData(3, 1, 222.0) tdSql.query("select avg(voltage) from st interval(1n, 15d)") tdSql.checkRows(4) diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py index 17027cf498ff9e87b558866cd4d1e6a8c865afc0..5ad49a265ea52279f7934476a4a1db2fbbdf5883 100644 --- a/tests/pytest/query/queryJoin.py +++ b/tests/pytest/query/queryJoin.py @@ -95,16 +95,18 @@ class TDTestCase: tdSql.error( "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id group by stb_t.id") tdSql.error( - "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.name;") - tdSql.error( - "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.location = stb_t.name") + "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.name;") tdSql.execute("alter table stb_t add tag pid int") tdSql.execute("alter table tb_t1 set tag pid=2") tdSql.execute("alter table tb_t2 set tag pid=1") + tdSql.query( + "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.location = stb_t.name") + tdSql.checkRows(0) + tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.pid") - tdSql.checkRows(3) + tdSql.checkRows(6) tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") tdSql.checkRows(6) diff --git a/tests/pytest/query/queryLike.py b/tests/pytest/query/queryLike.py new file mode 100644 index 0000000000000000000000000000000000000000..3c3b030f8f718371867f323f2496fb17b6b962e1 --- /dev/null +++ b/tests/pytest/query/queryLike.py @@ -0,0 +1,45 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdSql.execute("create table cars(ts timestamp, c nchar(2)) tags(t1 nchar(2))") + tdSql.execute("insert into car0 using cars tags('aa') values(now, 'bb');") + tdSql.query("select count(*) from cars where t1 like '%50 90 30 04 00 00%'") + tdSql.checkRows(0) + + tdSql.execute("create table test_cars(ts timestamp, c nchar(2)) tags(t1 nchar(20))") + tdSql.execute("insert into car1 using test_cars tags('150 90 30 04 00 002') values(now, 'bb');") + tdSql.query("select * from test_cars where t1 like '%50 90 30 04 00 00%'") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/queryPerformance.py b/tests/pytest/query/queryPerformance.py index a7fc08c5a39d7b9a59068463f521d83106865c2d..e09900acc4cde66b1f2505c0fb532889d2db81e4 100644 --- a/tests/pytest/query/queryPerformance.py +++ b/tests/pytest/query/queryPerformance.py @@ -11,6 +11,7 @@ # -*- coding: utf-8 -*- + import sys import os import taos @@ -32,17 +33,23 @@ class taosdemoQueryPerformace: def query(self): cursor = self.conn.cursor() - cursor.execute("use test") + cursor.execute("use test") totalTime = 0 - for i in range(100): - startTime = time.time() + for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") + startTime = time.time() cursor.execute("select count(*) from test.meters") totalTime += time.time() - startTime print("query time for: select count(*) from test.meters %f seconds" % (totalTime / 100)) totalTime = 0 for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") startTime = time.time() cursor.execute("select avg(f1), max(f2), min(f3) from test.meters") totalTime += time.time() - startTime @@ -50,6 +57,9 @@ class taosdemoQueryPerformace: totalTime = 0 for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") startTime = time.time() cursor.execute("select count(*) from test.meters where loc='beijing'") totalTime += time.time() - startTime @@ -57,6 +67,9 @@ class taosdemoQueryPerformace: totalTime = 0 for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") startTime = time.time() cursor.execute("select avg(f1), max(f2), min(f3) from test.meters where areaid=10") totalTime += time.time() - startTime @@ -64,6 +77,9 @@ class taosdemoQueryPerformace: totalTime = 0 for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") startTime = time.time() cursor.execute("select avg(f1), max(f2), min(f3) from test.t10 interval(10s)") totalTime += time.time() - startTime @@ -71,11 +87,34 @@ class taosdemoQueryPerformace: totalTime = 0 for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") startTime = time.time() cursor.execute("select last_row(*) from meters") totalTime += time.time() - startTime print("query time for: select last_row(*) from meters %f seconds" % (totalTime / 100)) + totalTime = 0 + for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") + startTime = time.time() + cursor.execute("select * from meters") + totalTime += time.time() - startTime + print("query time for: select * from meters %f seconds" % (totalTime / 100)) + + totalTime = 0 + for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") + startTime = time.time() + cursor.execute("select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'") + totalTime += time.time() - startTime + print("query time for: select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000' %f seconds" % (totalTime / 100)) + if __name__ == '__main__': perftest = taosdemoQueryPerformace() perftest.initConnection() diff --git a/tests/pytest/query/querySort.py b/tests/pytest/query/querySort.py index e5d3c8ce1f4eb9c1d2003bd659771562c9ea14e5..649e0dc1cb3191ba08b3f2da0a5edee3afc66575 100644 --- a/tests/pytest/query/querySort.py +++ b/tests/pytest/query/querySort.py @@ -96,6 +96,12 @@ class TDTestCase: tdSql.query("select * from st order by ts desc") self.checkColumnSorted(0, "desc") + print("======= step 2: verify order for special column =========") + + tdSql.query("select tbcol1 from st order by ts desc") + + tdSql.query("select tbcol6 from st order by ts desc") + for i in range(1, 10): tdSql.error("select * from st order by tbcol%d" % i) tdSql.error("select * from st order by tbcol%d asc" % i) diff --git a/tests/pytest/query/query_performance.py b/tests/pytest/query/query_performance.py new file mode 100644 index 0000000000000000000000000000000000000000..c31569ac1382aca17d3d5b37fbef981620c0df83 --- /dev/null +++ b/tests/pytest/query/query_performance.py @@ -0,0 +1,219 @@ +import time +import taos +import csv +import numpy as np +import random +import os +import requests +import json +import sys + +""" +需要第三方库: taos,requests,numpy +当前机器已经启动taosd服务 +使用方法见底部示例 +""" + + + +class Ding: + """ + 发送消息到钉钉, + urls: 钉钉群的token组成的list,可以发多个钉钉群,需要提前加白名单或其他放行策略 + at_mobiles: 需要@的人的手机号组成的list + msg: 要发送的str + """ + def __init__(self, url_list, at_mobiles): + self.urls = url_list + self.at_mobiles = at_mobiles + + def send_message(self, msg): + data1 = { + "msgtype": "text", + "text": { + "content": msg + }, + "at": { + "atMobiles": self.at_mobiles, + "isAtAll": False + } + } + + header = {'Content-Type': 'application/json; charset=utf-8'} + + for url in self.urls: + requests.post(url=url, data=json.dumps(data1), headers=header) + + + + +class TDConn: + def __init__(self, config:dict): + self.host = config['host'] + self.user = config['user'] + self.password = config['password'] + self.config = config['config'] + self.conn = None + self.cursor = None + + def connect(self): + conn = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config) + cursor = conn.cursor() + self.conn = conn + self.cursor = cursor + print('connect ...') + return self.cursor + + def close(self): + self.cursor.close() + self.conn.close() + print('close ... ') + + +class Tool: + """ + 可能有用 + """ + @staticmethod + def str_gen(num): + return ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', num)) + + @staticmethod + def float_gen(n, m): + return random.uniform(n, m) + + @staticmethod + def int_gen(n, m): + return random.randint(n, m) + +class Demo: + def __init__(self, engine): + self.engine = engine['engine'](engine['config']) + self.cursor = self.engine.connect() + + + def date_gen(self, db, number_per_table, type_of_cols, num_of_cols_per_record, num_of_tables): + """ + :目前都是 taosdemo 的参数 + :return: + """ + sql = 'yes | sudo taosdemo -d {db} -n {number_per_table} -b {type_of_cols} -l {num_of_cols_per_record} ' \ + '-t {num_of_tables}'.format(db=db, number_per_table=number_per_table, type_of_cols=type_of_cols, + num_of_cols_per_record=num_of_cols_per_record, num_of_tables=num_of_tables) + os.system(sql) + print('insert data completed') + + + # def main(self, db, circle, csv_name, case_func, result_csv, nums, ding_flag): + def main(self, every_num_per_table, result_csv, all_result_csv, values): + db = values['db_name'] + number_per_table = every_num_per_table + type_of_cols = values['col_type'] + num_of_cols_per_record = values['col_num'] + num_of_tables = values['table_num'] + self.date_gen(db=db, number_per_table=number_per_table, type_of_cols=type_of_cols, + num_of_cols_per_record=num_of_cols_per_record, num_of_tables=num_of_tables) + + circle = values['circle'] + # print(every_num_per_table, result_csv, values) + csv_name = result_csv + case_func = values['sql_func'] + nums = num_of_tables * number_per_table + ding_flag = values['ding_flag'] + + _data = [] + f = open(csv_name,'w',encoding='utf-8') + f1 = open(all_result_csv,'a',encoding='utf-8') + csv_writer = csv.writer(f) + csv_writer1 = csv.writer(f1) + csv_writer.writerow(["number", "elapse", 'sql']) + self.cursor.execute('use {db};'.format(db=db)) + + + for i in range(circle): + self.cursor.execute('reset query cache;') + sql = case_func() + start = time.time() + self.cursor.execute(sql) + self.cursor.fetchall() + end = time.time() + _data.append(end-start) + elapse = '%.4f' %(end -start) + print(sql, i, elapse, '\n') + csv_writer.writerow([i+1, elapse, sql]) + + # time.sleep(1) + _list = [nums, np.mean(_data)] + _str = '总数据: %s 条 , table数: %s , 每个table数据数: %s , 数据类型: %s \n' % \ + (nums, num_of_tables, number_per_table, type_of_cols) + # print('avg : ', np.mean(_data), '\n') + _str += '平均值 : %.4f 秒\n' % np.mean(_data) + for each in (50, 80, 90, 95): + _list.append(np.percentile(_data,each)) + _str += ' %d 分位数 : %.4f 秒\n' % (each , np.percentile(_data,each)) + + print(_str) + if ding_flag: + ding = Ding(values['ding_config']['urls'], values['ding_config']['at_mobiles']) + ding.send_message(_str) + csv_writer1.writerow(_list) + f.close() + f1.close() + self.engine.close() + + +def run(engine, test_cases: dict, result_dir): + for each_case, values in test_cases.items(): + for every_num_per_table in values['number_per_table']: + result_csv = result_dir + '{case}_table{table_num}_{number_per_table}.csv'.\ + format(case=each_case, table_num=values['table_num'], number_per_table=every_num_per_table) + all_result_csv = result_dir + '{case_all}_result.csv'.format(case_all=each_case) + d = Demo(engine) + # print(each_case, result_csv) + d.main(every_num_per_table, result_csv, all_result_csv, values) + + + +if __name__ == '__main__': + """ + 测试用例在test_cases中添加。 + result_dir: 报告生成目录,会生成每次测试结果,和具体某一用例的统计结果.需注意目录权限需要执行用户可写。 + case1、case2 : 具体用例名称 + engine: 数据库引擎,目前只有taosd。使用时需开启taosd服务。 + table_num: 造数据时的table数目 + circle: 循环测试次数,求平均值 + number_per_table:需要传list,多个数值代表会按照list内的数值逐个测试 + col_num:table col的数目 + col_type: 表中数据类型 + db_name: 造数据的db名,默认用test + sql_func: 当前测试的sql方法,需要自己定义 + ding_flag: 如果需要钉钉发送数据,flag设置真值, + ding_config: 如ding_flag 设置为真值,此项才有意义。ding_flag为假时此项可以为空。urls传入一list,内容为要发送的群的token, + 需提前设置白名单,at_mobiles传入一list,内容为在群内需要@的人的手机号 + """ + engine_dict = { + 'taosd': {'engine': TDConn, 'config': + {'host': '127.0.0.1', 'user': 'root', 'password': 'taosdata', 'config':'/etc/taos'}} + } + + def case1(): + return 'select * from meters where f1 = {n};'.format(n=random.randint(1,30)) + + def case2(): + return 'select * from meters where f1 = %.4f;' %random.uniform(1,30) + + + result_dir = '/usr/local/demo/benchmarktestdata/' + test_cases = { + 'case1': {'engine':'taosd', 'table_num': 10, 'circle': 100, 'number_per_table':[10, 100], 'col_num': 5, + 'col_type': 'INT', 'db_name': 'test', 'sql_func': case1, 'ding_flag': True, + 'ding_config': + {'urls': [r'https://oapi.dingtalk.com/robot/send?access_token=xxxxxxx0cd93'], + 'at_mobiles':[17000000000,],}}, + 'case2': {'engine':'taosd', 'table_num': 10, 'circle': 50, 'number_per_table':[10, 100], 'col_num': 5, + 'col_type': 'FLOAT', 'db_name': 'test', 'sql_func': case2, 'ding_flag': False, + 'ding_config': None + } + } + + run(engine_dict['taosd'], test_cases, result_dir) diff --git a/tests/pytest/query/sliding.py b/tests/pytest/query/sliding.py new file mode 100644 index 0000000000000000000000000000000000000000..810d90117a73bba81ea010f81aa605196a496dc4 --- /dev/null +++ b/tests/pytest/query/sliding.py @@ -0,0 +1,63 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +import random + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1500000000000 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table meters(ts timestamp, col1 int) tags(id int, loc nchar(20))") + sql = "insert into t0 using meters tags(1, 'beijing') values" + for i in range(100): + sql += "(%d, %d)" % (self.ts + i * 1000, random.randint(1, 100)) + tdSql.execute(sql) + + sql = "insert into t1 using meters tags(2, 'shanghai') values" + for i in range(100): + sql += "(%d, %d)" % (self.ts + i * 1000, random.randint(1, 100)) + tdSql.execute(sql) + + tdSql.query("select count(*) from meters interval(10s) sliding(5s)") + tdSql.checkRows(21) + + tdSql.error("select count(*) from meters sliding(5s)") + + tdSql.error("select count(*) from meters sliding(5s) interval(10s)") + + tdSql.error("select * from meters sliding(5s) order by ts desc") + + tdSql.query("select count(*) from meters group by loc") + tdSql.checkRows(2) + + tdSql.error("select * from meters group by loc sliding(5s)") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/stream/new.py b/tests/pytest/stream/new.py index eac93dc0e649f5d48481079d75851a27be270567..12ec6d4507710869632eac77d719217b3b0ed7b3 100644 --- a/tests/pytest/stream/new.py +++ b/tests/pytest/stream/new.py @@ -26,7 +26,6 @@ class TDTestCase: def run(self): rowNum = 200 - totalNum = 200 tdSql.prepare() tdLog.info("=============== step1") @@ -42,7 +41,9 @@ class TDTestCase: tdSql.execute("create table st as select count(*), count(tbcol), count(tbcol2) from mt interval(10s)") tdLog.info("=============== step3") + start = time.time() tdSql.waitedQuery("select * from st", 1, 120) + delay = int(time.time() - start) + 20 v = tdSql.getData(0, 3) if v >= 51: tdLog.exit("value is %d, which is larger than 51" % v) @@ -54,11 +55,18 @@ class TDTestCase: tdSql.execute("insert into tb%d values(now + %ds, %d, %d)" % (i, j, j, j)) tdLog.info("=============== step5") - tdLog.sleep(40) - tdSql.waitedQuery("select * from st order by ts desc", 1, 120) - v = tdSql.getData(0, 3) - if v <= 51: - tdLog.exit("value is %d, which is smaller than 51" % v) + maxValue = 0 + for i in range(delay): + time.sleep(1) + tdSql.query("select * from st order by ts desc") + v = tdSql.getData(0, 3) + if v > maxValue: + maxValue = v + if v > 51: + break + + if maxValue <= 51: + tdLog.exit("value is %d, which is smaller than 51" % maxValue) def stop(self): tdSql.close() diff --git a/tests/pytest/table/create_sensitive.py b/tests/pytest/table/create_sensitive.py new file mode 100644 index 0000000000000000000000000000000000000000..1934b662c7a57c13c2a1b8e8dfd65ab6ddbe13a4 --- /dev/null +++ b/tests/pytest/table/create_sensitive.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +import sys +import string +import random +import subprocess +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table TestSensitiveT(ts timestamp, i int)') + tdSql.execute('create table TestSensitiveT(ts timestamp, i int)') + tdLog.info('create table TestSensitiveSt(ts timestamp,i int) tags(j int)') + tdSql.execute('create table TestSensitiveSt(ts timestamp,i int) tags(j int)') + tdLog.info('create table Abcde using TestSensitiveSt tags(1)') + tdSql.execute('create table AbcdeFgh using TestSensitiveSt tags(1)') + tdLog.info('=============== step2') + tdLog.info('test normal table ') + tdSql.error('create table testsensitivet(ts timestamp, i int)') + tdSql.error('create table testsensitivet(ts timestamp, j int)') + tdSql.error('create table testsensItivet(ts timestamp, j int)') + tdSql.error('create table TESTSENSITIVET(ts timestamp, i int)') + tdLog.info('=============== step3') + tdLog.info('test super table ') + tdSql.error('create table testsensitivest(ts timestamp,i int) tags(j int)') + tdSql.error('create table testsensitivest(ts timestamp,i int) tags(k int)') + tdSql.error('create table TESTSENSITIVEST(ts timestamp,i int) tags(j int)') + tdSql.error('create table Testsensitivest(ts timestamp,i int) tags(j int)') + tdLog.info('=============== step4') + tdLog.info('test subtable ') + tdSql.error('create table abcdefgh using TestSensitiveSt tags(1)') + tdSql.error('create table ABCDEFGH using TestSensitiveSt tags(1)') + tdSql.error('create table Abcdefgh using TestSensitiveSt tags(1)') + tdSql.error('create table abcdeFgh using TestSensitiveSt tags(1)') + tdSql.error('insert into table abcdefgh using TestSensitiveSt tags(1) values(now,1)') + tdSql.error('insert into table ABCDEFGH using TestSensitiveSt tags(1) values(now,1)') + tdSql.error('insert into table Abcdefgh using TestSensitiveSt tags(1) values(now,1)') + tdSql.error('insert into table abcdeFgH using TestSensitiveSt tags(1) values(now,1)') + tdSql.query('show tables') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(2) + + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/test_data/__init__.py b/tests/pytest/test_data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1bd7af5b9db25bf22f960bfca6bf18b1518cc86f --- /dev/null +++ b/tests/pytest/test_data/__init__.py @@ -0,0 +1,15 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + + +""" +this directory contains test data files +""" \ No newline at end of file diff --git a/tests/pytest/test_data/disordered.csv b/tests/pytest/test_data/disordered.csv new file mode 100644 index 0000000000000000000000000000000000000000..0e6fd75e5b1a43da360541f57377236e0984093a --- /dev/null +++ b/tests/pytest/test_data/disordered.csv @@ -0,0 +1,500 @@ +"2020-03-01 20:01:49.493","130","7.595","128","8392704" +"2020-03-01 20:01:50.493","130","7.598","128","8392704" +"2020-03-01 20:01:51.493","130","7.602","128","8392704" +"2020-03-01 20:01:52.493","130","7.604","128","8392704" +"2020-03-01 20:01:53.493","130","7.604","128","8392704" +"2020-03-01 20:01:54.493","130","7.606","128","8392704" +"2020-03-01 20:01:55.493","130","7.607","128","8392704" +"2020-03-01 20:01:56.493","130","7.607","128","8392704" +"2020-03-01 20:01:57.493","130","7.607","128","8392704" +"2020-03-01 20:01:58.493","130","7.607","128","8392704" +"2020-03-01 20:01:59.493","130","7.606","128","8392704" +"2020-03-01 20:02:00.493","130","7.606","128","8392704" +"2020-03-01 20:02:01.493","130","7.606","128","8392704" +"2020-03-01 20:02:02.493","130","7.607","128","8392704" +"2020-03-01 20:02:03.493","130","7.608","128","8392704" +"2020-03-01 20:02:04.493","130","7.609","128","8392704" +"2020-03-01 20:02:05.493","130","7.609","128","8392704" +"2020-03-01 20:02:06.493","130","7.608","128","8392704" +"2020-03-01 20:02:07.493","130","7.606","128","8392704" +"2020-03-01 20:02:08.493","130","7.606","128","8392704" +"2020-03-01 20:02:09.493","130","7.607","128","8392704" +"2020-03-01 20:02:10.493","130","7.609","128","8392704" +"2020-03-01 20:02:11.493","130","7.61","128","8392704" +"2020-03-01 20:02:12.493","130","7.611","128","8392704" +"2020-03-01 20:02:13.493","130","7.61","128","8392704" +"2020-03-01 20:02:14.493","130","7.607","128","8392704" +"2020-03-01 20:02:15.493","130","7.605","128","8392704" +"2020-03-01 20:02:16.493","130","7.604","128","8392704" +"2020-03-01 20:02:17.493","130","7.603","128","8392704" +"2020-03-01 20:02:18.493","130","7.604","128","8392704" +"2020-03-01 20:02:19.493","130","7.604","128","8392704" +"2020-03-01 20:02:20.493","130","7.603","128","8392704" +"2020-03-01 20:02:21.493","130","7.601","128","8392704" +"2020-03-01 20:02:22.493","130","7.598","128","8392704" +"2020-03-01 20:02:23.493","130","7.595","128","8392704" +"2020-03-01 20:02:24.493","130","7.594","128","8392704" +"2020-03-01 20:02:25.493","130","7.594","128","8392704" +"2020-03-01 20:02:26.493","130","7.595","128","8392704" +"2020-03-01 20:02:27.493","130","7.596","128","8392704" +"2020-03-01 20:02:28.493","130","7.596","128","8392704" +"2020-03-01 20:02:29.493","130","7.598","128","8392704" +"2020-03-01 20:02:30.493","130","7.6","128","8392704" +"2020-03-01 20:02:31.493","130","7.6","128","8392704" +"2020-03-01 20:02:32.493","130","7.6","128","8392704" +"2020-03-01 20:02:33.493","130","7.601","128","8392704" +"2020-03-01 20:02:34.493","130","7.603","128","8392704" +"2020-03-01 20:02:35.493","130","7.604","128","8392704" +"2020-03-01 20:02:36.493","130","7.605","128","8392704" +"2020-03-01 20:02:37.493","130","7.606","128","8392704" +"2020-03-01 20:02:38.493","130","7.61","128","8392704" +"2020-03-01 20:02:39.493","130","7.611","128","8392704" +"2020-03-01 20:02:40.493","130","7.61","128","8392704" +"2020-03-01 20:02:41.493","130","7.609","128","8392704" +"2020-03-01 20:02:42.493","130","7.611","128","8392704" +"2020-03-01 20:02:43.493","130","7.61","128","8392704" +"2020-03-01 20:02:44.493","130","7.607","128","8392704" +"2020-03-01 20:02:45.493","130","7.605","128","8392704" +"2020-03-01 20:02:46.493","130","7.606","128","8392704" +"2020-03-01 20:02:47.493","130","7.604","128","8392704" +"2020-03-01 20:02:48.493","130","7.599","128","8392704" +"2020-03-01 20:02:49.493","130","7.595","128","8392704" +"2020-03-01 06:41:17.493","130","6.742","128","8392704" +"2020-03-01 06:41:18.493","130","6.741","128","8392704" +"2020-03-01 06:41:19.493","130","6.737","128","8392704" +"2020-03-01 06:41:20.493","130","6.734","128","8392704" +"2020-03-01 06:41:21.493","130","6.734","128","8392704" +"2020-03-01 06:41:22.493","130","6.733","128","8392704" +"2020-03-01 06:41:23.493","130","6.736","128","8392704" +"2020-03-01 06:41:24.493","130","6.739","128","8392704" +"2020-03-01 06:41:25.493","130","6.738","128","8392704" +"2020-03-01 06:41:26.493","130","6.74","128","8392704" +"2020-03-01 06:41:27.493","130","6.745","128","8392704" +"2020-03-01 06:41:28.493","130","6.749","128","8392704" +"2020-03-01 06:41:29.493","130","6.753","128","8392704" +"2020-03-01 06:41:30.493","130","6.753","128","8392704" +"2020-03-01 06:41:31.493","130","6.757","128","8392704" +"2020-03-01 06:41:32.493","130","6.763","128","8392704" +"2020-03-01 06:41:33.493","130","6.765","128","8392704" +"2020-03-01 06:41:34.493","130","6.764","128","8392704" +"2020-03-01 06:41:35.493","130","6.762","128","8392704" +"2020-03-01 06:41:36.493","130","6.758","128","8392704" +"2020-03-01 06:41:37.493","130","6.756","128","8392704" +"2020-03-01 06:41:38.493","130","6.755","128","8392704" +"2020-03-01 06:41:39.493","130","6.754","128","8392704" +"2020-03-01 06:41:40.493","130","6.755","128","8392704" +"2020-03-01 06:41:41.493","130","6.756","128","8392704" +"2020-03-01 06:41:42.493","130","6.757","128","8392704" +"2020-03-01 06:41:43.493","130","6.756","128","8392704" +"2020-03-01 06:41:44.493","130","6.756","128","8392704" +"2020-03-01 06:41:45.493","130","6.756","128","8392704" +"2020-03-01 06:41:46.493","130","6.759","128","8392704" +"2020-03-01 06:41:47.493","130","6.759","128","8392704" +"2020-03-01 06:41:48.493","130","6.758","128","8392704" +"2020-03-01 06:41:49.493","130","6.758","128","8392704" +"2020-03-01 06:41:50.493","130","6.756","128","8392704" +"2020-03-01 06:41:51.493","130","6.755","128","8392704" +"2020-03-01 06:41:52.493","130","6.755","128","8392704" +"2020-03-01 06:41:53.493","130","6.754","128","8392704" +"2020-03-01 06:41:54.493","130","6.751","128","8392704" +"2020-03-01 06:41:55.493","130","6.752","128","8392704" +"2020-03-01 06:41:56.493","130","6.753","128","8392704" +"2020-03-01 06:41:57.493","130","6.753","128","8392704" +"2020-03-01 06:41:58.493","130","6.753","128","8392704" +"2020-03-01 06:41:59.493","130","6.755","128","8392704" +"2020-03-01 06:42:00.493","130","6.752","128","8392704" +"2020-03-01 06:42:01.493","130","6.75","128","8392704" +"2020-03-01 06:42:02.493","130","6.75","128","8392704" +"2020-03-01 06:42:03.493","130","6.753","128","8392704" +"2020-03-01 06:42:04.493","130","6.755","128","8392704" +"2020-03-01 06:42:05.493","130","6.754","128","8392704" +"2020-03-01 06:42:06.493","130","6.754","128","8392704" +"2020-03-01 06:42:07.493","130","6.752","128","8392704" +"2020-03-01 06:42:08.493","130","6.748","128","8392704" +"2020-03-01 06:42:09.493","130","6.747","128","8392704" +"2020-03-01 06:42:10.493","130","6.747","128","8392704" +"2020-03-01 06:42:11.493","130","6.748","128","8392704" +"2020-03-01 06:42:12.493","130","6.748","128","8392704" +"2020-03-01 06:42:13.493","130","6.75","128","8392704" +"2020-03-01 06:42:14.493","130","6.754","128","8392704" +"2020-03-01 06:42:15.493","130","6.754","128","8392704" +"2020-03-01 06:42:16.493","130","6.756","128","8392704" +"2020-03-01 06:42:17.493","130","6.756","128","8392704" +"2020-03-01 06:42:18.493","130","6.757","128","8392704" +"2020-03-01 06:42:19.493","130","6.757","128","8392704" +"2020-03-01 06:42:20.493","130","6.76","128","8392704" +"2020-03-01 06:42:21.493","130","6.761","128","8392704" +"2020-03-01 06:42:22.493","130","6.76","128","8392704" +"2020-03-01 06:42:23.493","130","6.76","128","8392704" +"2020-03-01 06:42:24.493","130","6.76","128","8392704" +"2020-03-01 06:42:25.493","130","6.76","128","8392704" +"2020-03-01 06:42:26.493","130","6.758","128","8392704" +"2020-03-01 06:42:27.493","130","6.757","128","8392704" +"2020-03-01 06:42:28.493","130","6.752","128","8392704" +"2020-03-01 06:42:29.493","130","6.746","128","8392704" +"2020-03-01 06:42:30.493","130","6.742","128","8392704" +"2020-03-01 06:42:31.493","130","6.741","128","8392704" +"2020-03-01 06:42:32.493","130","6.739","128","8392704" +"2020-03-01 06:42:33.493","130","6.739","128","8392704" +"2020-03-01 06:42:34.493","130","6.737","128","8392704" +"2020-03-01 06:42:35.493","130","6.737","128","8392704" +"2020-03-01 06:42:36.493","130","6.738","128","8392704" +"2020-03-01 06:42:37.493","130","6.739","128","8392704" +"2020-03-01 06:42:38.493","130","6.743","128","8392704" +"2020-03-01 06:42:39.493","130","6.747","128","8392704" +"2020-03-01 06:42:40.493","130","6.748","128","8392704" +"2020-03-01 06:42:41.493","130","6.746","128","8392704" +"2020-03-01 06:42:42.493","130","6.746","128","8392704" +"2020-03-01 06:42:43.493","130","6.745","128","8392704" +"2020-03-01 06:42:44.493","130","6.742","128","8392704" +"2020-03-01 06:42:45.493","130","6.741","128","8392704" +"2020-03-01 06:42:46.493","130","6.74","128","8392704" +"2020-03-01 06:42:47.493","130","6.742","128","8392704" +"2020-03-01 06:42:48.493","130","6.743","128","8392704" +"2020-03-01 06:42:49.493","130","6.742","128","8392704" +"2020-03-01 06:42:50.493","130","6.742","128","8392704" +"2020-03-01 06:42:51.493","130","6.741","128","8392704" +"2020-03-01 06:42:52.493","130","6.741","128","8392704" +"2020-03-01 06:42:53.493","130","6.742","128","8392704" +"2020-03-01 06:42:54.493","130","6.742","128","8392704" +"2020-03-01 06:42:55.493","130","6.745","128","8392704" +"2020-03-01 06:42:56.493","130","6.747","128","8392704" +"2020-03-01 06:42:57.493","130","6.748","128","8392704" +"2020-03-01 06:42:58.493","130","6.75","128","8392704" +"2020-03-01 06:42:59.493","130","6.75","128","8392704" +"2020-03-01 06:43:00.493","130","6.748","128","8392704" +"2020-03-01 06:43:01.493","130","6.748","128","8392704" +"2020-03-01 06:43:02.493","130","6.746","128","8392704" +"2020-03-01 06:43:03.493","130","6.745","128","8392704" +"2020-03-01 06:43:04.493","130","6.745","128","8392704" +"2020-03-01 06:43:05.493","130","6.745","128","8392704" +"2020-03-01 06:43:06.493","130","6.744","128","8392704" +"2020-03-01 06:43:07.493","130","6.749","128","8392704" +"2020-03-01 06:43:08.493","130","6.756","128","8392704" +"2020-03-01 06:43:09.493","130","6.759","128","8392704" +"2020-03-01 06:43:10.493","130","6.759","128","8392704" +"2020-03-01 06:43:11.493","130","6.758","128","8392704" +"2020-03-01 06:43:12.493","130","6.758","128","8392704" +"2020-03-01 06:43:13.493","130","6.759","128","8392704" +"2020-03-01 06:43:14.493","130","6.759","128","8392704" +"2020-03-01 06:43:15.493","130","6.753","128","8392704" +"2020-03-01 06:43:16.493","130","6.751","128","8392704" +"2020-03-01 20:02:50.493","130","7.596","128","8392704" +"2020-03-01 20:02:51.493","130","7.597","128","8392704" +"2020-03-01 20:02:52.493","130","7.599","128","8392704" +"2020-03-01 20:02:53.493","130","7.6","128","8392704" +"2020-03-01 20:02:54.493","130","7.601","128","8392704" +"2020-03-01 20:02:55.493","130","7.603","128","8392704" +"2020-03-01 20:02:56.493","130","7.602","128","8392704" +"2020-03-01 20:02:57.493","130","7.601","128","8392704" +"2020-03-01 20:02:58.493","130","7.601","128","8392704" +"2020-03-01 20:02:59.493","130","7.6","128","8392704" +"2020-03-01 20:03:00.493","130","7.599","128","8392704" +"2020-03-01 20:03:01.493","130","7.599","128","8392704" +"2020-03-01 20:03:02.493","130","7.6","128","8392704" +"2020-03-01 20:03:03.493","130","7.601","128","8392704" +"2020-03-01 20:03:04.493","130","7.601","128","8392704" +"2020-03-01 20:03:05.493","130","7.601","128","8392704" +"2020-03-01 20:03:06.493","130","7.6","128","8392704" +"2020-03-01 20:03:07.493","130","7.602","128","8392704" +"2020-03-01 20:03:08.493","130","7.606","128","8392704" +"2020-03-01 20:03:09.493","130","7.609","128","8392704" +"2020-03-01 20:03:10.493","130","7.612","128","8392704" +"2020-03-01 20:03:11.493","130","7.614","128","8392704" +"2020-03-01 20:03:12.493","130","7.615","128","8392704" +"2020-03-01 20:03:13.493","130","7.614","128","8392704" +"2020-03-01 20:03:14.493","130","7.613","128","8392704" +"2020-03-01 20:03:15.493","130","7.614","128","8392704" +"2020-03-01 20:03:16.493","130","7.612","128","8392704" +"2020-03-01 20:03:17.493","130","7.609","128","8392704" +"2020-03-01 20:03:18.493","130","7.606","128","8392704" +"2020-03-01 20:03:19.493","130","7.604","128","8392704" +"2020-03-01 20:03:20.493","130","7.604","128","8392704" +"2020-03-01 20:03:21.493","130","7.605","128","8392704" +"2020-03-01 20:03:22.493","130","7.605","128","8392704" +"2020-03-01 20:03:23.493","130","7.605","128","8392704" +"2020-03-01 20:03:24.493","130","7.605","128","8392704" +"2020-03-01 20:03:25.493","130","7.604","128","8392704" +"2020-03-01 20:03:26.493","130","7.603","128","8392704" +"2020-03-01 20:03:27.493","130","7.604","128","8392704" +"2020-03-01 20:03:28.493","130","7.605","128","8392704" +"2020-03-01 20:03:29.493","130","7.607","128","8392704" +"2020-03-01 20:03:30.493","130","7.609","128","8392704" +"2020-03-01 20:03:31.493","130","7.609","128","8392704" +"2020-03-01 20:03:32.493","130","7.607","128","8392704" +"2020-03-01 20:03:33.493","130","7.606","128","8392704" +"2020-03-01 20:03:34.493","130","7.607","128","8392704" +"2020-03-01 20:03:35.493","130","7.608","128","8392704" +"2020-03-01 20:03:36.493","130","7.609","128","8392704" +"2020-03-01 20:03:37.493","130","7.609","128","8392704" +"2020-03-01 20:03:38.493","130","7.607","128","8392704" +"2020-03-01 20:03:39.493","130","7.602","128","8392704" +"2020-03-01 20:03:40.493","130","7.599","128","8392704" +"2020-03-01 20:03:41.493","130","7.598","128","8392704" +"2020-03-01 20:03:42.493","130","7.596","128","8392704" +"2020-03-01 20:03:43.493","130","7.595","128","8392704" +"2020-03-01 20:03:44.493","130","7.594","128","8392704" +"2020-03-01 20:03:45.493","130","7.595","128","8392704" +"2020-03-01 20:03:46.493","130","7.597","128","8392704" +"2020-03-01 20:03:47.493","130","7.596","128","8392704" +"2020-03-01 20:03:48.493","130","7.595","128","8392704" +"2020-03-01 20:03:49.493","130","7.596","128","8392704" +"2020-03-01 20:03:50.493","130","7.596","128","8392704" +"2020-03-01 20:03:51.493","130","7.595","128","8392704" +"2020-03-01 20:03:52.493","130","7.596","128","8392704" +"2020-03-01 20:03:53.493","130","7.597","128","8392704" +"2020-03-01 20:03:54.493","130","7.598","128","8392704" +"2020-03-01 20:03:55.493","130","7.596","128","8392704" +"2020-03-01 20:03:56.493","130","7.596","128","8392704" +"2020-03-01 20:03:57.493","130","7.599","128","8392704" +"2020-03-01 20:03:58.493","130","7.602","128","8392704" +"2020-03-01 20:03:59.493","130","7.603","128","8392704" +"2020-03-01 20:04:00.493","130","7.602","128","8392704" +"2020-03-01 20:04:01.493","130","7.6","128","8392704" +"2020-03-01 20:04:02.493","130","7.598","128","8392704" +"2020-03-01 20:04:03.493","130","7.595","128","8392704" +"2020-03-01 20:04:04.493","130","7.593","128","8392704" +"2020-03-01 20:04:05.493","130","7.592","128","8392704" +"2020-03-01 20:04:06.493","130","7.591","128","8392704" +"2020-03-01 20:04:07.493","130","7.591","128","8392704" +"2020-03-01 20:04:08.493","130","7.591","128","8392704" +"2020-03-01 20:04:09.493","130","7.592","128","8392704" +"2020-03-01 20:04:10.493","130","7.59","128","8392704" +"2020-03-01 20:04:11.493","130","7.587","128","8392704" +"2020-03-01 20:04:12.493","130","7.584","128","8392704" +"2020-03-01 20:04:13.493","130","7.583","128","8392704" +"2020-03-01 20:04:14.493","130","7.581","128","8392704" +"2020-03-01 20:04:15.493","130","7.578","128","8392704" +"2020-03-01 20:04:16.493","130","7.576","128","8392704" +"2020-03-01 20:04:17.493","130","7.577","128","8392704" +"2020-03-01 20:04:18.493","130","7.579","128","8392704" +"2020-03-01 20:04:19.493","130","7.583","128","8392704" +"2020-03-01 20:04:20.493","130","7.587","128","8392704" +"2020-03-01 20:04:21.493","130","7.588","128","8392704" +"2020-03-01 20:04:22.493","130","7.589","128","8392704" +"2020-03-01 20:04:23.493","130","7.59","128","8392704" +"2020-03-01 20:04:24.493","130","7.593","128","8392704" +"2020-03-01 20:04:25.493","130","7.597","128","8392704" +"2020-03-01 20:04:26.493","130","7.6","128","8392704" +"2020-03-01 20:04:27.493","130","7.603","128","8392704" +"2020-03-01 20:04:28.493","130","7.606","128","8392704" +"2020-03-01 20:04:29.493","130","7.608","128","8392704" +"2020-03-01 20:04:30.493","130","7.609","128","8392704" +"2020-03-01 20:04:31.493","130","7.607","128","8392704" +"2020-03-01 20:04:32.493","130","7.607","128","8392704" +"2020-03-01 20:04:33.493","130","7.607","128","8392704" +"2020-03-01 20:04:34.493","130","7.602","128","8392704" +"2020-03-01 20:04:35.493","130","7.599","128","8392704" +"2020-03-01 20:04:36.493","130","7.599","128","8392704" +"2020-03-01 20:04:37.493","130","7.599","128","8392704" +"2020-03-01 20:04:38.493","130","7.598","128","8392704" +"2020-03-01 20:04:39.493","130","7.596","128","8392704" +"2020-03-01 20:04:40.493","130","7.595","128","8392704" +"2020-03-01 20:04:41.493","130","7.592","128","8392704" +"2020-03-01 20:04:42.493","130","7.586","128","8392704" +"2020-03-01 20:04:43.493","130","7.582","128","8392704" +"2020-03-01 20:04:44.493","130","7.582","128","8392704" +"2020-03-01 20:04:45.493","130","7.584","128","8392704" +"2020-03-01 20:04:46.493","130","7.583","128","8392704" +"2020-03-01 20:04:47.493","130","7.582","128","8392704" +"2020-03-01 20:04:48.493","130","7.582","128","8392704" +"2020-03-01 20:04:49.493","130","7.585","128","8392704" +"2020-03-01 06:43:17.493","130","6.751","128","8392704" +"2020-03-01 06:43:18.493","130","6.75","128","8392704" +"2020-03-01 06:43:19.493","130","6.748","128","8392704" +"2020-03-01 06:43:20.493","130","6.751","128","8392704" +"2020-03-01 06:43:21.493","130","6.752","128","8392704" +"2020-03-01 06:43:22.493","130","6.751","128","8392704" +"2020-03-01 06:43:23.493","130","6.746","128","8392704" +"2020-03-01 06:43:24.493","130","6.739","128","8392704" +"2020-03-01 06:43:25.493","130","6.737","128","8392704" +"2020-03-01 06:43:26.493","130","6.735","128","8392704" +"2020-03-01 06:43:27.493","130","6.735","128","8392704" +"2020-03-01 06:43:28.493","130","6.734","128","8392704" +"2020-03-01 06:43:29.493","130","6.731","128","8392704" +"2020-03-01 06:43:30.493","130","6.729","128","8392704" +"2020-03-01 06:43:31.493","130","6.73","128","8392704" +"2020-03-01 06:43:32.493","130","6.736","128","8392704" +"2020-03-01 06:43:33.493","130","6.74","128","8392704" +"2020-03-01 06:43:34.493","130","6.741","128","8392704" +"2020-03-01 06:43:35.493","130","6.743","128","8392704" +"2020-03-01 06:43:36.493","130","6.743","128","8392704" +"2020-03-01 06:43:37.493","130","6.745","128","8392704" +"2020-03-01 06:43:38.493","130","6.747","128","8392704" +"2020-03-01 06:43:39.493","130","6.747","128","8392704" +"2020-03-01 06:43:40.493","130","6.746","128","8392704" +"2020-03-01 06:43:41.493","130","6.745","128","8392704" +"2020-03-01 06:43:42.493","130","6.743","128","8392704" +"2020-03-01 06:43:43.493","130","6.741","128","8392704" +"2020-03-01 06:43:44.493","130","6.737","128","8392704" +"2020-03-01 06:43:45.493","130","6.737","128","8392704" +"2020-03-01 06:43:46.493","130","6.74","128","8392704" +"2020-03-01 06:43:47.493","130","6.744","128","8392704" +"2020-03-01 06:43:48.493","130","6.746","128","8392704" +"2020-03-01 06:43:49.493","130","6.745","128","8392704" +"2020-03-01 06:43:50.493","130","6.743","128","8392704" +"2020-03-01 06:43:51.493","130","6.745","128","8392704" +"2020-03-01 06:43:52.493","130","6.747","128","8392704" +"2020-03-01 06:43:53.493","130","6.748","128","8392704" +"2020-03-01 06:43:54.493","130","6.748","128","8392704" +"2020-03-01 06:43:55.493","130","6.747","128","8392704" +"2020-03-01 06:43:56.493","130","6.746","128","8392704" +"2020-03-01 06:43:57.493","130","6.744","128","8392704" +"2020-03-01 06:43:58.493","130","6.742","128","8392704" +"2020-03-01 06:43:59.493","130","6.74","128","8392704" +"2020-03-01 06:44:00.493","130","6.739","128","8392704" +"2020-03-01 06:44:01.493","130","6.739","128","8392704" +"2020-03-01 06:44:02.493","130","6.742","128","8392704" +"2020-03-01 06:44:03.493","130","6.742","128","8392704" +"2020-03-01 06:44:04.493","130","6.756","128","8392704" +"2020-03-01 06:44:05.493","130","6.757","128","8392704" +"2020-03-01 06:44:06.493","130","6.757","128","8392704" +"2020-03-01 06:44:07.493","130","6.757","128","8392704" +"2020-03-01 06:44:08.493","130","6.759","128","8392704" +"2020-03-01 06:44:09.493","130","6.759","128","8392704" +"2020-03-01 06:44:10.493","130","6.75","128","8392704" +"2020-03-01 06:44:11.493","130","6.744","128","8392704" +"2020-03-01 06:44:12.493","130","6.739","128","8392704" +"2020-03-01 06:44:13.493","130","6.739","128","8392704" +"2020-03-01 06:44:14.493","130","6.736","128","8392704" +"2020-03-01 06:44:15.493","130","6.734","128","8392704" +"2020-03-01 06:44:16.493","130","6.735","128","8392704" +"2020-03-01 06:44:17.493","130","6.734","128","8392704" +"2020-03-01 06:44:18.493","130","6.736","128","8392704" +"2020-03-01 06:44:19.493","130","6.741","128","8392704" +"2020-03-01 06:44:20.493","130","6.744","128","8392704" +"2020-03-01 06:44:21.493","130","6.746","128","8392704" +"2020-03-01 06:44:22.493","130","6.746","128","8392704" +"2020-03-01 06:44:23.493","130","6.748","128","8392704" +"2020-03-01 06:44:24.493","130","6.751","128","8392704" +"2020-03-01 06:44:25.493","130","6.752","128","8392704" +"2020-03-01 06:44:26.493","130","6.752","128","8392704" +"2020-03-01 06:44:27.493","130","6.752","128","8392704" +"2020-03-01 06:44:28.493","130","6.753","128","8392704" +"2020-03-01 06:44:29.493","130","6.751","128","8392704" +"2020-03-01 06:44:30.493","130","6.751","128","8392704" +"2020-03-01 06:44:31.493","130","6.749","128","8392704" +"2020-03-01 06:44:32.493","130","6.747","128","8392704" +"2020-03-01 06:44:33.493","130","6.748","128","8392704" +"2020-03-01 06:44:34.493","130","6.749","128","8392704" +"2020-03-01 06:44:35.493","130","6.746","128","8392704" +"2020-03-01 06:44:36.493","130","6.742","128","8392704" +"2020-03-01 06:44:37.493","130","6.742","128","8392704" +"2020-03-01 06:44:38.493","130","6.743","128","8392704" +"2020-03-01 06:44:39.493","130","6.743","128","8392704" +"2020-03-01 06:44:40.493","130","6.743","128","8392704" +"2020-03-01 06:44:41.493","130","6.741","128","8392704" +"2020-03-01 06:44:42.493","130","6.741","128","8392704" +"2020-03-01 06:44:43.493","130","6.741","128","8392704" +"2020-03-01 06:44:44.493","130","6.74","128","8392704" +"2020-03-01 06:44:45.493","130","6.74","128","8392704" +"2020-03-01 06:44:46.493","130","6.739","128","8392704" +"2020-03-01 06:44:47.493","130","6.738","128","8392704" +"2020-03-01 06:44:48.493","130","6.738","128","8392704" +"2020-03-01 06:44:49.493","130","6.741","128","8392704" +"2020-03-01 06:44:50.493","130","6.749","128","8392704" +"2020-03-01 06:44:51.493","130","6.756","128","8392704" +"2020-03-01 06:44:52.493","130","6.763","128","8392704" +"2020-03-01 06:44:53.493","130","6.768","128","8392704" +"2020-03-01 06:44:54.493","130","6.771","128","8392704" +"2020-03-01 06:44:55.493","130","6.774","128","8392704" +"2020-03-01 06:44:56.493","130","6.774","128","8392704" +"2020-03-01 06:44:57.493","130","6.774","128","8392704" +"2020-03-01 06:44:58.493","130","6.765","128","8392704" +"2020-03-01 06:44:59.493","130","6.763","128","8392704" +"2020-03-01 06:45:00.493","130","6.761","128","8392704" +"2020-03-01 06:45:01.493","130","6.758","128","8392704" +"2020-03-01 06:45:02.493","130","6.756","128","8392704" +"2020-03-01 06:45:03.493","130","6.756","128","8392704" +"2020-03-01 06:45:04.493","130","6.756","128","8392704" +"2020-03-01 06:45:05.493","130","6.763","128","8392704" +"2020-03-01 06:45:06.493","130","6.763","128","8392704" +"2020-03-01 06:45:07.493","130","6.764","128","8392704" +"2020-03-01 06:45:08.493","130","6.762","128","8392704" +"2020-03-01 06:45:09.493","130","6.763","128","8392704" +"2020-03-01 06:45:10.493","130","6.764","128","8392704" +"2020-03-01 06:45:11.493","130","6.763","128","8392704" +"2020-03-01 06:45:12.493","130","6.76","128","8392704" +"2020-03-01 06:45:13.493","130","6.759","128","8392704" +"2020-03-01 06:45:14.493","130","6.758","128","8392704" +"2020-03-01 06:45:15.493","130","6.758","128","8392704" +"2020-03-01 06:45:16.493","130","6.755","128","8392704" +"2020-03-01 20:04:50.493","130","7.59","128","8392704" +"2020-03-01 20:04:51.493","130","7.592","128","8392704" +"2020-03-01 20:04:52.493","130","7.592","128","8392704" +"2020-03-01 20:04:53.493","130","7.593","128","8392704" +"2020-03-01 20:04:54.493","130","7.592","128","8392704" +"2020-03-01 20:04:55.493","130","7.592","128","8392704" +"2020-03-01 20:04:56.493","130","7.593","128","8392704" +"2020-03-01 20:04:57.493","130","7.593","128","8392704" +"2020-03-01 20:04:58.493","130","7.593","128","8392704" +"2020-03-01 20:04:59.493","130","7.594","128","8392704" +"2020-03-01 20:05:00.493","130","7.595","128","8392704" +"2020-03-01 20:05:01.493","130","7.596","128","8392704" +"2020-03-01 20:05:02.493","130","7.595","128","8392704" +"2020-03-01 20:05:03.493","130","7.595","128","8392704" +"2020-03-01 20:05:04.493","130","7.594","128","8392704" +"2020-03-01 20:05:05.493","130","7.595","128","8392704" +"2020-03-01 20:05:06.493","130","7.598","128","8392704" +"2020-03-01 20:05:07.493","130","7.597","128","8392704" +"2020-03-01 20:05:08.493","130","7.595","128","8392704" +"2020-03-01 20:05:09.493","130","7.597","128","8392704" +"2020-03-01 20:05:10.493","130","7.598","128","8392704" +"2020-03-01 20:05:11.493","130","7.598","128","8392704" +"2020-03-01 20:05:12.493","130","7.597","128","8392704" +"2020-03-01 20:05:13.493","130","7.595","128","8392704" +"2020-03-01 20:05:14.493","130","7.591","128","8392704" +"2020-03-01 20:05:15.493","130","7.589","128","8392704" +"2020-03-01 20:05:16.493","130","7.588","128","8392704" +"2020-03-01 20:05:17.493","130","7.589","128","8392704" +"2020-03-01 20:05:18.493","130","7.589","128","8392704" +"2020-03-01 20:05:19.493","130","7.589","128","8392704" +"2020-03-01 20:05:20.493","130","7.587","128","8392704" +"2020-03-01 20:05:21.493","130","7.584","128","8392704" +"2020-03-01 20:05:22.493","130","7.583","128","8392704" +"2020-03-01 20:05:23.493","130","7.585","128","8392704" +"2020-03-01 20:05:24.493","130","7.586","128","8392704" +"2020-03-01 20:05:25.493","130","7.586","128","8392704" +"2020-03-01 20:05:26.493","130","7.586","128","8392704" +"2020-03-01 20:05:27.493","130","7.586","128","8392704" +"2020-03-01 20:05:28.493","130","7.587","128","8392704" +"2020-03-01 20:05:29.493","130","7.585","128","8392704" +"2020-03-01 20:05:30.493","130","7.584","128","8392704" +"2020-03-01 20:05:31.493","130","7.586","128","8392704" +"2020-03-01 20:05:32.493","130","7.589","128","8392704" +"2020-03-01 20:05:33.493","130","7.59","128","8392704" +"2020-03-01 20:05:34.493","130","7.591","128","8392704" +"2020-03-01 20:05:35.493","130","7.591","128","8392704" +"2020-03-01 20:05:36.493","130","7.594","128","8392704" +"2020-03-01 20:05:37.493","130","7.599","128","8392704" +"2020-03-01 20:05:38.493","130","7.602","128","8392704" +"2020-03-01 20:05:39.493","130","7.604","128","8392704" +"2020-03-01 20:05:40.493","130","7.605","128","8392704" +"2020-03-01 20:05:41.493","130","7.607","128","8392704" +"2020-03-01 20:05:42.493","130","7.607","128","8392704" +"2020-03-01 20:05:43.493","130","7.604","128","8392704" +"2020-03-01 20:05:44.493","130","7.597","128","8392704" +"2020-03-01 20:05:45.493","130","7.592","128","8392704" +"2020-03-01 20:05:46.493","130","7.59","128","8392704" +"2020-03-01 20:05:47.493","130","7.59","128","8392704" +"2020-03-01 20:05:48.493","130","7.591","128","8392704" +"2020-03-01 20:05:49.493","130","7.591","128","8392704" +"2020-03-01 20:05:50.493","130","7.591","128","8392704" +"2020-03-01 20:05:51.493","130","7.594","128","8392704" +"2020-03-01 20:05:52.493","130","7.599","128","8392704" +"2020-03-01 20:05:53.493","130","7.601","128","8392704" +"2020-03-01 20:05:54.493","130","7.602","128","8392704" +"2020-03-01 20:05:55.493","130","7.602","128","8392704" +"2020-03-01 20:05:56.493","130","7.602","128","8392704" +"2020-03-01 20:05:57.493","130","7.603","128","8392704" +"2020-03-01 20:05:58.493","130","7.604","128","8392704" +"2020-03-01 20:05:59.493","130","7.604","128","8392704" +"2020-03-01 20:06:00.493","130","7.605","128","8392704" +"2020-03-01 20:06:01.493","130","7.606","128","8392704" +"2020-03-01 20:06:02.493","130","7.607","128","8392704" +"2020-03-01 20:06:03.493","130","7.605","128","8392704" +"2020-03-01 20:06:04.493","130","7.604","128","8392704" +"2020-03-01 20:06:05.493","130","7.603","128","8392704" +"2020-03-01 20:06:06.493","130","7.602","128","8392704" +"2020-03-01 20:06:07.493","130","7.603","128","8392704" +"2020-03-01 20:06:08.493","130","7.604","128","8392704" \ No newline at end of file diff --git a/tests/pytest/test_data/ordered.csv b/tests/pytest/test_data/ordered.csv new file mode 100644 index 0000000000000000000000000000000000000000..14da572d75e3c9bef32d6de7696ce65485b06d23 --- /dev/null +++ b/tests/pytest/test_data/ordered.csv @@ -0,0 +1,500 @@ +"2020-03-01 19:46:50.493","130","7.617","128","8392704" +"2020-03-01 19:46:51.493","130","7.615","128","8392704" +"2020-03-01 19:46:52.493","130","7.613","128","8392704" +"2020-03-01 19:46:53.493","130","7.612","128","8392704" +"2020-03-01 19:46:54.493","130","7.611","128","8392704" +"2020-03-01 19:46:55.493","130","7.612","128","8392704" +"2020-03-01 19:46:56.493","130","7.611","128","8392704" +"2020-03-01 19:46:57.493","130","7.61","128","8392704" +"2020-03-01 19:46:58.493","130","7.61","128","8392704" +"2020-03-01 19:46:59.493","130","7.613","128","8392704" +"2020-03-01 19:47:00.493","130","7.617","128","8392704" +"2020-03-01 19:47:01.493","130","7.618","128","8392704" +"2020-03-01 19:47:02.493","130","7.619","128","8392704" +"2020-03-01 19:47:03.493","130","7.62","128","8392704" +"2020-03-01 19:47:04.493","130","7.619","128","8392704" +"2020-03-01 19:47:05.493","130","7.62","128","8392704" +"2020-03-01 19:47:06.493","130","7.62","128","8392704" +"2020-03-01 19:47:07.493","130","7.618","128","8392704" +"2020-03-01 19:47:08.493","130","7.618","128","8392704" +"2020-03-01 19:47:09.493","130","7.616","128","8392704" +"2020-03-01 19:47:10.493","130","7.615","128","8392704" +"2020-03-01 19:47:11.493","130","7.614","128","8392704" +"2020-03-01 19:47:12.493","130","7.614","128","8392704" +"2020-03-01 19:47:13.493","130","7.615","128","8392704" +"2020-03-01 19:47:14.493","130","7.617","128","8392704" +"2020-03-01 19:47:15.493","130","7.617","128","8392704" +"2020-03-01 19:47:16.493","130","7.612","128","8392704" +"2020-03-01 19:47:17.493","130","7.609","128","8392704" +"2020-03-01 19:47:18.493","130","7.609","128","8392704" +"2020-03-01 19:47:19.493","130","7.609","128","8392704" +"2020-03-01 19:47:20.493","130","7.611","128","8392704" +"2020-03-01 19:47:21.493","130","7.613","128","8392704" +"2020-03-01 19:47:22.493","130","7.612","128","8392704" +"2020-03-01 19:47:23.493","130","7.612","128","8392704" +"2020-03-01 19:47:24.493","130","7.612","128","8392704" +"2020-03-01 19:47:25.493","130","7.613","128","8392704" +"2020-03-01 19:47:26.493","130","7.617","128","8392704" +"2020-03-01 19:47:27.493","130","7.62","128","8392704" +"2020-03-01 19:47:28.493","130","7.621","128","8392704" +"2020-03-01 19:47:29.493","130","7.621","128","8392704" +"2020-03-01 19:47:30.493","130","7.623","128","8392704" +"2020-03-01 19:47:31.493","130","7.624","128","8392704" +"2020-03-01 19:47:32.493","130","7.621","128","8392704" +"2020-03-01 19:47:33.493","130","7.619","128","8392704" +"2020-03-01 19:47:34.493","130","7.618","128","8392704" +"2020-03-01 19:47:35.493","130","7.616","128","8392704" +"2020-03-01 19:47:36.493","130","7.618","128","8392704" +"2020-03-01 19:47:37.493","130","7.618","128","8392704" +"2020-03-01 19:47:38.493","130","7.616","128","8392704" +"2020-03-01 19:47:39.493","130","7.615","128","8392704" +"2020-03-01 19:47:40.493","130","7.615","128","8392704" +"2020-03-01 19:47:41.493","130","7.614","128","8392704" +"2020-03-01 19:47:42.493","130","7.613","128","8392704" +"2020-03-01 19:47:43.493","130","7.612","128","8392704" +"2020-03-01 19:47:44.493","130","7.611","128","8392704" +"2020-03-01 19:47:45.493","130","7.612","128","8392704" +"2020-03-01 19:47:46.493","130","7.614","128","8392704" +"2020-03-01 19:47:47.493","130","7.618","128","8392704" +"2020-03-01 19:47:48.493","130","7.62","128","8392704" +"2020-03-01 19:47:49.493","130","7.62","128","8392704" +"2020-03-01 19:47:50.493","130","7.621","128","8392704" +"2020-03-01 19:47:51.493","130","7.62","128","8392704" +"2020-03-01 19:47:52.493","130","7.619","128","8392704" +"2020-03-01 19:47:53.493","130","7.621","128","8392704" +"2020-03-01 19:47:54.493","130","7.622","128","8392704" +"2020-03-01 19:47:55.493","130","7.622","128","8392704" +"2020-03-01 19:47:56.493","130","7.62","128","8392704" +"2020-03-01 19:47:57.493","130","7.617","128","8392704" +"2020-03-01 19:47:58.493","130","7.616","128","8392704" +"2020-03-01 19:47:59.493","130","7.618","128","8392704" +"2020-03-01 19:48:00.493","130","7.62","128","8392704" +"2020-03-01 19:48:01.493","130","7.62","128","8392704" +"2020-03-01 19:48:02.493","130","7.616","128","8392704" +"2020-03-01 19:48:03.493","130","7.612","128","8392704" +"2020-03-01 19:48:04.493","130","7.609","128","8392704" +"2020-03-01 19:48:05.493","130","7.608","128","8392704" +"2020-03-01 19:48:06.493","130","7.605","128","8392704" +"2020-03-01 19:48:07.493","130","7.604","128","8392704" +"2020-03-01 19:48:08.493","130","7.605","128","8392704" +"2020-03-01 19:48:09.493","130","7.604","128","8392704" +"2020-03-01 19:48:10.493","130","7.604","128","8392704" +"2020-03-01 19:48:11.493","130","7.608","128","8392704" +"2020-03-01 19:48:12.493","130","7.611","128","8392704" +"2020-03-01 19:48:13.493","130","7.614","128","8392704" +"2020-03-01 19:48:14.493","130","7.616","128","8392704" +"2020-03-01 19:48:15.493","130","7.618","128","8392704" +"2020-03-01 19:48:16.493","130","7.62","128","8392704" +"2020-03-01 19:48:17.493","130","7.617","128","8392704" +"2020-03-01 19:48:18.493","130","7.61","128","8392704" +"2020-03-01 19:48:19.493","130","7.607","128","8392704" +"2020-03-01 19:48:20.493","130","7.604","128","8392704" +"2020-03-01 19:48:21.493","130","7.601","128","8392704" +"2020-03-01 19:48:22.493","130","7.601","128","8392704" +"2020-03-01 19:48:23.493","130","7.601","128","8392704" +"2020-03-01 19:48:24.493","130","7.598","128","8392704" +"2020-03-01 19:48:25.493","130","7.598","128","8392704" +"2020-03-01 19:48:26.493","130","7.604","128","8392704" +"2020-03-01 19:48:27.493","130","7.608","128","8392704" +"2020-03-01 19:48:28.493","130","7.609","128","8392704" +"2020-03-01 19:48:29.493","130","7.61","128","8392704" +"2020-03-01 19:48:30.493","130","7.611","128","8392704" +"2020-03-01 19:48:31.493","130","7.614","128","8392704" +"2020-03-01 19:48:32.493","130","7.614","128","8392704" +"2020-03-01 19:48:33.493","130","7.611","128","8392704" +"2020-03-01 19:48:34.493","130","7.607","128","8392704" +"2020-03-01 19:48:35.493","130","7.601","128","8392704" +"2020-03-01 19:48:36.493","130","7.596","128","8392704" +"2020-03-01 19:48:37.493","130","7.593","128","8392704" +"2020-03-01 19:48:38.493","130","7.593","128","8392704" +"2020-03-01 19:48:39.493","130","7.593","128","8392704" +"2020-03-01 19:48:40.493","130","7.595","128","8392704" +"2020-03-01 19:48:41.493","130","7.596","128","8392704" +"2020-03-01 19:48:42.493","130","7.599","128","8392704" +"2020-03-01 19:48:43.493","130","7.603","128","8392704" +"2020-03-01 19:48:44.493","130","7.605","128","8392704" +"2020-03-01 19:48:45.493","130","7.607","128","8392704" +"2020-03-01 19:48:46.493","130","7.608","128","8392704" +"2020-03-01 19:48:47.493","130","7.609","128","8392704" +"2020-03-01 19:48:48.493","130","7.61","128","8392704" +"2020-03-01 19:48:49.493","130","7.608","128","8392704" +"2020-03-01 19:48:50.493","130","7.605","128","8392704" +"2020-03-01 19:48:51.493","130","7.605","128","8392704" +"2020-03-01 19:48:52.493","130","7.607","128","8392704" +"2020-03-01 19:48:53.493","130","7.608","128","8392704" +"2020-03-01 19:48:54.493","130","7.608","128","8392704" +"2020-03-01 19:48:55.493","130","7.608","128","8392704" +"2020-03-01 19:48:56.493","130","7.61","128","8392704" +"2020-03-01 19:48:57.493","130","7.613","128","8392704" +"2020-03-01 19:48:58.493","130","7.612","128","8392704" +"2020-03-01 19:48:59.493","130","7.61","128","8392704" +"2020-03-01 19:49:00.493","130","7.609","128","8392704" +"2020-03-01 19:49:01.493","130","7.61","128","8392704" +"2020-03-01 19:49:02.493","130","7.611","128","8392704" +"2020-03-01 19:49:03.493","130","7.61","128","8392704" +"2020-03-01 19:49:04.493","130","7.61","128","8392704" +"2020-03-01 19:49:05.493","130","7.613","128","8392704" +"2020-03-01 19:49:06.493","130","7.615","128","8392704" +"2020-03-01 19:49:07.493","130","7.614","128","8392704" +"2020-03-01 19:49:08.493","130","7.613","128","8392704" +"2020-03-01 19:49:09.493","130","7.613","128","8392704" +"2020-03-01 19:49:10.493","130","7.615","128","8392704" +"2020-03-01 19:49:11.493","130","7.619","128","8392704" +"2020-03-01 19:49:12.493","130","7.62","128","8392704" +"2020-03-01 19:49:13.493","130","7.618","128","8392704" +"2020-03-01 19:49:14.493","130","7.619","128","8392704" +"2020-03-01 19:49:15.493","130","7.618","128","8392704" +"2020-03-01 19:49:16.493","130","7.617","128","8392704" +"2020-03-01 19:49:17.493","130","7.617","128","8392704" +"2020-03-01 19:49:18.493","130","7.618","128","8392704" +"2020-03-01 19:49:19.493","130","7.617","128","8392704" +"2020-03-01 19:49:20.493","130","7.616","128","8392704" +"2020-03-01 19:49:21.493","130","7.615","128","8392704" +"2020-03-01 19:49:22.493","130","7.616","128","8392704" +"2020-03-01 19:49:23.493","130","7.617","128","8392704" +"2020-03-01 19:49:24.493","130","7.615","128","8392704" +"2020-03-01 19:49:25.493","130","7.613","128","8392704" +"2020-03-01 19:49:26.493","130","7.612","128","8392704" +"2020-03-01 19:49:27.493","130","7.613","128","8392704" +"2020-03-01 19:49:28.493","130","7.614","128","8392704" +"2020-03-01 19:49:29.493","130","7.612","128","8392704" +"2020-03-01 19:49:30.493","130","7.611","128","8392704" +"2020-03-01 19:49:31.493","130","7.611","128","8392704" +"2020-03-01 19:49:32.493","130","7.612","128","8392704" +"2020-03-01 19:49:33.493","130","7.613","128","8392704" +"2020-03-01 19:49:34.493","130","7.614","128","8392704" +"2020-03-01 19:49:35.493","130","7.612","128","8392704" +"2020-03-01 19:49:36.493","130","7.607","128","8392704" +"2020-03-01 19:49:37.493","130","7.603","128","8392704" +"2020-03-01 19:49:38.493","130","7.599","128","8392704" +"2020-03-01 19:49:39.493","130","7.599","128","8392704" +"2020-03-01 19:49:40.493","130","7.599","128","8392704" +"2020-03-01 19:49:41.493","130","7.599","128","8392704" +"2020-03-01 19:49:42.493","130","7.601","128","8392704" +"2020-03-01 19:49:43.493","130","7.605","128","8392704" +"2020-03-01 19:49:44.493","130","7.606","128","8392704" +"2020-03-01 19:49:45.493","130","7.606","128","8392704" +"2020-03-01 19:49:46.493","130","7.606","128","8392704" +"2020-03-01 19:49:47.493","130","7.604","128","8392704" +"2020-03-01 19:49:48.493","130","7.604","128","8392704" +"2020-03-01 19:49:49.493","130","7.603","128","8392704" +"2020-03-01 19:49:50.493","130","7.604","128","8392704" +"2020-03-01 19:49:51.493","130","7.608","128","8392704" +"2020-03-01 19:49:52.493","130","7.614","128","8392704" +"2020-03-01 19:49:53.493","130","7.618","128","8392704" +"2020-03-01 19:49:54.493","130","7.621","128","8392704" +"2020-03-01 19:49:55.493","130","7.623","128","8392704" +"2020-03-01 19:49:56.493","130","7.623","128","8392704" +"2020-03-01 19:49:57.493","130","7.624","128","8392704" +"2020-03-01 19:49:58.493","130","7.626","128","8392704" +"2020-03-01 19:49:59.493","130","7.628","128","8392704" +"2020-03-01 19:50:00.493","130","7.627","128","8392704" +"2020-03-01 19:50:01.493","130","7.625","128","8392704" +"2020-03-01 19:50:02.493","130","7.627","128","8392704" +"2020-03-01 19:50:03.493","130","7.63","128","8392704" +"2020-03-01 19:50:04.493","130","7.633","128","8392704" +"2020-03-01 19:50:05.493","130","7.635","128","8392704" +"2020-03-01 19:50:06.493","130","7.634","128","8392704" +"2020-03-01 19:50:07.493","130","7.632","128","8392704" +"2020-03-01 19:50:08.493","130","7.628","128","8392704" +"2020-03-01 19:50:09.493","130","7.625","128","8392704" +"2020-03-01 19:50:10.493","130","7.625","128","8392704" +"2020-03-01 19:50:11.493","130","7.623","128","8392704" +"2020-03-01 19:50:12.493","130","7.623","128","8392704" +"2020-03-01 19:50:13.493","130","7.623","128","8392704" +"2020-03-01 19:50:14.493","130","7.622","128","8392704" +"2020-03-01 19:50:15.493","130","7.621","128","8392704" +"2020-03-01 19:50:16.493","130","7.618","128","8392704" +"2020-03-01 19:50:17.493","130","7.618","128","8392704" +"2020-03-01 19:50:18.493","130","7.617","128","8392704" +"2020-03-01 19:50:19.493","130","7.616","128","8392704" +"2020-03-01 19:50:20.493","130","7.615","128","8392704" +"2020-03-01 19:50:21.493","130","7.615","128","8392704" +"2020-03-01 19:50:22.493","130","7.616","128","8392704" +"2020-03-01 19:50:23.493","130","7.619","128","8392704" +"2020-03-01 19:50:24.493","130","7.622","128","8392704" +"2020-03-01 19:50:25.493","130","7.624","128","8392704" +"2020-03-01 19:50:26.493","130","7.627","128","8392704" +"2020-03-01 19:50:27.493","130","7.627","128","8392704" +"2020-03-01 19:50:28.493","130","7.625","128","8392704" +"2020-03-01 19:50:29.493","130","7.625","128","8392704" +"2020-03-01 19:50:30.493","130","7.625","128","8392704" +"2020-03-01 19:50:31.493","130","7.624","128","8392704" +"2020-03-01 19:50:32.493","130","7.624","128","8392704" +"2020-03-01 19:50:33.493","130","7.624","128","8392704" +"2020-03-01 19:50:34.493","130","7.626","128","8392704" +"2020-03-01 19:50:35.493","130","7.627","128","8392704" +"2020-03-01 19:50:36.493","130","7.627","128","8392704" +"2020-03-01 19:50:37.493","130","7.626","128","8392704" +"2020-03-01 19:50:38.493","130","7.623","128","8392704" +"2020-03-01 19:50:39.493","130","7.619","128","8392704" +"2020-03-01 19:50:40.493","130","7.616","128","8392704" +"2020-03-01 19:50:41.493","130","7.616","128","8392704" +"2020-03-01 19:50:42.493","130","7.615","128","8392704" +"2020-03-01 19:50:43.493","130","7.613","128","8392704" +"2020-03-01 19:50:44.493","130","7.614","128","8392704" +"2020-03-01 19:50:45.493","130","7.614","128","8392704" +"2020-03-01 19:50:46.493","130","7.612","128","8392704" +"2020-03-01 19:50:47.493","130","7.611","128","8392704" +"2020-03-01 19:50:48.493","130","7.611","128","8392704" +"2020-03-01 19:50:49.493","130","7.611","128","8392704" +"2020-03-01 19:50:50.493","130","7.612","128","8392704" +"2020-03-01 19:50:51.493","130","7.613","128","8392704" +"2020-03-01 19:50:52.493","130","7.613","128","8392704" +"2020-03-01 19:50:53.493","130","7.615","128","8392704" +"2020-03-01 19:50:54.493","130","7.617","128","8392704" +"2020-03-01 19:50:55.493","130","7.617","128","8392704" +"2020-03-01 19:50:56.493","130","7.619","128","8392704" +"2020-03-01 19:50:57.493","130","7.622","128","8392704" +"2020-03-01 19:50:58.493","130","7.624","128","8392704" +"2020-03-01 19:50:59.493","130","7.625","128","8392704" +"2020-03-01 19:51:00.493","130","7.624","128","8392704" +"2020-03-01 19:51:01.493","130","7.624","128","8392704" +"2020-03-01 19:51:02.493","130","7.622","128","8392704" +"2020-03-01 19:51:03.493","130","7.62","128","8392704" +"2020-03-01 19:51:04.493","130","7.617","128","8392704" +"2020-03-01 19:51:05.493","130","7.617","128","8392704" +"2020-03-01 19:51:06.493","130","7.618","128","8392704" +"2020-03-01 19:51:07.493","130","7.618","128","8392704" +"2020-03-01 19:51:08.493","130","7.618","128","8392704" +"2020-03-01 19:51:09.493","130","7.62","128","8392704" +"2020-03-01 19:51:10.493","130","7.622","128","8392704" +"2020-03-01 19:51:11.493","130","7.623","128","8392704" +"2020-03-01 19:51:12.493","130","7.624","128","8392704" +"2020-03-01 19:51:13.493","130","7.625","128","8392704" +"2020-03-01 19:51:14.493","130","7.626","128","8392704" +"2020-03-01 19:51:15.493","130","7.626","128","8392704" +"2020-03-01 19:51:16.493","130","7.626","128","8392704" +"2020-03-01 19:51:17.493","130","7.627","128","8392704" +"2020-03-01 19:51:18.493","130","7.627","128","8392704" +"2020-03-01 19:51:19.493","130","7.629","128","8392704" +"2020-03-01 19:51:20.493","130","7.629","128","8392704" +"2020-03-01 19:51:21.493","130","7.626","128","8392704" +"2020-03-01 19:51:22.493","130","7.625","128","8392704" +"2020-03-01 19:51:23.493","130","7.625","128","8392704" +"2020-03-01 19:51:24.493","130","7.626","128","8392704" +"2020-03-01 19:51:25.493","130","7.626","128","8392704" +"2020-03-01 19:51:26.493","130","7.624","128","8392704" +"2020-03-01 19:51:27.493","130","7.623","128","8392704" +"2020-03-01 19:51:28.493","130","7.624","128","8392704" +"2020-03-01 19:51:29.493","130","7.624","128","8392704" +"2020-03-01 19:51:30.493","130","7.624","128","8392704" +"2020-03-01 19:51:31.493","130","7.624","128","8392704" +"2020-03-01 19:51:32.493","130","7.626","128","8392704" +"2020-03-01 19:51:33.493","130","7.626","128","8392704" +"2020-03-01 19:51:34.493","130","7.626","128","8392704" +"2020-03-01 19:51:35.493","130","7.625","128","8392704" +"2020-03-01 19:51:36.493","130","7.624","128","8392704" +"2020-03-01 19:51:37.493","130","7.623","128","8392704" +"2020-03-01 19:51:38.493","130","7.622","128","8392704" +"2020-03-01 19:51:39.493","130","7.62","128","8392704" +"2020-03-01 19:51:40.493","130","7.62","128","8392704" +"2020-03-01 19:51:41.493","130","7.62","128","8392704" +"2020-03-01 19:51:42.493","130","7.621","128","8392704" +"2020-03-01 19:51:43.493","130","7.62","128","8392704" +"2020-03-01 19:51:44.493","130","7.619","128","8392704" +"2020-03-01 19:51:45.493","130","7.62","128","8392704" +"2020-03-01 19:51:46.493","130","7.62","128","8392704" +"2020-03-01 19:51:47.493","130","7.618","128","8392704" +"2020-03-01 19:51:48.493","130","7.619","128","8392704" +"2020-03-01 19:51:49.493","130","7.62","128","8392704" +"2020-03-01 19:51:50.493","130","7.622","128","8392704" +"2020-03-01 19:51:51.493","130","7.622","128","8392704" +"2020-03-01 19:51:52.493","130","7.62","128","8392704" +"2020-03-01 19:51:53.493","130","7.62","128","8392704" +"2020-03-01 19:51:54.493","130","7.622","128","8392704" +"2020-03-01 19:51:55.493","130","7.624","128","8392704" +"2020-03-01 19:51:56.493","130","7.622","128","8392704" +"2020-03-01 19:51:57.493","130","7.616","128","8392704" +"2020-03-01 19:51:58.493","130","7.611","128","8392704" +"2020-03-01 19:51:59.493","130","7.61","128","8392704" +"2020-03-01 19:52:00.493","130","7.608","128","8392704" +"2020-03-01 19:52:01.493","130","7.606","128","8392704" +"2020-03-01 19:52:02.493","130","7.607","128","8392704" +"2020-03-01 19:52:03.493","130","7.608","128","8392704" +"2020-03-01 19:52:04.493","130","7.61","128","8392704" +"2020-03-01 19:52:05.493","130","7.612","128","8392704" +"2020-03-01 19:52:06.493","130","7.615","128","8392704" +"2020-03-01 19:52:07.493","130","7.62","128","8392704" +"2020-03-01 19:52:08.493","130","7.623","128","8392704" +"2020-03-01 19:52:09.493","130","7.624","128","8392704" +"2020-03-01 19:52:10.493","130","7.623","128","8392704" +"2020-03-01 19:52:11.493","130","7.623","128","8392704" +"2020-03-01 19:52:12.493","130","7.624","128","8392704" +"2020-03-01 19:52:13.493","130","7.622","128","8392704" +"2020-03-01 19:52:14.493","130","7.62","128","8392704" +"2020-03-01 19:52:15.493","130","7.621","128","8392704" +"2020-03-01 19:52:16.493","130","7.62","128","8392704" +"2020-03-01 19:52:17.493","130","7.622","128","8392704" +"2020-03-01 19:52:18.493","130","7.625","128","8392704" +"2020-03-01 19:52:19.493","130","7.627","128","8392704" +"2020-03-01 19:52:20.493","130","7.625","128","8392704" +"2020-03-01 19:52:21.493","130","7.621","128","8392704" +"2020-03-01 19:52:22.493","130","7.617","128","8392704" +"2020-03-01 19:52:23.493","130","7.617","128","8392704" +"2020-03-01 19:52:24.493","130","7.617","128","8392704" +"2020-03-01 19:52:25.493","130","7.616","128","8392704" +"2020-03-01 19:52:26.493","130","7.615","128","8392704" +"2020-03-01 19:52:27.493","130","7.616","128","8392704" +"2020-03-01 19:52:28.493","130","7.619","128","8392704" +"2020-03-01 19:52:29.493","130","7.621","128","8392704" +"2020-03-01 19:52:30.493","130","7.621","128","8392704" +"2020-03-01 19:52:31.493","130","7.621","128","8392704" +"2020-03-01 19:52:32.493","130","7.621","128","8392704" +"2020-03-01 19:52:33.493","130","7.621","128","8392704" +"2020-03-01 19:52:34.493","130","7.623","128","8392704" +"2020-03-01 19:52:35.493","130","7.621","128","8392704" +"2020-03-01 19:52:36.493","130","7.617","128","8392704" +"2020-03-01 19:52:37.493","130","7.615","128","8392704" +"2020-03-01 19:52:38.493","130","7.612","128","8392704" +"2020-03-01 19:52:39.493","130","7.609","128","8392704" +"2020-03-01 19:52:40.493","130","7.606","128","8392704" +"2020-03-01 19:52:41.493","130","7.606","128","8392704" +"2020-03-01 19:52:42.493","130","7.609","128","8392704" +"2020-03-01 19:52:43.493","130","7.612","128","8392704" +"2020-03-01 19:52:44.493","130","7.616","128","8392704" +"2020-03-01 19:52:45.493","130","7.619","128","8392704" +"2020-03-01 19:52:46.493","130","7.62","128","8392704" +"2020-03-01 19:52:47.493","130","7.622","128","8392704" +"2020-03-01 19:52:48.493","130","7.622","128","8392704" +"2020-03-01 19:52:49.493","130","7.621","128","8392704" +"2020-03-01 19:52:50.493","130","7.618","128","8392704" +"2020-03-01 19:52:51.493","130","7.616","128","8392704" +"2020-03-01 19:52:52.493","130","7.613","128","8392704" +"2020-03-01 19:52:53.493","130","7.612","128","8392704" +"2020-03-01 19:52:54.493","130","7.612","128","8392704" +"2020-03-01 19:52:55.493","130","7.611","128","8392704" +"2020-03-01 19:52:56.493","130","7.609","128","8392704" +"2020-03-01 19:52:57.493","130","7.608","128","8392704" +"2020-03-01 19:52:58.493","130","7.609","128","8392704" +"2020-03-01 19:52:59.493","130","7.611","128","8392704" +"2020-03-01 19:53:00.493","130","7.612","128","8392704" +"2020-03-01 19:53:01.493","130","7.614","128","8392704" +"2020-03-01 19:53:02.493","130","7.618","128","8392704" +"2020-03-01 19:53:03.493","130","7.62","128","8392704" +"2020-03-01 19:53:04.493","130","7.62","128","8392704" +"2020-03-01 19:53:05.493","130","7.62","128","8392704" +"2020-03-01 19:53:06.493","130","7.619","128","8392704" +"2020-03-01 19:53:07.493","130","7.617","128","8392704" +"2020-03-01 19:53:08.493","130","7.615","128","8392704" +"2020-03-01 19:53:09.493","130","7.612","128","8392704" +"2020-03-01 19:53:10.493","130","7.61","128","8392704" +"2020-03-01 19:53:11.493","130","7.609","128","8392704" +"2020-03-01 19:53:12.493","130","7.608","128","8392704" +"2020-03-01 19:53:13.493","130","7.61","128","8392704" +"2020-03-01 19:53:14.493","130","7.611","128","8392704" +"2020-03-01 19:53:15.493","130","7.609","128","8392704" +"2020-03-01 19:53:16.493","130","7.608","128","8392704" +"2020-03-01 19:53:17.493","130","7.608","128","8392704" +"2020-03-01 19:53:18.493","130","7.607","128","8392704" +"2020-03-01 19:53:19.493","130","7.607","128","8392704" +"2020-03-01 19:53:20.493","130","7.605","128","8392704" +"2020-03-01 19:53:21.493","130","7.603","128","8392704" +"2020-03-01 19:53:22.493","130","7.606","128","8392704" +"2020-03-01 19:53:23.493","130","7.611","128","8392704" +"2020-03-01 19:53:24.493","130","7.615","128","8392704" +"2020-03-01 19:53:25.493","130","7.618","128","8392704" +"2020-03-01 19:53:26.493","130","7.62","128","8392704" +"2020-03-01 19:53:27.493","130","7.622","128","8392704" +"2020-03-01 19:53:28.493","130","7.624","128","8392704" +"2020-03-01 19:53:29.493","130","7.626","128","8392704" +"2020-03-01 19:53:30.493","130","7.624","128","8392704" +"2020-03-01 19:53:31.493","130","7.617","128","8392704" +"2020-03-01 19:53:32.493","130","7.613","128","8392704" +"2020-03-01 19:53:33.493","130","7.613","128","8392704" +"2020-03-01 19:53:34.493","130","7.613","128","8392704" +"2020-03-01 19:53:35.493","130","7.61","128","8392704" +"2020-03-01 19:53:36.493","130","7.609","128","8392704" +"2020-03-01 19:53:37.493","130","7.611","128","8392704" +"2020-03-01 19:53:38.493","130","7.61","128","8392704" +"2020-03-01 19:53:39.493","130","7.609","128","8392704" +"2020-03-01 19:53:40.493","130","7.608","128","8392704" +"2020-03-01 19:53:41.493","130","7.605","128","8392704" +"2020-03-01 19:53:42.493","130","7.601","128","8392704" +"2020-03-01 19:53:43.493","130","7.6","128","8392704" +"2020-03-01 19:53:44.493","130","7.602","128","8392704" +"2020-03-01 19:53:45.493","130","7.604","128","8392704" +"2020-03-01 19:53:46.493","130","7.605","128","8392704" +"2020-03-01 19:53:47.493","130","7.606","128","8392704" +"2020-03-01 19:53:48.493","130","7.605","128","8392704" +"2020-03-01 19:53:49.493","130","7.605","128","8392704" +"2020-03-01 19:53:50.493","130","7.606","128","8392704" +"2020-03-01 19:53:51.493","130","7.606","128","8392704" +"2020-03-01 19:53:52.493","130","7.604","128","8392704" +"2020-03-01 19:53:53.493","130","7.606","128","8392704" +"2020-03-01 19:53:54.493","130","7.61","128","8392704" +"2020-03-01 19:53:55.493","130","7.612","128","8392704" +"2020-03-01 19:53:56.493","130","7.613","128","8392704" +"2020-03-01 19:53:57.493","130","7.613","128","8392704" +"2020-03-01 19:53:58.493","130","7.613","128","8392704" +"2020-03-01 19:53:59.493","130","7.615","128","8392704" +"2020-03-01 19:54:00.493","130","7.616","128","8392704" +"2020-03-01 19:54:01.493","130","7.616","128","8392704" +"2020-03-01 19:54:02.493","130","7.618","128","8392704" +"2020-03-01 19:54:03.493","130","7.621","128","8392704" +"2020-03-01 19:54:04.493","130","7.623","128","8392704" +"2020-03-01 19:54:05.493","130","7.623","128","8392704" +"2020-03-01 19:54:06.493","130","7.624","128","8392704" +"2020-03-01 19:54:07.493","130","7.624","128","8392704" +"2020-03-01 19:54:08.493","130","7.622","128","8392704" +"2020-03-01 19:54:09.493","130","7.62","128","8392704" +"2020-03-01 19:54:10.493","130","7.621","128","8392704" +"2020-03-01 19:54:11.493","130","7.62","128","8392704" +"2020-03-01 19:54:12.493","130","7.617","128","8392704" +"2020-03-01 19:54:13.493","130","7.617","128","8392704" +"2020-03-01 19:54:14.493","130","7.618","128","8392704" +"2020-03-01 19:54:15.493","130","7.617","128","8392704" +"2020-03-01 19:54:16.493","130","7.615","128","8392704" +"2020-03-01 19:54:17.493","130","7.613","128","8392704" +"2020-03-01 19:54:18.493","130","7.612","128","8392704" +"2020-03-01 19:54:19.493","130","7.609","128","8392704" +"2020-03-01 19:54:20.493","130","7.605","128","8392704" +"2020-03-01 19:54:21.493","130","7.604","128","8392704" +"2020-03-01 19:54:22.493","130","7.605","128","8392704" +"2020-03-01 19:54:23.493","130","7.608","128","8392704" +"2020-03-01 19:54:24.493","130","7.611","128","8392704" +"2020-03-01 19:54:25.493","130","7.613","128","8392704" +"2020-03-01 19:54:26.493","130","7.613","128","8392704" +"2020-03-01 19:54:27.493","130","7.611","128","8392704" +"2020-03-01 19:54:28.493","130","7.611","128","8392704" +"2020-03-01 19:54:29.493","130","7.611","128","8392704" +"2020-03-01 19:54:30.493","130","7.612","128","8392704" +"2020-03-01 19:54:31.493","130","7.614","128","8392704" +"2020-03-01 19:54:32.493","130","7.61","128","8392704" +"2020-03-01 19:54:33.493","130","7.603","128","8392704" +"2020-03-01 19:54:34.493","130","7.598","128","8392704" +"2020-03-01 19:54:35.493","130","7.594","128","8392704" +"2020-03-01 19:54:36.493","130","7.591","128","8392704" +"2020-03-01 19:54:37.493","130","7.591","128","8392704" +"2020-03-01 19:54:38.493","130","7.59","128","8392704" +"2020-03-01 19:54:39.493","130","7.588","128","8392704" +"2020-03-01 19:54:40.493","130","7.593","128","8392704" +"2020-03-01 19:54:41.493","130","7.599","128","8392704" +"2020-03-01 19:54:42.493","130","7.602","128","8392704" +"2020-03-01 19:54:43.493","130","7.604","128","8392704" +"2020-03-01 19:54:44.493","130","7.606","128","8392704" +"2020-03-01 19:54:45.493","130","7.609","128","8392704" +"2020-03-01 19:54:46.493","130","7.612","128","8392704" +"2020-03-01 19:54:47.493","130","7.614","128","8392704" +"2020-03-01 19:54:48.493","130","7.616","128","8392704" +"2020-03-01 19:54:49.493","130","7.617","128","8392704" +"2020-03-01 19:54:50.493","130","7.619","128","8392704" +"2020-03-01 19:54:51.493","130","7.623","128","8392704" +"2020-03-01 19:54:52.493","130","7.626","128","8392704" +"2020-03-01 19:54:53.493","130","7.626","128","8392704" +"2020-03-01 19:54:54.493","130","7.624","128","8392704" +"2020-03-01 19:54:55.493","130","7.623","128","8392704" +"2020-03-01 19:54:56.493","130","7.618","128","8392704" +"2020-03-01 19:54:57.493","130","7.613","128","8392704" +"2020-03-01 19:54:58.493","130","7.61","128","8392704" +"2020-03-01 19:54:59.493","130","7.605","128","8392704" +"2020-03-01 19:55:00.493","130","7.604","128","8392704" +"2020-03-01 19:55:01.493","130","7.603","128","8392704" +"2020-03-01 19:55:02.493","130","7.602","128","8392704" +"2020-03-01 19:55:03.493","130","7.602","128","8392704" +"2020-03-01 19:55:04.493","130","7.602","128","8392704" +"2020-03-01 19:55:05.493","130","7.603","128","8392704" +"2020-03-01 19:55:06.493","130","7.608","128","8392704" +"2020-03-01 19:55:07.493","130","7.609","128","8392704" +"2020-03-01 19:55:08.493","130","7.608","128","8392704" +"2020-03-01 19:55:09.493","130","7.609","128","8392704" \ No newline at end of file diff --git a/tests/pytest/update/allow_update-0.py b/tests/pytest/update/allow_update-0.py new file mode 100644 index 0000000000000000000000000000000000000000..61295ec4b7680b42519bf7c9e0a01e0e523aa38b --- /dev/null +++ b/tests/pytest/update/allow_update-0.py @@ -0,0 +1,170 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.numOfRecords = 10 + self.ts = 1604295582000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.execute("use udb") + + def run(self): + tdSql.prepare() + startTs = self.ts + + print("==============step1") + tdSql.execute("create database udb update 0") + tdSql.execute("use udb") + tdSql.execute("create table t (ts timestamp, a int)") + tdSql.execute("insert into t values (%d, 1)" % (startTs)) + tdSql.execute("insert into t values (%d, 1)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 1)" % (startTs + 3)) + + tdSql.query("select * from t") + tdSql.checkRows(3) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + + print("==============step2") + tdSql.execute("insert into t values (%d, 2)" % (startTs)) + tdSql.execute("insert into t values (%d, 2)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 2)" % (startTs + 3)) + + tdSql.query("select * from t") + tdSql.checkRows(3) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + + print("==============step3") + tdSql.execute("insert into t values (%d, 3)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 3)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 3)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 3)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(7) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 3) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 3) + + print("==============step4") + tdSql.execute("insert into t values (%d, 4)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 4)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 4)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 4)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(7) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 3) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 3) + + print("==============step5") + tdSql.execute("insert into t values (%d, 5)" % (startTs - 1)) + tdSql.execute("insert into t values (%d, 5)" % (startTs + 1)) + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 3) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 3) + + print("==============step6") + tdSql.execute("insert into t values (%d, 6)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 1)) + tdSql.execute("insert into t values (%d, 6)" % (startTs)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 1)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 3)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 3) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 3) + + # restart taosd to commit, and check + self.restartTaosd(); + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 3) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/allow_update.py b/tests/pytest/update/allow_update.py new file mode 100644 index 0000000000000000000000000000000000000000..5871197f0213dbcf0611b25685510a1d2e97610f --- /dev/null +++ b/tests/pytest/update/allow_update.py @@ -0,0 +1,266 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.numOfRecords = 10 + self.ts = 1604295582000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.execute("use udb") + + def run(self): + tdSql.prepare() + startTs = self.ts + + tdSql.execute("create database udb update 1") + tdSql.execute("use udb") + tdSql.execute("create table t (ts timestamp, a int)") + + print("==============step1") + tdSql.execute("insert into t values (%d, 1)" % (startTs)) + tdSql.execute("insert into t values (%d, 1)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 1)" % (startTs + 3)) + + tdSql.query("select * from t") + tdSql.checkRows(3) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + + print("==============step2") + tdSql.execute("insert into t values (%d, 2)" % (startTs)) + tdSql.execute("insert into t values (%d, 2)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 2)" % (startTs + 3)) + + tdSql.query("select * from t") + tdSql.checkRows(3) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 2) + tdSql.checkData(1, 0, 2) + tdSql.checkData(2, 0, 2) + + print("==============step3") + tdSql.execute("insert into t values (%d, 3)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 3)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 3)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 3)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(7) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 2) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 2) + tdSql.checkData(4, 0, 3) + tdSql.checkData(5, 0, 2) + tdSql.checkData(6, 0, 3) + + print("==============step4") + tdSql.execute("insert into t values (%d, 4)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 4)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 4)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 4)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(7) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 4) + tdSql.checkData(1, 0, 2) + tdSql.checkData(2, 0, 4) + tdSql.checkData(3, 0, 2) + tdSql.checkData(4, 0, 4) + tdSql.checkData(5, 0, 2) + tdSql.checkData(6, 0, 4) + + print("==============step5") + tdSql.execute("insert into t values (%d, 5)" % (startTs - 1)) + tdSql.execute("insert into t values (%d, 5)" % (startTs + 1)) + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 4) + tdSql.checkData(1, 0, 2) + tdSql.checkData(2, 0, 4) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 2) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 4) + tdSql.checkData(7, 0, 2) + tdSql.checkData(8, 0, 4) + + print("==============step6") + tdSql.execute("insert into t values (%d, 6)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 1)) + tdSql.execute("insert into t values (%d, 6)" % (startTs)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 1)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 3)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 6) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 6) + tdSql.checkData(3, 0, 6) + tdSql.checkData(4, 0, 6) + tdSql.checkData(5, 0, 6) + tdSql.checkData(6, 0, 6) + tdSql.checkData(7, 0, 6) + tdSql.checkData(8, 0, 6) + + # restart taosd to commit, and check + self.restartTaosd(); + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 6) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 6) + tdSql.checkData(3, 0, 6) + tdSql.checkData(4, 0, 6) + tdSql.checkData(5, 0, 6) + tdSql.checkData(6, 0, 6) + tdSql.checkData(7, 0, 6) + tdSql.checkData(8, 0, 6) + + tdSql.execute("create table subt (ts timestamp, a int, b float, c binary(16), d bool)") + + print("==============step7") + tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+0')" % (startTs)) + tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c-3')" % (startTs - 3)) + tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+3')" % (startTs + 3)) + + tdSql.query("select * from subt") + tdSql.checkRows(3) + + tdSql.query("select a,b,c,d from subt") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(2, 1, None) + tdSql.checkData(0, 2, 'c-3') + tdSql.checkData(1, 2, 'c+0') + tdSql.checkData(2, 2, 'c+3') + tdSql.checkData(0, 3, None) + tdSql.checkData(1, 3, None) + tdSql.checkData(2, 3, None) + + print("==============step8") + tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs)) + tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs - 3)) + tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, false)" % (startTs + 3)) + + tdSql.query("select * from subt") + tdSql.checkRows(3) + + tdSql.query("select a,b,c,d from subt") + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(0, 1, 2.0) + tdSql.checkData(1, 1, 2.0) + tdSql.checkData(2, 1, 2.0) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(0, 3, 1) + tdSql.checkData(1, 3, 1) + tdSql.checkData(2, 3, 0) + + # restart taosd to commit, and check + self.restartTaosd(); + + tdSql.query("select * from subt") + tdSql.checkRows(3) + + tdSql.query("select a,b,c,d from subt") + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(0, 1, 2.0) + tdSql.checkData(1, 1, 2.0) + tdSql.checkData(2, 1, 2.0) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(0, 3, 1) + tdSql.checkData(1, 3, 1) + tdSql.checkData(2, 3, 0) + + + + tdSql.execute("create table ct (ts timestamp, a int, b float, c binary(128))") + + print("==============step9") + insertRows = 20000 + for i in range(0, insertRows): + tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i, i)) + + tdSql.query("select * from ct") + tdSql.checkRows(insertRows) + + for i in range(0, insertRows): + tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i+insertRows, i+insertRows)) + + tdSql.query("select * from ct") + tdSql.checkRows(insertRows) + + tdSql.query("select a,b from ct limit 3") + tdSql.checkData(0, 0, insertRows+0) + tdSql.checkData(1, 0, insertRows+1) + tdSql.checkData(2, 0, insertRows+2) + + tdSql.checkData(0, 1, insertRows+0) + tdSql.checkData(1, 1, insertRows+1) + tdSql.checkData(2, 1, insertRows+2) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/append_commit_data-0.py b/tests/pytest/update/append_commit_data-0.py new file mode 100644 index 0000000000000000000000000000000000000000..b844a50a086dc52d7edb5250801ee87cf68ee28f --- /dev/null +++ b/tests/pytest/update/append_commit_data-0.py @@ -0,0 +1,84 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + print("==========step1") + print("create table && insert data") + s = 'reset query cache' + tdSql.execute(s) + s = 'drop database if exists db' + tdSql.execute(s) + s = 'create database db' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t1 (ts timestamp, a int)') + + insertRows = 200 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t1 values (%d , 1)' % + (t0+i)) + print("==========step2") + print("restart to commit ") + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from db.t1") + tdSql.checkRows(insertRows) + for k in range(0,100): + tdLog.info("insert %d rows" % (insertRows)) + for i in range (0,insertRows): + ret = tdSql.execute( + 'insert into db.t1 values(%d,1)' % + (t0+k*200+i) + ) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from db.t1") + tdSql.checkRows(insertRows+200*k) + print("==========step2") + print("insert into another table ") + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t2 (ts timestamp, a int)') + insertRows = 20000 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t2 values (%d, 1)' % + (t0+i)) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py new file mode 100644 index 0000000000000000000000000000000000000000..3169b748e0843c720beb54eced28a14f1ca747a6 --- /dev/null +++ b/tests/pytest/update/append_commit_data.py @@ -0,0 +1,84 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + print("==========step1") + print("create table && insert data") + s = 'reset query cache' + tdSql.execute(s) + s = 'drop database if exists db' + tdSql.execute(s) + s = 'create database db update 1' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t1 (ts timestamp, a int)') + + insertRows = 200 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t1 values (%d , 1)' % + (t0+i)) + print("==========step2") + print("restart to commit ") + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from db.t1") + tdSql.checkRows(insertRows) + for k in range(0,100): + tdLog.info("insert %d rows" % (insertRows)) + for i in range (0,insertRows): + ret = tdSql.execute( + 'insert into db.t1 values(%d,1)' % + (t0+k*200+i) + ) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from db.t1") + tdSql.checkRows(insertRows+200*k) + print("==========step2") + print("insert into another table ") + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t2 (ts timestamp, a int)') + insertRows = 20000 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t2 values (%d, 1)' % + (t0+i)) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/append_commit_last-0.py b/tests/pytest/update/append_commit_last-0.py new file mode 100644 index 0000000000000000000000000000000000000000..c884207f2bba5dd0da09cf4aae501d27caef7aab --- /dev/null +++ b/tests/pytest/update/append_commit_last-0.py @@ -0,0 +1,90 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1604298064000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use db") + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table t1 (ts timestamp, a int)") + + for i in range(10): + tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t1") + tdSql.checkRows(i + 1) + tdSql.query("select sum(a) from t1") + tdSql.checkData(0, 0, i + 1) + + print("==============step2") + tdSql.execute("create table t2 (ts timestamp, a int)") + tdSql.execute("insert into t2 values(%d, 1)" % self.ts) + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + for i in range(1, 151): + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(151) + tdSql.query("select sum(a) from t2") + tdSql.checkData(0, 0, 151) + + + print("==============step3") + tdSql.execute("create table t3 (ts timestamp, a int)") + tdSql.execute("insert into t3 values(%d, 1)" % self.ts) + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + for i in range(8): + for j in range(1, 11): + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j)) + + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(81) + tdSql.query("select sum(a) from t3") + tdSql.checkData(0, 0, 81) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/append_commit_last.py b/tests/pytest/update/append_commit_last.py new file mode 100644 index 0000000000000000000000000000000000000000..013983f9402292d03d26bd998c96eaf39b26a8fd --- /dev/null +++ b/tests/pytest/update/append_commit_last.py @@ -0,0 +1,85 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1604298064000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use udb") + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create database udb update 1") + tdSql.execute("use udb") + tdSql.execute("create table t1 (ts timestamp, a int)") + + for i in range(10): + tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t1") + tdSql.checkRows(i + 1) + + + print("==============step2") + tdSql.execute("create table t2 (ts timestamp, a int)") + tdSql.execute("insert into t2 values(%d, 1)" % self.ts) + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(1) + + for i in range(1, 151): + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(151) + + + print("==============step3") + tdSql.execute("create table t3 (ts timestamp, a int)") + tdSql.execute("insert into t3 values(%d, 1)" % self.ts) + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(1) + + for i in range(8): + for j in range(1, 11): + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j)) + + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(81) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_data-0.py b/tests/pytest/update/merge_commit_data-0.py new file mode 100644 index 0000000000000000000000000000000000000000..14d435f7f20d9e04565fdb7036da043d948b1dcf --- /dev/null +++ b/tests/pytest/update/merge_commit_data-0.py @@ -0,0 +1,351 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + print("==========step1") + print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY") + s = 'reset query cache' + tdSql.execute(s) + s = 'drop database if exists db' + tdSql.execute(s) + s = 'create database db days 30' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t1 (ts timestamp, a int)') + + insertRows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t1 values (%d , 1)' % + (t0 + i)) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + + for k in range(0,10): + for i in range (0,insertRows): + ret = tdSql.execute( + 'insert into t1 values(%d,1)' % + (t0+i) + ) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + print("==========step2") + print("PREPEND DATA ") + ret = tdSql.execute('create table t2 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t2 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + for i in range(-100,0): + ret = tdSql.execute( + 'insert into t2 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows+100) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows+100) + print("==========step3") + print("PREPEND MASSIVE DATA ") + ret = tdSql.execute('create table t3 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t3 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows) + for i in range(-6000,0): + ret = tdSql.execute( + 'insert into t3 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows+6000) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows+6000) + print("==========step4") + print("APPEND DATA") + ret = tdSql.execute('create table t4 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t4 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows) + for i in range(0,100): + ret = tdSql.execute( + 'insert into t4 values (%d , 1)' % + (t0+200+i)) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows+100) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows+100) + print("==========step5") + print("APPEND DATA") + ret = tdSql.execute('create table t5 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t5 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows) + for i in range(0,6000): + ret = tdSql.execute( + 'insert into t5 values (%d , 1)' % + (t0+200+i)) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows+6000) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows+6000) + print("==========step6") + print("UPDATE BLOCK IN TWO STEP") + ret = tdSql.execute('create table t6 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t6 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + for i in range(0,100): + ret = tdSql.execute( + 'insert into t6 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'200') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'200') + for i in range(0,200): + ret = tdSql.execute( + 'insert into t6 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'200') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'200') + print("==========step7") + print("UPDATE LAST HALF AND INSERT LITTLE DATA") + ret = tdSql.execute('create table t7 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t7 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows) + for i in range(100,300): + ret = tdSql.execute( + 'insert into t7 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t7") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0,0,'400') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t7") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0,0,'400') + print("==========step8") + print("UPDATE LAST HALF AND INSERT MASSIVE DATA") + ret = tdSql.execute('create table t8 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t8 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t8") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t8") + tdSql.checkRows(insertRows) + for i in range(6000): + ret = tdSql.execute( + 'insert into t8 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0,0,'11800') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0,0,'11800') + print("==========step9") + print("UPDATE FIRST HALF AND PREPEND LITTLE DATA") + ret = tdSql.execute('create table t9 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t9 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t9") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t9") + tdSql.checkRows(insertRows) + for i in range(-100,100): + ret = tdSql.execute( + 'insert into t9 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t9") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0,0,'400') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t9") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0,0,'400') + print("==========step10") + print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA") + ret = tdSql.execute('create table t10 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t10 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t10") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t10") + tdSql.checkRows(insertRows) + for i in range(-6000,100): + ret = tdSql.execute( + 'insert into t10 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t10") + tdSql.checkRows(6200) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0,0,'12200') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t10") + tdSql.checkRows(6200) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0,0,'12200') + print("==========step11") + print("UPDATE FIRST HALF AND APPEND MASSIVE DATA") + ret = tdSql.execute('create table t11 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t11 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t11") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t11") + tdSql.checkRows(insertRows) + for i in range(100): + ret = tdSql.execute( + 'insert into t11 values (%d , 2)' % + (t0+i)) + for i in range(200,6000): + ret = tdSql.execute( + 'insert into t11 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t11") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t11") + tdSql.checkData(0,0,'11800') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t11") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t11") + tdSql.checkData(0,0,'11800') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_data.py b/tests/pytest/update/merge_commit_data.py new file mode 100644 index 0000000000000000000000000000000000000000..4fb6765361e8099acb0f1f861623a88fd6b8e466 --- /dev/null +++ b/tests/pytest/update/merge_commit_data.py @@ -0,0 +1,351 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + print("==========step1") + print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY") + s = 'reset query cache' + tdSql.execute(s) + s = 'drop database if exists db' + tdSql.execute(s) + s = 'create database db update 1 days 30' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t1 (ts timestamp, a int)') + + insertRows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t1 values (%d , 1)' % + (t0 + i)) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + + for k in range(0,10): + for i in range (0,insertRows): + ret = tdSql.execute( + 'insert into t1 values(%d,1)' % + (t0+i) + ) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + print("==========step2") + print("PREPEND DATA ") + ret = tdSql.execute('create table t2 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t2 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + for i in range(-100,0): + ret = tdSql.execute( + 'insert into t2 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows+100) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows+100) + print("==========step3") + print("PREPEND MASSIVE DATA ") + ret = tdSql.execute('create table t3 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t3 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows) + for i in range(-6000,0): + ret = tdSql.execute( + 'insert into t3 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows+6000) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows+6000) + print("==========step4") + print("APPEND DATA") + ret = tdSql.execute('create table t4 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t4 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows) + for i in range(0,100): + ret = tdSql.execute( + 'insert into t4 values (%d , 1)' % + (t0+200+i)) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows+100) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows+100) + print("==========step5") + print("APPEND DATA") + ret = tdSql.execute('create table t5 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t5 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows) + for i in range(0,6000): + ret = tdSql.execute( + 'insert into t5 values (%d , 1)' % + (t0+200+i)) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows+6000) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows+6000) + print("==========step6") + print("UPDATE BLOCK IN TWO STEP") + ret = tdSql.execute('create table t6 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t6 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + for i in range(0,100): + ret = tdSql.execute( + 'insert into t6 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'300') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'300') + for i in range(0,200): + ret = tdSql.execute( + 'insert into t6 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'400') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'400') + print("==========step7") + print("UPDATE LAST HALF AND INSERT LITTLE DATA") + ret = tdSql.execute('create table t7 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t7 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows) + for i in range(100,300): + ret = tdSql.execute( + 'insert into t7 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t7") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0,0,'500') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t7") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0,0,'500') + print("==========step8") + print("UPDATE LAST HALF AND INSERT MASSIVE DATA") + ret = tdSql.execute('create table t8 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t8 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t8") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t8") + tdSql.checkRows(insertRows) + for i in range(6000): + ret = tdSql.execute( + 'insert into t8 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0,0,'12000') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0,0,'12000') + print("==========step9") + print("UPDATE FIRST HALF AND PREPEND LITTLE DATA") + ret = tdSql.execute('create table t9 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t9 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t9") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t9") + tdSql.checkRows(insertRows) + for i in range(-100,100): + ret = tdSql.execute( + 'insert into t9 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t9") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0,0,'500') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t9") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0,0,'500') + print("==========step10") + print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA") + ret = tdSql.execute('create table t10 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t10 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t10") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t10") + tdSql.checkRows(insertRows) + for i in range(-6000,100): + ret = tdSql.execute( + 'insert into t10 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t10") + tdSql.checkRows(6200) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0,0,'12300') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t10") + tdSql.checkRows(6200) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0,0,'12300') + print("==========step11") + print("UPDATE FIRST HALF AND APPEND MASSIVE DATA") + ret = tdSql.execute('create table t11 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t11 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t11") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t11") + tdSql.checkRows(insertRows) + for i in range(100): + ret = tdSql.execute( + 'insert into t11 values (%d , 2)' % + (t0+i)) + for i in range(200,6000): + ret = tdSql.execute( + 'insert into t11 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t11") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t11") + tdSql.checkData(0,0,'11900') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t11") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t11") + tdSql.checkData(0,0,'11900') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_data2.py b/tests/pytest/update/merge_commit_data2.py new file mode 100644 index 0000000000000000000000000000000000000000..3f0fc718ad83244353bf88da905e6ac0ff800cb5 --- /dev/null +++ b/tests/pytest/update/merge_commit_data2.py @@ -0,0 +1,352 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import time + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def restart_taosd(self,db): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use %s;" % db) + + def date_to_timestamp_microseconds(self, date): + datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f") + obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0) + return obj_stamp + + def timestamp_microseconds_to_date(self, timestamp): + d = datetime.datetime.fromtimestamp(timestamp/1000) + str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f") + return str1 + + + + def run(self): + print("==========step1") + print("create table && insert data") + sql = 'reset query cache' + tdSql.execute(sql) + sql = 'drop database if exists db' + tdSql.execute(sql) + sql = 'create database db update 1 days 30;' + tdSql.execute(sql) + sql = 'use db;' + tdSql.execute(sql) + tdSql.execute('create table t1 (ts timestamp, a int)') + + + print("==================================1 start") + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i)) + print("==========step2") + print("restart to commit ") + self.restart_taosd('db') + + print('check query result after restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows): + tdSql.checkData(i, 1, 1) + + print("==========step3") + print('insert data') + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000)) + print('check query result before restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + print("==========step4") + print('insert data') + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 2)' %(t0+i)) + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t1;') + print(tdSql.queryResult) + for i in range(insert_rows): + tdSql.checkData(i, 1, 2) + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t1;') + # print(tdSql.queryResult) + for i in range(insert_rows): + tdSql.checkData(i, 1, 2) + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + print("==================================2 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t2 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t2 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + for i in range(insert_rows): + tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for k in range(10): + for i in range(10): + tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i)) + print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i)) + + + print("==========step2") + print('check query result before restart') + tdSql.query('select * from db.t2;') + for i in range(insert_rows*2+100): + tdSql.checkData(i, 1, 1) + # print(tdSql.queryResult) + print('restart to commit') + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t2;') + for i in range(insert_rows*2+100): + tdSql.checkData(i, 1, 1) + + + print("==================================3 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t3 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t3 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(5200): + tdSql.execute('insert into t3 values (%d , 2)' %(t0+i)) + + print("==========step2") + print('check query result before restart') + tdSql.query('select * from db.t3;') + for i in range(5200): + tdSql.checkData(i, 1, 2) + # print(tdSql.queryResult) + print('restart to commit') + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t3;') + for i in range(5200): + tdSql.checkData(i, 1, 2) + + print("==================================4 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t4 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(100): + tdSql.execute('insert into t4 values (%d , 2)' %(t0+i)) + + for i in range(200, 5000): + tdSql.execute('insert into t4 values (%d , 2)' %(t0+i)) + + for i in range(100): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t4;') + for i in range(100): + tdSql.checkData(i, 1, 2) + for i in range(100, 200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t4;') + for i in range(100): + tdSql.checkData(i, 1, 2) + for i in range(100, 200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print("==================================5 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t5 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t5 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(100, 200): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i)) + + for i in range(200, 5000): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i)) + + for i in range(100, 200): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t5;') + for i in range(100): + tdSql.checkData(i, 1, 1) + for i in range(100, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5100): + tdSql.checkData(i, 1, 1) + for i in range(5100, 5200): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t5;') + for i in range(100): + tdSql.checkData(i, 1, 1) + for i in range(100, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5100): + tdSql.checkData(i, 1, 1) + for i in range(5100, 5200): + tdSql.checkData(i, 1, 2) + + print("==================================6 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t6 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t6 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(-1000, 10000): + tdSql.execute('insert into t6 values (%d , 2)' %(t0+i)) + + print('check query result before restart') + tdSql.query('select * from db.t6;') + tdSql.checkRows(11000) + for i in range(11000): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t6;') + tdSql.checkRows(11000) + for i in range(11000): + tdSql.checkData(i, 1, 2) + + + print("==================================7 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t7 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t7 values (%d , 1)' %(t0+i)) + + for i in range(insert_rows): + tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(-1000, 10000): + tdSql.execute('insert into t7 values (%d , 2)' %(t0+i)) + + print('check query result before restart') + tdSql.query('select * from db.t7;') + tdSql.checkRows(11000) + for i in range(11000): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t7;') + tdSql.checkRows(11000) + for i in range(11000): + tdSql.checkData(i, 1, 2) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_data2_update0.py b/tests/pytest/update/merge_commit_data2_update0.py new file mode 100644 index 0000000000000000000000000000000000000000..def50e04661b1752668202359eec7dd89df9b6f0 --- /dev/null +++ b/tests/pytest/update/merge_commit_data2_update0.py @@ -0,0 +1,384 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import time + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def restart_taosd(self,db): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use %s;" % db) + + def date_to_timestamp_microseconds(self, date): + datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f") + obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0) + return obj_stamp + + def timestamp_microseconds_to_date(self, timestamp): + d = datetime.datetime.fromtimestamp(timestamp/1000) + str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f") + return str1 + + + + def run(self): + print("==========step1") + print("create table && insert data") + sql = 'reset query cache' + tdSql.execute(sql) + sql = 'drop database if exists db' + tdSql.execute(sql) + sql = 'create database db update 0 days 30;' + tdSql.execute(sql) + sql = 'use db;' + tdSql.execute(sql) + tdSql.execute('create table t1 (ts timestamp, a int)') + + + print("==================================1 start") + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i)) + print("==========step2") + print("restart to commit ") + self.restart_taosd('db') + + print('check query result after restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows): + tdSql.checkData(i, 1, 1) + + print("==========step3") + print('insert data') + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000)) + print('check query result before restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + print("==========step4") + print('insert data') + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 2)' %(t0+i)) + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t1;') + print(tdSql.queryResult) + for i in range(insert_rows): + tdSql.checkData(i, 1, 1) + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t1;') + # print(tdSql.queryResult) + for i in range(insert_rows): + tdSql.checkData(i, 1, 1) + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + print("==================================2 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t2 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t2 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + for i in range(insert_rows): + tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for k in range(10): + for i in range(10): + tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i)) + # print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i)) + + + print("==========step2") + print('check query result before restart') + tdSql.query('select * from db.t2;') + for i in range(insert_rows*2+100): + tdSql.checkData(i, 1, 1) + # print(tdSql.queryResult) + print('restart to commit') + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t2;') + for i in range(insert_rows*2+100): + tdSql.checkData(i, 1, 1) + + + print("==================================3 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t3 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t3 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(5200): + tdSql.execute('insert into t3 values (%d , 2)' %(t0+i)) + + print("==========step2") + print('check query result before restart') + tdSql.query('select * from db.t3;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + # print(tdSql.queryResult) + print('restart to commit') + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t3;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print("==================================4 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t4 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(100): + tdSql.execute('insert into t4 values (%d , 2)' %(t0+i)) + + for i in range(200, 5000): + tdSql.execute('insert into t4 values (%d , 2)' %(t0+i)) + + for i in range(100): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t4;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t4;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + # + print("==================================5 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t5 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t5 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(100, 200): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i)) + + for i in range(200, 5000): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i)) + + for i in range(100, 200): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t5;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t5;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print("==================================6 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t6 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t6 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(-1000, 10000): + tdSql.execute('insert into t6 values (%d , 2)' %(t0+i)) + + print('check query result before restart') + tdSql.query('select * from db.t6;') + tdSql.checkRows(11000) + for i in range(1000): + tdSql.checkData(i, 1, 2) + for i in range(1000,1200): + tdSql.checkData(i, 1, 1) + for i in range(1200,6000): + tdSql.checkData(i, 1, 2) + for i in range(6000,6200): + tdSql.checkData(i, 1, 1) + for i in range(6200, 11000): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t6;') + tdSql.checkRows(11000) + for i in range(1000): + tdSql.checkData(i, 1, 2) + for i in range(1000,1200): + tdSql.checkData(i, 1, 1) + for i in range(1200,6000): + tdSql.checkData(i, 1, 2) + for i in range(6000,6200): + tdSql.checkData(i, 1, 1) + for i in range(6200, 11000): + tdSql.checkData(i, 1, 2) + + + print("==================================7 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t7 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t7 values (%d , 1)' %(t0+i)) + + for i in range(insert_rows): + tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(-1000, 10000): + tdSql.execute('insert into t7 values (%d , 2)' %(t0+i)) + + print('check query result before restart') + tdSql.query('select * from db.t7;') + tdSql.checkRows(11000) + for i in range(1000): + tdSql.checkData(i, 1, 2) + for i in range(1000,1200): + tdSql.checkData(i, 1, 1) + for i in range(1200,6000): + tdSql.checkData(i, 1, 2) + for i in range(6000,6200): + tdSql.checkData(i, 1, 1) + for i in range(6200, 11000): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t7;') + tdSql.checkRows(11000) + for i in range(1000): + tdSql.checkData(i, 1, 2) + for i in range(1000,1200): + tdSql.checkData(i, 1, 1) + for i in range(1200,6000): + tdSql.checkData(i, 1, 2) + for i in range(6000,6200): + tdSql.checkData(i, 1, 1) + for i in range(6200, 11000): + tdSql.checkData(i, 1, 2) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_last-0.py b/tests/pytest/update/merge_commit_last-0.py new file mode 100644 index 0000000000000000000000000000000000000000..8a247f38091467f69c74c57f00341adde0e15992 --- /dev/null +++ b/tests/pytest/update/merge_commit_last-0.py @@ -0,0 +1,309 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1603152000000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use db") + + def run(self): + tdSql.prepare() + + print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY") + tdSql.execute("create table t1 (ts timestamp, a int)") + + for i in range(5): + tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i)) + self.restartTaosd() + tdSql.query("select * from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 0) + + print("==============step 2: UPDATE THE WHOLE LAST BLOCK") + tdSql.execute("create table t2 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t2") + tdSql.checkData(0, 0, 50) + + for i in range(50): + tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i)) + tdSql.query("select * from t2") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t2") + tdSql.checkData(0, 0, 50) + + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t2") + tdSql.checkData(0, 0, 50) + + print("==============step 3: UPDATE PART OF THE LAST BLOCK") + tdSql.execute("create table t3 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t3") + tdSql.checkData(0, 0, 50) + + for i in range(25): + tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t3") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t3") + tdSql.checkData(0, 0, 50) + + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t3") + tdSql.checkData(0, 0, 50) + + print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA") + tdSql.execute("create table t4 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t4") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t4") + tdSql.checkData(0, 0, 50) + + for i in range(25): + tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i)) + + for i in range(50, 60): + tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t4") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t4") + tdSql.checkData(0, 0, 70) + + self.restartTaosd() + tdSql.query("select * from t4") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t4") + tdSql.checkData(0, 0, 70) + + print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA") + tdSql.execute("create table t5 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 50) + + for i in range(-10, 0): + tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i)) + + for i in range(25): + tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 70) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 70) + + for i in range(-10, 0): + tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i)) + + for i in range(25, 50): + tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i)) + + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 70) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 70) + + + print("==============step 6: INSERT AHEAD A LOT OF DATA") + tdSql.execute("create table t6 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t6") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 50) + + for i in range(-1000, 0): + tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t6") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 2050) + + self.restartTaosd() + tdSql.query("select * from t6") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 2050) + + print("==============step 7: INSERT AHEAD A LOT AND UPDATE") + tdSql.execute("create table t7 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t7") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 50) + + for i in range(-1000, 25): + tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t7") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 2050) + + self.restartTaosd() + tdSql.query("select * from t7") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 2050) + + print("==============step 8: INSERT AFTER A LOT AND UPDATE") + tdSql.execute("create table t8 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t8") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 50) + + for i in range(25, 6000): + tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 11950) + + self.restartTaosd() + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 11950) + + print("==============step 9: UPDATE ONLY MIDDLE") + tdSql.execute("create table t9 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 50) + + for i in range(20, 30): + tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 50) + + self.restartTaosd() + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 50) + + print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK") + tdSql.execute("create table t10 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t10") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 50) + + for i in range(-4000, 4000): + tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t10") + tdSql.checkRows(8000) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 15950) + + self.restartTaosd() + tdSql.query("select * from t10") + tdSql.checkRows(8000) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 15950) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_last.py b/tests/pytest/update/merge_commit_last.py new file mode 100644 index 0000000000000000000000000000000000000000..183cca0a1e40fd995daaed0f271bb5083838a78f --- /dev/null +++ b/tests/pytest/update/merge_commit_last.py @@ -0,0 +1,321 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1603152000000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use udb") + + def run(self): + tdSql.prepare() + + tdSql.execute("create database udb update 1 days 30") + tdSql.execute("use udb") + + print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY") + tdSql.execute("create table t1 (ts timestamp, a int)") + + for i in range(5): + tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i)) + self.restartTaosd() + tdSql.query("select * from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, i) + + print("==============step 2: UPDATE THE WHOLE LAST BLOCK") + tdSql.execute("create table t2 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 1) + + for i in range(50): + tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i)) + tdSql.query("select * from t2") + for i in range(50): + tdSql.checkData(i, 1, 2) + + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 2) + + print("==============step 3: UPDATE PART OF THE LAST BLOCK") + tdSql.execute("create table t3 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 1) + + for i in range(25): + tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t3") + for i in range(25): + tdSql.checkData(i, 1, 2) + for i in range(25, 50): + tdSql.checkData(i, 1, 1) + + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(50) + for i in range(25): + tdSql.checkData(i, 1, 2) + for i in range(25, 50): + tdSql.checkData(i, 1, 1) + + print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA") + tdSql.execute("create table t4 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t4") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 1) + + for i in range(25): + tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i)) + + for i in range(50, 60): + tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t4") + tdSql.checkRows(60) + for i in range(25): + tdSql.checkData(i, 1, 2) + for i in range(25, 50): + tdSql.checkData(i, 1, 1) + for i in range(50, 60): + tdSql.checkData(i, 1, 2) + + self.restartTaosd() + tdSql.query("select * from t4") + tdSql.checkRows(60) + for i in range(25): + tdSql.checkData(i, 1, 2) + for i in range(25, 50): + tdSql.checkData(i, 1, 1) + for i in range(50, 60): + tdSql.checkData(i, 1, 2) + + print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA") + tdSql.execute("create table t5 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 1) + + for i in range(-10, 0): + tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i)) + + for i in range(25): + tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 95) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 95) + + for i in range(-10, 0): + tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i)) + + for i in range(25, 50): + tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i)) + + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 155) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 155) + + + print("==============step 6: INSERT AHEAD A LOT OF DATA") + tdSql.execute("create table t6 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t6") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 50) + + for i in range(-1000, 0): + tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t6") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 2050) + + self.restartTaosd() + tdSql.query("select * from t6") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 2050) + + print("==============step 7: INSERT AHEAD A LOT AND UPDATE") + tdSql.execute("create table t7 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t7") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 50) + + for i in range(-1000, 25): + tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t7") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 2075) + + self.restartTaosd() + tdSql.query("select * from t7") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 2075) + + print("==============step 8: INSERT AFTER A LOT AND UPDATE") + tdSql.execute("create table t8 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t8") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 50) + + for i in range(25, 6000): + tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 11975) + + self.restartTaosd() + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 11975) + + print("==============step 9: UPDATE ONLY MIDDLE") + tdSql.execute("create table t9 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 50) + + for i in range(20, 30): + tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 60) + + self.restartTaosd() + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 60) + + print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK") + tdSql.execute("create table t10 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t10") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 50) + + for i in range(-4000, 4000): + tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t10") + tdSql.checkRows(8000) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 16000) + + self.restartTaosd() + tdSql.query("select * from t10") + tdSql.checkRows(8000) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 16000) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 1ac492bb3ad2733c7b2ebb46560edbb7204e8951..757399b4a262dff7b11619791d3c82686fb293e8 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -15,6 +15,7 @@ import sys import os import os.path import subprocess +from time import sleep from util.log import * @@ -210,6 +211,7 @@ class TDDnode: (self.index, self.cfgPath)) def getBuildPath(self): + buildPath = "" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): @@ -256,6 +258,35 @@ class TDDnode: tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) time.sleep(5) + + def startWithoutSleep(self): + buildPath = self.getBuildPath() + + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + + binPath = buildPath + "/build/bin/taosd" + + if self.deployed == 0: + tdLog.exit("dnode:%d is not deployed" % (self.index)) + + if self.valgrind == 0: + cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( + binPath, self.cfgDir) + else: + valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" + + cmd = "nohup %s %s -c %s 2>&1 & " % ( + valgrindCmdline, binPath, self.cfgDir) + + print(cmd) + + if os.system(cmd) != 0: + tdLog.exit(cmd) + self.running = 1 + tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) def stop(self): if self.valgrind == 0: @@ -425,6 +456,10 @@ class TDDnodes: def start(self, index): self.check(index) self.dnodes[index - 1].start() + + def startWithoutSleep(self, index): + self.check(index) + self.dnodes[index - 1].startWithoutSleep() def stop(self, index): self.check(index) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 9abec354c6d58507ff3bcc74d1c0dc03f691440c..b2ed6212fd643c158f7ed6f4cc6cb2449a512a2e 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -25,7 +25,7 @@ class TDSql: self.queryCols = 0 self.affectedRows = 0 - def init(self, cursor, log=True): + def init(self, cursor, log=False): self.cursor = cursor if (log): diff --git a/tests/pytest/wal/addOldWalTest.py b/tests/pytest/wal/addOldWalTest.py new file mode 100644 index 0000000000000000000000000000000000000000..2f4dcd5ce807cf7bbadfa480af6ed6342058a78a --- /dev/null +++ b/tests/pytest/wal/addOldWalTest.py @@ -0,0 +1,70 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def createOldDir(self): + oldDir = tdDnodes.getDnodesRootDir() + "dnode1/data/vnode/vnode2/wal/old" + os.system("sudo mkdir -p %s" % oldDir) + + def createOldDirAndAddWal(self): + oldDir = tdDnodes.getDnodesRootDir() + "dnode1/data/vnode/vnode2/wal/old" + os.system("sudo echo 'test' >> %s/wal" % oldDir) + + + def run(self): + tdSql.prepare() + + tdSql.execute("create table t1(ts timestamp, a int)") + tdSql.execute("insert into t1 values(now, 1)") + + # create old dir only + self.createOldDir() + os.system("sudo kill -9 $(pgrep taosd)") + tdDnodes.start(1) + + tdSql.execute("use db") + tdSql.query("select * from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + # create old dir and add wal under old dir + self.createOldDir() + self.createOldDirAndAddWal() + os.system("sudo kill -9 $(pgrep taosd)") + tdDnodes.start(1) + + tdSql.query("select * from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/script/general/db/nosuchfile.sim b/tests/script/general/db/nosuchfile.sim new file mode 100644 index 0000000000000000000000000000000000000000..98ac4ec012dc694357878a61ca0dbc11259f0a9e --- /dev/null +++ b/tests/script/general/db/nosuchfile.sim @@ -0,0 +1,66 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 2 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 3000 + +print ========== step3 +sql create database d1 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 35) +sql insert into d1.t1 values(now+2s, 34) +sql insert into d1.t1 values(now+3s, 33) +sql insert into d1.t1 values(now+4s, 32) +sql insert into d1.t1 values(now+5s, 31) + +print ========== step4 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +sleep 3000 + +print ========== step5 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +print ========== step6 +system_content rm -rf ../../../sim/dnode1/data/vnode/vnode2/tsdb/data + +print ========== step7 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != null then + return -1 +endi +if $data11 != null then + return -1 +endi +if $data21 != null then + return -1 +endi +if $data31 != null then + return -1 +endi +if $data41 != null then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 8d2f1a7c00304c42c91311ae703bcef97aa6ace0..94ecb59f75304d99f48ebdb644be432370f86f2a 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -81,7 +81,7 @@ print =============== step2 - no db #11 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql print 11-> $system_content -if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","update","status"],"data":[],"rows":0}@ then return -1 endi diff --git a/tests/script/general/insert/basic.sim b/tests/script/general/insert/basic.sim index ba8cff83fa8b470af1b1f7fccf74d0e1d042049e..3f0f25a95ba68c8099c02e6daab933411089a632 100644 --- a/tests/script/general/insert/basic.sim +++ b/tests/script/general/insert/basic.sim @@ -8,8 +8,8 @@ sleep 3000 sql connect $i = 0 -$dbPrefix = tb_in_db -$tbPrefix = tb_in_tb +$dbPrefix = d +$tbPrefix = t $db = $dbPrefix . $i $tb = $tbPrefix . $i @@ -22,28 +22,27 @@ sql create table $tb (ts timestamp, speed int) $x = 0 while $x < 10 - $ms = $x . m - sql insert into $tb values (now + $ms , $x ) + $cc = $x * 60000 + $ms = 1601481600000 + $cc + + sql insert into $tb values ($ms , $x ) $x = $x + 1 endw print =============== step 2 -sql insert into $tb values (now - 5m , 10) -sql insert into $tb values (now - 6m , 10) -sql insert into $tb values (now - 7m , 10) -sql insert into $tb values (now - 8m , 10) +$x = 0 +while $x < 5 + $cc = $x * 60000 + $ms = 1551481600000 + $cc + + sql insert into $tb values ($ms , $x ) + $x = $x + 1 +endw sql select * from $tb print $rows points data are retrieved -if $rows != 14 then - return -1 -endi - -sql drop database $db -sleep 1000 -sql show databases -if $rows != 0 then +if $rows != 15 then return -1 endi diff --git a/tests/script/general/parser/fill.sim b/tests/script/general/parser/fill.sim index f89c27d71fd8ba02105f502afd4df26a1870ffb5..92aa6a922cc901c88ad68f87f61e8befbed5e87c 100644 --- a/tests/script/general/parser/fill.sim +++ b/tests/script/general/parser/fill.sim @@ -116,7 +116,7 @@ if $data81 != 1 then endi # avg_with_fill -print avg_witt_constant_fill +print avg_with_constant_fill sql select avg(c1), avg(c2), avg(c3), avg(c4), avg(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6) if $rows != 9 then return -1 @@ -371,12 +371,10 @@ if $data11 != 99 then endi sql select * from $tb -#print data08 = $data08 if $data08 != NCHAR then + print expect NCHAR, actual:$data08 return -1 endi -#return -1 - # fill_into_nonarithmetic_fieds sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index bd0d3c1a12c77570c19ea1ef061395912ad9f93a..145c4a008f1ae26378baeabdfd5103cc807266f3 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -27,7 +27,7 @@ $mt = $mtPrefix . $i $tstart = 100000 -sql drop database if exits $db -x step1 +sql drop database if exists $db -x step1 step1: sql create database if not exists $db keep 36500 sql use $db @@ -435,53 +435,53 @@ sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.00 sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ; sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ; -sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2; -if $rows != 40 then - return -1 -endi - -if $data01 != 1.000000000 then - return -1 -endi -if $data02 != t1 then - return -1 -endi -if $data03 != 1 then - return -1 -endi -if $data04 != 1 then - return -1 -endi - -if $data11 != 1.000000000 then - return -1 -endi -if $data12 != t1 then - return -1 -endi -if $data13 != 1 then - return -1 -endi -if $data14 != 1 then - return -1 -endi - -sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2 limit 1; -if $rows != 2 then - return -1 -endi - -if $data11 != 1.000000000 then - return -1 -endi -if $data12 != t2 then - return -1 -endi -if $data13 != 1 then - return -1 -endi -if $data14 != 2 then - return -1 -endi +#sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2; +#if $rows != 40 then +# return -1 +#endi +# +#if $data01 != 1.000000000 then +# return -1 +#endi +#if $data02 != t1 then +# return -1 +#endi +#if $data03 != 1 then +# return -1 +#endi +#if $data04 != 1 then +# return -1 +#endi +# +#if $data11 != 1.000000000 then +# return -1 +#endi +#if $data12 != t1 then +# return -1 +#endi +#if $data13 != 1 then +# return -1 +#endi +#if $data14 != 1 then +# return -1 +#endi +# +#sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2 limit 1; +#if $rows != 2 then +# return -1 +#endi +# +#if $data11 != 1.000000000 then +# return -1 +#endi +#if $data12 != t2 then +# return -1 +#endi +#if $data13 != 1 then +# return -1 +#endi +#if $data14 != 2 then +# return -1 +#endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 254571bda103957fbaeaf0e311e7be03b4dcfc35..624c2e72c40bfee35f8b0b17646c97149eda65c1 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -24,7 +24,7 @@ $mt = $mtPrefix . $i $tstart = 100000 -sql drop database if exits $db -x step1 +sql drop database if exists $db -x step1 step1: sql create database if not exists $db keep 36500 sql use $db @@ -444,6 +444,9 @@ if $rows != $val then return -1 endi +#=============================================================== +sql select first(join_tb0.c8),first(join_tb0.c9) from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts <= 100002 and join_tb0.c7 = true + #====================group by========================================= diff --git a/tests/script/general/parser/join_multivnode.sim b/tests/script/general/parser/join_multivnode.sim index 51f1ef11c7fc9f8cfff60ebe86ad00104266e7ad..76230f79f0136975006e81cee4236a2fa9521214 100644 --- a/tests/script/general/parser/join_multivnode.sim +++ b/tests/script/general/parser/join_multivnode.sim @@ -22,7 +22,7 @@ $mt = $mtPrefix . $i $tstart = 100000 -sql drop database if exits $db -x step1 +sql drop database if exists $db -x step1 step1: sql create database if not exists $db keep 36500 sql use $db @@ -132,4 +132,84 @@ sql select join_mt0.ts, join_mt1.t1, join_mt0.t1, join_mt1.tbname, join_mt0.tbna sql select join_mt0.ts, join_mt1.t1, join_mt0.t1, join_mt1.tbname, join_mt0.tbname from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1 limit 1 +#1970-01-01 08:01:40.800 | 10 | 45.000000000 | 0 | true | false | 0 | +#1970-01-01 08:01:40.790 | 10 | 945.000000000 | 90 | true | true | 0 | +sql_error select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc limit 20 offset 19; + + +sql select count(join_mt0.c1), sum(join_mt0.c2)/count(*), avg(c2), first(join_mt0.c5), last(c7) from join_mt0 interval(10a) group by join_mt0.t1 order by join_mt0.ts desc; +if $rows != 100 then + return -1 +endi + +if $data00 != @70-01-01 08:01:40.990@ then + print expect 0, actual: $data00 + return -1 +endi + +if $data01 != 30 then + return -1 +endi + +if $data02 != 94.500000000 then + print expect 94.500000000, actual $data02 + return -1 +endi + +if $data03 != 94.500000000 then + return -1 +endi + +if $data04 != 90 then + return -1 +endi + +if $data05 != 1 then + return -1 +endi + +if $data06 != 2 then + return -1 +endi + +if $data10 != @70-01-01 08:01:40.980@ then + print expect 70-01-01 08:01:40.980, actual: $data10 + return -1 +endi + +if $data11 != 30 then + return -1 +endi + +if $data12 != 84.500000000 then + print expect 84.500000000, actual $data12 + return -1 +endi + +if $data13 != 84.500000000 then + return -1 +endi + +if $data14 != 80 then + return -1 +endi + +if $data15 != 1 then + return -1 +endi + +if $data16 != 2 then + return -1 +endi + +# this function will cause shell crash +sql_error select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc; + +sql_error select last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10m) group by join_mt0.t1 order by join_mt0.ts asc; + +sql_error select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); +sql select count(join_mt0.c1), first(join_mt0.c1)/count(*), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); +sql select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); +sql select last(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/projection_limit_offset.sim b/tests/script/general/parser/projection_limit_offset.sim index fbff99d58f5b6355863b90172c2fb14c1f2ba393..7c83ca0c2ff3d44bf487f8a94750736767834cfa 100644 --- a/tests/script/general/parser/projection_limit_offset.sim +++ b/tests/script/general/parser/projection_limit_offset.sim @@ -21,7 +21,7 @@ $mt = $mtPrefix . $i $tstart = 100000 -sql drop database if exits $db -x step1 +sql drop database if exists $db -x step1 step1: sql create database if not exists $db keep 36500 sql use $db @@ -324,8 +324,22 @@ sql create table tm0 using m1 tags(1); sql create table tm1 using m1 tags(2); sql insert into tm0 values(10000, 1) (20000, 2)(30000, 3) (40000, NULL) (50000, 2) tm1 values(10001, 2)(20000,4)(90000,9); -sql select count(*),first(k),last(k) from m1 where tbname in ('tm0') interval(1s) order by ts desc; +#=============================tbase-1205 +sql select count(*) from tm1 where ts= now -1d interval(1h) fill(NULL); + +print ===================>TD-1834 +sql select * from tm0 where ts>11000 and ts< 20000 order by ts asc +if $rows != 0 then + return -1 +endi + +sql select * from tm0 where ts>11000 and ts< 20000 order by ts desc +if $rows != 0 then + return -1 +endi + +sql select count(*),first(k),last(k) from m1 where tbname in ('tm0') interval(1s) order by ts desc; if $row != 5 then return -1 endi @@ -386,7 +400,25 @@ sql_error select k+1,sum(k) from tm0; sql_error select k, sum(k) from tm0; sql_error select k, sum(k)+1 from tm0; +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 5000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed + #=============================tbase-1205 sql select count(*) from tm1 where ts= now -1d interval(1h) fill(NULL); +print ===================>TD-1834 +sql select * from tm0 where ts>11000 and ts< 20000 order by ts asc +if $rows != 0 then + return -1 +endi + +sql select * from tm0 where ts>11000 and ts< 20000 order by ts desc +if $rows != 0 then + return -1 +endi + + diff --git a/tests/script/general/parser/sliding.sim b/tests/script/general/parser/sliding.sim index f85211beb83e575e2a73518d89f4b7d989486f83..ec0e31311afe1a08644aa28515071bced71ae0f0 100644 --- a/tests/script/general/parser/sliding.sim +++ b/tests/script/general/parser/sliding.sim @@ -26,7 +26,7 @@ $i = 0 $db = $dbPrefix . $i $mt = $mtPrefix . $i -sql drop database if exits $db -x step1 +sql drop database if exists $db -x step1 step1: sql create database if not exists $db maxtables 4 keep 36500 sql use $db diff --git a/tests/script/general/parser/stream_on_sys.sim b/tests/script/general/parser/stream_on_sys.sim index 5507b4db484006fc8ae7887a47bcd83df5208c50..845a484488137e509a79bbb057f104378843cb8f 100644 --- a/tests/script/general/parser/stream_on_sys.sim +++ b/tests/script/general/parser/stream_on_sys.sim @@ -22,12 +22,12 @@ $i = 0 sql use $db -sql create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s) sliding(2s) -sql create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s) sliding(2s) -sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s) sliding(2s) -sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s) sliding(2s) -sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s) sliding(2s) -sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s) sliding(2s) +sql create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s) +sql create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s) +sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s) +sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s) +sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s) +sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s) sleep 120000 sql select * from cpustrm if $rows <= 0 then diff --git a/tests/script/general/parser/union.sim b/tests/script/general/parser/union.sim index 4af482bde073000d9f2ba098b469f8ec33d7f419..024b9c76efe6b8ddd69c723ffc0e1e0b0d82e075 100644 --- a/tests/script/general/parser/union.sim +++ b/tests/script/general/parser/union.sim @@ -27,7 +27,7 @@ $j = 1 $mt1 = $mtPrefix . $j -sql drop database if exits $db -x step1 +sql drop database if exists $db -x step1 step1: sql create database if not exists $db sql use $db diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim index 5cac3f47235c1321d740e373d0f5828f076c31c7..066fac43ad9ee617a2cb9a91a62a6ebf984be6f0 100644 --- a/tests/script/general/parser/where.sim +++ b/tests/script/general/parser/where.sim @@ -20,7 +20,7 @@ $i = 0 $db = $dbPrefix . $i $mt = $mtPrefix . $i -sql drop database if exits $db -x step1 +sql drop database if exists $db -x step1 step1: sql create database if not exists $db sql use $db diff --git a/tests/script/general/wal/kill.sim b/tests/script/general/wal/kill.sim new file mode 100644 index 0000000000000000000000000000000000000000..7f103874a561c2bb2534996ab30d30ab0e8907a3 --- /dev/null +++ b/tests/script/general/wal/kill.sim @@ -0,0 +1,77 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ============== deploy +system sh/exec.sh -n dnode1 -s start +sleep 3001 +sql connect + +sql create database d1 +sql use d1 + +sql create table t1 (ts timestamp, i int) +sql insert into t1 values(now, 1); + +print =============== step3 +sleep 3000 +sql select * from t1; +print rows: $rows +if $rows != 1 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +sleep 3000 + +print =============== step4 +system sh/exec.sh -n dnode1 -s start -x SIGKILL +sleep 3000 +sql select * from t1; +print rows: $rows +if $rows != 1 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +sleep 3000 + +print =============== step5 +system sh/exec.sh -n dnode1 -s start -x SIGKILL +sleep 3000 +sql select * from t1; +print rows: $rows +if $rows != 1 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +sleep 3000 + +print =============== step6 +system sh/exec.sh -n dnode1 -s start -x SIGKILL +sleep 3000 +sql select * from t1; +print rows: $rows +if $rows != 1 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +sleep 3000 + +print =============== step7 +system sh/exec.sh -n dnode1 -s start -x SIGKILL +sleep 3000 +sql select * from t1; +print rows: $rows +if $rows != 1 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +sleep 3000 + +print =============== step8 +system sh/exec.sh -n dnode1 -s start -x SIGKILL +sleep 3000 +sql select * from t1; +print rows: $rows +if $rows != 1 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGKILL diff --git a/tests/script/general/wal/maxtables.sim b/tests/script/general/wal/maxtables.sim new file mode 100644 index 0000000000000000000000000000000000000000..e504c7e92e3447f7d29dbb2dc03456c3775ced2c --- /dev/null +++ b/tests/script/general/wal/maxtables.sim @@ -0,0 +1,46 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode1 -c tableIncStepPerVnode -v 2 + + +print ============== deploy +system sh/exec.sh -n dnode1 -s start +sleep 3001 +sql connect + +sql create database d1 +sql use d1 +sql create table st (ts timestamp, tbcol int) TAGS(tgcol int) + +$i = 0 +while $i < 100 + $tb = t . $i + sql create table $tb using st tags( $i ) + sql insert into $tb values (now , $i ) + $i = $i + 1 +endw + +sql_error sql create table tt (ts timestamp, i int) + +print =============== step3 +sql select * from st; +if $rows != 100 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +sleep 3000 + +print =============== step4 +system sh/exec.sh -n dnode1 -s start +sleep 3000 + +sql select * from st; +if $rows != 100 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/wal/sync.sim b/tests/script/general/wal/sync.sim new file mode 100644 index 0000000000000000000000000000000000000000..abaf22f91921e5bcc8effbe6fc1c66766ff92a3f --- /dev/null +++ b/tests/script/general/wal/sync.sim @@ -0,0 +1,124 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c http -v 1 +system sh/cfg.sh -n dnode2 -c http -v 1 +system sh/cfg.sh -n dnode3 -c http -v 1 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000 + +system sh/cfg.sh -n dnode1 -c replica -v 3 +system sh/cfg.sh -n dnode2 -c replica -v 3 +system sh/cfg.sh -n dnode3 -c replica -v 3 + +system sh/cfg.sh -n dnode1 -c maxSQLLength -v 940032 +system sh/cfg.sh -n dnode2 -c maxSQLLength -v 940032 +system sh/cfg.sh -n dnode3 -c maxSQLLength -v 940032 + +print ============== deploy + +system sh/exec.sh -n dnode1 -s start +sleep 5001 +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +print =============== step1 +$x = 0 +show1: + $x = $x + 1 + sleep 2000 + if $x == 5 then + return -1 + endi +sql show mnodes -x show1 +$mnode1Role = $data2_1 +print mnode1Role $mnode1Role +$mnode2Role = $data2_2 +print mnode2Role $mnode2Role +$mnode3Role = $data2_3 +print mnode3Role $mnode3Role + +if $mnode1Role != master then + goto show1 +endi +if $mnode2Role != slave then + goto show1 +endi +if $mnode3Role != slave then + goto show1 +endi + +print =============== step2 +sql create database d1 replica 3 +sql use d1 + +sql create table table_rest (ts timestamp, i int) +print sql length is 870KB +restful d1 table_rest 1591072800 30000 +restful d1 table_rest 1591172800 30000 +restful d1 table_rest 1591272800 30000 +restful d1 table_rest 1591372800 30000 +restful d1 table_rest 1591472800 30000 +restful d1 table_rest 1591572800 30000 +restful d1 table_rest 1591672800 30000 +restful d1 table_rest 1591772800 30000 +restful d1 table_rest 1591872800 30000 +restful d1 table_rest 1591972800 30000 + +sql select * from table_rest; +print rows: $rows +if $rows != 300000 then + return -1 +endi + +print =============== step3 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 5000 +sql select * from table_rest; +print rows: $rows +if $rows != 300000 then + return -1 +endi +system sh/exec.sh -n dnode1 -s start -x SIGINT +sleep 5000 + +print =============== step4 +system sh/exec.sh -n dnode2 -s stop -x SIGINT +sleep 5000 +sql select * from table_rest; +print rows: $rows +if $rows != 300000 then + return -1 +endi +system sh/exec.sh -n dnode2 -s start -x SIGINT +sleep 5000 + +print =============== step5 +system sh/exec.sh -n dnode3 -s stop -x SIGINT +sleep 5000 +sql select * from table_rest; +print rows: $rows +if $rows != 300000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 4e68d1566a76468b06115a80aace70871665c9cb..daf92679bd61d8317b648b0285e46a4c956a9d94 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -64,6 +64,7 @@ cd ../../../debug; make ./test.sh -f general/db/repeat.sim ./test.sh -f general/db/tables.sim ./test.sh -f general/db/vnodes.sim +./test.sh -f general/db/nosuchfile.sim ./test.sh -f general/field/2.sim ./test.sh -f general/field/3.sim @@ -235,6 +236,10 @@ cd ../../../debug; make ./test.sh -f general/vector/table_query.sim ./test.sh -f general/vector/table_time.sim +./test.sh -f general/wal/sync.sim +./test.sh -f general/wal/kill.sim +./test.sh -f general/wal/maxtables.sim + ./test.sh -f unique/account/account_create.sim ./test.sh -f unique/account/account_delete.sim ./test.sh -f unique/account/account_len.sim @@ -277,6 +282,7 @@ cd ../../../debug; make ./test.sh -f unique/dnode/balance2.sim ./test.sh -f unique/dnode/balance3.sim ./test.sh -f unique/dnode/balancex.sim +./test.sh -f unique/dnode/data1.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim ./test.sh -f unique/dnode/reason.sim diff --git a/tests/script/unique/dnode/data1.sim b/tests/script/unique/dnode/data1.sim new file mode 100644 index 0000000000000000000000000000000000000000..61a991148b21b471aef906223a5f600f7db38f5f --- /dev/null +++ b/tests/script/unique/dnode/data1.sim @@ -0,0 +1,137 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/cfg.sh -n dnode2 -c wallevel -v 2 +system sh/cfg.sh -n dnode3 -c wallevel -v 2 +system sh/cfg.sh -n dnode4 -c wallevel -v 2 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 3000 + +print ========== step2 +sql create dnode $hostname2 +system sh/exec.sh -n dnode2 -s start +sql create dnode $hostname3 +system sh/exec.sh -n dnode3 -s start +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start + +$x = 0 +show2: + $x = $x + 1 + sleep 3000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +if $data2_1 != 0 then + goto show2 +endi +if $data2_2 != 0 then + goto show2 +endi +if $data2_3 != 0 then + goto show2 +endi +if $data2_4 != 0 then + goto show2 +endi + +print ========== step3 +sql create database d1 replica 3 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 35) +sql insert into d1.t1 values(now+2s, 34) +sql insert into d1.t1 values(now+3s, 33) +sql insert into d1.t1 values(now+4s, 32) +sql insert into d1.t1 values(now+5s, 31) + +$x = 0 +show3: + $x = $x + 1 + sleep 3000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +if $data2_1 != 0 then + goto show3 +endi +if $data2_2 != 1 then + goto show3 +endi +if $data2_3 != 1 then + goto show3 +endi +if $data2_4 != 1 then + goto show3 +endi + +print ========== step4 +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT + +print ========== step5 +system_content rm -rf ../../../sim/dnode4/data/vnode/vnode2/tsdb/data + +print ========== step6 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sleep 10000 + +print ========== step7 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/offline1.sim b/tests/script/unique/dnode/offline1.sim index 02d03dee97be0f2de62b3f8eb18194c595e7b050..beebbfda60c03d24a774347c8ecf8c2f9a4c6c9e 100644 --- a/tests/script/unique/dnode/offline1.sim +++ b/tests/script/unique/dnode/offline1.sim @@ -49,7 +49,7 @@ print dnode1 $data4_2 if $data4_1 != ready then return -1 endi -if $data4_2 != offline then +if $data4_2 == ready then return -1 endi diff --git a/tests/test-all.sh b/tests/test-all.sh index e45dd15fedc999c08037544254f13f78607ee638..214360f3b8d438198bbeadedf63296cbfbaa05a9 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -9,7 +9,7 @@ NC='\033[0m' function runSimCaseOneByOne { while read -r line; do - if [[ $line =~ ^./test.sh* ]]; then + if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then case=`echo $line | grep sim$ |awk '{print $NF}'` start_time=`date +%s` @@ -26,7 +26,12 @@ function runPyCaseOneByOne { while read -r line; do if [[ $line =~ ^python.* ]]; then if [[ $line != *sleep* ]]; then - case=`echo $line|awk '{print $NF}'` + + if [[ $line =~ '-r' ]];then + case=`echo $line|awk '{print $4}'` + else + case=`echo $line|awk '{print $NF}'` + fi start_time=`date +%s` $line > /dev/null 2>&1 && \ echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log || \ diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 58e58a442cedca564864578604faa6cea28ede68..01e5016557f2987741fdec6c6fd54facd885ea8a 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -100,7 +100,7 @@ typedef struct _cmd_t { int16_t cmdno; int16_t nlen; char name[MAX_SIM_CMD_NAME_LEN]; - bool (*parseCmd)(char *, struct _cmd_t *, int); + bool (*parseCmd)(char *, struct _cmd_t *, int32_t); bool (*executeCmd)(struct _script_t *script, char *option); struct _cmd_t *next; } SCommand; @@ -111,7 +111,7 @@ typedef struct { int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ int16_t lineNum; // correspodning line number in original file - int optionOffset; // relative option offset + int32_t optionOffset;// relative option offset } SCmdLine; typedef struct _var_t { @@ -121,59 +121,56 @@ typedef struct _var_t { } SVariable; typedef struct _script_t { - int type; - bool killed; - - void *taos; - char rows[12]; // number of rows data retrieved - char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM] - [MAX_QUERY_VALUE_LEN]; // query results - char system_exit_code[12]; - char system_ret_content[MAX_SYSTEM_RESULT_LEN]; - - int varLen; - int linePos; // current cmd position - int numOfLines; // number of lines in the script - int bgScriptLen; - char fileName[MAX_FILE_NAME_LEN]; // script file name - char error[MAX_ERROR_LEN]; - char *optionBuffer; + int32_t type; + bool killed; + void * taos; + char rows[12]; // number of rows data retrieved + char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results + char system_exit_code[12]; + char system_ret_content[MAX_SYSTEM_RESULT_LEN]; + int32_t varLen; + int32_t linePos; // current cmd position + int32_t numOfLines; // number of lines in the script + int32_t bgScriptLen; + char fileName[MAX_FILE_NAME_LEN]; // script file name + char error[MAX_ERROR_LEN]; + char * optionBuffer; SCmdLine *lines; // command list SVariable variables[MAX_VAR_LEN]; + pthread_t bgPid; + char auth[128]; struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM]; - char auth[128]; } SScript; extern SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; extern SCommand simCmdList[]; -extern int simScriptPos; -extern int simScriptSucced; -extern int simDebugFlag; -extern char tsScriptDir[]; -extern bool simAsyncQuery; +extern int32_t simScriptPos; +extern int32_t simScriptSucced; +extern int32_t simDebugFlag; +extern char tsScriptDir[]; +extern bool simAsyncQuery; SScript *simParseScript(char *fileName); - SScript *simProcessCallOver(SScript *script); -void *simExecuteScript(void *script); -void simInitsimCmdList(); -bool simSystemInit(); -void simSystemCleanUp(); -char *simGetVariable(SScript *script, char *varName, int varLen); -bool simExecuteExpCmd(SScript *script, char *option); -bool simExecuteTestCmd(SScript *script, char *option); -bool simExecuteGotoCmd(SScript *script, char *option); -bool simExecuteRunCmd(SScript *script, char *option); -bool simExecuteRunBackCmd(SScript *script, char *option); -bool simExecuteSystemCmd(SScript *script, char *option); -bool simExecuteSystemContentCmd(SScript *script, char *option); -bool simExecutePrintCmd(SScript *script, char *option); -bool simExecuteSleepCmd(SScript *script, char *option); -bool simExecuteReturnCmd(SScript *script, char *option); -bool simExecuteSqlCmd(SScript *script, char *option); -bool simExecuteSqlErrorCmd(SScript *script, char *rest); -bool simExecuteSqlSlowCmd(SScript *script, char *option); -bool simExecuteRestfulCmd(SScript *script, char *rest); -void simVisuallizeOption(SScript *script, char *src, char *dst); +void * simExecuteScript(void *script); +void simInitsimCmdList(); +bool simSystemInit(); +void simSystemCleanUp(); +char * simGetVariable(SScript *script, char *varName, int32_t varLen); +bool simExecuteExpCmd(SScript *script, char *option); +bool simExecuteTestCmd(SScript *script, char *option); +bool simExecuteGotoCmd(SScript *script, char *option); +bool simExecuteRunCmd(SScript *script, char *option); +bool simExecuteRunBackCmd(SScript *script, char *option); +bool simExecuteSystemCmd(SScript *script, char *option); +bool simExecuteSystemContentCmd(SScript *script, char *option); +bool simExecutePrintCmd(SScript *script, char *option); +bool simExecuteSleepCmd(SScript *script, char *option); +bool simExecuteReturnCmd(SScript *script, char *option); +bool simExecuteSqlCmd(SScript *script, char *option); +bool simExecuteSqlErrorCmd(SScript *script, char *rest); +bool simExecuteSqlSlowCmd(SScript *script, char *option); +bool simExecuteRestfulCmd(SScript *script, char *rest); +void simVisuallizeOption(SScript *script, char *src, char *dst); #endif \ No newline at end of file diff --git a/tests/tsim/inc/simParse.h b/tests/tsim/inc/simParse.h index d3f92add71f3dc6ddb205c1810e1a3caace47e36..ef7d8e5ce72cc9bf0ae52380089d36576e84bd28 100644 --- a/tests/tsim/inc/simParse.h +++ b/tests/tsim/inc/simParse.h @@ -50,6 +50,6 @@ typedef struct { char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */ } SBlock; -bool simParseExpression(char *token, int lineNum); +bool simParseExpression(char *token, int32_t lineNum); #endif \ No newline at end of file diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 7f786dfaa9de5f6a93d3a181ddd53e37e8694310..2db750cdd302c63522e008e3ea324230dd87b6e5 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "sim.h" #include "taos.h" @@ -38,30 +39,28 @@ void simLogSql(char *sql, bool useSharp) { } else { fprintf(fp, "%s;\n", sql); } - + fflush(fp); } char *simParseArbitratorName(char *varName); char *simParseHostName(char *varName); -char *simGetVariable(SScript *script, char *varName, int varLen) { +char *simGetVariable(SScript *script, char *varName, int32_t varLen) { if (strncmp(varName, "hostname", 8) == 0) { return simParseHostName(varName); } if (strncmp(varName, "arbitrator", 10) == 0) { - return simParseArbitratorName(varName); + return simParseArbitratorName(varName); } if (strncmp(varName, "error", varLen) == 0) return script->error; if (strncmp(varName, "rows", varLen) == 0) return script->rows; - if (strncmp(varName, "system_exit", varLen) == 0) - return script->system_exit_code; + if (strncmp(varName, "system_exit", varLen) == 0) return script->system_exit_code; - if (strncmp(varName, "system_content", varLen) == 0) - return script->system_ret_content; + if (strncmp(varName, "system_content", varLen) == 0) return script->system_ret_content; // variable like data2_192.168.0.1 if (strncmp(varName, "data", 4) == 0) { @@ -70,16 +69,16 @@ char *simGetVariable(SScript *script, char *varName, int varLen) { } if (varName[5] == '_') { - int col = varName[4] - '0'; + int32_t col = varName[4] - '0'; if (col < 0 || col >= MAX_QUERY_COL_NUM) { return "null"; } - char *keyName; - int keyLen; + char * keyName; + int32_t keyLen; paGetToken(varName + 6, &keyName, &keyLen); - for (int i = 0; i < MAX_QUERY_ROW_NUM; ++i) { + for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { if (strncmp(keyName, script->data[i][0], keyLen) == 0) { simDebug("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); return script->data[i][col]; @@ -87,16 +86,16 @@ char *simGetVariable(SScript *script, char *varName, int varLen) { } return "null"; } else if (varName[6] == '_') { - int col = (varName[4] - '0') * 10 + (varName[5] - '0'); + int32_t col = (varName[4] - '0') * 10 + (varName[5] - '0'); if (col < 0 || col >= MAX_QUERY_COL_NUM) { return "null"; } - char *keyName; - int keyLen; + char * keyName; + int32_t keyLen; paGetToken(varName + 7, &keyName, &keyLen); - for (int i = 0; i < MAX_QUERY_ROW_NUM; ++i) { + for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { if (strncmp(keyName, script->data[i][0], keyLen) == 0) { simTrace("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); return script->data[i][col]; @@ -104,8 +103,8 @@ char *simGetVariable(SScript *script, char *varName, int varLen) { } return "null"; } else { - int row = varName[4] - '0'; - int col = varName[5] - '0'; + int32_t row = varName[4] - '0'; + int32_t col = varName[5] - '0'; if (row < 0 || row >= MAX_QUERY_ROW_NUM) { return "null"; } @@ -118,7 +117,7 @@ char *simGetVariable(SScript *script, char *varName, int varLen) { } } - for (int i = 0; i < script->varLen; ++i) { + for (int32_t i = 0; i < script->varLen; ++i) { SVariable *var = &script->variables[i]; if (var->varNameLen != varLen) { continue; @@ -144,11 +143,11 @@ char *simGetVariable(SScript *script, char *varName, int varLen) { return var->varValue; } -int simExecuteExpression(SScript *script, char *exp) { - char *op1, *op2, *var1, *var2, *var3, *rest; - int op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; - char t0[512], t1[512], t2[512], t3[1024]; - int result; +int32_t simExecuteExpression(SScript *script, char *exp) { + char * op1, *op2, *var1, *var2, *var3, *rest; + int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; + char t0[512], t1[512], t2[512], t3[1024]; + int32_t result; rest = paGetToken(exp, &var1, &var1Len); rest = paGetToken(rest, &op1, &op1Len); @@ -234,7 +233,7 @@ bool simExecuteExpCmd(SScript *script, char *option) { } bool simExecuteTestCmd(SScript *script, char *option) { - int result; + int32_t result; result = simExecuteExpression(script, option); if (result >= 0) @@ -285,13 +284,12 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { sprintf(script->error, "lineNum:%d. parse file:%s error", script->lines[script->linePos].lineNum, fileName); return false; } - simInfo("script:%s, start to execute in background", newScript->fileName); newScript->type = SIM_SCRIPT_TYPE_BACKGROUND; script->bgScripts[script->bgScriptLen++] = newScript; + simInfo("script:%s, start to execute in background,", newScript->fileName); - pthread_t pid; - if (pthread_create(&pid, NULL, simExecuteScript, (void *)newScript) != 0) { + if (pthread_create(&newScript->bgPid, NULL, simExecuteScript, (void *)newScript) != 0) { sprintf(script->error, "lineNum:%d. create background thread failed", script->lines[script->linePos].lineNum); return false; } @@ -307,13 +305,13 @@ bool simExecuteSystemCmd(SScript *script, char *option) { simVisuallizeOption(script, option, buf + strlen(buf)); simLogSql(buf, true); - int code = system(buf); - int repeatTimes = 0; + int32_t code = system(buf); + int32_t repeatTimes = 0; while (code < 0) { - simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", - script->fileName, buf, code, errno, strerror(errno), repeatTimes); + simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", script->fileName, buf, code, + errno, strerror(errno), repeatTimes); taosMsleep(1000); -#ifdef LINUX +#ifdef LINUX signal(SIGCHLD, SIG_DFL); #endif if (repeatTimes++ >= 10) { @@ -368,8 +366,8 @@ bool simExecutePrintCmd(SScript *script, char *rest) { } bool simExecuteSleepCmd(SScript *script, char *option) { - int delta; - char buf[1024]; + int32_t delta; + char buf[1024]; simVisuallizeOption(script, option, buf); option = buf; @@ -395,7 +393,7 @@ bool simExecuteReturnCmd(SScript *script, char *option) { simVisuallizeOption(script, option, buf); option = buf; - int ret = 1; + int32_t ret = 1; if (option && option[0] != 0) ret = atoi(option); if (ret < 0) { @@ -411,8 +409,8 @@ bool simExecuteReturnCmd(SScript *script, char *option) { } void simVisuallizeOption(SScript *script, char *src, char *dst) { - char *var, *token, *value; - int dstLen, srcLen, tokenLen; + char * var, *token, *value; + int32_t dstLen, srcLen, tokenLen; dst[0] = 0, dstLen = 0; @@ -420,14 +418,14 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) { var = strchr(src, '$'); if (var == NULL) break; if (var && ((var - src - 1) > 0) && *(var - 1) == '\\') { - srcLen = (int)(var - src - 1); + srcLen = (int32_t)(var - src - 1); memcpy(dst + dstLen, src, srcLen); dstLen += srcLen; src = var; break; } - srcLen = (int)(var - src); + srcLen = (int32_t)(var - src); memcpy(dst + dstLen, src, srcLen); dstLen += srcLen; @@ -435,13 +433,13 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) { value = simGetVariable(script, token, tokenLen); strcpy(dst + dstLen, value); - dstLen += (int)strlen(value); + dstLen += (int32_t)strlen(value); } strcpy(dst + dstLen, src); } -void simCloseRestFulConnect(SScript *script) { +void simCloseRestFulConnect(SScript *script) { memset(script->auth, 0, sizeof(script->auth)); } @@ -465,7 +463,7 @@ void simCloseTaosdConnect(SScript *script) { // {"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"} // {"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1} // {"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10} -int simParseHttpCommandResult(SScript *script, char *command) { +int32_t simParseHttpCommandResult(SScript *script, char *command) { cJSON* root = cJSON_Parse(command); if (root == NULL) { simError("script:%s, failed to parse json, response:%s", script->fileName, command); @@ -492,14 +490,15 @@ int simParseHttpCommandResult(SScript *script, char *command) { cJSON_Delete(root); return -1; } - int retcode = (int)code->valueint; + int32_t retcode = (int32_t)code->valueint; if (retcode != 1017) { - simError("script:%s, json:status:%s not equal to succ, response:%s", script->fileName, status->valuestring, command); + simError("script:%s, json:status:%s not equal to succ, response:%s", script->fileName, status->valuestring, + command); cJSON_Delete(root); return retcode; } else { simDebug("script:%s, json:status:%s not equal to succ, but code is %d, response:%s", script->fileName, - status->valuestring, retcode, command); + status->valuestring, retcode, command); cJSON_Delete(root); return 0; } @@ -524,27 +523,27 @@ int simParseHttpCommandResult(SScript *script, char *command) { return -1; } - int rowsize = cJSON_GetArraySize(data); + int32_t rowsize = cJSON_GetArraySize(data); if (rowsize < 0) { simError("script:%s, failed to parse json:data, data size %d, response:%s", script->fileName, rowsize, command); cJSON_Delete(root); return -1; } - int rowIndex = 0; + int32_t rowIndex = 0; sprintf(script->rows, "%d", rowsize); - for (int r = 0; r < rowsize; ++r) { + for (int32_t r = 0; r < rowsize; ++r) { cJSON *row = cJSON_GetArrayItem(data, r); if (row == NULL) continue; if (rowIndex++ >= 10) break; - int colsize = cJSON_GetArraySize(row); + int32_t colsize = cJSON_GetArraySize(row); if (colsize < 0) { break; } colsize = MIN(10, colsize); - for (int c = 0; c < colsize; ++c) { + for (int32_t c = 0; c < colsize; ++c) { cJSON *col = cJSON_GetArrayItem(row, c); if (col->valuestring != NULL) { strcpy(script->data[r][c], col->valuestring); @@ -561,7 +560,7 @@ int simParseHttpCommandResult(SScript *script, char *command) { return 0; } -int simExecuteRestFulCommand(SScript *script, char *command) { +int32_t simExecuteRestFulCommand(SScript *script, char *command) { char buf[5000] = {0}; sprintf(buf, "%s 2>/dev/null", command); @@ -571,13 +570,13 @@ int simExecuteRestFulCommand(SScript *script, char *command) { return -1; } - int mallocSize = 2000; - int alreadyReadSize = 0; - char* content = malloc(mallocSize); + int32_t mallocSize = 2000; + int32_t alreadyReadSize = 0; + char * content = malloc(mallocSize); while (!feof(fp)) { - int availSize = mallocSize - alreadyReadSize; - int len = (int)fread(content + alreadyReadSize, 1, availSize, fp); + int32_t availSize = mallocSize - alreadyReadSize; + int32_t len = (int32_t)fread(content + alreadyReadSize, 1, availSize, fp); if (len >= availSize) { alreadyReadSize += len; mallocSize *= 2; @@ -595,10 +594,11 @@ bool simCreateRestFulConnect(SScript *script, char *user, char *pass) { sprintf(command, "curl 127.0.0.1:6041/rest/login/%s/%s", user, pass); bool success = false; - for (int attempt = 0; attempt < 10; ++attempt) { + for (int32_t attempt = 0; attempt < 10; ++attempt) { success = simExecuteRestFulCommand(script, command) == 0; if (!success) { - simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), attempt); + simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), + attempt); taosMsleep(1000); } else { simDebug("script:%s, user:%s connect taosd successed, attempt:%d", script->fileName, user, attempt); @@ -607,7 +607,8 @@ bool simCreateRestFulConnect(SScript *script, char *user, char *pass) { } if (!success) { - sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, taos_errstr(NULL)); + sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, + taos_errstr(NULL)); return false; } @@ -619,10 +620,11 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) { simCloseTaosdConnect(script); void *taos = NULL; taosMsleep(2000); - for (int attempt = 0; attempt < 10; ++attempt) { + for (int32_t attempt = 0; attempt < 10; ++attempt) { taos = taos_connect(NULL, user, pass, NULL, tsDnodeShellPort); if (taos == NULL) { - simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), attempt); + simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), + attempt); taosMsleep(1000); } else { simDebug("script:%s, user:%s connect taosd successed, attempt:%d", script->fileName, user, attempt); @@ -631,7 +633,8 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) { } if (taos == NULL) { - sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, taos_errstr(NULL)); + sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, + taos_errstr(NULL)); return false; } @@ -642,9 +645,9 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) { } bool simCreateTaosdConnect(SScript *script, char *rest) { - char *user = TSDB_DEFAULT_USER; - char *token; - int tokenLen; + char * user = TSDB_DEFAULT_USER; + char * token; + int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); rest = paGetToken(rest, &token, &tokenLen); if (tokenLen != 0) { @@ -659,26 +662,27 @@ bool simCreateTaosdConnect(SScript *script, char *rest) { } bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { - char timeStr[30] = {0}; - time_t tt; + char timeStr[30] = {0}; + time_t tt; struct tm *tp; - SCmdLine *line = &script->lines[script->linePos]; - int ret = -1; + SCmdLine * line = &script->lines[script->linePos]; + int32_t ret = -1; + + TAOS_RES *pSql = NULL; - TAOS_RES* pSql = NULL; - - for (int attempt = 0; attempt < 10; ++attempt) { + for (int32_t attempt = 0; attempt < 10; ++attempt) { simLogSql(rest, false); pSql = taos_query(script->taos, rest); ret = taos_errno(pSql); - + if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { - simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret)); + simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, + tstrerror(ret)); ret = 0; break; } else if (ret != 0) { - simDebug("script:%s, taos:%p, %s failed, ret:%d:%s, error:%s", - script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret), taos_errstr(pSql)); + simDebug("script:%s, taos:%p, %s failed, ret:%d:%s, error:%s", script->fileName, script->taos, rest, ret & 0XFFFF, + tstrerror(ret), taos_errstr(pSql)); if (line->errorJump == SQL_JUMP_TRUE) { script->linePos = line->jump; @@ -689,7 +693,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { } else { break; } - + taos_free_result(pSql); } @@ -698,8 +702,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { return false; } - int numOfRows = 0; - int num_fields = taos_field_count(pSql); + int32_t numOfRows = 0; + int32_t num_fields = taos_field_count(pSql); if (num_fields != 0) { if (pSql == NULL) { simDebug("script:%s, taos:%p, %s failed, result is null", script->fileName, script->taos, rest); @@ -717,9 +721,9 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { while ((row = taos_fetch_row(pSql))) { if (numOfRows < MAX_QUERY_ROW_NUM) { TAOS_FIELD *fields = taos_fetch_fields(pSql); - int* length = taos_fetch_lengths(pSql); - - for (int i = 0; i < num_fields; i++) { + int32_t * length = taos_fetch_lengths(pSql); + + for (int32_t i = 0; i < num_fields; i++) { char *value = NULL; if (i < MAX_QUERY_COL_NUM) { value = script->data[numOfRows][i]; @@ -735,8 +739,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { switch (fields[i].type) { case TSDB_DATA_TYPE_BOOL: - sprintf(value, "%s", - ((((int)(*((char *)row[i]))) == 1) ? "1" : "0")); + sprintf(value, "%s", ((((int32_t)(*((char *)row[i]))) == 1) ? "1" : "0")); break; case TSDB_DATA_TYPE_TINYINT: sprintf(value, "%d", *((int8_t *)row[i])); @@ -779,9 +782,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { tp = localtime(&tt); strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", tp); - sprintf(value, "%s.%03d", timeStr, - (int)(*((int64_t *)row[i]) % 1000)); - + sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000)); + break; default: break; @@ -814,17 +816,16 @@ bool simExecuteRestFulSqlCommand(SScript *script, char *rest) { char command[4096]; sprintf(command, "curl -H 'Authorization: Taosd %s' -d \"%s\" 127.0.0.1:6041/rest/sql", script->auth, rest); - int ret = -1; - for (int attempt = 0; attempt < 10; ++attempt) { + int32_t ret = -1; + for (int32_t attempt = 0; attempt < 10; ++attempt) { ret = simExecuteRestFulCommand(script, command); - if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || - ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { - simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret)); + if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { + simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, + tstrerror(ret)); ret = 0; break; } else if (ret != 0) { - simDebug("script:%s, taos:%p, %s failed, ret:%d", - script->fileName, script->taos, rest, ret); + simDebug("script:%s, taos:%p, %s failed, ret:%d", script->fileName, script->taos, rest, ret); if (line->errorJump == SQL_JUMP_TRUE) { script->linePos = line->jump; @@ -854,8 +855,8 @@ bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) { simDebug("script:%s, exec:%s", script->fileName, rest); strcpy(script->rows, "-1"); - for (int row = 0; row < MAX_QUERY_ROW_NUM; ++row) { - for (int col = 0; col < MAX_QUERY_COL_NUM; ++col) { + for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) { + for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) { strcpy(script->data[row][col], "null"); } } @@ -903,23 +904,23 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) { bool simExecuteRestfulCmd(SScript *script, char *rest) { FILE *fp = NULL; - char filename[256]; - sprintf(filename, "%s/tmp.sql", tsScriptDir); + char filename[256]; + sprintf(filename, "%s/tmp.sql", tsScriptDir); fp = fopen(filename, "w"); if (fp == NULL) { fprintf(stderr, "ERROR: failed to open file: %s\n", filename); return false; } - char db[64] = {0}; - char tb[64] = {0}; - char gzip[32] = {0}; + char db[64] = {0}; + char tb[64] = {0}; + char gzip[32] = {0}; int32_t ts; int32_t times; sscanf(rest, "%s %s %d %d %s", db, tb, &ts, ×, gzip); - + fprintf(fp, "insert into %s.%s values ", db, tb); - for (int i = 0; i < times; ++i) { + for (int32_t i = 0; i < times; ++i) { fprintf(fp, "(%d000, %d)", ts + i, ts); } fprintf(fp, " \n"); @@ -951,8 +952,8 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { simDebug("script:%s, exec:%s", script->fileName, rest); strcpy(script->rows, "-1"); - for (int row = 0; row < MAX_QUERY_ROW_NUM; ++row) { - for (int col = 0; col < MAX_QUERY_COL_NUM; ++col) { + for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) { + for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) { strcpy(script->data[row][col], "null"); } } @@ -981,27 +982,27 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { return true; } - int ret; - TAOS_RES* pSql = NULL; + int32_t ret; + TAOS_RES *pSql = NULL; if (simAsyncQuery) { char command[4096]; sprintf(command, "curl -H 'Authorization: Taosd %s' -d '%s' 127.0.0.1:6041/rest/sql", script->auth, rest); ret = simExecuteRestFulCommand(script, command); - } - else { + } else { pSql = taos_query(script->taos, rest); ret = taos_errno(pSql); taos_free_result(pSql); } if (ret != TSDB_CODE_SUCCESS) { - simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s", - script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret)); + simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s", script->fileName, script->taos, + rest, ret & 0XFFFF, tstrerror(ret)); script->linePos++; return true; } - - sprintf(script->error, "lineNum:%d. sql:%s expect failed, but success, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, tstrerror(ret)); + + sprintf(script->error, "lineNum:%d. sql:%s expect failed, but success, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, + tstrerror(ret)); return false; } diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index ef1a488f602959b740cbe863cc22654a24b2d41b..33fd24dd5823b8ec7bf818cec7c36ddac783bc24 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "tglobal.h" #include "sim.h" @@ -20,15 +21,15 @@ bool simAsyncQuery = false; -void simHandleSignal(int signo) { +void simHandleSignal(int32_t signo) { simSystemCleanUp(); exit(1); } -int main(int argc, char *argv[]) { +int32_t main(int32_t argc, char *argv[]) { char scriptFile[MAX_FILE_NAME_LEN] = "sim_main_test.sim"; - for (int i = 1; i < argc; ++i) { + for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0 && i < argc - 1) { tstrncpy(configDir, argv[++i], MAX_FILE_NAME_LEN); } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { @@ -37,8 +38,7 @@ int main(int argc, char *argv[]) { simAsyncQuery = true; } else { printf("usage: %s [options] \n", argv[0]); - printf(" [-c config]: config directory, default is: %s\n", - configDir); + printf(" [-c config]: config directory, default is: %s\n", configDir); printf(" [-f script]: script filename\n"); exit(0); } diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 2e6121304f18022a74b9e4f09a8043b5b4beb909..b909f5bd8fc10bea09afd65dc504ae35d6de3505 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -57,6 +57,7 @@ * */ +#define _DEFAULT_SOURCE #include "os.h" #include "sim.h" #include "simParse.h" @@ -64,16 +65,16 @@ #undef TAOS_MEM_CHECK static SCommand *cmdHashList[MAX_NUM_CMD]; -static SCmdLine cmdLine[MAX_CMD_LINES]; -static char parseErr[MAX_ERROR_LEN]; -static char optionBuffer[MAX_OPTION_BUFFER]; -static int numOfLines, optionOffset; -static SLabel label, dest; -static SBlock block; +static SCmdLine cmdLine[MAX_CMD_LINES]; +static char parseErr[MAX_ERROR_LEN]; +static char optionBuffer[MAX_OPTION_BUFFER]; +static int32_t numOfLines, optionOffset; +static SLabel label, dest; +static SBlock block; -int simHashCmd(char *token, int tokenLen) { - int i; - int hash = 0; +int32_t simHashCmd(char *token, int32_t tokenLen) { + int32_t i; + int32_t hash = 0; for (i = 0; i < tokenLen; ++i) hash += token[i]; @@ -82,8 +83,8 @@ int simHashCmd(char *token, int tokenLen) { return hash; } -SCommand *simCheckCmd(char *token, int tokenLen) { - int hash; +SCommand *simCheckCmd(char *token, int32_t tokenLen) { + int32_t hash; SCommand *node; hash = simHashCmd(token, tokenLen); @@ -102,10 +103,10 @@ SCommand *simCheckCmd(char *token, int tokenLen) { } void simAddCmdIntoHash(SCommand *pCmd) { - int hash; + int32_t hash; SCommand *node; - hash = simHashCmd(pCmd->name, (int)strlen(pCmd->name)); + hash = simHashCmd(pCmd->name, (int32_t)strlen(pCmd->name)); node = cmdHashList[hash]; pCmd->next = node; cmdHashList[hash] = pCmd; @@ -122,7 +123,7 @@ void simResetParser() { } SScript *simBuildScriptObj(char *fileName) { - int i, destPos; + int32_t i, destPos; /* process labels */ @@ -176,11 +177,11 @@ SScript *simBuildScriptObj(char *fileName) { } SScript *simParseScript(char *fileName) { - FILE *fd; - int tokenLen, lineNum = 0; - char buffer[MAX_LINE_LEN], name[128], *token, *rest; + FILE * fd; + int32_t tokenLen, lineNum = 0; + char buffer[MAX_LINE_LEN], name[128], *token, *rest; SCommand *pCmd; - SScript *script; + SScript * script; if ((fileName[0] == '.') || (fileName[0] == '/')) { strcpy(name, fileName); @@ -199,12 +200,13 @@ SScript *simParseScript(char *fileName) { if (fgets(buffer, sizeof(buffer), fd) == NULL) continue; lineNum++; - int cmdlen = (int)strlen(buffer); - if (buffer[cmdlen - 1] == '\r' || buffer[cmdlen - 1] == '\n') + int32_t cmdlen = (int32_t)strlen(buffer); + if (buffer[cmdlen - 1] == '\r' || buffer[cmdlen - 1] == '\n') { buffer[cmdlen - 1] = 0; + } rest = buffer; - for (int i = 0; i < cmdlen; ++i) { + for (int32_t i = 0; i < cmdlen; ++i) { if (buffer[i] == '\r' || buffer[i] == '\n') { buffer[i] = ' '; } @@ -249,9 +251,9 @@ SScript *simParseScript(char *fileName) { return script; } -int simCheckExpression(char *exp) { - char *op1, *op2, *op, *rest; - int op1Len, op2Len, opLen; +int32_t simCheckExpression(char *exp) { + char * op1, *op2, *op, *rest; + int32_t op1Len, op2Len, opLen; rest = paGetToken(exp, &op1, &op1Len); if (op1Len == 0) { @@ -282,8 +284,7 @@ int simCheckExpression(char *exp) { return -1; } } else if (opLen == 2) { - if (op[1] != '=' || - (op[0] != '=' && op[0] != '<' && op[0] != '>' && op[0] != '!')) { + if (op[1] != '=' || (op[0] != '=' && op[0] != '<' && op[0] != '>' && op[0] != '!')) { sprintf(parseErr, "left side of assignment must be variable"); return -1; } @@ -294,10 +295,10 @@ int simCheckExpression(char *exp) { rest = paGetToken(rest, &op, &opLen); - if (opLen == 0) return (int)(rest - exp); + if (opLen == 0) return (int32_t)(rest - exp); /* if it is key word "then" */ - if (strncmp(op, "then", 4) == 0) return (int)(op - exp); + if (strncmp(op, "then", 4) == 0) return (int32_t)(op - exp); rest = paGetToken(rest, &op2, &op2Len); if (op2Len == 0) { @@ -310,16 +311,15 @@ int simCheckExpression(char *exp) { return -1; } - if (op[0] == '+' || op[0] == '-' || op[0] == '*' || op[0] == '/' || - op[0] == '.') { - return (int)(rest - exp); + if (op[0] == '+' || op[0] == '-' || op[0] == '*' || op[0] == '/' || op[0] == '.') { + return (int32_t)(rest - exp); } return -1; } -bool simParseExpression(char *token, int lineNum) { - int expLen; +bool simParseExpression(char *token, int32_t lineNum) { + int32_t expLen; expLen = simCheckExpression(token); if (expLen <= 0) return -1; @@ -335,9 +335,9 @@ bool simParseExpression(char *token, int lineNum) { return true; } -bool simParseIfCmd(char *rest, SCommand *pCmd, int lineNum) { - char *ret; - int expLen; +bool simParseIfCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * ret; + int32_t expLen; expLen = simCheckExpression(rest); @@ -364,8 +364,8 @@ bool simParseIfCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseElifCmd(char *rest, SCommand *pCmd, int lineNum) { - int expLen; +bool simParseElifCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; expLen = simCheckExpression(rest); @@ -382,8 +382,7 @@ bool simParseElifCmd(char *rest, SCommand *pCmd, int lineNum) { } cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; - block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = - &(cmdLine[numOfLines].jump); + block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); block.numJump[block.top - 1]++; numOfLines++; @@ -402,7 +401,7 @@ bool simParseElifCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseElseCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseElseCmd(char *rest, SCommand *pCmd, int32_t lineNum) { if (block.top < 1) { sprintf(parseErr, "no matching if"); return false; @@ -414,8 +413,7 @@ bool simParseElseCmd(char *rest, SCommand *pCmd, int lineNum) { } cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; - block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = - &(cmdLine[numOfLines].jump); + block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); block.numJump[block.top - 1]++; numOfLines++; @@ -426,8 +424,8 @@ bool simParseElseCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseEndiCmd(char *rest, SCommand *pCmd, int lineNum) { - int i; +bool simParseEndiCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t i; if (block.top < 1) { sprintf(parseErr, "no matching if"); @@ -441,8 +439,9 @@ bool simParseEndiCmd(char *rest, SCommand *pCmd, int lineNum) { if (block.pos[block.top - 1]) *(block.pos[block.top - 1]) = numOfLines; - for (i = 0; i < block.numJump[block.top - 1]; ++i) + for (i = 0; i < block.numJump[block.top - 1]; ++i) { *(block.jump[block.top - 1][i]) = numOfLines; + } block.numJump[block.top - 1] = 0; block.top--; @@ -450,8 +449,8 @@ bool simParseEndiCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseWhileCmd(char *rest, SCommand *pCmd, int lineNum) { - int expLen; +bool simParseWhileCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; expLen = simCheckExpression(rest); @@ -473,8 +472,8 @@ bool simParseWhileCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseEndwCmd(char *rest, SCommand *pCmd, int lineNum) { - int i; +bool simParseEndwCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t i; if (block.top < 1) { sprintf(parseErr, "no matching while"); @@ -493,17 +492,18 @@ bool simParseEndwCmd(char *rest, SCommand *pCmd, int lineNum) { *(block.pos[block.top - 1]) = numOfLines; - for (i = 0; i < block.numJump[block.top - 1]; ++i) + for (i = 0; i < block.numJump[block.top - 1]; ++i) { *(block.jump[block.top - 1][i]) = numOfLines; + } block.top--; return true; } -bool simParseSwitchCmd(char *rest, SCommand *pCmd, int lineNum) { - char *token; - int tokenLen; +bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * token; + int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); if (tokenLen == 0) { @@ -524,9 +524,9 @@ bool simParseSwitchCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseCaseCmd(char *rest, SCommand *pCmd, int lineNum) { - char *token; - int tokenLen; +bool simParseCaseCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * token; + int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); if (tokenLen == 0) { @@ -544,16 +544,16 @@ bool simParseCaseCmd(char *rest, SCommand *pCmd, int lineNum) { return false; } - if (block.pos[block.top - 1] != NULL) + if (block.pos[block.top - 1] != NULL) { *(block.pos[block.top - 1]) = numOfLines; + } block.pos[block.top - 1] = &(cmdLine[numOfLines].jump); cmdLine[numOfLines].cmdno = SIM_CMD_TEST; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - memcpy(optionBuffer + optionOffset, block.sexp[block.top - 1], - block.sexpLen[block.top - 1]); + memcpy(optionBuffer + optionOffset, block.sexp[block.top - 1], block.sexpLen[block.top - 1]); optionOffset += block.sexpLen[block.top - 1]; *(optionBuffer + optionOffset++) = ' '; *(optionBuffer + optionOffset++) = '='; @@ -567,20 +567,18 @@ bool simParseCaseCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseBreakCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseBreakCmd(char *rest, SCommand *pCmd, int32_t lineNum) { if (block.top < 1) { sprintf(parseErr, "no blcok exists"); return false; } - if (block.type[block.top - 1] != BLOCK_SWITCH && - block.type[block.top - 1] != BLOCK_WHILE) { + if (block.type[block.top - 1] != BLOCK_SWITCH && block.type[block.top - 1] != BLOCK_WHILE) { sprintf(parseErr, "not in switch or while block"); return false; } - block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = - &(cmdLine[numOfLines].jump); + block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); block.numJump[block.top - 1]++; cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; @@ -590,7 +588,7 @@ bool simParseBreakCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseDefaultCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseDefaultCmd(char *rest, SCommand *pCmd, int32_t lineNum) { if (block.top < 1) { sprintf(parseErr, "no matching switch"); return false; @@ -601,14 +599,15 @@ bool simParseDefaultCmd(char *rest, SCommand *pCmd, int lineNum) { return false; } - if (block.pos[block.top - 1] != NULL) + if (block.pos[block.top - 1] != NULL) { *(block.pos[block.top - 1]) = numOfLines; + } return true; } -bool simParseEndsCmd(char *rest, SCommand *pCmd, int lineNum) { - int i; +bool simParseEndsCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t i; if (block.top < 1) { sprintf(parseErr, "no matching switch"); @@ -620,8 +619,9 @@ bool simParseEndsCmd(char *rest, SCommand *pCmd, int lineNum) { return false; } - for (i = 0; i < block.numJump[block.top - 1]; ++i) + for (i = 0; i < block.numJump[block.top - 1]; ++i) { *(block.jump[block.top - 1][i]) = numOfLines; + } block.numJump[block.top - 1] = 0; block.top--; @@ -629,7 +629,7 @@ bool simParseEndsCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseContinueCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseContinueCmd(char *rest, SCommand *pCmd, int32_t lineNum) { if (block.top < 1) { sprintf(parseErr, "no matching while"); return false; @@ -648,14 +648,14 @@ bool simParseContinueCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParsePrintCmd(char *rest, SCommand *pCmd, int lineNum) { - int expLen; +bool simParsePrintCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; rest++; cmdLine[numOfLines].cmdno = SIM_CMD_PRINT; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int)strlen(rest); + expLen = (int32_t)strlen(rest); memcpy(optionBuffer + optionOffset, rest, expLen); optionOffset += expLen + 1; *(optionBuffer + optionOffset - 1) = 0; @@ -665,8 +665,8 @@ bool simParsePrintCmd(char *rest, SCommand *pCmd, int lineNum) { } void simCheckSqlOption(char *rest) { - int valueLen; - char *value, *xpos; + int32_t valueLen; + char * value, *xpos; xpos = strstr(rest, " -x"); // need a blank if (xpos) { @@ -682,15 +682,15 @@ void simCheckSqlOption(char *rest) { } } -bool simParseSqlCmd(char *rest, SCommand *pCmd, int lineNum) { - int expLen; +bool simParseSqlCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; rest++; simCheckSqlOption(rest); cmdLine[numOfLines].cmdno = SIM_CMD_SQL; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int)strlen(rest); + expLen = (int32_t)strlen(rest); memcpy(optionBuffer + optionOffset, rest, expLen); optionOffset += expLen + 1; *(optionBuffer + optionOffset - 1) = 0; @@ -699,14 +699,14 @@ bool simParseSqlCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int lineNum) { - int expLen; +bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; rest++; cmdLine[numOfLines].cmdno = SIM_CMD_SQL_ERROR; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int)strlen(rest); + expLen = (int32_t)strlen(rest); memcpy(optionBuffer + optionOffset, rest, expLen); optionOffset += expLen + 1; *(optionBuffer + optionOffset - 1) = 0; @@ -715,26 +715,26 @@ bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int32_t lineNum) { simParseSqlCmd(rest, pCmd, lineNum); cmdLine[numOfLines - 1].cmdno = SIM_CMD_SQL_SLOW; return true; } -bool simParseRestfulCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseRestfulCmd(char *rest, SCommand *pCmd, int32_t lineNum) { simParseSqlCmd(rest, pCmd, lineNum); cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL; return true; } -bool simParseSystemCmd(char *rest, SCommand *pCmd, int lineNum) { - int expLen; +bool simParseSystemCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; rest++; cmdLine[numOfLines].cmdno = SIM_CMD_SYSTEM; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int)strlen(rest); + expLen = (int32_t)strlen(rest); memcpy(optionBuffer + optionOffset, rest, expLen); optionOffset += expLen + 1; *(optionBuffer + optionOffset - 1) = 0; @@ -743,15 +743,15 @@ bool simParseSystemCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseSystemContentCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseSystemContentCmd(char *rest, SCommand *pCmd, int32_t lineNum) { simParseSystemCmd(rest, pCmd, lineNum); cmdLine[numOfLines - 1].cmdno = SIM_CMD_SYSTEM_CONTENT; return true; } -bool simParseSleepCmd(char *rest, SCommand *pCmd, int lineNum) { - char *token; - int tokenLen; +bool simParseSleepCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * token; + int32_t tokenLen; cmdLine[numOfLines].cmdno = SIM_CMD_SLEEP; cmdLine[numOfLines].lineNum = lineNum; @@ -768,9 +768,9 @@ bool simParseSleepCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseReturnCmd(char *rest, SCommand *pCmd, int lineNum) { - char *token; - int tokenLen; +bool simParseReturnCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * token; + int32_t tokenLen; cmdLine[numOfLines].cmdno = SIM_CMD_RETURN; cmdLine[numOfLines].lineNum = lineNum; @@ -787,9 +787,9 @@ bool simParseReturnCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseGotoCmd(char *rest, SCommand *pCmd, int lineNum) { - char *token; - int tokenLen; +bool simParseGotoCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * token; + int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); @@ -810,9 +810,9 @@ bool simParseGotoCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseRunCmd(char *rest, SCommand *pCmd, int lineNum) { - char *token; - int tokenLen; +bool simParseRunCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * token; + int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); @@ -832,14 +832,14 @@ bool simParseRunCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseRunBackCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseRunBackCmd(char *rest, SCommand *pCmd, int32_t lineNum) { simParseRunCmd(rest, pCmd, lineNum); cmdLine[numOfLines - 1].cmdno = SIM_CMD_RUN_BACK; return true; } void simInitsimCmdList() { - int cmdno; + int32_t cmdno; memset(simCmdList, 0, SIM_CMD_END * sizeof(SCommand)); /* internal command */ diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 17df7f306a47545f7283df1848f98ce18b87db79..693ade7b35a095b499198db9dcc27335f73e14bd 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "sim.h" #include "taos.h" @@ -24,11 +25,11 @@ SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; SCommand simCmdList[SIM_CMD_END]; -int simScriptPos = -1; -int simScriptSucced = 0; -int simDebugFlag = 135; -void simCloseTaosdConnect(SScript *script); -char simHostName[128]; +int32_t simScriptPos = -1; +int32_t simScriptSucced = 0; +int32_t simDebugFlag = 135; +void simCloseTaosdConnect(SScript *script); +char simHostName[128]; char *simParseArbitratorName(char *varName) { static char hostName[140]; @@ -39,8 +40,8 @@ char *simParseArbitratorName(char *varName) { char *simParseHostName(char *varName) { static char hostName[140]; - int index = atoi(varName + 8); - int port = 7100; + int32_t index = atoi(varName + 8); + int32_t port = 7100; switch (index) { case 1: port = 7100; @@ -70,9 +71,9 @@ char *simParseHostName(char *varName) { port = 7900; break; } - + sprintf(hostName, "'%s:%d'", simHostName, port); - //simInfo("hostName:%s", hostName); + // simInfo("hostName:%s", hostName); return hostName; } @@ -88,39 +89,45 @@ void simSystemCleanUp() {} void simFreeScript(SScript *script) { if (script->type == SIM_SCRIPT_TYPE_MAIN) { - for (int i = 0; i < script->bgScriptLen; ++i) { + simInfo("script:%s, background script num:%d, stop them", script->fileName, script->bgScriptLen); + + for (int32_t i = 0; i < script->bgScriptLen; ++i) { SScript *bgScript = script->bgScripts[i]; + simInfo("script:%s, set stop flag", script->fileName); bgScript->killed = true; + if (taosCheckPthreadValid(bgScript->bgPid)) { + pthread_join(bgScript->bgPid, NULL); + } } } + simDebug("script:%s, is freed", script->fileName); taos_close(script->taos); - taosTFree(script->lines); - taosTFree(script->optionBuffer); - taosTFree(script); + tfree(script->lines); + tfree(script->optionBuffer); + tfree(script); } SScript *simProcessCallOver(SScript *script) { if (script->type == SIM_SCRIPT_TYPE_MAIN) { if (script->killed) { - simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX - "failed" FAILED_POSTFIX ", error:%s", - script->fileName, script->error); + simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX "failed" FAILED_POSTFIX ", error:%s", + script->fileName, script->error); exit(-1); } else { - simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX - "success" SUCCESS_POSTFIX, - script->fileName); + simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX "success" SUCCESS_POSTFIX, + script->fileName); simCloseTaosdConnect(script); simScriptSucced++; simScriptPos--; + + simFreeScript(script); if (simScriptPos == -1) { simInfo("----------------------------------------------------------------------"); simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced); exit(0); } - simFreeScript(script); return simScriptList[simScriptPos]; } } else { @@ -143,11 +150,11 @@ void *simExecuteScript(void *inputScript) { if (script == NULL) break; } else { SCmdLine *line = &script->lines[script->linePos]; - char *option = script->optionBuffer + line->optionOffset; + char * option = script->optionBuffer + line->optionOffset; simDebug("script:%s, line:%d with option \"%s\"", script->fileName, line->lineNum, option); SCommand *cmd = &simCmdList[line->cmdno]; - int ret = (*(cmd->executeCmd))(script, option); + int32_t ret = (*(cmd->executeCmd))(script, option); if (!ret) { script->killed = true; }