未验证 提交 c42cfea1 编写于 作者: S Shengliang Guan 提交者: GitHub

Merge pull request #3971 from taosdata/release/s104

Release/s104
...@@ -32,6 +32,8 @@ matrix: ...@@ -32,6 +32,8 @@ matrix:
- python3-setuptools - python3-setuptools
- valgrind - valgrind
- psmisc - psmisc
- unixodbc
- unixodbc-dev
before_script: before_script:
- export TZ=Asia/Harbin - export TZ=Asia/Harbin
...@@ -54,6 +56,7 @@ matrix: ...@@ -54,6 +56,7 @@ matrix:
py3ver=`python3 --version|awk '{print $2}'|cut -d "." -f 1,2` && apt install python$py3ver-dev py3ver=`python3 --version|awk '{print $2}'|cut -d "." -f 1,2` && apt install python$py3ver-dev
pip3 install psutil pip3 install psutil
pip3 install guppy3
pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
cd ${TRAVIS_BUILD_DIR}/tests cd ${TRAVIS_BUILD_DIR}/tests
......
pipeline { pipeline {
agent any agent none
environment{
WK = '/var/lib/jenkins/workspace/TDinternal'
WKC= '/var/lib/jenkins/workspace/TDinternal/community'
}
stages { stages {
stage('build TDengine') { stage('Parallel test stage') {
parallel {
stage('pytest') {
agent{label 'master'}
steps { steps {
sh '''cd ${WORKSPACE} sh '''
export TZ=Asia/Harbin date
date cd ${WKC}
rm -rf ${WORKSPACE}/debug git checkout develop
mkdir debug git pull
cd debug git submodule update
cmake .. > /dev/null cd ${WK}
make > /dev/null git checkout develop
cd ${WORKSPACE}/debug''' git pull
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests
#./test-all.sh smoke
./test-all.sh pytest
date'''
} }
} }
stage('test_b1') {
stage('test_tsim') { agent{label '184'}
parallel {
stage('test') {
steps { steps {
sh '''cd ${WORKSPACE}/tests sh '''
#./test-all.sh smoke cd ${WKC}
sudo ./test-all.sh full''' git checkout develop
git pull
git submodule update
cd ${WK}
git checkout develop
git pull
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests
#./test-all.sh smoke
./test-all.sh b1
date'''
} }
} }
stage('test_crash_gen') { stage('test_crash_gen') {
agent{label "185"}
steps { steps {
sh '''cd ${WORKSPACE}/tests/pytest sh '''
sudo ./crash_gen.sh -a -p -t 4 -s 2000''' cd ${WKC}
git checkout develop
git pull
git submodule update
cd ${WK}
git checkout develop
git pull
export TZ=Asia/Harbin
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests/pytest
./crash_gen.sh -a -p -t 4 -s 2000
date
cd ${WKC}/tests
./test-all.sh b2
date
'''
} }
} }
stage('test_valgrind') { stage('test_valgrind') {
agent{label "186"}
steps { steps {
sh '''cd ${WORKSPACE}/tests/pytest sh '''
sudo ./valgrind-test.sh 2>&1 > mem-error-out.log cd ${WKC}
grep \'start to execute\\|ERROR SUMMARY\' mem-error-out.log|grep -v \'grep\'|uniq|tee uniq-mem-error-out.log git checkout develop
git pull
for memError in `grep \'ERROR SUMMARY\' uniq-mem-error-out.log | awk \'{print $4}\'` git submodule update
do cd ${WK}
if [ -n "$memError" ]; then git checkout develop
if [ "$memError" -gt 12 ]; then git pull
echo -e "${RED} ## Memory errors number valgrind reports is $memError.\\ export TZ=Asia/Harbin
More than our threshold! ## ${NC}" date
travis_terminate $memError rm -rf ${WK}/debug
fi mkdir debug
fi cd debug
done cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests/pytest
./valgrind-test.sh 2>&1 > mem-error-out.log
./handle_val_log.sh
grep \'start to execute\\|definitely lost:\' mem-error-out.log|grep -v \'grep\'|uniq|tee uniq-definitely-lost-out.log date
for defiMemError in `grep \'definitely lost:\' uniq-definitely-lost-out.log | awk \'{print $7}\'` cd ${WKC}/tests
do ./test-all.sh b3
if [ -n "$defiMemError" ]; then date'''
if [ "$defiMemError" -gt 13 ]; then }
echo -e "${RED} ## Memory errors number valgrind reports \\ }
Definitely lost is $defiMemError. More than our threshold! ## ${NC}" stage('connector'){
travis_terminate $defiMemError agent{label "release"}
fi steps{
fi sh'''
done''' cd ${WORKSPACE}
git checkout develop
cd tests/gotest
bash batchtest.sh
cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
mvn clean package assembly:single >/dev/null
java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
python3 PythonChecker.py
'''
} }
} }
...@@ -67,4 +139,5 @@ done''' ...@@ -67,4 +139,5 @@ done'''
} }
} }
} }
\ No newline at end of file
...@@ -45,7 +45,7 @@ IF (${CMAKE_BUILD_TYPE} MATCHES "Debug") ...@@ -45,7 +45,7 @@ IF (${CMAKE_BUILD_TYPE} MATCHES "Debug")
ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release") ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release")
MESSAGE(STATUS "Build Release Version") MESSAGE(STATUS "Build Release Version")
ELSE () ELSE ()
IF (TD_WINDOWS_64) IF (TD_WINDOWS)
SET(CMAKE_BUILD_TYPE "Release") SET(CMAKE_BUILD_TYPE "Release")
MESSAGE(STATUS "Build Release Version in Windows as default") MESSAGE(STATUS "Build Release Version in Windows as default")
ELSE () ELSE ()
......
...@@ -16,6 +16,7 @@ ELSEIF (TD_WINDOWS) ...@@ -16,6 +16,7 @@ ELSEIF (TD_WINDOWS)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include) INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)
INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taoserror.h DESTINATION include)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver)
...@@ -30,7 +31,7 @@ ELSEIF (TD_WINDOWS) ...@@ -30,7 +31,7 @@ ELSEIF (TD_WINDOWS)
#INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS taos RUNTIME DESTINATION driver)
#INSTALL(TARGETS shell RUNTIME DESTINATION .) #INSTALL(TARGETS shell RUNTIME DESTINATION .)
IF (TD_MVN_INSTALLED) IF (TD_MVN_INSTALLED)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.0-dist.jar DESTINATION connector/jdbc) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.8-dist.jar DESTINATION connector/jdbc)
ENDIF () ENDIF ()
ELSEIF (TD_DARWIN) ELSEIF (TD_DARWIN)
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
......
...@@ -4,7 +4,7 @@ PROJECT(TDengine) ...@@ -4,7 +4,7 @@ PROJECT(TDengine)
IF (DEFINED VERNUMBER) IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER}) SET(TD_VER_NUMBER ${VERNUMBER})
ELSE () ELSE ()
SET(TD_VER_NUMBER "2.0.5.1") SET(TD_VER_NUMBER "2.0.6.0")
ENDIF () ENDIF ()
IF (DEFINED VERCOMPATIBLE) IF (DEFINED VERCOMPATIBLE)
......
...@@ -472,7 +472,7 @@ typedef uint64_t z_crc_t; ...@@ -472,7 +472,7 @@ typedef uint64_t z_crc_t;
#endif #endif
#ifndef Z_SOLO #ifndef Z_SOLO
#if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE) #if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
#if (_WIN64) #if defined(_WIN64) || defined(_WIN32)
#include <io.h> #include <io.h>
#include <process.h> #include <process.h>
#else #else
......
...@@ -472,7 +472,7 @@ typedef uLong FAR uLongf; ...@@ -472,7 +472,7 @@ typedef uLong FAR uLongf;
#endif #endif
#ifndef Z_SOLO #ifndef Z_SOLO
# if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE) # if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
#if (_WIN64) #if defined(_WIN64) || defined(_WIN32)
#include <io.h> #include <io.h>
#include <process.h> #include <process.h>
#else #else
......
...@@ -51,8 +51,8 @@ INTERVAL(1M)</code></pre> ...@@ -51,8 +51,8 @@ INTERVAL(1M)</code></pre>
<li><p>mseconds:查询数据库更新的时间间隔,单位为毫秒。一般设置为1000毫秒。返回值为指向TDengine_SUB 结构的指针,如果返回为空,表示失败。</p></li> <li><p>mseconds:查询数据库更新的时间间隔,单位为毫秒。一般设置为1000毫秒。返回值为指向TDengine_SUB 结构的指针,如果返回为空,表示失败。</p></li>
</ul><li><p><code>TAOS_ROW taos_consume(TAOS_SUB *tsub)</code> </ul><li><p><code>TAOS_ROW taos_consume(TAOS_SUB *tsub)</code>
</p><p>该函数用来获取订阅的结果,用户应用程序将其置于一个无限循环语句。如果数据库有新记录到达,该API将返回该最新的记录。如果没有新的记录,该API将阻塞。如果返回值为空,说明系统出错。参数说明:</p></li><ul><li><p>tsub:taos_subscribe的结构体指针。</p></li></ul><li><p><code>void taos_unsubscribe(TAOS_SUB *tsub)</code></p><p>取消订阅。应用程序退出时,务必调用该函数以避免资源泄露。</p></li> </p><p>该函数用来获取订阅的结果,用户应用程序将其置于一个无限循环语句。如果数据库有新记录到达,该API将返回该最新的记录。如果没有新的记录,该API将阻塞。如果返回值为空,说明系统出错。参数说明:</p></li><ul><li><p>tsub:taos_subscribe的结构体指针。</p></li></ul><li><p><code>void taos_unsubscribe(TAOS_SUB *tsub)</code></p><p>取消订阅。应用程序退出时,务必调用该函数以避免资源泄露。</p></li>
<li><p><code>int taos_num_subfields(TAOS_SUB *tsub)</code></p><p>获取返回的一行记录中数据包含多少列。</p></li> <li><p><code>int taos_num_fields(TAOS_SUB *tsub)</code></p><p>获取返回的一行记录中数据包含多少列。</p></li>
<li><p><code>TAOS_FIELD *taos_fetch_subfields(TAOS_SUB *tsub)</code></p><p>获取每列数据的属性(数据类型、名字、长度),与taos_num_subfileds配合使用,可解析返回的每行数据。</p></li></ul> <li><p><code>TAOS_FIELD *taos_fetch_fields(TAOS_SUB *tsub)</code></p><p>获取每列数据的属性(数据类型、名字、长度),与taos_num_subfileds配合使用,可解析返回的每行数据。</p></li></ul>
<p>示例代码:请看安装包中的的示范程序</p> <p>示例代码:请看安装包中的的示范程序</p>
<a class='anchor' id='缓存-(Cache)'></a><h2>缓存 (Cache)</h2> <a class='anchor' id='缓存-(Cache)'></a><h2>缓存 (Cache)</h2>
<p>TDengine采用时间驱动缓存管理策略(First-In-First-Out,FIFO),又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式(Least-Recent-Use,LRU),直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候,将最早的数据批量写入磁盘。一般意义上来说,对于物联网数据的使用,用户最为关心最近产生的数据,即当前状态。TDengine充分利用了这一特性,将最近到达的(当前状态)数据保存在缓存中。</p> <p>TDengine采用时间驱动缓存管理策略(First-In-First-Out,FIFO),又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式(Least-Recent-Use,LRU),直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候,将最早的数据批量写入磁盘。一般意义上来说,对于物联网数据的使用,用户最为关心最近产生的数据,即当前状态。TDengine充分利用了这一特性,将最近到达的(当前状态)数据保存在缓存中。</p>
......
...@@ -64,9 +64,9 @@ ...@@ -64,9 +64,9 @@
<p>该API用来获取最新消息,应用程序一般会将其置于一个无限循环语句中。其中参数tsub是taos_subscribe的返回值。如果数据库有新的记录,该API将返回,返回参数是一行记录。如果没有新的记录,该API将阻塞。如果返回值为空,说明系统出错,需要检查系统是否还在正常运行。</p></li> <p>该API用来获取最新消息,应用程序一般会将其置于一个无限循环语句中。其中参数tsub是taos_subscribe的返回值。如果数据库有新的记录,该API将返回,返回参数是一行记录。如果没有新的记录,该API将阻塞。如果返回值为空,说明系统出错,需要检查系统是否还在正常运行。</p></li>
<li><p><code>void taos_unsubscribe(TAOS_SUB *tsub)</code></p> <li><p><code>void taos_unsubscribe(TAOS_SUB *tsub)</code></p>
<p>该API用于取消订阅,参数tsub是taos_subscribe的返回值。应用程序退出时,需要调用该API,否则有资源泄露。</p></li> <p>该API用于取消订阅,参数tsub是taos_subscribe的返回值。应用程序退出时,需要调用该API,否则有资源泄露。</p></li>
<li><p><code>int taos_num_subfields(TAOS_SUB *tsub)</code></p> <li><p><code>int taos_num_fields(TAOS_SUB *tsub)</code></p>
<p>该API用来获取返回的一排数据中数据的列数</p></li> <p>该API用来获取返回的一排数据中数据的列数</p></li>
<li><p><code>TAOS_FIELD *taos_fetch_subfields(TAOS_RES *res)</code></p> <li><p><code>TAOS_FIELD *taos_fetch_fields(TAOS_RES *res)</code></p>
<p>该API用来获取每列数据的属性(数据类型、名字、字节数),与taos_num_subfileds配合使用,可用来解析返回的一排数据。</p></li> <p>该API用来获取每列数据的属性(数据类型、名字、字节数),与taos_num_subfileds配合使用,可用来解析返回的一排数据。</p></li>
</ul> </ul>
<a class='anchor' id='Java-Connector'></a><h2>Java Connector</h2> <a class='anchor' id='Java-Connector'></a><h2>Java Connector</h2>
......
...@@ -72,9 +72,9 @@ The API is used to start a subscription session by given a handle. The parameter ...@@ -72,9 +72,9 @@ The API is used to start a subscription session by given a handle. The parameter
The API used to get the new data from a TDengine server. It should be put in an infinite loop. The parameter <em>tsub</em> is the handle returned by <em>taos_subscribe</em>. If new data are updated, the API will return a row of the result. Otherwise, the API is blocked until new data arrives. If <em>NULL</em> pointer is returned, it means an error occurs.</p></li> The API used to get the new data from a TDengine server. It should be put in an infinite loop. The parameter <em>tsub</em> is the handle returned by <em>taos_subscribe</em>. If new data are updated, the API will return a row of the result. Otherwise, the API is blocked until new data arrives. If <em>NULL</em> pointer is returned, it means an error occurs.</p></li>
<li><p><code>void taos_unsubscribe(TAOS_SUB *tsub)</code> <li><p><code>void taos_unsubscribe(TAOS_SUB *tsub)</code>
Stop a subscription session by the handle returned by <em>taos_subscribe</em>.</p></li> Stop a subscription session by the handle returned by <em>taos_subscribe</em>.</p></li>
<li><p><code>int taos_num_subfields(TAOS_SUB *tsub)</code> <li><p><code>int taos_num_fields(TAOS_SUB *tsub)</code>
The API used to get the number of fields in a row.</p></li> The API used to get the number of fields in a row.</p></li>
<li><p><code>TAOS_FIELD *taos_fetch_subfields(TAOS_RES *res)</code> <li><p><code>TAOS_FIELD *taos_fetch_fields(TAOS_RES *res)</code>
The API used to get the description of each column.</p></li> The API used to get the description of each column.</p></li>
</ul> </ul>
<a class='anchor' id='Java-Connector'></a><h2>Java Connector</h2> <a class='anchor' id='Java-Connector'></a><h2>Java Connector</h2>
......
...@@ -11,7 +11,7 @@ TDengine的模块之一是时序数据库。但除此之外,为减少研发的 ...@@ -11,7 +11,7 @@ TDengine的模块之一是时序数据库。但除此之外,为减少研发的
* __全栈时序数据处理引擎__:将数据库、消息队列、缓存、流式计算等功能融为一体,应用无需再集成Kafka/Redis/HBase/Spark/HDFS等软件,大幅降低应用开发和维护的复杂度成本。 * __全栈时序数据处理引擎__:将数据库、消息队列、缓存、流式计算等功能融为一体,应用无需再集成Kafka/Redis/HBase/Spark/HDFS等软件,大幅降低应用开发和维护的复杂度成本。
* __强大的分析功能__:无论是十年前还是一秒钟前的数据,指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell, Python, R, Matlab随时进行。 * __强大的分析功能__:无论是十年前还是一秒钟前的数据,指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell, Python, R, Matlab随时进行。
* __与第三方工具无缝连接__:不用一行代码,即可与Telegraf, Grafana, EMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。 * __与第三方工具无缝连接__:不用一行代码,即可与Telegraf, Grafana, EMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。
* __零运维成本、零学习成本__:安装、集群一秒搞定,无需分库分表,实时备份。标准SQL,支持JDBC, RESTful, 支持Python/Java/C/C++/Go, 与MySQL相似,零学习成本。 * __零运维成本、零学习成本__:安装集群简单快捷,无需分库分表,实时备份。类似标准SQL,支持RESTful, 支持Python/Java/C/C++/C#/Go/Node.js, 与MySQL相似,零学习成本。
采用TDengine,可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是,因充分利用了物联网时序数据的特点,它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。 采用TDengine,可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是,因充分利用了物联网时序数据的特点,它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。
......
...@@ -30,13 +30,13 @@ TDengine软件分为服务器、客户端和报警模块三部分,目前2.0版 ...@@ -30,13 +30,13 @@ TDengine软件分为服务器、客户端和报警模块三部分,目前2.0版
- TDengine-alert-2.0.0-Linux-x64.tar.gz (8.1M) - TDengine-alert-2.0.0-Linux-x64.tar.gz (8.1M)
目前,TDengine只支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装。其他linux系统的支持正在开发中。用`which`命令来检测系统中是否存在`systemd`: 目前,TDengine只支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装。其他linux系统的支持正在开发中。用`which systemctl`命令来检测系统中是否存在`systemd`:
```cmd ```cmd
which systemd which systemctl
``` ```
如果系统中不存在`systemd`命令,请考虑[通过源码安装](#通过源码安装)TDengine。 如果系统中不存在`systemd`,请考虑[通过源码安装](#通过源码安装)TDengine。
具体的安装过程,请参见<a href="https://www.taosdata.com/blog/2019/08/09/566.html">TDengine多种安装包的安装和卸载</a> 具体的安装过程,请参见<a href="https://www.taosdata.com/blog/2019/08/09/566.html">TDengine多种安装包的安装和卸载</a>
......
...@@ -16,13 +16,13 @@ Three different packages are provided, please pick up the one you like. ...@@ -16,13 +16,13 @@ Three different packages are provided, please pick up the one you like.
<li><a id='tdengine-deb' style='color:var(--b2)'>TDengine DEB package (1.7M)</a></li> <li><a id='tdengine-deb' style='color:var(--b2)'>TDengine DEB package (1.7M)</a></li>
<li><a id='tdengine-tar' style='color:var(--b2)'>TDengine Tarball (3.0M)</a></li> <li><a id='tdengine-tar' style='color:var(--b2)'>TDengine Tarball (3.0M)</a></li>
</ul> </ul>
For the time being, TDengine only supports installation on Linux systems using [`systemd`](https://en.wikipedia.org/wiki/Systemd) as the service manager. To check if your system has *systemd*, use the _which_ command. For the time being, TDengine only supports installation on Linux systems using [`systemd`](https://en.wikipedia.org/wiki/Systemd) as the service manager. To check if your system has *systemd* package, use the _which systemctl_ command.
```cmd ```cmd
which systemd which systemctl
``` ```
If the `systemd` command is not found, please [install from source code](#Install-from-Source). If the `systemd` package is not found, please [install from source code](#Install-from-Source).
### Running TDengine ### Running TDengine
......
...@@ -29,23 +29,9 @@ Query OK, 2 row(s) in set (0.001100s) ...@@ -29,23 +29,9 @@ Query OK, 2 row(s) in set (0.001100s)
具体的查询语法请看<a href="https://www.taosdata.com/cn/documentation20/taos-sql/">TAOS SQL </a> 具体的查询语法请看<a href="https://www.taosdata.com/cn/documentation20/taos-sql/">TAOS SQL </a>
## 多表聚合查询 ## 多表聚合查询
物联网场景中,往往同一个类型的数据采集点有多个。TDengine采用超级表(STable)的概念来描述某一个类型的数据采集点,一张普通的表来描述一个具体的数据采集点。同时TDengine使用标签来描述数据采集点的静态属性,一个具体的数据采集点有具体的标签值。通过指定标签的过滤条件,TDengine提供了一高效的方法将超级表(某一类型的数据采集点)所属的子表进行聚合查询。对普通表的聚合函数以及绝大部分操作都适用于超级表,语法完全一样。
TDengine对每个数据采集点单独建表,但在实际应用中经常需要对不同的采集点数据进行聚合。为高效的进行聚合操作,TDengine引入超级表(STable)的概念。超级表用来代表一特定类型的数据采集点,它是包含多张表的表集合,集合里每张表的模式(schema)完全一致,但每张表都带有自己的静态标签,标签可以多个,可以随时增加、删除和修改。 **示例1**:在TAOS Shell,查找北京所有智能电表采集的电压平均值,并按照location分组
应用可通过指定标签的过滤条件,对一个STable下的全部或部分表进行聚合或统计操作,这样大大简化应用的开发。其具体流程如下图所示:
<center> <img src="../assets/stable.png"> </center>
<center> 多表聚合查询原理图 </center>
1:应用将一个查询条件发往系统;2: taosc将超级表的名字发往 Meta Node(管理节点);3:管理节点将超级表所拥有的 vnode 列表发回 taosc;4:taosc将计算的请求连同标签过滤条件发往这些vnode对应的多个数据节点;5:每个vnode先在内存里查找出自己节点里符合标签过滤条件的表的集合,然后扫描存储的时序数据,完成相应的聚合计算,将结果返回给taosc;6:taosc将多个数据节点返回的结果做最后的聚合,将其返回给应用。
由于TDengine在vnode内将标签数据与时序数据分离存储,通过先在内存里过滤标签数据,将需要扫描的数据集大幅减少,大幅提升聚合计算速度。同时,由于数据分布在多个vnode/dnode,聚合计算操作在多个vnode里并发进行,又进一步提升了聚合的速度。
对普通表的聚合函数以及绝大部分操作都适用于超级表,语法完全一样,细节请看 TAOS SQL。
比如:在TAOS Shell,查找所有智能电表采集的电压平均值,并按照location分组
```mysql ```mysql
taos> SELECT AVG(voltage) FROM meters GROUP BY location; taos> SELECT AVG(voltage) FROM meters GROUP BY location;
avg(voltage) | location | avg(voltage) | location |
...@@ -55,6 +41,18 @@ taos> SELECT AVG(voltage) FROM meters GROUP BY location; ...@@ -55,6 +41,18 @@ taos> SELECT AVG(voltage) FROM meters GROUP BY location;
Query OK, 2 row(s) in set (0.002136s) Query OK, 2 row(s) in set (0.002136s)
``` ```
**示例2**:在TAOS shell, 查找groupId为2的所有智能电表过去24小时的记录条数,电流的最大值
```mysql
taos> SELECT count(*), max(current) FROM meters where groupId = 2 and ts > now - 24h;
cunt(*) | max(current) |
==================================
5 | 13.4 |
Query OK, 1 row(s) in set (0.002136s)
```
TDengine仅容许对属于同一个超级表的表之间进行聚合查询,不同超级表之间的聚合查询不支持。在<a href="https://www.taosdata.com/cn/documentation20/taos-sql/">TAOS SQL </a>一章,查询类操作都会注明是否支持超级表。
## 降采样查询、插值 ## 降采样查询、插值
物联网场景里,经常需要通过降采样(down sampling)将采集的数据按时间段进行聚合。TDengine 提供了一个简便的关键词 interval 让按照时间窗口的查询操作变得极为简单。比如,将智能电表 d1001 采集的电流值每10秒钟求和 物联网场景里,经常需要通过降采样(down sampling)将采集的数据按时间段进行聚合。TDengine 提供了一个简便的关键词 interval 让按照时间窗口的查询操作变得极为简单。比如,将智能电表 d1001 采集的电流值每10秒钟求和
...@@ -66,9 +64,9 @@ taos> SELECT sum(current) FROM d1001 INTERVAL(10s); ...@@ -66,9 +64,9 @@ taos> SELECT sum(current) FROM d1001 INTERVAL(10s);
2018-10-03 14:38:10.000 | 24.900000572 | 2018-10-03 14:38:10.000 | 24.900000572 |
Query OK, 2 row(s) in set (0.000883s) Query OK, 2 row(s) in set (0.000883s)
``` ```
降采样操作也适用于超级表,比如:将所有智能电表采集的电流值每秒钟求和 降采样操作也适用于超级表,比如:将北京所有智能电表采集的电流值每秒钟求和
```mysql ```mysql
taos> SELECT SUM(current) FROM meters INTERVAL(1s); taos> SELECT SUM(current) FROM meters where location like "Beijing%" INTERVAL(1s);
ts | sum(current) | ts | sum(current) |
====================================================== ======================================================
2018-10-03 14:38:04.000 | 10.199999809 | 2018-10-03 14:38:04.000 | 10.199999809 |
......
...@@ -125,6 +125,7 @@ TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMic ...@@ -125,6 +125,7 @@ TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMic
1) 表的第一个字段必须是TIMESTAMP,并且系统自动将其设为主键; 1) 表的第一个字段必须是TIMESTAMP,并且系统自动将其设为主键;
2) 表名最大长度为193; 2) 表名最大长度为193;
3) 表的每行长度不能超过16k个字符; 3) 表的每行长度不能超过16k个字符;
4) 子表名只能由字母、数字和下划线组成,且不能以数字开头
5) 使用数据类型binary或nchar,需指定其最长的字节数,如binary(20),表示20字节; 5) 使用数据类型binary或nchar,需指定其最长的字节数,如binary(20),表示20字节;
- **删除数据表** - **删除数据表**
......
...@@ -82,8 +82,7 @@ TDengine系统后台服务由taosd提供,可以在配置文件taos.cfg里修 ...@@ -82,8 +82,7 @@ TDengine系统后台服务由taosd提供,可以在配置文件taos.cfg里修
下面仅仅列出一些重要的配置参数,更多的参数请看配置文件里的说明。各个参数的详细介绍及作用请看前述章节,而且这些参数的缺省配置都是工作的,一般无需设置。**注意:配置修改后,需要重启*taosd*服务才能生效。** 下面仅仅列出一些重要的配置参数,更多的参数请看配置文件里的说明。各个参数的详细介绍及作用请看前述章节,而且这些参数的缺省配置都是工作的,一般无需设置。**注意:配置修改后,需要重启*taosd*服务才能生效。**
- firstEp: taosd启动时,主动连接的集群中第一个dnode的end point, 默认值为localhost:6030。 - firstEp: taosd启动时,主动连接的集群中首个dnode的end point, 默认值为localhost:6030。
- secondEp: taosd启动时,如果first连接不上,尝试连接集群中第二个dnode的end point, 默认值为空。
- fqdn:数据节点的FQDN,缺省为操作系统配置的第一个hostname。如果习惯IP地址访问,可设置为该节点的IP地址。 - fqdn:数据节点的FQDN,缺省为操作系统配置的第一个hostname。如果习惯IP地址访问,可设置为该节点的IP地址。
- serverPort:taosd启动后,对外服务的端口号,默认值为6030。 - serverPort:taosd启动后,对外服务的端口号,默认值为6030。
- httpPort: RESTful服务使用的端口号,所有的HTTP请求(TCP)都需要向该接口发起查询/写入请求, 默认值为6041。 - httpPort: RESTful服务使用的端口号,所有的HTTP请求(TCP)都需要向该接口发起查询/写入请求, 默认值为6041。
...@@ -156,76 +155,80 @@ TDengine系统的前台交互客户端应用程序为taos,它与taosd共享同 ...@@ -156,76 +155,80 @@ TDengine系统的前台交互客户端应用程序为taos,它与taosd共享同
客户端配置参数 客户端配置参数
- firstEp: taos启动时,主动连接的集群中第一个taosd实例的end point, 缺省值为 localhost:6030。 - firstEp: taos启动时,主动连接的集群中第一个taosd实例的end point, 缺省值为 localhost:6030。
- secondEp: taos启动时,如果first连接不上,尝试连接集群中第二个taosd实例的end point, 缺省值为空。
- locale - locale
> 默认值:系统中动态获取,如果自动获取失败,需要用户在配置文件设置或通过API设置 默认值:系统中动态获取,如果自动获取失败,需要用户在配置文件设置或通过API设置
TDengine为存储中文、日文、韩文等非ASCII编码的宽字符,提供一种专门的字段类型nchar。写入nchar字段的数据将统一采用UCS4-LE格式进行编码并发送到服务器。需要注意的是,编码正确性是客户端来保证。因此,如果用户想要正常使用nchar字段来存储诸如中文、日文、韩文等非ASCII字符,需要正确设置客户端的编码格式。 TDengine为存储中文、日文、韩文等非ASCII编码的宽字符,提供一种专门的字段类型nchar。写入nchar字段的数据将统一采用UCS4-LE格式进行编码并发送到服务器。需要注意的是,编码正确性是客户端来保证。因此,如果用户想要正常使用nchar字段来存储诸如中文、日文、韩文等非ASCII字符,需要正确设置客户端的编码格式。
客户端的输入的字符均采用操作系统当前默认的编码格式,在Linux系统上多为UTF-8,部分中文系统编码则可能是GB18030或GBK等。在docker环境中默认的编码是POSIX。在中文版Windows系统中,编码则是CP936。客户端需要确保正确设置自己所使用的字符集,即客户端运行的操作系统当前编码字符集,才能保证nchar中的数据正确转换为UCS4-LE编码格式。 客户端的输入的字符均采用操作系统当前默认的编码格式,在Linux系统上多为UTF-8,部分中文系统编码则可能是GB18030或GBK等。在docker环境中默认的编码是POSIX。在中文版Windows系统中,编码则是CP936。客户端需要确保正确设置自己所使用的字符集,即客户端运行的操作系统当前编码字符集,才能保证nchar中的数据正确转换为UCS4-LE编码格式。
在 Linux 中 locale 的命名规则为: <语言>_<地区>.<字符集编码> 如:zh_CN.UTF-8,zh代表中文,CN代表大陆地区,UTF-8表示字符集。字符集编码为客户端正确解析本地字符串提供编码转换的说明。Linux系统与 Mac OSX 系统可以通过设置locale来确定系统的字符编码,由于Windows使用的locale中不是POSIX标准的locale格式,因此在Windows下需要采用另一个配置参数charset来指定字符编码。在Linux 系统中也可以使用charset来指定字符编码。 在 Linux 中 locale 的命名规则为: <语言>_<地区>.<字符集编码> 如:zh_CN.UTF-8,zh代表中文,CN代表大陆地区,UTF-8表示字符集。字符集编码为客户端正确解析本地字符串提供编码转换的说明。Linux系统与 Mac OSX 系统可以通过设置locale来确定系统的字符编码,由于Windows使用的locale中不是POSIX标准的locale格式,因此在Windows下需要采用另一个配置参数charset来指定字符编码。在Linux 系统中也可以使用charset来指定字符编码。
- charset - charset
> 默认值:系统中动态获取,如果自动获取失败,需要用户在配置文件设置或通过API设置 默认值:系统中动态获取,如果自动获取失败,需要用户在配置文件设置或通过API设置
如果配置文件中不设置charset,在Linux系统中,taos在启动时候,自动读取系统当前的locale信息,并从locale信息中解析提取charset编码格式。如果自动读取locale信息失败,则尝试读取charset配置,如果读取charset配置也失败,则中断启动过程。 如果配置文件中不设置charset,在Linux系统中,taos在启动时候,自动读取系统当前的locale信息,并从locale信息中解析提取charset编码格式。如果自动读取locale信息失败,则尝试读取charset配置,如果读取charset配置也失败,则中断启动过程。
在Linux系统中,locale信息包含了字符编码信息,因此正确设置了Linux系统locale以后可以不用再单独设置charset。例如: 在Linux系统中,locale信息包含了字符编码信息,因此正确设置了Linux系统locale以后可以不用再单独设置charset。例如:
``` ```
locale zh_CN.UTF-8 locale zh_CN.UTF-8
``` ```
在Windows系统中,无法从locale获取系统当前编码。如果无法从配置文件中读取字符串编码信息,taos默认设置为字符编码为CP936。其等效在配置文件中添加如下配置: 在Windows系统中,无法从locale获取系统当前编码。如果无法从配置文件中读取字符串编码信息,taos默认设置为字符编码为CP936。其等效在配置文件中添加如下配置:
``` ```
charset CP936 charset CP936
``` ```
如果需要调整字符编码,请查阅当前操作系统使用的编码,并在配置文件中正确设置。 如果需要调整字符编码,请查阅当前操作系统使用的编码,并在配置文件中正确设置。
在Linux系统中,如果用户同时设置了locale和字符集编码charset,并且locale和charset的不一致,后设置的值将覆盖前面设置的值。 在Linux系统中,如果用户同时设置了locale和字符集编码charset,并且locale和charset的不一致,后设置的值将覆盖前面设置的值。
``` ```
locale zh_CN.UTF-8 locale zh_CN.UTF-8
charset GBK charset GBK
``` ```
则charset的有效值是GBK。 则charset的有效值是GBK。
``` ```
charset GBK charset GBK
locale zh_CN.UTF-8 locale zh_CN.UTF-8
``` ```
charset的有效值是UTF-8。 charset的有效值是UTF-8。
日志的配置参数,与server 的配置参数完全一样。 日志的配置参数,与server 的配置参数完全一样。
- timezone - timezone
默认值:从系统中动态获取当前的时区设置 默认值:从系统中动态获取当前的时区设置
客户端运行系统所在的时区。为应对多时区的数据写入和查询问题,TDengine 采用 Unix 时间戳(Unix Timestamp)来记录和存储时间戳。Unix 时间戳的特点决定了任一时刻不论在任何时区,产生的时间戳均一致。需要注意的是,Unix时间戳是在客户端完成转换和记录。为了确保客户端其他形式的时间转换为正确的 Unix 时间戳,需要设置正确的时区。 客户端运行系统所在的时区。为应对多时区的数据写入和查询问题,TDengine 采用 Unix 时间戳(Unix Timestamp)来记录和存储时间戳。Unix 时间戳的特点决定了任一时刻不论在任何时区,产生的时间戳均一致。需要注意的是,Unix时间戳是在客户端完成转换和记录。为了确保客户端其他形式的时间转换为正确的 Unix 时间戳,需要设置正确的时区。
在Linux系统中,客户端会自动读取系统设置的时区信息。用户也可以采用多种方式在配置文件设置时区。例如: 在Linux系统中,客户端会自动读取系统设置的时区信息。用户也可以采用多种方式在配置文件设置时区。例如:
``` ```
timezone UTC-8 timezone UTC-8
timezone GMT-8 timezone GMT-8
timezone Asia/Shanghai timezone Asia/Shanghai
``` ```
均是合法的设置东八区时区的格式。 均是合法的设置东八区时区的格式。
时区的设置对于查询和写入SQL语句中非Unix时间戳的内容(时间戳字符串、关键词now的解析)产生影响。例如: 时区的设置对于查询和写入SQL语句中非Unix时间戳的内容(时间戳字符串、关键词now的解析)产生影响。例如:
``` ```
SELECT count(*) FROM table_name WHERE TS<'2019-04-11 12:01:08'; SELECT count(*) FROM table_name WHERE TS<'2019-04-11 12:01:08';
``` ```
在东八区,SQL语句等效于 在东八区,SQL语句等效于
``` ```
SELECT count(*) FROM table_name WHERE TS<1554955268000; SELECT count(*) FROM table_name WHERE TS<1554955268000;
``` ```
在UTC时区,SQL语句等效于 在UTC时区,SQL语句等效于
``` ```
SELECT count(*) FROM table_name WHERE TS<1554984068000; SELECT count(*) FROM table_name WHERE TS<1554984068000;
``` ```
为了避免使用字符串时间格式带来的不确定性,也可以直接使用Unix时间戳。此外,还可以在SQL语句中使用带有时区的时间戳字符串,例如:RFC3339格式的时间戳字符串,2013-04-12T15:52:01.123+08:00或者ISO-8601格式时间戳字符串2013-04-12T15:52:01.123+0800。上述两个字符串转化为Unix时间戳不受系统所在时区的影响。 为了避免使用字符串时间格式带来的不确定性,也可以直接使用Unix时间戳。此外,还可以在SQL语句中使用带有时区的时间戳字符串,例如:RFC3339格式的时间戳字符串,2013-04-12T15:52:01.123+08:00或者ISO-8601格式时间戳字符串2013-04-12T15:52:01.123+0800。上述两个字符串转化为Unix时间戳不受系统所在时区的影响。
启动taos时,也可以从命令行指定一个taosd实例的end point,否则就从taos.cfg读取。
- maxBinaryDisplayWidth
Shell中binary 和 nchar字段的显示宽度上限,超过此限制的部分将被隐藏。默认值:30。可在 shell 中通过命令 set max_binary_display_width nn 动态修改此选项。
启动taos时,也可以从命令行指定一个taosd实例的end point,否则就从taos.cfg读取。
## 用户管理 ## 用户管理
...@@ -378,7 +381,7 @@ KILL STREAM <stream-id>; ...@@ -378,7 +381,7 @@ KILL STREAM <stream-id>;
## 系统监控 ## 系统监控
TDengine启动后,会自动创建一个监测数据库SYS,并自动将服务器的CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入该数据库。TDengine还将重要的系统操作(比如登录、创建、删除数据库等)日志以及各种错误报警信息记录下来存放在SYS库里。系统管理员可以从CLI直接查看这个数据库,也可以在WEB通过图形化界面查看这些监测信息。 TDengine启动后,会自动创建一个监测数据库log,并自动将服务器的CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入该数据库。TDengine还将重要的系统操作(比如登录、创建、删除数据库等)日志以及各种错误报警信息记录下来存放在log库里。系统管理员可以从CLI直接查看这个数据库,也可以在WEB通过图形化界面查看这些监测信息。
这些监测信息的采集缺省是打开的,但可以修改配置文件里的选项enableMonitor将其关闭或打开。 这些监测信息的采集缺省是打开的,但可以修改配置文件里的选项enableMonitor将其关闭或打开。
...@@ -408,5 +411,4 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 ...@@ -408,5 +411,4 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。 您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。
##
...@@ -65,24 +65,24 @@ TDengine 的设计是基于单个硬件、软件系统不可靠,基于任何 ...@@ -65,24 +65,24 @@ TDengine 的设计是基于单个硬件、软件系统不可靠,基于任何
TDengine 分布式架构的逻辑结构图如下: TDengine 分布式架构的逻辑结构图如下:
<center> <img src="../assets/structure.png"> </center> <center> <img src="../assets/structure.png"> </center>
<center> 图 1 TDengine架构示意图 </center> <center> 图 1 TDengine架构示意图 </center>
一个完整的 TDengine 系统是运行在一到多个物理节点上的,逻辑上,它包含数据节点(dnode)、TDengine客户端(taosc)以及应用(app)。系统中存在一到多个数据节点,这些数据节点组成一个集群(cluster)。应用通过taosc的API与TDengine集群进行互动。下面对每个逻辑单元进行简要介绍。 一个完整的 TDengine 系统是运行在一到多个物理节点上的,逻辑上,它包含数据节点(dnode)、TDengine应用驱动(taosc)以及应用(app)。系统中存在一到多个数据节点,这些数据节点组成一个集群(cluster)。应用通过taosc的API与TDengine集群进行互动。下面对每个逻辑单元进行简要介绍。
**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机,可以是安装有OS的物理机、虚拟机或容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯,如果不了解FQDN,请看博文《[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)》。 **物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机,可以是安装有OS的物理机、虚拟机或Docker容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯,如果不了解FQDN,请看博文《[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)》。
**数据节点(dnode):** dnode 是 TDengine 服务器侧执行代码 taosd 在物理节点上的一个运行实例,一个工作的系统必须有至少一个数据节点。dnode包含零到多个逻辑的虚拟节点(VNODE),零或者至多一个逻辑的管理节点(mnode)。dnode在系统中的唯一标识由实例的End Point (EP )决定。EP是dnode所在物理节点的FQDN (Fully Qualified Domain Name)和系统所配置的网络端口号(Port)的组合。通过配置不同的端口,一个物理节点(一台物理机、虚拟机或容器)可以运行多个实例,或有多个数据节点。 **数据节点(dnode):** dnode 是 TDengine 服务器侧执行代码 taosd 在物理节点上的一个运行实例,一个工作的系统必须有至少一个数据节点。dnode包含零到多个逻辑的虚拟节点(VNODE),零或者至多一个逻辑的管理节点(mnode)。dnode在系统中的唯一标识由实例的End Point (EP )决定。EP是dnode所在物理节点的FQDN (Fully Qualified Domain Name)和系统所配置的网络端口号(Port)的组合。通过配置不同的端口,一个物理节点(一台物理机、虚拟机或容器)可以运行多个实例,或有多个数据节点。
**虚拟节点(vnode)**: 为更好的支持数据分片、负载均衡,防止数据过热或倾斜,数据节点被虚拟化成多个虚拟节点(vnode,图中V2, V3, V4等)。每个 vnode 都是一个相对独立的工作单元,是时序数据存储的基本单元,具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表(数据采集点)。当创建一张新表时,系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个DB,但一个DB可以有多个 vnode。一个 vnode 除存储的时序数据外,也保存有所包含的表的SCHEMA、标签值等。一个虚拟节点由所属的数据节点的EP,以及所属的VGroup ID在系统内唯一标识,由管理节点创建并管理。 **虚拟节点(vnode)**: 为更好的支持数据分片、负载均衡,防止数据过热或倾斜,数据节点被虚拟化成多个虚拟节点(vnode,图中V2, V3, V4等)。每个 vnode 都是一个相对独立的工作单元,是时序数据存储的基本单元,具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表(数据采集点)。当创建一张新表时,系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个DB,但一个DB可以有多个 vnode。一个 vnode 除存储的时序数据外,也保存有所包含的表的schema、标签值等。一个虚拟节点由所属的数据节点的EP,以及所属的VGroup ID在系统内唯一标识,由管理节点创建并管理。
**管理节点(mnode):** 一个虚拟的逻辑单元,负责所有数据节点运行状态的监控和维护,以及节点之间的负载均衡(图中M)。同时,管理节点也负责元数据(包括用户、数据库、表、静态标签等)的存储和管理,因此也称为 Meta Node。TDengine 集群中可配置多个(最多不超过5个) mnode,它们自动构建成为一个虚拟管理节点组(图中M0, M1, M2)。mnode 间采用 master/slave 的机制进行管理,而且采取强一致方式进行数据同步, 任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成,无需人工干预。每个dnode上至多有一个mnode,由所属的数据节点的EP来唯一标识。每个dnode通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的EP。 **管理节点(mnode):** 一个虚拟的逻辑单元,负责所有数据节点运行状态的监控和维护,以及节点之间的负载均衡(图中M)。同时,管理节点也负责元数据(包括用户、数据库、表、静态标签等)的存储和管理,因此也称为 Meta Node。TDengine 集群中可配置多个(最多不超过5个) mnode,它们自动构建成为一个虚拟管理节点组(图中M0, M1, M2)。mnode 间采用 master/slave 的机制进行管理,而且采取强一致方式进行数据同步, 任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成,无需人工干预。每个dnode上至多有一个mnode,由所属的数据节点的EP来唯一标识。每个dnode通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的EP。
**虚拟节点组(VGroup):** 不同数据节点上的 vnode 可以组成一个虚拟节点组(vnode group)来保证系统的高可靠。虚拟节点组内采取master/slave的方式进行管理。写操作只能在 master vnode 上进行,系统采用异步复制的方式将数据同步到 slave vnode,这样确保了一份数据在多个物理节点上有拷贝。一个 vgroup 里虚拟节点个数就是数据的副本数。如果一个DB的副本数为N,系统必须有至少N个数据节点。副本数在创建DB时通过参数 replica 可以指定,缺省为1。使用 TDengine 的多副本特性,可以不再需要昂贵的磁盘阵列等存储设备,就可以获得同样的数据高可靠性。虚拟节点组由管理节点创建、管理,并且由管理节点分配一个系统唯一的ID,VGroup ID。如果两个虚拟节点的vnode group ID相同,说明他们属于同一个组,数据互为备份。虚拟节点组里虚拟节点的个数是可以动态改变的,容许只有一个,也就是没有数据复制。VGroup ID是永远不变的,即使一个虚拟节点组被删除,它的ID也不会被收回重复利用。 **虚拟节点组(VGroup):** 不同数据节点上的 vnode 可以组成一个虚拟节点组(vnode group)来保证系统的高可靠。虚拟节点组内采取master/slave的方式进行管理。写操作只能在 master vnode 上进行,系统采用异步复制的方式将数据同步到 slave vnode,这样确保了一份数据在多个物理节点上有拷贝。一个 vgroup 里虚拟节点个数就是数据的副本数。如果一个DB的副本数为N,系统必须有至少N个数据节点。副本数在创建DB时通过参数 replica 可以指定,缺省为1。使用 TDengine 的多副本特性,可以不再需要昂贵的磁盘阵列等存储设备,就可以获得同样的数据高可靠性。虚拟节点组由管理节点创建、管理,并且由管理节点分配一个系统唯一的ID,VGroup ID。如果两个虚拟节点的vnode group ID相同,说明他们属于同一个组,数据互为备份。虚拟节点组里虚拟节点的个数是可以动态改变的,容许只有一个,也就是没有数据复制。VGroup ID是永远不变的,即使一个虚拟节点组被删除,它的ID也不会被收回重复利用。
**TAOSC:** taosc是TDengine给应用提供的驱动程序(driver),负责处理应用与集群的接口交互,内嵌于JDBC、ODBC driver中,或者C、Python、Go语言连接库里。应用都是通过taosc而不是直接连接集群中的数据节点与整个集群进行交互的。这个模块负责获取并缓存元数据;将插入、查询等请求转发到正确的数据节点;在把结果返回给应用时,还需要负责最后一级的聚合、排序、过滤等操作。对于JDBC, ODBC, C/C++接口而言,这个模块是在应用所处的物理节点上运行,但消耗的资源很小。同时,为支持全分布式的RESTful接口,taosc在TDengine集群的每个dnode上都有一运行实例。 **TAOSC:** taosc是TDengine给应用提供的驱动程序(driver),负责处理应用与集群的接口交互,提供C/C++语言原生接口,内嵌于JDBC、C#、Python、Go、Node.js语言连接库里。应用都是通过taosc而不是直接连接集群中的数据节点与整个集群进行交互的。这个模块负责获取并缓存元数据;将插入、查询等请求转发到正确的数据节点;在把结果返回给应用时,还需要负责最后一级的聚合、排序、过滤等操作。对于JDBC, C/C++/C#/Python/Go/Node.js接口而言,这个模块是在应用所处的物理节点上运行。同时,为支持全分布式的RESTful接口,taosc在TDengine集群的每个dnode上都有一运行实例。
### 节点之间的通讯 ### 节点之间的通讯
**通讯方式:**TDengine系统的各个节点之间的通讯是通过TCP/UDP进行的。因为考虑到物联网场景,数据写入的包一般不大,因此TDengine 除采用TCP做传输之外,还采用UDP方式,因为UDP 更加高效,而且不受连接数的限制。TDengine实现了自己的超时、重传、确认等机制,以确保UDP的可靠传输。对于数据量不到15K的数据包,采取UDP的方式进行传输,超过15K的,或者是查询类的操作,自动采取TCP的方式进行传输。同时,TDengine根据配置和数据包,会自动对数据进行压缩/解压缩,数字签名/认证等处理。对于数据节点之间的数据复制,只采用TCP方式进行数据传输。 **通讯方式:**TDengine系统的各个数据节点之间,以及应用驱动与各数据节点之间的通讯是通过TCP/UDP进行的。因为考虑到物联网场景,数据写入的包一般不大,因此TDengine 除采用TCP做传输之外,还采用UDP方式,因为UDP 更加高效,而且不受连接数的限制。TDengine实现了自己的超时、重传、确认等机制,以确保UDP的可靠传输。对于数据量不到15K的数据包,采取UDP的方式进行传输,超过15K的,或者是查询类的操作,自动采取TCP的方式进行传输。同时,TDengine根据配置和数据包,会自动对数据进行压缩/解压缩,数字签名/认证等处理。对于数据节点之间的数据复制,只采用TCP方式进行数据传输。
**FQDN配置**:一个数据节点有一个或多个FQDN,可以在系统配置文件taos.cfg通过参数“fqdn"进行指定,如果没有指定,系统将自动获取FQDN。如果节点没有配置FQDN,可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP,因为IP地址可变,一旦变化,将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN,需要保证DNS服务正常工作,或者在节点以及应用所在的节点配置好hosts文件。 **FQDN配置**:一个数据节点有一个或多个FQDN,可以在系统配置文件taos.cfg通过参数“fqdn"进行指定,如果没有指定,系统将自动获取计算机的hostname作为其FQDN。如果节点没有配置FQDN,可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP,因为IP地址可变,一旦变化,将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN,需要保证DNS服务正常工作,或者在节点以及应用所在的节点配置好hosts文件。
**端口配置:**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定,对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口,是serverPort+10. 为支持多线程高效的处理UDP数据,每个对内和对外的UDP连接,都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10,总共11个TCP/UDP端口。使用时,需要确保防火墙将这些端口打开。每个数据节点可以配置不同的serverPort。 **端口配置:**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定,对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口,是serverPort+10. 为支持多线程高效的处理UDP数据,每个对内和对外的UDP连接,都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10,总共11个TCP/UDP端口。使用时,需要确保防火墙将这些端口打开。每个数据节点可以配置不同的serverPort。
...@@ -153,6 +153,7 @@ TDengine除vnode分片之外,还对时序数据按照时间段进行分区。 ...@@ -153,6 +153,7 @@ TDengine除vnode分片之外,还对时序数据按照时间段进行分区。
当新的数据节点被添加进集群,因为新的计算和存储被添加进来,系统也将自动启动负载均衡流程。 当新的数据节点被添加进集群,因为新的计算和存储被添加进来,系统也将自动启动负载均衡流程。
负载均衡过程无需任何人工干预,应用也无需重启,将自动连接新的节点,完全透明。 负载均衡过程无需任何人工干预,应用也无需重启,将自动连接新的节点,完全透明。
**提示:负载均衡由参数balance控制,决定开启/关闭自动负载均衡。**
## 数据写入与复制流程 ## 数据写入与复制流程
如果一个数据库有N个副本,那一个虚拟节点组就有N个虚拟节点,但是只有一个是Master,其他都是slave。当应用将新的记录写入系统时,只有Master vnode能接受写的请求。如果slave vnode收到写的请求,系统将通知taosc需要重新定向。 如果一个数据库有N个副本,那一个虚拟节点组就有N个虚拟节点,但是只有一个是Master,其他都是slave。当应用将新的记录写入系统时,只有Master vnode能接受写的请求。如果slave vnode收到写的请求,系统将通知taosc需要重新定向。
...@@ -192,7 +193,8 @@ Master Vnode遵循下面的写入流程: ...@@ -192,7 +193,8 @@ Master Vnode遵循下面的写入流程:
理论上,只要是异步复制,就无法保证100%不丢失。但是这个窗口极小,mater与slave要同时发生故障,而且发生在刚给应用确认写入成功之后。 理论上,只要是异步复制,就无法保证100%不丢失。但是这个窗口极小,mater与slave要同时发生故障,而且发生在刚给应用确认写入成功之后。
注:异地容灾、IDC无中断迁移,仅仅企业版支持 注:异地容灾、IDC无中断迁移,仅仅企业版支持。
**提示:该功能暂未提供**
### 主从选择 ### 主从选择
Vnode会保持一个数据版本号(Version),对内存数据进行持久化存储时,对该版本号也进行持久化存储。每个数据更新操作,无论是采集的时序数据还是元数据,这个版本号将增一。 Vnode会保持一个数据版本号(Version),对内存数据进行持久化存储时,对该版本号也进行持久化存储。每个数据更新操作,无论是采集的时序数据还是元数据,这个版本号将增一。
...@@ -259,6 +261,7 @@ dataDir /mnt/disk6/taos 2 ...@@ -259,6 +261,7 @@ dataDir /mnt/disk6/taos 2
挂载的盘也可以是非本地的网络盘,只要系统能访问即可。 挂载的盘也可以是非本地的网络盘,只要系统能访问即可。
注:多级存储功能仅企业版支持 注:多级存储功能仅企业版支持
**提示:该功能暂未提供**
## 数据查询 ## 数据查询
TDengine提供了多种多样针对表和超级表的查询处理功能,除了常规的聚合查询之外,还提供针对时序数据的窗口查询、统计聚合等功能。TDengine的查询处理需要客户端、vnode, mnode节点协同完成。 TDengine提供了多种多样针对表和超级表的查询处理功能,除了常规的聚合查询之外,还提供针对时序数据的窗口查询、统计聚合等功能。TDengine的查询处理需要客户端、vnode, mnode节点协同完成。
...@@ -289,11 +292,18 @@ select count(*) from d1001 interval(1h) fill(prev); ...@@ -289,11 +292,18 @@ select count(*) from d1001 interval(1h) fill(prev);
针对d1001设备采集数据统计每小时记录数,如果某一个小时不存在数据,这返回之前一个小时的统计数据。TDengine提供前向插值(prev)、线性插值(linear)、NULL值填充(NULL)、特定值填充(value)。 针对d1001设备采集数据统计每小时记录数,如果某一个小时不存在数据,这返回之前一个小时的统计数据。TDengine提供前向插值(prev)、线性插值(linear)、NULL值填充(NULL)、特定值填充(value)。
### 多表聚合查询 ### 多表聚合查询
多表聚合查询与单表查询的整体流程相同,但是存在如下的差异: TDengine对每个数据采集点单独建表,但在实际应用中经常需要对不同的采集点数据进行聚合。为高效的进行聚合操作,TDengine引入超级表(STable)的概念。超级表用来代表一特定类型的数据采集点,它是包含多张表的表集合,集合里每张表的模式(schema)完全一致,但每张表都带有自己的静态标签,标签可以多个,可以随时增加、删除和修改。 应用可通过指定标签的过滤条件,对一个STable下的全部或部分表进行聚合或统计操作,这样大大简化应用的开发。其具体流程如下图所示:
<center> <img src="../assets/multi_tables.png"> </center>
- 由于多表可能分布在不同的节点(dnode),因此多表的聚合查询需要首先获得表所在的全部数据节点的信息,并且同时向相关的dnode发出查询请求。
- 每个vnode的计算获得的中间结果(partial results)需要进行第二阶段的聚合才能形成最终结果,第二阶段的聚合过程在客户端完成。 <center> 图 5 多表聚合查询原理图 </center>
- 由于表标签信息存储在vnode中,因此针对标签信息的查询也需要vnode完成。客户端将标签的过滤表达式封装在查询请求结构体中发送给vnode,由vnode的查询执行线程从中抽取出标签查询条件,然后执行查询。标签查询与过滤是在针对表的查询之前完成。标签查询完成以后,将符合条件的表纳入到接下来的查询处理流程中。 1:应用将一个查询条件发往系统;
2: taosc将超级表的名字发往 Meta Node(管理节点);
3:管理节点将超级表所拥有的 vnode 列表发回 taosc;
4:taosc将计算的请求连同标签过滤条件发往这些vnode对应的多个数据节点;
5:每个vnode先在内存里查找出自己节点里符合标签过滤条件的表的集合,然后扫描存储的时序数据,完成相应的聚合计算,将结果返回给taosc;
6:taosc将多个数据节点返回的结果做最后的聚合,将其返回给应用。
由于TDengine在vnode内将标签数据与时序数据分离存储,通过在内存里过滤标签数据,先找到需要参与聚合操作的表的集合,将需要扫描的数据集大幅减少,大幅提升聚合计算速度。同时,由于数据分布在多个vnode/dnode,聚合计算操作在多个vnode里并发进行,又进一步提升了聚合的速度。 对普通表的聚合函数以及绝大部分操作都适用于超级表,语法完全一样,细节请看 TAOS SQL。
### 预计算 ### 预计算
为有效提升查询处理的性能,针对物联网数据的不可更改的特点,在数据块头部记录该数据块中存储数据的统计信息:包括最大值、最小值、和。我们称之为预计算单元。如果查询处理涉及整个数据块的全部数据,直接使用预计算结果,完全不需要读取数据块的内容。由于预计算数据量远小于磁盘上存储的数据块数据的大小,对于磁盘IO为瓶颈的查询处理,使用预计算结果可以极大地减小读取IO压力,加速查询处理的流程。预计算机制与Postgre SQL的索引BRIN(block range index)有异曲同工之妙。 为有效提升查询处理的性能,针对物联网数据的不可更改的特点,在数据块头部记录该数据块中存储数据的统计信息:包括最大值、最小值、和。我们称之为预计算单元。如果查询处理涉及整个数据块的全部数据,直接使用预计算结果,完全不需要读取数据块的内容。由于预计算数据量远小于磁盘上存储的数据块数据的大小,对于磁盘IO为瓶颈的查询处理,使用预计算结果可以极大地减小读取IO压力,加速查询处理的流程。预计算机制与Postgre SQL的索引BRIN(block range index)有异曲同工之妙。
...@@ -8,7 +8,7 @@ TDengine的集群管理极其简单,除添加和删除节点需要人工干预 ...@@ -8,7 +8,7 @@ TDengine的集群管理极其简单,除添加和删除节点需要人工干预
## 准备工作 ## 准备工作
**第零步**如果没有部署DNS服务,请规划集群所有物理节点的FQDN,然后按照《[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)》里的步骤,将所有集群物理节点的IP与FQDN的对应关系添加好。 **第零步**规划集群所有物理节点的FQDN,将规划好的FQDN分别添加到每个物理节点的/etc/hostname;修改每个物理节点的/etc/hosts,将所有集群物理节点的IP与FQDN的对应添加好。【如部署了DNS,请联系网络管理员在DNS上做好相关配置】
**第一步**:如果搭建集群的物理节点中,存有之前的测试数据、装过1.X的版本,或者装过其他版本的TDengine,请先将其删除,并清空所有数据,具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) **第一步**:如果搭建集群的物理节点中,存有之前的测试数据、装过1.X的版本,或者装过其他版本的TDengine,请先将其删除,并清空所有数据,具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html )
**注意1:**因为FQDN的信息会写进文件,如果之前没有配置或者更改FQDN,且启动了TDengine。请一定在确保数据无用或者备份的前提下,清理一下之前的数据(rm -rf /var/lib/taos/); **注意1:**因为FQDN的信息会写进文件,如果之前没有配置或者更改FQDN,且启动了TDengine。请一定在确保数据无用或者备份的前提下,清理一下之前的数据(rm -rf /var/lib/taos/);
...@@ -16,9 +16,9 @@ TDengine的集群管理极其简单,除添加和删除节点需要人工干预 ...@@ -16,9 +16,9 @@ TDengine的集群管理极其简单,除添加和删除节点需要人工干预
**第二步**:建议关闭所有物理节点的防火墙,至少保证端口:6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙,集群搭建完毕之后,再来配置端口; **第二步**:建议关闭所有物理节点的防火墙,至少保证端口:6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙,集群搭建完毕之后,再来配置端口;
**第三步**:在所有节点安装TDengine,且版本必须是一致的,**但不要启动taosd**。安装时,提示输入是否要加入一个已经存在的TDengine集群时,第一个物理节点直接回车创建新集群,后续物理节点则输入该集群任何一个在线的物理节点的FQDN:端口号(默认6030); **第三步**:在所有物理节点安装TDengine,且版本必须是一致的,**但不要启动taosd**。安装时,提示输入是否要加入一个已经存在的TDengine集群时,第一个物理节点直接回车创建新集群,后续物理节点则输入该集群任何一个在线的物理节点的FQDN:端口号(默认6030);
**第四步**:检查所有数据节点,以及应用所在物理节点的网络设置: **第四步**:检查所有数据节点,以及应用程序所在物理节点的网络设置:
1. 每个物理节点上执行命令`hostname -f`,查看和确认所有节点的hostname是不相同的(应用驱动所在节点无需做此项检查); 1. 每个物理节点上执行命令`hostname -f`,查看和确认所有节点的hostname是不相同的(应用驱动所在节点无需做此项检查);
2. 每个物理节点上执行`ping host`, 其中host是其他物理节点的hostname, 看能否ping通其它物理节点; 如果不能ping通,需要检查网络设置, 或/etc/hosts文件(Windows系统默认路径为C:\Windows\system32\drivers\etc\hosts),或DNS的配置。如果无法ping通,是无法组成集群的; 2. 每个物理节点上执行`ping host`, 其中host是其他物理节点的hostname, 看能否ping通其它物理节点; 如果不能ping通,需要检查网络设置, 或/etc/hosts文件(Windows系统默认路径为C:\Windows\system32\drivers\etc\hosts),或DNS的配置。如果无法ping通,是无法组成集群的;
...@@ -213,6 +213,6 @@ SHOW MNODES; ...@@ -213,6 +213,6 @@ SHOW MNODES;
## Arbitrator的使用 ## Arbitrator的使用
如果副本数为偶数,当一个vnode group里一半或超过一半的vnode不工作时,是无法从中选出master的。同理,一半或超过一半的mnode不工作时,是无法选出mnode的master的,因为存在“split brain”问题。为解决这个问题,TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作,但只简单的负责网络连接,不处理任何数据插入或访问。只要包含arbitrator在内,超过半数的vnode或mnode工作,那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形,如果一个节点A离线,但另外一个节点B正常,而且能连接到arbitrator, 那么节点B就能正常工作。 如果副本数为偶数,当一个vnode group里一半vnode不工作时,是无法从中选出master的。同理,一半mnode不工作时,是无法选出mnode的master的,因为存在“split brain”问题。为解决这个问题,TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作,但只简单的负责网络连接,不处理任何数据插入或访问。只要包含arbitrator在内,超过半数的vnode或mnode工作,那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形,如果一个节点A离线,但另外一个节点B正常,而且能连接到arbitrator, 那么节点B就能正常工作。
TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求,只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号,缺省是6042。配置每个taosd实例时,可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了,当副本数为偶数数,系统将自动连接配置的arbitrator TDengine提供一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。请点击[安装包下载](https://www.taosdata.com/cn/all-downloads/),在TDengine Arbitrator Linux一节中,选择适合的版本下载并安装。该程序对系统资源几乎没有要求,只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号,缺省是6042。配置每个taosd实例时,可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了,当副本数为偶数数,系统将自动连接配置的arbitrator。如果副本数为奇数,即使配置了arbitrator, 系统也不会去建立连接
...@@ -72,38 +72,34 @@ maven 项目中使用如下 pom.xml 配置即可: ...@@ -72,38 +72,34 @@ maven 项目中使用如下 pom.xml 配置即可:
### 获取连接 ### 获取连接
如下所示配置即可获取 TDengine Connection: #### 通过JdbcUrl获取连接
通过指定的jdbcUrl获取连接,如下所示:
```java ```java
Class.forName("com.taosdata.jdbc.TSDBDriver"); Class.forName("com.taosdata.jdbc.TSDBDriver");
String jdbcUrl = "jdbc:TAOS://127.0.0.1:6030/log?user=root&password=taosdata"; String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata";
Connection conn = DriverManager.getConnection(jdbcUrl); Connection conn = DriverManager.getConnection(jdbcUrl);
``` ```
> 端口 6030 为默认连接端口,JDBC URL 中的 log 为系统本身的监控数据库 以上示例,建立了到hostname为taosdemo.com,端口为6030(TDengine的默认端口),数据库名为test的连接。这个url中指定用户名(user)为root,密码(password)为taosdata
TDengine 的 JDBC URL 规范格式为: TDengine 的 JDBC URL 规范格式为:
`jdbc:TAOS://{host_ip}:{port}/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]` `jdbc:TAOS://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]`
url中的配置参数如下:
其中,`{}` 中的内容必须,`[]` 中为可选。配置参数说明如下:
* user:登录 TDengine 用户名,默认值 root。 * user:登录 TDengine 用户名,默认值 root。
* password:用户登录密码,默认值 taosdata。 * password:用户登录密码,默认值 taosdata。
* charset:客户端使用的字符集,默认值为系统字符集。
* cfgdir:客户端配置文件目录路径,Linux OS 上默认值 /etc/taos ,Windows OS 上默认值 C:/TDengine/cfg。 * cfgdir:客户端配置文件目录路径,Linux OS 上默认值 /etc/taos ,Windows OS 上默认值 C:/TDengine/cfg。
* charset:客户端使用的字符集,默认值为系统字符集。
* locale:客户端语言环境,默认值系统当前 locale。 * locale:客户端语言环境,默认值系统当前 locale。
* timezone:客户端使用的时区,默认值为系统当前时区。 * timezone:客户端使用的时区,默认值为系统当前时区。
以上参数可以在 3 处配置,`优先级由高到低`分别如下: #### 使用JdbcUrl和Properties获取连接
1. JDBC URL 参数
如上所述,可以在 JDBC URL 的参数中指定。 除了通过指定的jdbcUrl获取连接,还可以使用Properties指定建立连接时的参数,如下所示:
2. java.sql.DriverManager.getConnection(String jdbcUrl, Properties connProps)
```java ```java
public Connection getConn() throws Exception{ public Connection getConn() throws Exception{
Class.forName("com.taosdata.jdbc.TSDBDriver"); Class.forName("com.taosdata.jdbc.TSDBDriver");
String jdbcUrl = "jdbc:TAOS://127.0.0.1:0/log?user=root&password=taosdata"; String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata";
Properties connProps = new Properties(); Properties connProps = new Properties();
connProps.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR, "/etc/taos");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
...@@ -111,16 +107,39 @@ public Connection getConn() throws Exception{ ...@@ -111,16 +107,39 @@ public Connection getConn() throws Exception{
return conn; return conn;
} }
``` ```
以上示例,建立一个到hostname为taosdemo.com,端口为6030,数据库名为test的连接。这个连接在url中指定了用户名(user)为root,密码(password)为taosdata,并在connProps中指定了使用的字符集、语言环境、时区等信息。
properties中的配置参数如下:
* TSDBDriver.PROPERTY_KEY_USER:登录 TDengine 用户名,默认值 root。
* TSDBDriver.PROPERTY_KEY_PASSWORD:用户登录密码,默认值 taosdata。
* TSDBDriver.PROPERTY_KEY_CONFIG_DIR:客户端配置文件目录路径,Linux OS 上默认值 /etc/taos ,Windows OS 上默认值 C:/TDengine/cfg。
* TSDBDriver.PROPERTY_KEY_CHARSET:客户端使用的字符集,默认值为系统字符集。
* TSDBDriver.PROPERTY_KEY_LOCALE:客户端语言环境,默认值系统当前 locale。
* TSDBDriver.PROPERTY_KEY_TIME_ZONE:客户端使用的时区,默认值为系统当前时区。
#### 使用客户端配置文件建立连接
当使用JDBC连接TDengine集群时,可以使用客户端配置文件,在客户端配置文件中指定集群的firstEp、secondEp参数。
如下所示:
1. 在java中不指定hostname和port
```java
public Connection getConn() throws Exception{
Class.forName("com.taosdata.jdbc.TSDBDriver");
String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata";
Properties connProps = new Properties();
connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
Connection conn = DriverManager.getConnection(jdbcUrl, connProps);
return conn;
}
```
2. 在配置文件中指定firstEp和secondEp
```
# first fully qualified domain name (FQDN) for TDengine system
firstEp cluster_node1:6030
3. 客户端配置文件 taos.cfg # second fully qualified domain name (FQDN) for TDengine system, for cluster only
secondEp cluster_node2:6030
linux 系统默认配置文件为 /var/lib/taos/taos.cfg,windows 系统默认配置文件路径为 C:\TDengine\cfg\taos.cfg。
```properties
# client default username
# defaultUser root
# client default password
# defaultPass taosdata
# default system charset # default system charset
# charset UTF-8 # charset UTF-8
...@@ -128,6 +147,19 @@ public Connection getConn() throws Exception{ ...@@ -128,6 +147,19 @@ public Connection getConn() throws Exception{
# system locale # system locale
# locale en_US.UTF-8 # locale en_US.UTF-8
``` ```
以上示例,jdbc会使用客户端的配置文件,建立到hostname为cluster_node1,端口为6030,数据库名为test的连接。当集群中firstEp节点失效时,JDBC会尝试使用secondEp连接集群。
TDengine中,只要保证firstEp和secondEp中一个节点有效,就可以正常建立到集群的连接。
> 注意:这里的配置文件指的是调用JDBC Connector的应用程序所在机器上的配置文件,Linux OS 上默认值 /etc/taos/taos.cfg ,Windows OS 上默认值 C://TDengine/cfg/taos.cfg。
#### 配置参数的优先级
通过以上3种方式获取连接,如果配置参数在url、Properties、客户端配置文件中有重复,则参数的`优先级由高到低`分别如下:
1. JDBC URL 参数,如上所述,可以在 JDBC URL 的参数中指定。
2. Properties connProps
3. 客户端配置文件 taos.cfg
例如:在url中指定了password为taosdata,在Properties中指定了password为taosdemo,那么,JDBC会使用url中的password建立连接。
> 更多详细配置请参考[客户端配置][13] > 更多详细配置请参考[客户端配置][13]
### 创建数据库和表 ### 创建数据库和表
......
...@@ -22,7 +22,7 @@ INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, ...@@ -22,7 +22,7 @@ INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6,
**Tips:** **Tips:**
- 要提高写入效率,需要批量写入。一批写入的记录条数越多,插入效率就越高。但一条记录不能超过16K,一条SQL语句总长度不能超过64K(可通过参数maxSQLLength配置,最大可配置为8M)。 - 要提高写入效率,需要批量写入。一批写入的记录条数越多,插入效率就越高。但一条记录不能超过16K,一条SQL语句总长度不能超过64K(可通过参数maxSQLLength配置,最大可配置为1M)。
- TDengine支持多线程同时写入,要进一步提高写入速度,一个客户端需要打开20个以上的线程同时写。但线程数达到一定数量后,无法再提高,甚至还会下降,因为线程切频繁切换,带来额外开销。 - TDengine支持多线程同时写入,要进一步提高写入速度,一个客户端需要打开20个以上的线程同时写。但线程数达到一定数量后,无法再提高,甚至还会下降,因为线程切频繁切换,带来额外开销。
- 对同一张表,如果新插入记录的时间戳已经存在,新记录将被直接抛弃,也就是说,在一张表里,时间戳必须是唯一的。如果应用自动生成记录,很有可能生成的时间戳是一样的,这样,成功插入的记录条数会小于应用插入的记录条数。 - 对同一张表,如果新插入记录的时间戳已经存在,新记录将被直接抛弃,也就是说,在一张表里,时间戳必须是唯一的。如果应用自动生成记录,很有可能生成的时间戳是一样的,这样,成功插入的记录条数会小于应用插入的记录条数。
- 写入的数据的时间戳必须大于当前时间减去配置参数keep的时间。如果keep配置为3650天,那么无法写入比3650天还老的数据。写入数据的时间戳也不能大于当前时间加配置参数days。如果days配置为2,那么无法写入比当前时间还晚2天的数据。 - 写入的数据的时间戳必须大于当前时间减去配置参数keep的时间。如果keep配置为3650天,那么无法写入比3650天还老的数据。写入数据的时间戳也不能大于当前时间加配置参数days。如果days配置为2,那么无法写入比当前时间还晚2天的数据。
......
...@@ -61,7 +61,7 @@ vnode与其子模块是通过API直接调用,而不是通过消息队列传递 ...@@ -61,7 +61,7 @@ vnode与其子模块是通过API直接调用,而不是通过消息队列传递
mnode是整个系统的大脑,负责整个系统的资源调度,负责meta data的管理与存储。 mnode是整个系统的大脑,负责整个系统的资源调度,负责meta data的管理与存储。
一个运行的系统里,只有一个mnode,但它有多个副本(由系统配置参数numOfMpeers控制)。这些副本分布在不同的dnode里,目的是保证系统的高可靠运行。副本之间的数据复制是采用同步而非异步的方式,以确保数据的一致性,确保数据不会丢失。这些副本会自动选举一个Master,其他副本是slave。所有数据更新类的操作,都只能在master上进行,而查询类的可以在slave节点上进行。代码实现上,同步模块与vnode共享,但mnode被分配一个特殊的vgroup ID: 1,而且quorum大于1。整个集群系统是由多个dnode组成的,运行的mnode的副本数不可能超过dnode的个数,但不会超过配置的副本数。如果某个mnode副本宕机一段时间,只要超过半数的mnode副本仍在运行,运行的mnode会自动根据整个系统的资源情况,在其他dnode里再启动一个mnode, 以保证运行的副本数。 一个运行的系统里,只有一个mnode,但它有多个副本(由系统配置参数numOfMnodes控制)。这些副本分布在不同的dnode里,目的是保证系统的高可靠运行。副本之间的数据复制是采用同步而非异步的方式,以确保数据的一致性,确保数据不会丢失。这些副本会自动选举一个Master,其他副本是slave。所有数据更新类的操作,都只能在master上进行,而查询类的可以在slave节点上进行。代码实现上,同步模块与vnode共享,但mnode被分配一个特殊的vgroup ID: 1,而且quorum大于1。整个集群系统是由多个dnode组成的,运行的mnode的副本数不可能超过dnode的个数,但不会超过配置的副本数。如果某个mnode副本宕机一段时间,只要超过半数的mnode副本仍在运行,运行的mnode会自动根据整个系统的资源情况,在其他dnode里再启动一个mnode, 以保证运行的副本数。
各个dnode通过信息交换,保存有mnode各个副本的End Point列表,并向其中的master节点定时(间隔由系统配置参数statusInterval控制)发送status消息,消息体里包含该dnode的CPU、内存、剩余存储空间、vnode个数,以及各个vnode的状态(存储空间、原始数据大小、记录条数、角色等)。这样mnode就了解整个系统的资源情况,如果用户创建新的表,就可以决定需要在哪个dnode创建;如果增加或删除dnode, 或者监测到某dnode数据过热、或离线太长,就可以决定需要挪动那些vnode,以实现负载均衡。 各个dnode通过信息交换,保存有mnode各个副本的End Point列表,并向其中的master节点定时(间隔由系统配置参数statusInterval控制)发送status消息,消息体里包含该dnode的CPU、内存、剩余存储空间、vnode个数,以及各个vnode的状态(存储空间、原始数据大小、记录条数、角色等)。这样mnode就了解整个系统的资源情况,如果用户创建新的表,就可以决定需要在哪个dnode创建;如果增加或删除dnode, 或者监测到某dnode数据过热、或离线太长,就可以决定需要挪动那些vnode,以实现负载均衡。
......
...@@ -6,10 +6,7 @@ ...@@ -6,10 +6,7 @@
######################################################## ########################################################
# first fully qualified domain name (FQDN) for TDengine system # first fully qualified domain name (FQDN) for TDengine system
# firstEp hostname1:6030 # firstEp hostname:6030
# second fully qualified domain name (FQDN) for TDengine system, for cluster only
# secondEp cluster_hostname2:6030
# local fully qualified domain name (FQDN) # local fully qualified domain name (FQDN)
# fqdn hostname # fqdn hostname
...@@ -29,12 +26,18 @@ ...@@ -29,12 +26,18 @@
# number of threads per CPU core # number of threads per CPU core
# numOfThreadsPerCore 1.0 # numOfThreadsPerCore 1.0
# the proportion of total threads responsible for query
# ratioOfQueryThreads 0.5
# number of management nodes in the system # number of management nodes in the system
# numOfMnodes 3 # numOfMnodes 3
# enable/disable backuping vnode directory when removing dnode # enable/disable backuping vnode directory when removing dnode
# vnodeBak 1 # vnodeBak 1
# if report installation / use information
# telemetryReporting 1
# enable/disable load balancing # enable/disable load balancing
# balance 1 # balance 1
...@@ -89,9 +92,6 @@ ...@@ -89,9 +92,6 @@
# max number of tables per vnode # max number of tables per vnode
# maxTablesPerVnode 1000000 # maxTablesPerVnode 1000000
# step size of increasing table number in a vnode
# tableIncStepPerVnode 1000
# cache block size (Mbyte) # cache block size (Mbyte)
# cache 16 # cache 16
...@@ -110,6 +110,9 @@ ...@@ -110,6 +110,9 @@
# maximum rows of records in file block # maximum rows of records in file block
# maxRows 4096 # maxRows 4096
# the number of acknowledgments required for successful data writing
# quorum 1
# enable/disable compression # enable/disable compression
# comp 2 # comp 2
...@@ -122,15 +125,6 @@ ...@@ -122,15 +125,6 @@
# number of replications, for cluster only # number of replications, for cluster only
# replica 1 # replica 1
# mqtt hostname
# mqttHostName test.mosquitto.org
# mqtt port
# mqttPort 1883
# mqtt topic
# mqttTopic /test
# the compressed rpc message, option: # the compressed rpc message, option:
# -1 (no compression) # -1 (no compression)
# 0 (all message compressed), # 0 (all message compressed),
...@@ -167,12 +161,12 @@ ...@@ -167,12 +161,12 @@
# stop writing data when the disk size of the log folder is less than this value # stop writing data when the disk size of the log folder is less than this value
# minimalDataDirGB 0.1 # minimalDataDirGB 0.1
# One mnode is equal to the number of vnode consumed
# mnodeEqualVnodeNum 4
# enbale/disable http service # enbale/disable http service
# http 1 # http 1
# enable/disable muqq service
# mqtt 0
# enable/disable system monitor # enable/disable system monitor
# monitor 1 # monitor 1
...@@ -189,11 +183,12 @@ ...@@ -189,11 +183,12 @@
# max number of rows per log filters # max number of rows per log filters
# numOfLogLines 10000000 # numOfLogLines 10000000
# enable/disable async log
# asyncLog 1
# time of keeping log files, days # time of keeping log files, days
# logKeepDays 0 # logKeepDays 0
# enable/disable async log
# asyncLog 1
# The following parameters are used for debug purpose only. # The following parameters are used for debug purpose only.
# debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR # debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR
...@@ -231,18 +226,12 @@ ...@@ -231,18 +226,12 @@
# debug flag for JNI # debug flag for JNI
# jniDebugflag 131 # jniDebugflag 131
# debug flag for ODBC
# odbcDebugflag 131
# debug flag for storage # debug flag for storage
# uDebugflag 131 # uDebugflag 131
# debug flag for http server # debug flag for http server
# httpDebugFlag 131 # httpDebugFlag 131
# debug flag for mqtt
# mqttDebugFlag 131
# debug flag for monitor # debug flag for monitor
# monitorDebugFlag 131 # monitorDebugFlag 131
...@@ -255,6 +244,9 @@ ...@@ -255,6 +244,9 @@
# debug flag for http server # debug flag for http server
# tsdbDebugFlag 131 # tsdbDebugFlag 131
# debug flag for continue query
# cqDebugFlag 131
# enable/disable recording the SQL in taos client # enable/disable recording the SQL in taos client
# tscEnableRecordSql 0 # tscEnableRecordSql 0
......
FROM centos:7 FROM ubuntu:20.04
WORKDIR /root WORKDIR /root
...@@ -7,7 +7,7 @@ RUN echo $version ...@@ -7,7 +7,7 @@ RUN echo $version
COPY tdengine.tar.gz /root/ COPY tdengine.tar.gz /root/
RUN tar -zxf tdengine.tar.gz RUN tar -zxf tdengine.tar.gz
WORKDIR /root/TDengine-server-$version/ WORKDIR /root/TDengine-server-$version/
RUN sh install.sh -e no RUN /bin/bash install.sh -e no
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib"
......
#!/bin/bash
set -x
docker build --rm -f "Dockerfile" -t tdengine/tdengine-aarch64:$1 "." --build-arg version=$1
docker login -u tdengine -p $2 #replace the docker registry username and password
docker push tdengine/tdengine-aarch64:$1
...@@ -272,6 +272,29 @@ function install_config() { ...@@ -272,6 +272,29 @@ function install_config() {
break break
fi fi
done done
# user email
#EMAIL_PATTERN='^[A-Za-z0-9\u4e00-\u9fa5]+@[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)+$'
#EMAIL_PATTERN='^[\w-]+(\.[\w-]+)*@[\w-]+(\.[\w-]+)+$'
#EMAIL_PATTERN="^[\w-]+(\.[\w-]+)*@[\w-]+(\.[\w-]+)+$"
echo
echo -e -n "${GREEN}Enter your email address for priority support or enter empty to skip${NC}: "
read emailAddr
while true; do
if [ ! -z "$emailAddr" ]; then
# check the format of the emailAddr
#if [[ "$emailAddr" =~ $EMAIL_PATTERN ]]; then
# Write the email address to temp file
email_file="${install_main_dir}/email"
${csudo} bash -c "echo $emailAddr > ${email_file}"
break
#else
# read -p "Please enter the correct email address: " emailAddr
#fi
else
break
fi
done
} }
......
name: tdengine name: tdengine
base: core18 base: core18
version: 'RELEASE_VERSION' version: '2.0.5.1'
icon: snap/gui/t-dengine.svg icon: snap/gui/t-dengine.svg
summary: an open-source big data platform designed and optimized for IoT. summary: an open-source big data platform designed and optimized for IoT.
description: | description: |
...@@ -72,7 +72,7 @@ parts: ...@@ -72,7 +72,7 @@ parts:
- usr/bin/taosd - usr/bin/taosd
- usr/bin/taos - usr/bin/taos
- usr/bin/taosdemo - usr/bin/taosdemo
- usr/lib/libtaos.so.RELEASE_VERSION - usr/lib/libtaos.so.2.0.5.1
- usr/lib/libtaos.so.1 - usr/lib/libtaos.so.1
- usr/lib/libtaos.so - usr/lib/libtaos.so
......
...@@ -20,4 +20,6 @@ ADD_SUBDIRECTORY(tsdb) ...@@ -20,4 +20,6 @@ ADD_SUBDIRECTORY(tsdb)
ADD_SUBDIRECTORY(wal) ADD_SUBDIRECTORY(wal)
ADD_SUBDIRECTORY(cq) ADD_SUBDIRECTORY(cq)
ADD_SUBDIRECTORY(dnode) ADD_SUBDIRECTORY(dnode)
ADD_SUBDIRECTORY(connector/odbc)
ADD_SUBDIRECTORY(connector/jdbc) ADD_SUBDIRECTORY(connector/jdbc)
...@@ -126,6 +126,8 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { ...@@ -126,6 +126,8 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
balanceAccquireDnodeList(); balanceAccquireDnodeList();
mDebug("db:%s, try alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes,
mnodeGetDnodesNum(), tsBalanceDnodeListSize);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
for (; dnode < tsBalanceDnodeListSize; ++dnode) { for (; dnode < tsBalanceDnodeListSize; ++dnode) {
SDnodeObj *pDnode = tsBalanceDnodeList[dnode]; SDnodeObj *pDnode = tsBalanceDnodeList[dnode];
...@@ -135,17 +137,33 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { ...@@ -135,17 +137,33 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
pVnodeGid->pDnode = pDnode; pVnodeGid->pDnode = pDnode;
dnode++; dnode++;
vnodes++; vnodes++;
mDebug("dnode:%d, is selected, vnodeIndex:%d", pDnode->dnodeId, i);
break; break;
} else {
mDebug("dnode:%d, is not selected, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId,
mnodeGetDnodeStatusStr(pDnode->status), pDnode->openVnodes, pDnode->diskAvailable,
pDnode->alternativeRole);
} }
} }
} }
if (vnodes != pVgroup->numOfVnodes) { if (vnodes != pVgroup->numOfVnodes) {
mDebug("vgId:%d, db:%s need vnodes:%d, but alloc:%d, free them", pVgroup->vgId, pVgroup->dbName,
pVgroup->numOfVnodes, vnodes);
balanceReleaseDnodeList(); balanceReleaseDnodeList();
balanceUnLock(); balanceUnLock();
mDebug("db:%s, need vnodes:%d, but alloc:%d", pVgroup->dbName, pVgroup->numOfVnodes, vnodes);
void * pIter = NULL;
SDnodeObj *pDnode = NULL;
while (1) {
pIter = mnodeGetNextDnode(pIter, &pDnode);
if (pDnode == NULL) break;
mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status),
pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
mnodeDecDnodeRef(pDnode);
}
sdbFreeIter(pIter);
if (mnodeGetOnlineDnodesNum() == 0) { if (mnodeGetOnlineDnodesNum() == 0) {
return TSDB_CODE_MND_NOT_READY; return TSDB_CODE_MND_NOT_READY;
} else { } else {
...@@ -198,8 +216,8 @@ static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) { ...@@ -198,8 +216,8 @@ static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
SVnodeGid *pVnode = pVgroup->vnodeGid + i; SVnodeGid *pVnode = pVgroup->vnodeGid + i;
if (pVnode == pRmVnode) continue; if (pVnode == pRmVnode) continue;
mTrace("vgId:%d, change vgroup status, dnode:%d status:%d", pVgroup->vgId, pVnode->pDnode->dnodeId, mTrace("vgId:%d, check vgroup status, dnode:%d status:%d, vnode role:%s", pVgroup->vgId, pVnode->pDnode->dnodeId,
pVnode->pDnode->status); pVnode->pDnode->status, syncRole[pVnode->role]);
if (pVnode->pDnode->status == TAOS_DN_STATUS_DROPPING) continue; if (pVnode->pDnode->status == TAOS_DN_STATUS_DROPPING) continue;
if (pVnode->pDnode->status == TAOS_DN_STATUS_OFFLINE) continue; if (pVnode->pDnode->status == TAOS_DN_STATUS_OFFLINE) continue;
...@@ -553,7 +571,8 @@ static void balanceCheckDnodeAccess() { ...@@ -553,7 +571,8 @@ static void balanceCheckDnodeAccess() {
if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) { if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) {
pDnode->status = TAOS_DN_STATUS_OFFLINE; pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT; pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
mInfo("dnode:%d, set to offline state", pDnode->dnodeId); mInfo("dnode:%d, set to offline state, access seq:%d, last seq:%d", pDnode->dnodeId, tsAccessSquence,
pDnode->lastAccess);
balanceSetVgroupOffline(pDnode); balanceSetVgroupOffline(pDnode);
} }
} }
...@@ -937,6 +956,7 @@ static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, ...@@ -937,6 +956,7 @@ static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows,
mnodeDecDnodeRef(pDnode); mnodeDecDnodeRef(pDnode);
} }
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
return numOfRows; return numOfRows;
} }
...@@ -957,11 +977,16 @@ static void balanceMonitorDnodeModule() { ...@@ -957,11 +977,16 @@ static void balanceMonitorDnodeModule() {
continue; continue;
} }
mLInfo("dnode:%d, numOfMnodes:%d expect:%d, add mnode in this dnode", pDnode->dnodeId, numOfMnodes, tsNumOfMnodes); mLInfo("dnode:%d, numOfMnodes:%d expect:%d, create mnode in this dnode", pDnode->dnodeId, numOfMnodes, tsNumOfMnodes);
mnodeAddMnode(pDnode->dnodeId); mnodeCreateMnode(pDnode->dnodeId, pDnode->dnodeEp, true);
#if 0
// Only create one mnode each time
return;
#else
numOfMnodes = mnodeGetMnodesNum(); numOfMnodes = mnodeGetMnodesNum();
if (numOfMnodes >= tsNumOfMnodes) return; if (numOfMnodes >= tsNumOfMnodes) return;
#endif
} }
} }
......
...@@ -72,17 +72,10 @@ typedef struct SLocalReducer { ...@@ -72,17 +72,10 @@ typedef struct SLocalReducer {
bool orderPrjOnSTable; // projection query on stable bool orderPrjOnSTable; // projection query on stable
} SLocalReducer; } SLocalReducer;
typedef struct SSubqueryState {
int32_t numOfRemain; // the number of remain unfinished subquery
int32_t numOfTotal; // the number of total sub-queries
uint64_t numOfRetrievedRows; // total number of points in this query
} SSubqueryState;
typedef struct SRetrieveSupport { typedef struct SRetrieveSupport {
tExtMemBuffer ** pExtMemBuffer; // for build loser tree tExtMemBuffer ** pExtMemBuffer; // for build loser tree
tOrderDescriptor *pOrderDescriptor; tOrderDescriptor *pOrderDescriptor;
SColumnModel * pFinalColModel; // colModel for final result SColumnModel * pFinalColModel; // colModel for final result
SSubqueryState * pState;
int32_t subqueryIndex; // index of current vnode in vnode list int32_t subqueryIndex; // index of current vnode in vnode list
SSqlObj * pParentSql; SSqlObj * pParentSql;
tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to
......
...@@ -28,7 +28,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql); ...@@ -28,7 +28,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql);
void tscSetupOutputColumnIndex(SSqlObj* pSql); void tscSetupOutputColumnIndex(SSqlObj* pSql);
void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code); void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code);
SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index); SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index);
void tscHandleMasterJoinQuery(SSqlObj* pSql); void tscHandleMasterJoinQuery(SSqlObj* pSql);
...@@ -39,7 +39,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql); ...@@ -39,7 +39,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql);
int32_t tscHandleInsertRetry(SSqlObj* pSql); int32_t tscHandleInsertRetry(SSqlObj* pSql);
void tscBuildResFromSubqueries(SSqlObj *pSql); void tscBuildResFromSubqueries(SSqlObj *pSql);
void **doSetResultRowData(SSqlObj *pSql, bool finalResult); TAOS_ROW doSetResultRowData(SSqlObj *pSql, bool finalResult);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
...@@ -66,7 +66,6 @@ typedef struct STidTags { ...@@ -66,7 +66,6 @@ typedef struct STidTags {
#pragma pack(pop) #pragma pack(pop)
typedef struct SJoinSupporter { typedef struct SJoinSupporter {
SSubqueryState* pState;
SSqlObj* pObj; // parent SqlObj SSqlObj* pObj; // parent SqlObj
int32_t subqueryIndex; // index of sub query int32_t subqueryIndex; // index of sub query
SInterval interval; SInterval interval;
...@@ -110,7 +109,6 @@ SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint ...@@ -110,7 +109,6 @@ SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint
void* tscDestroyBlockArrayList(SArray* pDataBlockList); void* tscDestroyBlockArrayList(SArray* pDataBlockList);
int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock); int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock);
void tscFreeUnusedDataBlocks(SArray* pDataBlockList);
int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pDataList); int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pDataList);
int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t id, int32_t size, int32_t tscGetDataBlockFromList(void* pHashList, SArray* pDataBlockList, int64_t id, int32_t size,
int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta, int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta,
...@@ -151,14 +149,13 @@ int tscAllocPayload(SSqlCmd* pCmd, int size); ...@@ -151,14 +149,13 @@ int tscAllocPayload(SSqlCmd* pCmd, int size);
TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes); TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes);
SFieldSupInfo* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField); SInternalField* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField);
SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field); SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field);
SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index); SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index);
TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index); TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index);
void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo); void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo);
void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src);
void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo); void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo);
int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index);
...@@ -207,8 +204,6 @@ void tscTagCondRelease(STagCond* pCond); ...@@ -207,8 +204,6 @@ void tscTagCondRelease(STagCond* pCond);
void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo); void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo);
void tscSetFreeHeatBeat(STscObj* pObj);
bool tscShouldFreeHeartBeat(SSqlObj* pHb);
bool tscShouldBeFreed(SSqlObj* pSql); bool tscShouldBeFreed(SSqlObj* pSql);
STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex, int32_t tableIndex); STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex, int32_t tableIndex);
...@@ -235,10 +230,11 @@ int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); ...@@ -235,10 +230,11 @@ int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo);
int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists);
void tscResetForNextRetrieve(SSqlRes* pRes); void tscResetForNextRetrieve(SSqlRes* pRes);
void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t tableIndex);
void tscDoQuery(SSqlObj* pSql); void tscDoQuery(SSqlObj* pSql);
SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *pInfo);
void* tscVgroupInfoClear(SVgroupsInfo *pInfo);
void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src);
/** /**
* The create object function must be successful expect for the out of memory issue. * The create object function must be successful expect for the out of memory issue.
* *
......
...@@ -80,6 +80,8 @@ enum { ...@@ -80,6 +80,8 @@ enum {
DATA_FROM_DATA_FILE = 2, DATA_FROM_DATA_FILE = 2,
}; };
typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows);
typedef struct STableComInfo { typedef struct STableComInfo {
uint8_t numOfTags; uint8_t numOfTags;
uint8_t precision; uint8_t precision;
...@@ -91,7 +93,7 @@ typedef struct SCMCorVgroupInfo { ...@@ -91,7 +93,7 @@ typedef struct SCMCorVgroupInfo {
int32_t version; int32_t version;
int8_t inUse; int8_t inUse;
int8_t numOfEps; int8_t numOfEps;
SEpAddr epAddr[TSDB_MAX_REPLICA]; SEpAddr1 epAddr[TSDB_MAX_REPLICA];
} SCMCorVgroupInfo; } SCMCorVgroupInfo;
typedef struct STableMeta { typedef struct STableMeta {
...@@ -140,16 +142,17 @@ typedef struct SColumnIndex { ...@@ -140,16 +142,17 @@ typedef struct SColumnIndex {
int16_t columnIndex; int16_t columnIndex;
} SColumnIndex; } SColumnIndex;
typedef struct SFieldSupInfo { typedef struct SInternalField {
TAOS_FIELD field;
bool visible; bool visible;
SExprInfo *pArithExprInfo; SExprInfo *pArithExprInfo;
SSqlExpr *pSqlExpr; SSqlExpr *pSqlExpr;
} SFieldSupInfo; } SInternalField;
typedef struct SFieldInfo { typedef struct SFieldInfo {
int16_t numOfOutput; // number of column in result int16_t numOfOutput; // number of column in result
SArray *pFields; // SArray<TAOS_FIELD> TAOS_FIELD* final;
SArray *pSupportInfo; // SArray<SFieldSupInfo> SArray *internalField; // SArray<SInternalField>
} SFieldInfo; } SFieldInfo;
typedef struct SColumn { typedef struct SColumn {
...@@ -226,7 +229,7 @@ typedef struct STableDataBlocks { ...@@ -226,7 +229,7 @@ typedef struct STableDataBlocks {
typedef struct SQueryInfo { typedef struct SQueryInfo {
int16_t command; // the command may be different for each subclause, so keep it seperately. int16_t command; // the command may be different for each subclause, so keep it seperately.
uint32_t type; // query/insert type uint32_t type; // query/insert type
// TODO refactor
STimeWindow window; // query time window STimeWindow window; // query time window
SInterval interval; SInterval interval;
...@@ -306,7 +309,7 @@ typedef struct { ...@@ -306,7 +309,7 @@ typedef struct {
int32_t numOfGroups; int32_t numOfGroups;
SResRec * pGroupRec; SResRec * pGroupRec;
char * data; char * data;
void ** tsrow; TAOS_ROW tsrow;
int32_t* length; // length for each field for current row int32_t* length; // length for each field for current row
char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t)
SColumnIndex * pColumnIndex; SColumnIndex * pColumnIndex;
...@@ -334,6 +337,12 @@ typedef struct STscObj { ...@@ -334,6 +337,12 @@ typedef struct STscObj {
T_REF_DECLARE() T_REF_DECLARE()
} STscObj; } STscObj;
typedef struct SSubqueryState {
int32_t numOfRemain; // the number of remain unfinished subquery
int32_t numOfSub; // the number of total sub-queries
uint64_t numOfRetrievedRows; // total number of points in this query
} SSubqueryState;
typedef struct SSqlObj { typedef struct SSqlObj {
void *signature; void *signature;
pthread_t owner; // owner of sql object, by which it is executed pthread_t owner; // owner of sql object, by which it is executed
...@@ -355,10 +364,11 @@ typedef struct SSqlObj { ...@@ -355,10 +364,11 @@ typedef struct SSqlObj {
tsem_t rspSem; tsem_t rspSem;
SSqlCmd cmd; SSqlCmd cmd;
SSqlRes res; SSqlRes res;
uint16_t numOfSubs;
SSubqueryState subState;
struct SSqlObj **pSubs; struct SSqlObj **pSubs;
struct SSqlObj * prev, *next;
struct SSqlObj *prev, *next;
struct SSqlObj **self; struct SSqlObj **self;
} SSqlObj; } SSqlObj;
...@@ -433,19 +443,20 @@ void tscPartiallyFreeSqlObj(SSqlObj *pSql); ...@@ -433,19 +443,20 @@ void tscPartiallyFreeSqlObj(SSqlObj *pSql);
* @param pObj * @param pObj
*/ */
void tscFreeSqlObj(SSqlObj *pSql); void tscFreeSqlObj(SSqlObj *pSql);
void tscFreeRegisteredSqlObj(void *pSql);
void tscFreeSqlObjInCache(void *pSql); void tscFreeTableMetaHelper(void *pTableMeta);
void tscCloseTscObj(STscObj *pObj); void tscCloseTscObj(STscObj *pObj);
// todo move to taos? or create a new file: taos_internal.h
TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
void *param, void **taos); void *param, TAOS **taos);
void waitForQueryRsp(void *param, TAOS_RES *tres, int code) ; TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res);
void waitForQueryRsp(void *param, TAOS_RES *tres, int code);
void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen); void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen);
void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql); void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql);
void tscKillSTableQuery(SSqlObj *pSql);
void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen); void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen);
bool tscIsUpdateQuery(SSqlObj* pSql); bool tscIsUpdateQuery(SSqlObj* pSql);
bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes);
...@@ -458,7 +469,7 @@ int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* s ...@@ -458,7 +469,7 @@ int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* s
int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo);
static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) { static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) {
SFieldSupInfo* pInfo = (SFieldSupInfo*) TARRAY_GET_ELEM(pFieldInfo->pSupportInfo, columnIndex); SInternalField* pInfo = (SInternalField*) TARRAY_GET_ELEM(pFieldInfo->internalField, columnIndex);
assert(pInfo->pSqlExpr != NULL); assert(pInfo->pSqlExpr != NULL);
int32_t type = pInfo->pSqlExpr->resType; int32_t type = pInfo->pSqlExpr->resType;
...@@ -471,11 +482,11 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField ...@@ -471,11 +482,11 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField
if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) {
pData = pInfo->pSqlExpr->param[1].pz; pData = pInfo->pSqlExpr->param[1].pz;
pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen; pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen;
pRes->tsrow[columnIndex] = (pInfo->pSqlExpr->param[1].nType == TSDB_DATA_TYPE_NULL) ? NULL : pData; pRes->tsrow[columnIndex] = (pInfo->pSqlExpr->param[1].nType == TSDB_DATA_TYPE_NULL) ? NULL : (unsigned char*)pData;
} else { } else {
assert(bytes == tDataTypeDesc[type].nSize); assert(bytes == tDataTypeDesc[type].nSize);
pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : &pInfo->pSqlExpr->param[1].i64Key; pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : (unsigned char*)&pInfo->pSqlExpr->param[1].i64Key;
pRes->length[columnIndex] = bytes; pRes->length[columnIndex] = bytes;
} }
} else { } else {
...@@ -483,7 +494,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField ...@@ -483,7 +494,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField
int32_t realLen = varDataLen(pData); int32_t realLen = varDataLen(pData);
assert(realLen <= bytes - VARSTR_HEADER_SIZE); assert(realLen <= bytes - VARSTR_HEADER_SIZE);
pRes->tsrow[columnIndex] = (isNull(pData, type)) ? NULL : ((tstr *)pData)->data; pRes->tsrow[columnIndex] = (isNull(pData, type)) ? NULL : (unsigned char*)((tstr *)pData)->data;
if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor
*(pData + realLen + VARSTR_HEADER_SIZE) = 0; *(pData + realLen + VARSTR_HEADER_SIZE) = 0;
} }
...@@ -492,7 +503,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField ...@@ -492,7 +503,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField
} else { } else {
assert(bytes == tDataTypeDesc[type].nSize); assert(bytes == tDataTypeDesc[type].nSize);
pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : pData; pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : (unsigned char*)pData;
pRes->length[columnIndex] = bytes; pRes->length[columnIndex] = bytes;
} }
} }
...@@ -510,8 +521,6 @@ extern SRpcCorEpSet tscMgmtEpSet; ...@@ -510,8 +521,6 @@ extern SRpcCorEpSet tscMgmtEpSet;
extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows);
int32_t tscCompareTidTags(const void* p1, const void* p2); int32_t tscCompareTidTags(const void* p1, const void* p2);
void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables); void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables);
......
...@@ -40,7 +40,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo ...@@ -40,7 +40,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows); static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows);
static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows); static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows);
void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) { void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* param, const char* sqlstr, size_t sqlLen) {
SSqlCmd* pCmd = &pSql->cmd; SSqlCmd* pCmd = &pSql->cmd;
pSql->signature = pSql; pSql->signature = pSql;
...@@ -327,7 +327,7 @@ void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows) { ...@@ -327,7 +327,7 @@ void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows) {
} }
for (int i = 0; i < pCmd->numOfCols; ++i){ for (int i = 0; i < pCmd->numOfCols; ++i){
SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); SInternalField* pSup = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i);
if (pSup->pSqlExpr != NULL) { if (pSup->pSqlExpr != NULL) {
// pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pSup->pSqlExpr->resBytes * pRes->row; // pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pSup->pSqlExpr->resBytes * pRes->row;
} else { } else {
...@@ -348,7 +348,7 @@ void tscProcessFetchRow(SSchedMsg *pMsg) { ...@@ -348,7 +348,7 @@ void tscProcessFetchRow(SSchedMsg *pMsg) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
for (int i = 0; i < pCmd->numOfCols; ++i) { for (int i = 0; i < pCmd->numOfCols; ++i) {
SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); SInternalField* pSup = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i);
if (pSup->pSqlExpr != NULL) { if (pSup->pSqlExpr != NULL) {
tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i);
...@@ -361,15 +361,6 @@ void tscProcessFetchRow(SSchedMsg *pMsg) { ...@@ -361,15 +361,6 @@ void tscProcessFetchRow(SSchedMsg *pMsg) {
(*pSql->fetchFp)(pSql->param, pSql, pRes->tsrow); (*pSql->fetchFp)(pSql->param, pSql, pRes->tsrow);
} }
void tscProcessAsyncRes(SSchedMsg *pMsg) {
SSqlObj *pSql = (SSqlObj *)pMsg->ahandle;
SSqlRes *pRes = &pSql->res;
assert(pSql->fp != NULL && pSql->fetchFp != NULL);
pSql->fp = pSql->fetchFp;
(*pSql->fp)(pSql->param, pSql, pRes->code);
}
// this function will be executed by queue task threads, so the terrno is not valid // this function will be executed by queue task threads, so the terrno is not valid
static void tscProcessAsyncError(SSchedMsg *pMsg) { static void tscProcessAsyncError(SSchedMsg *pMsg) {
void (*fp)() = pMsg->ahandle; void (*fp)() = pMsg->ahandle;
...@@ -393,22 +384,15 @@ void tscQueueAsyncRes(SSqlObj *pSql) { ...@@ -393,22 +384,15 @@ void tscQueueAsyncRes(SSqlObj *pSql) {
if (pSql == NULL || pSql->signature != pSql) { if (pSql == NULL || pSql->signature != pSql) {
tscDebug("%p SqlObj is freed, not add into queue async res", pSql); tscDebug("%p SqlObj is freed, not add into queue async res", pSql);
return; return;
} else {
tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code));
} }
SSchedMsg schedMsg = { 0 }; tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code));
schedMsg.fp = tscProcessAsyncRes;
schedMsg.ahandle = pSql;
schedMsg.thandle = (void *)1;
schedMsg.msg = NULL;
taosScheduleTask(tscQhandle, &schedMsg);
}
void tscProcessAsyncFree(SSchedMsg *pMsg) { SSqlRes *pRes = &pSql->res;
SSqlObj *pSql = (SSqlObj *)pMsg->ahandle; assert(pSql->fp != NULL && pSql->fetchFp != NULL);
tscDebug("%p sql is freed", pSql);
taos_free_result(pSql); pSql->fp = pSql->fetchFp;
(*pSql->fp)(pSql->param, pSql, pRes->code);
} }
int tscSendMsgToServer(SSqlObj *pSql); int tscSendMsgToServer(SSqlObj *pSql);
...@@ -421,11 +405,11 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { ...@@ -421,11 +405,11 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
pRes->code = code; pRes->code = code;
const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta";
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tscError("%p get tableMeta failed, code:%s", pSql, tstrerror(code)); tscError("%p get %s failed, code:%s", pSql, msg, tstrerror(code));
goto _error; goto _error;
} else { } else {
const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta";
tscDebug("%p get %s successfully", pSql, msg); tscDebug("%p get %s successfully", pSql, msg);
} }
......
...@@ -2445,8 +2445,8 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) { ...@@ -2445,8 +2445,8 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
// in the first round, get the min-max value of all involved data // in the first round, get the min-max value of all involved data
SResultInfo *pResInfo = GET_RES_INFO(pCtx); SResultInfo *pResInfo = GET_RES_INFO(pCtx);
SPercentileInfo *pInfo = pResInfo->interResultBuf; SPercentileInfo *pInfo = pResInfo->interResultBuf;
pInfo->minval = DBL_MAX; SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX);
pInfo->maxval = -DBL_MAX; SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX);
pInfo->numOfElems = 0; pInfo->numOfElems = 0;
return true; return true;
...@@ -2461,12 +2461,12 @@ static void percentile_function(SQLFunctionCtx *pCtx) { ...@@ -2461,12 +2461,12 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
// the first stage, only acquire the min/max value // the first stage, only acquire the min/max value
if (pInfo->stage == 0) { if (pInfo->stage == 0) {
if (pCtx->preAggVals.isSet) { if (pCtx->preAggVals.isSet) {
if (pInfo->minval > pCtx->preAggVals.statis.min) { if (GET_DOUBLE_VAL(&pInfo->minval) > pCtx->preAggVals.statis.min) {
pInfo->minval = (double)pCtx->preAggVals.statis.min; SET_DOUBLE_VAL(&pInfo->minval, (double)pCtx->preAggVals.statis.min);
} }
if (pInfo->maxval < pCtx->preAggVals.statis.max) { if (GET_DOUBLE_VAL(&pInfo->maxval) < pCtx->preAggVals.statis.max) {
pInfo->maxval = (double)pCtx->preAggVals.statis.max; SET_DOUBLE_VAL(&pInfo->maxval, (double)pCtx->preAggVals.statis.max);
} }
pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull); pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull);
...@@ -2500,12 +2500,12 @@ static void percentile_function(SQLFunctionCtx *pCtx) { ...@@ -2500,12 +2500,12 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
break; break;
} }
if (v < pInfo->minval) { if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
pInfo->minval = v; SET_DOUBLE_VAL(&pInfo->minval, v);
} }
if (v > pInfo->maxval) { if (v > GET_DOUBLE_VAL(&pInfo->maxval)) {
pInfo->maxval = v; SET_DOUBLE_VAL(&pInfo->maxval, v);
} }
pInfo->numOfElems += 1; pInfo->numOfElems += 1;
...@@ -2564,12 +2564,12 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { ...@@ -2564,12 +2564,12 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
break; break;
} }
if (v < pInfo->minval) { if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
pInfo->minval = v; SET_DOUBLE_VAL(&pInfo->minval, v);
} }
if (v > pInfo->maxval) { if (v > GET_DOUBLE_VAL(&pInfo->maxval)) {
pInfo->maxval = v; SET_DOUBLE_VAL(&pInfo->maxval, v);
} }
pInfo->numOfElems += 1; pInfo->numOfElems += 1;
...@@ -2609,7 +2609,7 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) { ...@@ -2609,7 +2609,7 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) {
} }
pInfo->stage += 1; pInfo->stage += 1;
pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, GET_DOUBLE_VAL(&pInfo->minval), GET_DOUBLE_VAL(&pInfo->maxval));
} else { } else {
pResInfo->complete = true; pResInfo->complete = true;
} }
......
...@@ -239,7 +239,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, ...@@ -239,7 +239,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE}; TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE};
tstrncpy(f.name, "Field", sizeof(f.name)); tstrncpy(f.name, "Field", sizeof(f.name));
SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY,
(TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, (TSDB_COL_NAME_LEN - 1), false); (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, (TSDB_COL_NAME_LEN - 1), false);
...@@ -296,7 +296,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { ...@@ -296,7 +296,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) {
return tscSetValueToResObj(pSql, rowLen); return tscSetValueToResObj(pSql, rowLen);
} }
static int32_t tscGetNthFieldResult(TAOS_ROW row, TAOS_FIELD* fields, int *lengths, int idx, char *result) { static int32_t tscGetNthFieldResult(TAOS_ROW row, TAOS_FIELD* fields, int *lengths, int idx, char *result) {
const char *val = row[idx]; const char *val = (const char*)row[idx];
if (val == NULL) { if (val == NULL) {
sprintf(result, "%s", TSDB_DATA_NULL_STR); sprintf(result, "%s", TSDB_DATA_NULL_STR);
return -1; return -1;
...@@ -485,7 +485,7 @@ static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const ...@@ -485,7 +485,7 @@ static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const
tstrncpy(f.name, "Database", sizeof(f.name)); tstrncpy(f.name, "Database", sizeof(f.name));
} }
SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY,
f.bytes, f.bytes - VARSTR_HEADER_SIZE, false); f.bytes, f.bytes - VARSTR_HEADER_SIZE, false);
...@@ -922,19 +922,17 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa ...@@ -922,19 +922,17 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa
pQueryInfo->order.order = TSDB_ORDER_ASC; pQueryInfo->order.order = TSDB_ORDER_ASC;
tscFieldInfoClear(&pQueryInfo->fieldsInfo); tscFieldInfoClear(&pQueryInfo->fieldsInfo);
pQueryInfo->fieldsInfo.pFields = taosArrayInit(1, sizeof(TAOS_FIELD)); pQueryInfo->fieldsInfo.internalField = taosArrayInit(1, sizeof(SInternalField));
pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(1, sizeof(SFieldSupInfo));
TAOS_FIELD f = tscCreateField((int8_t)type, columnName, (int16_t)valueLength); TAOS_FIELD f = tscCreateField((int8_t)type, columnName, (int16_t)valueLength);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
tscInitResObjForLocalQuery(pSql, 1, (int32_t)valueLength); tscInitResObjForLocalQuery(pSql, 1, (int32_t)valueLength);
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, 0);
SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, 0);
pInfo->pSqlExpr = taosArrayGetP(pQueryInfo->exprList, 0); pInfo->pSqlExpr = taosArrayGetP(pQueryInfo->exprList, 0);
memcpy(pRes->data, val, pField->bytes); memcpy(pRes->data, val, pInfo->field.bytes);
} }
int tscProcessLocalCmd(SSqlObj *pSql) { int tscProcessLocalCmd(SSqlObj *pSql) {
......
...@@ -510,7 +510,8 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { ...@@ -510,7 +510,8 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
taosTFree(pLocalReducer->pResultBuf); taosTFree(pLocalReducer->pResultBuf);
if (pLocalReducer->pResInfo != NULL) { if (pLocalReducer->pResInfo != NULL) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { size_t num = tscSqlExprNumOfExprs(pQueryInfo);
for (int32_t i = 0; i < num; ++i) {
taosTFree(pLocalReducer->pResInfo[i].interResultBuf); taosTFree(pLocalReducer->pResInfo[i].interResultBuf);
} }
...@@ -554,27 +555,48 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm ...@@ -554,27 +555,48 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm
numOfGroupByCols++; numOfGroupByCols++;
} }
int32_t *orderIdx = (int32_t *)calloc(numOfGroupByCols, sizeof(int32_t)); int32_t *orderColIndexList = (int32_t *)calloc(numOfGroupByCols, sizeof(int32_t));
if (orderIdx == NULL) { if (orderColIndexList == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
if (numOfGroupByCols > 0) { if (numOfGroupByCols > 0) {
if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) {
int32_t startCols = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols; int32_t startCols = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols;
// tags value locate at the last columns // the last "pQueryInfo->groupbyExpr.numOfGroupCols" columns are order-by columns
for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) {
orderIdx[i] = startCols++; orderColIndexList[i] = startCols++;
} }
if (pQueryInfo->interval.interval != 0) { if (pQueryInfo->interval.interval != 0) {
// the first column is the timestamp, handles queries like "interval(10m) group by tags" // the first column is the timestamp, handles queries like "interval(10m) group by tags"
orderIdx[numOfGroupByCols - 1] = PRIMARYKEY_TIMESTAMP_COL_INDEX; orderColIndexList[numOfGroupByCols - 1] = PRIMARYKEY_TIMESTAMP_COL_INDEX; //TODO ???
}
} else {
/*
* 1. the orderby ts asc/desc projection query for the super table
* 2. interval query without groupby clause
*/
if (pQueryInfo->interval.interval != 0) {
orderColIndexList[0] = PRIMARYKEY_TIMESTAMP_COL_INDEX;
} else {
size_t size = tscSqlExprNumOfExprs(pQueryInfo);
for (int32_t i = 0; i < size; ++i) {
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i);
if (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
orderColIndexList[0] = i;
}
} }
} }
*pOrderDesc = tOrderDesCreate(orderIdx, numOfGroupByCols, pModel, pQueryInfo->order.order); assert(pQueryInfo->order.orderColId == PRIMARYKEY_TIMESTAMP_COL_INDEX);
taosTFree(orderIdx); }
}
*pOrderDesc = tOrderDesCreate(orderColIndexList, numOfGroupByCols, pModel, pQueryInfo->order.order);
taosTFree(orderColIndexList);
if (*pOrderDesc == NULL) { if (*pOrderDesc == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
...@@ -588,7 +610,6 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage ...@@ -588,7 +610,6 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage
// disable merge procedure for column projection query // disable merge procedure for column projection query
int16_t functionId = pReducer->pCtx[0].functionId; int16_t functionId = pReducer->pCtx[0].functionId;
assert(functionId != TSDB_FUNC_ARITHM);
if (pReducer->orderPrjOnSTable) { if (pReducer->orderPrjOnSTable) {
return true; return true;
} }
...@@ -606,7 +627,7 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage ...@@ -606,7 +627,7 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage
return true; return true;
} }
if (orderInfo->pData[numOfCols - 1] == PRIMARYKEY_TIMESTAMP_COL_INDEX) { if (orderInfo->colIndex[numOfCols - 1] == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
/* /*
* super table interval query * super table interval query
* if the order columns is the primary timestamp, all result data belongs to one group * if the order columns is the primary timestamp, all result data belongs to one group
...@@ -620,7 +641,7 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage ...@@ -620,7 +641,7 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage
} }
// only one row exists // only one row exists
int32_t index = orderInfo->pData[0]; int32_t index = orderInfo->colIndex[0];
int32_t offset = (pOrderDesc->pColumnModel)->pFields[index].offset; int32_t offset = (pOrderDesc->pColumnModel)->pFields[index].offset;
int32_t ret = memcmp(pPrev + offset, tmpBuffer->data + offset, pOrderDesc->pColumnModel->rowSize - offset); int32_t ret = memcmp(pPrev + offset, tmpBuffer->data + offset, pOrderDesc->pColumnModel->rowSize - offset);
...@@ -639,7 +660,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr ...@@ -639,7 +660,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
(*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pSql->numOfSubs); (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pSql->subState.numOfSub);
if (*pMemBuffer == NULL) { if (*pMemBuffer == NULL) {
tscError("%p failed to allocate memory", pSql); tscError("%p failed to allocate memory", pSql);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
...@@ -661,7 +682,6 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr ...@@ -661,7 +682,6 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
pSchema[i].bytes = pExpr->resBytes; pSchema[i].bytes = pExpr->resBytes;
pSchema[i].type = (int8_t)pExpr->resType; pSchema[i].type = (int8_t)pExpr->resType;
rlen += pExpr->resBytes; rlen += pExpr->resBytes;
} }
...@@ -701,12 +721,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr ...@@ -701,12 +721,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
int16_t type = -1; int16_t type = -1;
int16_t bytes = 0; int16_t bytes = 0;
// if ((pExpr->functionId >= TSDB_FUNC_FIRST_DST && pExpr->functionId <= TSDB_FUNC_LAST_DST) ||
// (pExpr->functionId >= TSDB_FUNC_SUM && pExpr->functionId <= TSDB_FUNC_MAX) ||
// pExpr->functionId == TSDB_FUNC_LAST_ROW) {
// the final result size and type in the same as query on single table. // the final result size and type in the same as query on single table.
// so here, set the flag to be false; // so here, set the flag to be false;
int32_t functionId = pExpr->functionId; int32_t functionId = pExpr->functionId;
if (functionId >= TSDB_FUNC_TS && functionId <= TSDB_FUNC_DIFF) { if (functionId >= TSDB_FUNC_TS && functionId <= TSDB_FUNC_DIFF) {
type = pModel->pFields[i].field.type; type = pModel->pFields[i].field.type;
...@@ -742,6 +758,7 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe ...@@ -742,6 +758,7 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe
int32_t numOfVnodes) { int32_t numOfVnodes) {
destroyColumnModel(pFinalModel); destroyColumnModel(pFinalModel);
tOrderDescDestroy(pDesc); tOrderDescDestroy(pDesc);
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]); pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]);
} }
......
...@@ -266,6 +266,388 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { ...@@ -266,6 +266,388 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (1) {
// allow user bind param data with different type
short size = 0;
union {
int8_t v1;
int16_t v2;
int32_t v4;
int64_t v8;
float f4;
double f8;
unsigned char buf[32*1024];
} u;
switch (param->type) {
case TSDB_DATA_TYPE_BOOL: {
switch (bind->buffer_type) {
case TSDB_DATA_TYPE_BOOL: {
u.v1 = *(int8_t*)bind->buffer;
if (u.v1==0 || u.v1==1) break;
} break;
case TSDB_DATA_TYPE_TINYINT: {
u.v1 = *(int8_t*)bind->buffer;
if (u.v1==0 || u.v1==1) break;
} break;
case TSDB_DATA_TYPE_SMALLINT: {
u.v1 = (int8_t)*(int16_t*)bind->buffer;
if (u.v1==0 || u.v1==1) break;
} break;
case TSDB_DATA_TYPE_INT: {
u.v1 = (int8_t)*(int32_t*)bind->buffer;
if (u.v1==0 || u.v1==1) break;
} break;
case TSDB_DATA_TYPE_BIGINT: {
u.v1 = (int8_t)*(int64_t*)bind->buffer;
if (u.v1==0 || u.v1==1) break;
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: {
// "0", "1" convertible
if (strncmp((const char*)bind->buffer, "0", *bind->length)==0) {
u.v1 = 0;
break;
}
if (strncmp((const char*)bind->buffer, "1", *bind->length)==0) {
u.v1 = 1;
break;
}
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
case TSDB_DATA_TYPE_TIMESTAMP:
default: {
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
}
memcpy(data + param->offset, &u.v1, sizeof(u.v1));
return TSDB_CODE_SUCCESS;
} break;
case TSDB_DATA_TYPE_TINYINT: {
switch (bind->buffer_type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: {
int8_t v = *(int8_t*)bind->buffer;
u.v1 = v;
if (v >= SCHAR_MIN && v <= SCHAR_MAX) break;
} break;
case TSDB_DATA_TYPE_SMALLINT: {
int16_t v = *(int16_t*)bind->buffer;
u.v1 = (int8_t)v;
if (v >= SCHAR_MIN && v <= SCHAR_MAX) break;
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_INT: {
int32_t v = *(int32_t*)bind->buffer;
u.v1 = (int8_t)v;
if (v >= SCHAR_MIN && v <= SCHAR_MAX) break;
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_BIGINT: {
int64_t v = *(int64_t*)bind->buffer;
u.v1 = (int8_t)v;
if (v >= SCHAR_MIN && v <= SCHAR_MAX) break;
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: {
int64_t v;
int n,r;
r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n);
if (r==1 && n==strlen((const char*)bind->buffer)) {
u.v1 = (int8_t)v;
if (v >= SCHAR_MIN && v <= SCHAR_MAX) break;
}
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
case TSDB_DATA_TYPE_TIMESTAMP:
default: {
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
}
memcpy(data + param->offset, &u.v1, sizeof(u.v1));
return TSDB_CODE_SUCCESS;
} break;
case TSDB_DATA_TYPE_SMALLINT: {
switch (bind->buffer_type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT: {
int v = *(int16_t*)bind->buffer;
u.v2 = (int16_t)v;
} break;
case TSDB_DATA_TYPE_INT: {
int32_t v = *(int32_t*)bind->buffer;
u.v2 = (int16_t)v;
if (v >= SHRT_MIN && v <= SHRT_MAX) break;
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_BIGINT: {
int64_t v = *(int64_t*)bind->buffer;
u.v2 = (int16_t)v;
if (v >= SHRT_MIN && v <= SHRT_MAX) break;
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: {
int64_t v;
int n,r;
r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n);
if (r==1 && n==strlen((const char*)bind->buffer)) {
u.v2 = (int16_t)v;
if (v >= SHRT_MIN && v <= SHRT_MAX) break;
}
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
case TSDB_DATA_TYPE_TIMESTAMP:
default: {
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
}
memcpy(data + param->offset, &u.v2, sizeof(u.v2));
return TSDB_CODE_SUCCESS;
} break;
case TSDB_DATA_TYPE_INT: {
switch (bind->buffer_type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT: {
u.v4 = *(int32_t*)bind->buffer;
} break;
case TSDB_DATA_TYPE_BIGINT: {
int64_t v = *(int64_t*)bind->buffer;
u.v4 = (int32_t)v;
if (v >= INT_MIN && v <= INT_MAX) break;
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: {
int64_t v;
int n,r;
r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n);
if (r==1 && n==strlen((const char*)bind->buffer)) {
u.v4 = (int32_t)v;
if (v >= INT_MIN && v <= INT_MAX) break;
}
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
case TSDB_DATA_TYPE_TIMESTAMP:
default: {
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
}
memcpy(data + param->offset, &u.v2, sizeof(u.v2));
return TSDB_CODE_SUCCESS;
} break;
case TSDB_DATA_TYPE_FLOAT: {
switch (bind->buffer_type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: {
u.f4 = *(int8_t*)bind->buffer;
} break;
case TSDB_DATA_TYPE_SMALLINT: {
u.f4 = *(int16_t*)bind->buffer;
} break;
case TSDB_DATA_TYPE_INT: {
u.f4 = (float)*(int32_t*)bind->buffer;
// shall we check equality?
} break;
case TSDB_DATA_TYPE_BIGINT: {
u.f4 = (float)*(int64_t*)bind->buffer;
// shall we check equality?
} break;
case TSDB_DATA_TYPE_FLOAT: {
u.f4 = *(float*)bind->buffer;
} break;
case TSDB_DATA_TYPE_DOUBLE: {
u.f4 = *(float*)bind->buffer;
// shall we check equality?
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: {
float v;
int n,r;
r = sscanf((const char*)bind->buffer, "%f%n", &v, &n);
if (r==1 && n==strlen((const char*)bind->buffer)) {
u.f4 = v;
break;
}
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_TIMESTAMP:
default: {
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
}
memcpy(data + param->offset, &u.f4, sizeof(u.f4));
return TSDB_CODE_SUCCESS;
} break;
case TSDB_DATA_TYPE_BIGINT: {
switch (bind->buffer_type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: {
u.v8 = *(int8_t*)bind->buffer;
} break;
case TSDB_DATA_TYPE_SMALLINT: {
u.v8 = *(int16_t*)bind->buffer;
} break;
case TSDB_DATA_TYPE_INT: {
u.v8 = *(int32_t*)bind->buffer;
} break;
case TSDB_DATA_TYPE_BIGINT: {
u.v8 = *(int64_t*)bind->buffer;
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: {
int64_t v;
int n,r;
r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n);
if (r==1 && n==strlen((const char*)bind->buffer)) {
u.v8 = v;
break;
}
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
case TSDB_DATA_TYPE_TIMESTAMP:
default: {
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
}
memcpy(data + param->offset, &u.v8, sizeof(u.v8));
return TSDB_CODE_SUCCESS;
} break;
case TSDB_DATA_TYPE_DOUBLE: {
switch (bind->buffer_type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: {
u.f8 = *(int8_t*)bind->buffer;
} break;
case TSDB_DATA_TYPE_SMALLINT: {
u.f8 = *(int16_t*)bind->buffer;
} break;
case TSDB_DATA_TYPE_INT: {
u.f8 = *(int32_t*)bind->buffer;
} break;
case TSDB_DATA_TYPE_BIGINT: {
u.f8 = (double)*(int64_t*)bind->buffer;
} break;
case TSDB_DATA_TYPE_FLOAT: {
u.f8 = *(float*)bind->buffer;
} break;
case TSDB_DATA_TYPE_DOUBLE: {
u.f8 = *(double*)bind->buffer;
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: {
double v;
int n,r;
r = sscanf((const char*)bind->buffer, "%lf%n", &v, &n);
if (r==1 && n==strlen((const char*)bind->buffer)) {
u.f8 = v;
break;
}
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_TIMESTAMP:
default: {
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
} break;
memcpy(data + param->offset, &u.f8, sizeof(u.f8));
return TSDB_CODE_SUCCESS;
} break;
case TSDB_DATA_TYPE_TIMESTAMP: {
switch (bind->buffer_type) {
case TSDB_DATA_TYPE_TIMESTAMP: {
u.v8 = *(int64_t*)bind->buffer;
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: {
// is this the correct way to call taosParseTime?
int32_t len = (int32_t)*bind->length;
if (taosParseTime(bind->buffer, &u.v8, len, 3, tsDaylight) == TSDB_CODE_SUCCESS) {
break;
}
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_DOUBLE:
default: {
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
} break;
memcpy(data + param->offset, &u.v8, sizeof(u.v8));
return TSDB_CODE_SUCCESS;
} break;
case TSDB_DATA_TYPE_BINARY: {
switch (bind->buffer_type) {
case TSDB_DATA_TYPE_BINARY: {
if ((*bind->length) > (uintptr_t)param->bytes) {
return TSDB_CODE_TSC_INVALID_VALUE;
}
size = (short)*bind->length;
STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer, size);
return TSDB_CODE_SUCCESS;
} break;
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_DOUBLE:
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_NCHAR:
default: {
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
}
} break;
case TSDB_DATA_TYPE_NCHAR: {
switch (bind->buffer_type) {
case TSDB_DATA_TYPE_NCHAR: {
size_t output = 0;
if (!taosMbsToUcs4(bind->buffer, *bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) {
return TSDB_CODE_TSC_INVALID_VALUE;
}
varDataSetLen(data + param->offset, output);
return TSDB_CODE_SUCCESS;
} break;
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_DOUBLE:
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BINARY:
default: {
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
}
} break;
default: {
return TSDB_CODE_TSC_INVALID_VALUE;
} break;
}
}
if (bind->buffer_type != param->type) { if (bind->buffer_type != param->type) {
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }
...@@ -637,3 +1019,80 @@ TAOS_RES *taos_stmt_use_result(TAOS_STMT* stmt) { ...@@ -637,3 +1019,80 @@ TAOS_RES *taos_stmt_use_result(TAOS_STMT* stmt) {
pStmt->pSql = NULL; pStmt->pSql = NULL;
return result; return result;
} }
int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert) {
STscStmt* pStmt = (STscStmt*)stmt;
if (stmt == NULL || pStmt->taos == NULL || pStmt->pSql == NULL) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return TSDB_CODE_TSC_DISCONNECTED;
}
if (insert) *insert = pStmt->isInsert;
return TSDB_CODE_SUCCESS;
}
int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) {
STscStmt* pStmt = (STscStmt*)stmt;
if (stmt == NULL || pStmt->taos == NULL || pStmt->pSql == NULL) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return TSDB_CODE_TSC_DISCONNECTED;
}
if (pStmt->isInsert) {
SSqlObj* pSql = pStmt->pSql;
SSqlCmd *pCmd = &pSql->cmd;
*nums = pCmd->numOfParams;
return TSDB_CODE_SUCCESS;
} else {
SNormalStmt* normal = &pStmt->normal;
*nums = normal->numParams;
return TSDB_CODE_SUCCESS;
}
}
int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes) {
STscStmt* pStmt = (STscStmt*)stmt;
if (stmt == NULL || pStmt->taos == NULL || pStmt->pSql == NULL) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return TSDB_CODE_TSC_DISCONNECTED;
}
if (pStmt->isInsert) {
SSqlObj* pSql = pStmt->pSql;
SSqlCmd *pCmd = &pSql->cmd;
STableDataBlocks* pBlock = taosArrayGetP(pCmd->pDataBlocks, 0);
assert(pCmd->numOfParams == pBlock->numOfParams);
if (idx < 0 || idx >= pBlock->numOfParams) return -1;
SParamInfo* param = pBlock->params + idx;
if (type) *type = param->type;
if (bytes) *bytes = param->bytes;
return TSDB_CODE_SUCCESS;
} else {
return TSDB_CODE_TSC_APP_ERROR;
}
}
const char *taos_data_type(int type) {
switch (type) {
case TSDB_DATA_TYPE_NULL: return "TSDB_DATA_TYPE_NULL";
case TSDB_DATA_TYPE_BOOL: return "TSDB_DATA_TYPE_BOOL";
case TSDB_DATA_TYPE_TINYINT: return "TSDB_DATA_TYPE_TINYINT";
case TSDB_DATA_TYPE_SMALLINT: return "TSDB_DATA_TYPE_SMALLINT";
case TSDB_DATA_TYPE_INT: return "TSDB_DATA_TYPE_INT";
case TSDB_DATA_TYPE_BIGINT: return "TSDB_DATA_TYPE_BIGINT";
case TSDB_DATA_TYPE_FLOAT: return "TSDB_DATA_TYPE_FLOAT";
case TSDB_DATA_TYPE_DOUBLE: return "TSDB_DATA_TYPE_DOUBLE";
case TSDB_DATA_TYPE_BINARY: return "TSDB_DATA_TYPE_BINARY";
case TSDB_DATA_TYPE_TIMESTAMP: return "TSDB_DATA_TYPE_TIMESTAMP";
case TSDB_DATA_TYPE_NCHAR: return "TSDB_DATA_TYPE_NCHAR";
default: return "UNKNOWN";
}
}
...@@ -20,8 +20,10 @@ ...@@ -20,8 +20,10 @@
#include "tutil.h" #include "tutil.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "taos.h"
void tscSaveSlowQueryFp(void *handle, void *tmrId); void tscSaveSlowQueryFp(void *handle, void *tmrId);
void *tscSlowQueryConn = NULL; TAOS *tscSlowQueryConn = NULL;
bool tscSlowQueryConnInitialized = false; bool tscSlowQueryConnInitialized = false;
void tscInitConnCb(void *param, TAOS_RES *result, int code) { void tscInitConnCb(void *param, TAOS_RES *result, int code) {
...@@ -151,10 +153,12 @@ void tscKillQuery(STscObj *pObj, uint32_t killId) { ...@@ -151,10 +153,12 @@ void tscKillQuery(STscObj *pObj, uint32_t killId) {
pthread_mutex_unlock(&pObj->mutex); pthread_mutex_unlock(&pObj->mutex);
if (pSql == NULL) return; if (pSql == NULL) {
tscError("failed to kill query, id:%d, it may have completed/terminated", killId);
} else {
tscDebug("%p query is killed, queryId:%d", pSql, killId); tscDebug("%p query is killed, queryId:%d", pSql, killId);
taos_stop_query(pSql); taos_stop_query(pSql);
}
} }
void tscAddIntoStreamList(SSqlStream *pStream) { void tscAddIntoStreamList(SSqlStream *pStream) {
...@@ -242,6 +246,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { ...@@ -242,6 +246,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
pQdesc->stime = htobe64(pSql->stime); pQdesc->stime = htobe64(pSql->stime);
pQdesc->queryId = htonl(pSql->queryId); pQdesc->queryId = htonl(pSql->queryId);
pQdesc->useconds = htobe64(pSql->res.useconds); pQdesc->useconds = htobe64(pSql->res.useconds);
pQdesc->qHandle = htobe64(pSql->res.qhandle);
pHeartbeat->numOfQueries++; pHeartbeat->numOfQueries++;
pQdesc++; pQdesc++;
......
...@@ -122,7 +122,7 @@ static int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSql ...@@ -122,7 +122,7 @@ static int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSql
static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo);
static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo);
static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index); static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index);
static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols); static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid);
/* /*
* Used during parsing query sql. Since the query sql usually small in length, error position * Used during parsing query sql. Since the query sql usually small in length, error position
...@@ -190,7 +190,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { ...@@ -190,7 +190,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
SSqlRes* pRes = &pSql->res; SSqlRes* pRes = &pSql->res;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (!pInfo->valid) { if (!pInfo->valid || terrno == TSDB_CODE_TSC_SQL_SYNTAX_ERROR) {
terrno = TSDB_CODE_SUCCESS; // clear the error number
return tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), NULL, pInfo->pzErrMsg); return tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), NULL, pInfo->pzErrMsg);
} }
...@@ -626,6 +627,11 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ ...@@ -626,6 +627,11 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// orderby column not set yet, set it to be the primary timestamp column
if (pQueryInfo->order.orderColId == INT32_MIN) {
pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
}
// interval is not null // interval is not null
SStrToken* t = &pQuerySql->interval; SStrToken* t = &pQuerySql->interval;
if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.interval, &pQueryInfo->interval.intervalUnit) != TSDB_CODE_SUCCESS) { if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.interval, &pQueryInfo->interval.intervalUnit) != TSDB_CODE_SUCCESS) {
...@@ -1200,6 +1206,10 @@ int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStr ...@@ -1200,6 +1206,10 @@ int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStr
return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL; return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL;
} }
static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) {
SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
tscColumnListInsert(pQueryInfo->colList, &tsCol);
}
static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) { static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) {
const char* msg1 = "invalid column name, or illegal column type"; const char* msg1 = "invalid column name, or illegal column type";
const char* msg2 = "invalid arithmetic expression in select clause"; const char* msg2 = "invalid arithmetic expression in select clause";
...@@ -1240,7 +1250,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t ...@@ -1240,7 +1250,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
tExprNode* pNode = NULL; tExprNode* pNode = NULL;
SArray* colList = taosArrayInit(10, sizeof(SColIndex)); SArray* colList = taosArrayInit(10, sizeof(SColIndex));
int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList); int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo, colList, NULL);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
taosArrayDestroy(colList); taosArrayDestroy(colList);
tExprTreeDestroy(&pNode, NULL); tExprTreeDestroy(&pNode, NULL);
...@@ -1275,6 +1285,8 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t ...@@ -1275,6 +1285,8 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, (int32_t)len, index.tableIndex); addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, (int32_t)len, index.tableIndex);
insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr); insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr);
// add ts column
tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
tbufCloseWriter(&bw); tbufCloseWriter(&bw);
taosArrayDestroy(colList); taosArrayDestroy(colList);
...@@ -1294,7 +1306,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t ...@@ -1294,7 +1306,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, aliasName, NULL); insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, aliasName, NULL);
int32_t slot = tscNumOfFields(pQueryInfo) - 1; int32_t slot = tscNumOfFields(pQueryInfo) - 1;
SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, slot);
if (pInfo->pSqlExpr == NULL) { if (pInfo->pSqlExpr == NULL) {
SExprInfo* pArithExprInfo = calloc(1, sizeof(SExprInfo)); SExprInfo* pArithExprInfo = calloc(1, sizeof(SExprInfo));
...@@ -1304,7 +1316,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t ...@@ -1304,7 +1316,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
pArithExprInfo->interBytes = sizeof(double); pArithExprInfo->interBytes = sizeof(double);
pArithExprInfo->type = TSDB_DATA_TYPE_DOUBLE; pArithExprInfo->type = TSDB_DATA_TYPE_DOUBLE;
int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL); int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo, NULL, &pArithExprInfo->uid);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
tExprTreeDestroy(&pArithExprInfo->pExpr, NULL); tExprTreeDestroy(&pArithExprInfo->pExpr, NULL);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause");
...@@ -1317,10 +1329,6 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t ...@@ -1317,10 +1329,6 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) {
SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
tscColumnListInsert(pQueryInfo->colList, &tsCol);
}
static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) {
SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex); SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex);
...@@ -1345,6 +1353,32 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn ...@@ -1345,6 +1353,32 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn
insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr); insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr);
} }
static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo) {
// primary timestamp column has been added already
size_t size = tscSqlExprNumOfExprs(pQueryInfo);
for (int32_t i = 0; i < size; ++i) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
if (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
return;
}
}
SColumnIndex index = {0};
// set the constant column value always attached to first table.
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX);
// add the timestamp column into the output columns
int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL);
SInternalField* pSupInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfCols);
pSupInfo->visible = false;
pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY;
}
int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery) { int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery) {
assert(pSelection != NULL && pCmd != NULL); assert(pSelection != NULL && pCmd != NULL);
...@@ -1398,20 +1432,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel ...@@ -1398,20 +1432,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
// there is only one user-defined column in the final result field, add the timestamp column. // there is only one user-defined column in the final result field, add the timestamp column.
size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList);
if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) {
SColumnIndex index = {0}; addPrimaryTsColIntoResult(pQueryInfo);
// set the constant column value always attached to first table.
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, clauseIndex, 0);
SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX);
// add the timestamp column into the output columns
int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL);
SFieldSupInfo* pSupInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, numOfCols);
pSupInfo->visible = false;
pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY;
} }
if (!functionCompatibleCheck(pQueryInfo, joinQuery)) { if (!functionCompatibleCheck(pQueryInfo, joinQuery)) {
...@@ -1449,7 +1470,7 @@ int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnLi ...@@ -1449,7 +1470,7 @@ int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnLi
} }
TAOS_FIELD f = tscCreateField(type, fieldName, bytes); TAOS_FIELD f = tscCreateField(type, fieldName, bytes);
SFieldSupInfo* pInfo = tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f); SInternalField* pInfo = tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f);
pInfo->pSqlExpr = pSqlExpr; pInfo->pSqlExpr = pSqlExpr;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -1603,8 +1624,8 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t ...@@ -1603,8 +1624,8 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, char* aliasName, static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc,
int32_t resColIdx, SColumnIndex* pColIndex) { char* aliasName, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) {
const char* msg1 = "not support column types"; const char* msg1 = "not support column types";
int16_t type = 0; int16_t type = 0;
...@@ -1650,8 +1671,13 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS ...@@ -1650,8 +1671,13 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
tscColumnListInsert(pQueryInfo->colList, &index); tscColumnListInsert(pQueryInfo->colList, &index);
// if it is not in the final result, do not add it
SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex);
if (finalResult) {
insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, columnName, pExpr); insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, columnName, pExpr);
} else {
tscColumnListInsert(pQueryInfo->colList, &(ids.ids[0]));
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -1926,7 +1952,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col ...@@ -1926,7 +1952,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) {
index.columnIndex = j; index.columnIndex = j;
if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex++, &index) != 0) { if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex++, &index, finalResult) != 0) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
} }
...@@ -1943,7 +1969,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col ...@@ -1943,7 +1969,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) { if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
} }
if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex + i, &index) != 0) {
if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex + i, &index, finalResult) != 0) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
...@@ -1980,7 +2007,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col ...@@ -1980,7 +2007,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) {
SColumnIndex index = {.tableIndex = j, .columnIndex = i}; SColumnIndex index = {.tableIndex = j, .columnIndex = i};
if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex, &index) != 0) { if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex, &index, finalResult) != 0) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
...@@ -3358,10 +3385,26 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer ...@@ -3358,10 +3385,26 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer
tSQLExprItem item = {.pNode = pExpr, .aliasName = NULL}; tSQLExprItem item = {.pNode = pExpr, .aliasName = NULL};
// sql function in selection clause, append sql function info in pSqlCmd structure sequentially // sql function list in selection clause.
// Append the sqlExpr into exprList of pQueryInfo structure sequentially
if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, false) != TSDB_CODE_SUCCESS) { if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, false) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
// It is invalid in case of more than one sqlExpr, such as first(ts, k) - last(ts, k)
int32_t inc = (int32_t) tscSqlExprNumOfExprs(pQueryInfo) - outputIndex;
if (inc > 1) {
return TSDB_CODE_TSC_INVALID_SQL;
}
// Not supported data type in arithmetic expression
for(int32_t i = 0; i < inc; ++i) {
SSqlExpr* p1 = tscSqlExprGet(pQueryInfo, i + outputIndex);
int16_t t = p1->resType;
if (t == TSDB_DATA_TYPE_BINARY || t == TSDB_DATA_TYPE_NCHAR || t == TSDB_DATA_TYPE_BOOL || t == TSDB_DATA_TYPE_TIMESTAMP) {
return TSDB_CODE_TSC_INVALID_SQL;
}
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -4036,7 +4079,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE ...@@ -4036,7 +4079,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
tExprNode* p = NULL; tExprNode* p = NULL;
SArray* colList = taosArrayInit(10, sizeof(SColIndex)); SArray* colList = taosArrayInit(10, sizeof(SColIndex));
ret = exprTreeFromSqlExpr(pCmd, &p, p1, NULL, pQueryInfo, colList); ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL);
SBufferWriter bw = tbufInitWriter(NULL, false); SBufferWriter bw = tbufInitWriter(NULL, false);
TRY(0) { TRY(0) {
...@@ -4363,14 +4406,13 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery ...@@ -4363,14 +4406,13 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) {
/* set default timestamp order information for all queries */ /* set default timestamp order information for all queries */
pQueryInfo->order.order = TSDB_ORDER_ASC;
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
if (isTopBottomQuery(pQueryInfo)) {
pQueryInfo->order.order = TSDB_ORDER_ASC; pQueryInfo->order.order = TSDB_ORDER_ASC;
if (isTopBottomQuery(pQueryInfo)) {
pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
} else { } else { // in case of select tbname from super_table, the defualt order column can not be the primary ts column
pQueryInfo->order.orderColId = -1; pQueryInfo->order.orderColId = INT32_MIN;
} }
/* for super table query, set default ascending order for group output */ /* for super table query, set default ascending order for group output */
...@@ -4474,6 +4516,11 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu ...@@ -4474,6 +4516,11 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
} else { } else {
pQueryInfo->order.order = pSortorder->a[0].sortOrder; pQueryInfo->order.order = pSortorder->a[0].sortOrder;
pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
// orderby ts query on super table
if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
addPrimaryTsColIntoResult(pQueryInfo);
}
} }
} }
...@@ -4703,7 +4750,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { ...@@ -4703,7 +4750,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
// validate the length of binary // validate the length of binary
if ((pTagsSchema->type == TSDB_DATA_TYPE_BINARY || pTagsSchema->type == TSDB_DATA_TYPE_NCHAR) && if ((pTagsSchema->type == TSDB_DATA_TYPE_BINARY || pTagsSchema->type == TSDB_DATA_TYPE_NCHAR) &&
(pVarList->a[1].pVar.nLen + VARSTR_HEADER_SIZE) > pTagsSchema->bytes) { varDataTLen(pAlterSQL->tagData.data) > pTagsSchema->bytes) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg14); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg14);
} }
...@@ -5229,26 +5276,6 @@ int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { ...@@ -5229,26 +5276,6 @@ int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
//void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t tableIndex) {
// // the first column not timestamp column, add it
// SSqlExpr* pExpr = NULL;
// if (tscSqlExprNumOfExprs(pQueryInfo) > 0) {
// pExpr = tscSqlExprGet(pQueryInfo, 0);
// }
//
// if (pExpr == NULL || pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX || pExpr->functionId != functionId) {
// SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX};
//
// pExpr = tscSqlExprInsert(pQueryInfo, 0, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE, false);
// pExpr->colInfo.flag = TSDB_COL_NORMAL;
//
// // NOTE: tag column does not add to source column list
// SColumnList ids = getColumnList(1, tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX);
//
// insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, "ts", pExpr);
// }
//}
void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex) { void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex) {
SQueryInfo* pParentQueryInfo = tscGetQueryInfoDetail(&pParentObj->cmd, subClauseIndex); SQueryInfo* pParentQueryInfo = tscGetQueryInfoDetail(&pParentObj->cmd, subClauseIndex);
...@@ -5305,7 +5332,7 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { ...@@ -5305,7 +5332,7 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) {
tscAddSpecialColumnForSelect(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL); tscAddSpecialColumnForSelect(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL);
SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, (int32_t)size); SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, (int32_t)size);
doLimitOutputNormalColOfGroupby(pInfo->pSqlExpr); doLimitOutputNormalColOfGroupby(pInfo->pSqlExpr);
pInfo->visible = false; pInfo->visible = false;
} }
...@@ -6269,6 +6296,11 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { ...@@ -6269,6 +6296,11 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
// set order by info
if (parseOrderbyClause(pCmd, pQueryInfo, pQuerySql, tscGetTableSchema(pTableMetaInfo->pTableMeta)) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
// set interval value // set interval value
if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
...@@ -6279,11 +6311,6 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { ...@@ -6279,11 +6311,6 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
} }
} }
// set order by info
if (parseOrderbyClause(pCmd, pQueryInfo, pQuerySql, tscGetTableSchema(pTableMetaInfo->pTableMeta)) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
// user does not specified the query time window, twa is not allowed in such case. // user does not specified the query time window, twa is not allowed in such case.
if ((pQueryInfo->window.skey == INT64_MIN || pQueryInfo->window.ekey == INT64_MAX || if ((pQueryInfo->window.skey == INT64_MIN || pQueryInfo->window.ekey == INT64_MAX ||
(pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) { (pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) {
...@@ -6350,19 +6377,19 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { ...@@ -6350,19 +6377,19 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
return TSDB_CODE_SUCCESS; // Does not build query message here return TSDB_CODE_SUCCESS; // Does not build query message here
} }
int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols) { int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid) {
tExprNode* pLeft = NULL; tExprNode* pLeft = NULL;
tExprNode* pRight= NULL; tExprNode* pRight= NULL;
if (pSqlExpr->pLeft != NULL) { if (pSqlExpr->pLeft != NULL) {
int32_t ret = exprTreeFromSqlExpr(pCmd, &pLeft, pSqlExpr->pLeft, pExprInfo, pQueryInfo, pCols); int32_t ret = exprTreeFromSqlExpr(pCmd, &pLeft, pSqlExpr->pLeft, pQueryInfo, pCols, uid);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
return ret; return ret;
} }
} }
if (pSqlExpr->pRight != NULL) { if (pSqlExpr->pRight != NULL) {
int32_t ret = exprTreeFromSqlExpr(pCmd, &pRight, pSqlExpr->pRight, pExprInfo, pQueryInfo, pCols); int32_t ret = exprTreeFromSqlExpr(pCmd, &pRight, pSqlExpr->pRight, pQueryInfo, pCols, uid);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
return ret; return ret;
} }
...@@ -6389,14 +6416,19 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS ...@@ -6389,14 +6416,19 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
strncpy((*pExpr)->pSchema->name, pSqlExpr->operand.z, pSqlExpr->operand.n); strncpy((*pExpr)->pSchema->name, pSqlExpr->operand.z, pSqlExpr->operand.n);
// set the input column data byte and type. // set the input column data byte and type.
size_t size = taosArrayGetSize(pExprInfo); size_t size = taosArrayGetSize(pQueryInfo->exprList);
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
SSqlExpr* p1 = taosArrayGetP(pExprInfo, i); SSqlExpr* p1 = taosArrayGetP(pQueryInfo->exprList, i);
if (strcmp((*pExpr)->pSchema->name, p1->aliasName) == 0) { if (strcmp((*pExpr)->pSchema->name, p1->aliasName) == 0) {
(*pExpr)->pSchema->type = (uint8_t)p1->resType; (*pExpr)->pSchema->type = (uint8_t)p1->resType;
(*pExpr)->pSchema->bytes = p1->resBytes; (*pExpr)->pSchema->bytes = p1->resBytes;
if (uid != NULL) {
*uid = p1->uid;
}
break; break;
} }
} }
......
...@@ -145,10 +145,11 @@ static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo ...@@ -145,10 +145,11 @@ static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo
corVgroupInfo->inUse = 0; corVgroupInfo->inUse = 0;
corVgroupInfo->numOfEps = vgroupInfo->numOfEps; corVgroupInfo->numOfEps = vgroupInfo->numOfEps;
for (int32_t i = 0; i < corVgroupInfo->numOfEps; i++) { for (int32_t i = 0; i < corVgroupInfo->numOfEps; i++) {
strncpy(corVgroupInfo->epAddr[i].fqdn, vgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); corVgroupInfo->epAddr[i].fqdn = strdup(vgroupInfo->epAddr[i].fqdn);
corVgroupInfo->epAddr[i].port = vgroupInfo->epAddr[i].port; corVgroupInfo->epAddr[i].port = vgroupInfo->epAddr[i].port;
} }
} }
STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) { STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) {
assert(pTableMetaMsg != NULL); assert(pTableMetaMsg != NULL);
...@@ -162,9 +163,19 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size ...@@ -162,9 +163,19 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
.numOfColumns = pTableMetaMsg->numOfColumns, .numOfColumns = pTableMetaMsg->numOfColumns,
}; };
pTableMeta->id.tid = pTableMetaMsg->sid; pTableMeta->id.tid = pTableMetaMsg->tid;
pTableMeta->id.uid = pTableMetaMsg->uid; pTableMeta->id.uid = pTableMetaMsg->uid;
pTableMeta->vgroupInfo = pTableMetaMsg->vgroup;
SCMVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo;
pVgroupInfo->numOfEps = pTableMetaMsg->vgroup.numOfEps;
pVgroupInfo->vgId = pTableMetaMsg->vgroup.vgId;
for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) {
SEpAddrMsg* pEpMsg = &pTableMetaMsg->vgroup.epAddr[i];
pVgroupInfo->epAddr[i].fqdn = strndup(pEpMsg->fqdn, tListLen(pEpMsg->fqdn));
pVgroupInfo->epAddr[i].port = pEpMsg->port;
}
tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo); tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo);
......
...@@ -24,7 +24,6 @@ ...@@ -24,7 +24,6 @@
#include "tschemautil.h" #include "tschemautil.h"
#include "tsclient.h" #include "tsclient.h"
#include "ttimer.h" #include "ttimer.h"
#include "tutil.h"
#include "tlockfree.h" #include "tlockfree.h"
SRpcCorEpSet tscMgmtEpSet; SRpcCorEpSet tscMgmtEpSet;
...@@ -125,9 +124,11 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { ...@@ -125,9 +124,11 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) {
pVgroupInfo->inUse = pEpSet->inUse; pVgroupInfo->inUse = pEpSet->inUse;
pVgroupInfo->numOfEps = pEpSet->numOfEps; pVgroupInfo->numOfEps = pEpSet->numOfEps;
for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) { for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) {
tstrncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); taosTFree(pVgroupInfo->epAddr[i].fqdn);
pVgroupInfo->epAddr[i].fqdn = strndup(pEpSet->fqdn[i], tListLen(pEpSet->fqdn[i]));
pVgroupInfo->epAddr[i].port = pEpSet->port[i]; pVgroupInfo->epAddr[i].port = pEpSet->port[i];
} }
tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse); tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse);
taosCorEndWrite(&pVgroupInfo->version); taosCorEndWrite(&pVgroupInfo->version);
} }
...@@ -198,16 +199,20 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { ...@@ -198,16 +199,20 @@ void tscProcessActivityTimer(void *handle, void *tmrId) {
return; return;
} }
if (tscShouldFreeHeartBeat(pHB)) { void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE));
tscDebug("%p free HB object and release connection", pHB); if (p == NULL) {
pObj->pHb = 0; tscWarn("%p HB object has been released already", pHB);
taos_free_result(pHB); return;
} else { }
assert(*pHB->self == pHB);
int32_t code = tscProcessSql(pHB); int32_t code = tscProcessSql(pHB);
taosCacheRelease(tscObjCache, (void**) &p, false);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code)); tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code));
} }
}
} }
int tscSendMsgToServer(SSqlObj *pSql) { int tscSendMsgToServer(SSqlObj *pSql) {
...@@ -464,35 +469,6 @@ int tscProcessSql(SSqlObj *pSql) { ...@@ -464,35 +469,6 @@ int tscProcessSql(SSqlObj *pSql) {
return doProcessSql(pSql); return doProcessSql(pSql);
} }
void tscKillSTableQuery(SSqlObj *pSql) {
SSqlCmd* pCmd = &pSql->cmd;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
return;
}
pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
for (int i = 0; i < pSql->numOfSubs; ++i) {
// NOTE: pSub may have been released already here
SSqlObj *pSub = pSql->pSubs[i];
if (pSub == NULL) {
continue;
}
pSub->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
if (pSub->pRpcCtx != NULL) {
rpcCancelRequest(pSub->pRpcCtx);
pSub->pRpcCtx = NULL;
}
tscQueueAsyncRes(pSub); // async res? not other functions?
}
tscDebug("%p super table query cancelled", pSql);
}
int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload; SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload;
pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle); pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle);
...@@ -1392,7 +1368,7 @@ static int tscSetResultPointer(SQueryInfo *pQueryInfo, SSqlRes *pRes) { ...@@ -1392,7 +1368,7 @@ static int tscSetResultPointer(SQueryInfo *pQueryInfo, SSqlRes *pRes) {
for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i); int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i);
pRes->tsrow[i] = ((char*) pRes->data + offset * pRes->numOfRows); pRes->tsrow[i] = (unsigned char*)((char*) pRes->data + offset * pRes->numOfRows);
} }
return 0; return 0;
...@@ -1451,7 +1427,7 @@ int tscProcessLocalRetrieveRsp(SSqlObj *pSql) { ...@@ -1451,7 +1427,7 @@ int tscProcessLocalRetrieveRsp(SSqlObj *pSql) {
int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) {
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd* pCmd = &pSql->cmd;
int32_t code = pRes->code; int32_t code = pRes->code;
if (pRes->code != TSDB_CODE_SUCCESS) { if (pRes->code != TSDB_CODE_SUCCESS) {
...@@ -1494,6 +1470,7 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { ...@@ -1494,6 +1470,7 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMConnectMsg *pConnect = (SCMConnectMsg*)pCmd->payload; SCMConnectMsg *pConnect = (SCMConnectMsg*)pCmd->payload;
// TODO refactor full_name
char *db; // ugly code to move the space char *db; // ugly code to move the space
db = strstr(pObj->db, TS_PATH_DELIMITER); db = strstr(pObj->db, TS_PATH_DELIMITER);
db = (db == NULL) ? pObj->db : db + 1; db = (db == NULL) ? pObj->db : db + 1;
...@@ -1501,6 +1478,9 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { ...@@ -1501,6 +1478,9 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion));
tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion));
pConnect->pid = htonl(taosGetPId());
taosGetCurrentAPPName(pConnect->appName, NULL);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -1653,6 +1633,10 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { ...@@ -1653,6 +1633,10 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload; SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload;
pHeartbeat->numOfQueries = numOfQueries; pHeartbeat->numOfQueries = numOfQueries;
pHeartbeat->numOfStreams = numOfStreams; pHeartbeat->numOfStreams = numOfStreams;
pHeartbeat->pid = htonl(taosGetPId());
taosGetCurrentAPPName(pHeartbeat->appName, NULL);
int msgLen = tscBuildQueryStreamDesc(pHeartbeat, pObj); int msgLen = tscBuildQueryStreamDesc(pHeartbeat, pObj);
pthread_mutex_unlock(&pObj->mutex); pthread_mutex_unlock(&pObj->mutex);
...@@ -1666,7 +1650,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { ...@@ -1666,7 +1650,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int tscProcessTableMetaRsp(SSqlObj *pSql) { int tscProcessTableMetaRsp(SSqlObj *pSql) {
STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp;
pMetaMsg->sid = htonl(pMetaMsg->sid); pMetaMsg->tid = htonl(pMetaMsg->tid);
pMetaMsg->sversion = htons(pMetaMsg->sversion); pMetaMsg->sversion = htons(pMetaMsg->sversion);
pMetaMsg->tversion = htons(pMetaMsg->tversion); pMetaMsg->tversion = htons(pMetaMsg->tversion);
pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId);
...@@ -1676,9 +1660,9 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { ...@@ -1676,9 +1660,9 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns);
if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) &&
(pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { (pMetaMsg->tid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) {
tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId,
pMetaMsg->sid, pMetaMsg->tableId); pMetaMsg->tid, pMetaMsg->tableId);
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }
...@@ -1857,22 +1841,30 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { ...@@ -1857,22 +1841,30 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
SSqlCmd* pCmd = &parent->cmd; SSqlCmd* pCmd = &parent->cmd;
for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) {
STableMetaInfo *pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i); STableMetaInfo *pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i);
SVgroupsInfo * pVgroupInfo = (SVgroupsInfo *)pMsg;
pVgroupInfo->numOfVgroups = htonl(pVgroupInfo->numOfVgroups);
size_t size = sizeof(SCMVgroupInfo) * pVgroupInfo->numOfVgroups + sizeof(SVgroupsInfo); SVgroupsMsg * pVgroupMsg = (SVgroupsMsg *) pMsg;
pInfo->vgroupList = calloc(1, size); pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups);
size_t size = sizeof(SCMVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg);
size_t vgroupsz = sizeof(SCMVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo);
pInfo->vgroupList = calloc(1, vgroupsz);
assert(pInfo->vgroupList != NULL); assert(pInfo->vgroupList != NULL);
memcpy(pInfo->vgroupList, pVgroupInfo, size); pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups;
for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) {
//just init, no need to lock //just init, no need to lock
SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j];
pVgroups->vgId = htonl(pVgroups->vgId);
assert(pVgroups->numOfEps >= 1); SCMVgroupMsg *vmsg = &pVgroupMsg->vgroups[j];
pVgroups->vgId = htonl(vmsg->vgId);
pVgroups->numOfEps = vmsg->numOfEps;
assert(pVgroups->numOfEps >= 1 && pVgroups->vgId >= 1);
for (int32_t k = 0; k < pVgroups->numOfEps; ++k) { for (int32_t k = 0; k < pVgroups->numOfEps; ++k) {
pVgroups->epAddr[k].port = htons(pVgroups->epAddr[k].port); pVgroups->epAddr[k].port = htons(vmsg->epAddr[k].port);
pVgroups->epAddr[k].fqdn = strndup(vmsg->epAddr[k].fqdn, tListLen(vmsg->epAddr[k].fqdn));
} }
} }
...@@ -1908,7 +1900,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { ...@@ -1908,7 +1900,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns); pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns);
pSchema = pMetaMsg->schema; pSchema = pMetaMsg->schema;
pMetaMsg->sid = ntohs(pMetaMsg->sid); pMetaMsg->tid = ntohs(pMetaMsg->tid);
for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { for (int i = 0; i < pMetaMsg->numOfColumns; ++i) {
pSchema->bytes = htons(pSchema->bytes); pSchema->bytes = htons(pSchema->bytes);
pSchema++; pSchema++;
...@@ -1942,7 +1934,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { ...@@ -1942,7 +1934,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
tscColumnListInsert(pQueryInfo->colList, &index); tscColumnListInsert(pQueryInfo->colList, &index);
TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes);
SFieldSupInfo* pInfo = tscFieldInfoAppend(pFieldInfo, &f); SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f);
pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index,
pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes, false); pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes, false);
......
...@@ -26,7 +26,9 @@ ...@@ -26,7 +26,9 @@
#include "tsclient.h" #include "tsclient.h"
#include "ttokendef.h" #include "ttokendef.h"
#include "tutil.h" #include "tutil.h"
#include "ttimer.h"
#include "tscProfile.h" #include "tscProfile.h"
#include "ttimer.h"
static bool validImpl(const char* str, size_t maxsize) { static bool validImpl(const char* str, size_t maxsize) {
if (str == NULL) { if (str == NULL) {
...@@ -49,8 +51,8 @@ static bool validPassword(const char* passwd) { ...@@ -49,8 +51,8 @@ static bool validPassword(const char* passwd) {
return validImpl(passwd, TSDB_PASSWORD_LEN - 1); return validImpl(passwd, TSDB_PASSWORD_LEN - 1);
} }
SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *auth, const char *db, static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *auth, const char *db,
uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos) {
taos_init(); taos_init();
if (!validUserName(user)) { if (!validUserName(user)) {
...@@ -241,25 +243,39 @@ static void asyncConnCallback(void *param, TAOS_RES *tres, int code) { ...@@ -241,25 +243,39 @@ static void asyncConnCallback(void *param, TAOS_RES *tres, int code) {
} }
TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
void *param, void **taos) { void *param, TAOS **taos) {
SSqlObj* pSql = taosConnectImpl(ip, user, pass, NULL, db, port, asyncConnCallback, param, taos); STscObj *pObj = NULL;
SSqlObj *pSql = taosConnectImpl(ip, user, pass, NULL, db, port, asyncConnCallback, param, (void **)&pObj);
if (pSql == NULL) { if (pSql == NULL) {
return NULL; return NULL;
} }
if (taos) *taos = pObj;
pSql->fetchFp = fp; pSql->fetchFp = fp;
pSql->res.code = tscProcessSql(pSql); pSql->res.code = tscProcessSql(pSql);
tscDebug("%p DB async connection is opening", taos); tscDebug("%p DB async connection is opening", taos);
return taos; return pObj;
} }
void taos_close(TAOS *taos) { void taos_close(TAOS *taos) {
STscObj *pObj = (STscObj *)taos; STscObj *pObj = (STscObj *)taos;
if (pObj == NULL || pObj->signature != pObj) { if (pObj == NULL) {
tscDebug("(null) try to free tscObj and close dnodeConn");
return;
}
tscDebug("%p try to free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
if (pObj->signature != pObj) {
tscDebug("%p already closed or invalid tscObj", pObj);
return; return;
} }
// make sure that the close connection can only be executed once.
pObj->signature = NULL;
taosTmrStopA(&(pObj->pTimer));
SSqlObj* pHb = pObj->pHb; SSqlObj* pHb = pObj->pHb;
if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) { if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
if (pHb->pRpcCtx != NULL) { // wait for rsp from dnode if (pHb->pRpcCtx != NULL) { // wait for rsp from dnode
...@@ -295,7 +311,7 @@ static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) { ...@@ -295,7 +311,7 @@ static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) {
tsem_post(&pSql->rspSem); tsem_post(&pSql->rspSem);
} }
TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) { TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES** res) {
STscObj *pObj = (STscObj *)taos; STscObj *pObj = (STscObj *)taos;
if (pObj == NULL || pObj->signature != pObj) { if (pObj == NULL || pObj->signature != pObj) {
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
...@@ -320,12 +336,20 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) { ...@@ -320,12 +336,20 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) {
tsem_init(&pSql->rspSem, 0, 0); tsem_init(&pSql->rspSem, 0, 0);
doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen);
if (res != NULL) {
*res = pSql;
}
tsem_wait(&pSql->rspSem); tsem_wait(&pSql->rspSem);
return pSql; return pSql;
} }
TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) { TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) {
return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr)); return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr), NULL);
}
TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res) {
return taos_query_c(taos, sqlstr, (uint32_t) strlen(sqlstr), res);
} }
int taos_result_precision(TAOS_RES *res) { int taos_result_precision(TAOS_RES *res) {
...@@ -349,7 +373,7 @@ int taos_num_fields(TAOS_RES *res) { ...@@ -349,7 +373,7 @@ int taos_num_fields(TAOS_RES *res) {
size_t numOfCols = tscNumOfFields(pQueryInfo); size_t numOfCols = tscNumOfFields(pQueryInfo);
for(int32_t i = 0; i < numOfCols; ++i) { for(int32_t i = 0; i < numOfCols; ++i) {
SFieldSupInfo* pInfo = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); SInternalField* pInfo = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i);
if (pInfo->visible) { if (pInfo->visible) {
num++; num++;
} }
...@@ -386,7 +410,23 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { ...@@ -386,7 +410,23 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
return NULL; return NULL;
} }
return pQueryInfo->fieldsInfo.pFields->pData; SFieldInfo *pFieldInfo = &pQueryInfo->fieldsInfo;
if (pFieldInfo->final == NULL) {
TAOS_FIELD* f = calloc(pFieldInfo->numOfOutput, sizeof(TAOS_FIELD));
int32_t j = 0;
for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) {
SInternalField* pField = tscFieldInfoGetInternalField(pFieldInfo, i);
if (pField->visible) {
f[j++] = pField->field;
}
}
pFieldInfo->final = f;
}
return pFieldInfo->final;
} }
int taos_retrieve(TAOS_RES *res) { int taos_retrieve(TAOS_RES *res) {
...@@ -459,6 +499,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { ...@@ -459,6 +499,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
if (pRes->qhandle == 0 || if (pRes->qhandle == 0 ||
pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED ||
pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
pCmd->command == TSDB_SQL_INSERT) { pCmd->command == TSDB_SQL_INSERT) {
return NULL; return NULL;
...@@ -522,7 +563,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { ...@@ -522,7 +563,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
pRes->numOfClauseTotal = 0; pRes->numOfClauseTotal = 0;
pRes->rspType = 0; pRes->rspType = 0;
pSql->numOfSubs = 0; pSql->subState.numOfSub = 0;
taosTFree(pSql->pSubs); taosTFree(pSql->pSubs);
assert(pSql->fp == NULL); assert(pSql->fp == NULL);
...@@ -566,7 +607,8 @@ static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) { ...@@ -566,7 +607,8 @@ static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) {
return true; return true;
} }
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) { if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
return true; return true;
} }
...@@ -676,6 +718,46 @@ int* taos_fetch_lengths(TAOS_RES *res) { ...@@ -676,6 +718,46 @@ int* taos_fetch_lengths(TAOS_RES *res) {
char *taos_get_client_info() { return version; } char *taos_get_client_info() { return version; }
static void tscKillSTableQuery(SSqlObj *pSql) {
SSqlCmd* pCmd = &pSql->cmd;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
return;
}
// set the master sqlObj flag to cancel query
pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
for (int i = 0; i < pSql->subState.numOfSub; ++i) {
// NOTE: pSub may have been released already here
SSqlObj *pSub = pSql->pSubs[i];
if (pSub == NULL) {
continue;
}
void** p = taosCacheAcquireByKey(tscObjCache, &pSub, sizeof(TSDB_CACHE_PTR_TYPE));
if (p == NULL) {
continue;
}
SSqlObj* pSubObj = (SSqlObj*) (*p);
assert(pSubObj->self == (SSqlObj**) p);
pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
if (pSubObj->pRpcCtx != NULL) {
rpcCancelRequest(pSubObj->pRpcCtx);
pSubObj->pRpcCtx = NULL;
}
tscQueueAsyncRes(pSubObj);
taosCacheRelease(tscObjCache, (void**) &p, false);
}
tscDebug("%p super table query cancelled", pSql);
}
void taos_stop_query(TAOS_RES *res) { void taos_stop_query(TAOS_RES *res) {
SSqlObj *pSql = (SSqlObj *)res; SSqlObj *pSql = (SSqlObj *)res;
if (pSql == NULL || pSql->signature != pSql) { if (pSql == NULL || pSql->signature != pSql) {
...@@ -685,23 +767,27 @@ void taos_stop_query(TAOS_RES *res) { ...@@ -685,23 +767,27 @@ void taos_stop_query(TAOS_RES *res) {
tscDebug("%p start to cancel query", res); tscDebug("%p start to cancel query", res);
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
// TODO there are multi-thread problem.
// It may have been released by the other thread already.
// The ref count may fix this problem.
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
// set the error code for master pSqlObj firstly // set the error code for master pSqlObj firstly
pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
assert(pSql->pRpcCtx == NULL); assert(pSql->pRpcCtx == NULL);
tscKillSTableQuery(pSql); tscKillSTableQuery(pSql);
} else { } else {
if (pSql->cmd.command < TSDB_SQL_LOCAL) { if (pSql->cmd.command < TSDB_SQL_LOCAL) {
/*
* There is multi-thread problem here, since pSql->pRpcCtx may have been
* reset and freed in the processMsgFromServer function, and causes the invalid
* write problem for rpcCancelRequest.
*/
if (pSql->pRpcCtx != NULL) { if (pSql->pRpcCtx != NULL) {
rpcCancelRequest(pSql->pRpcCtx); rpcCancelRequest(pSql->pRpcCtx);
pSql->pRpcCtx = NULL; pSql->pRpcCtx = NULL;
} }
tscQueueAsyncRes(pSql);
} }
} }
...@@ -814,7 +900,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { ...@@ -814,7 +900,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
if (pSql->sqlstr == NULL) { if (pSql->sqlstr == NULL) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscError("%p failed to malloc sql string buffer", pSql); tscError("%p failed to malloc sql string buffer", pSql);
tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(pSql), pObj);
taosTFree(pSql); taosTFree(pSql);
return pRes->code; return pRes->code;
} }
...@@ -830,6 +916,8 @@ int taos_validate_sql(TAOS *taos, const char *sql) { ...@@ -830,6 +916,8 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
pSql->fp = asyncCallback; pSql->fp = asyncCallback;
pSql->fetchFp = asyncCallback; pSql->fetchFp = asyncCallback;
pSql->param = pSql; pSql->param = pSql;
registerSqlObj(pSql);
int code = tsParseSql(pSql, true); int code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
tsem_wait(&pSql->rspSem); tsem_wait(&pSql->rspSem);
...@@ -837,7 +925,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { ...@@ -837,7 +925,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
} }
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, code, taos_errstr(taos), pObj); tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, code, taos_errstr(pSql), pObj);
} }
taos_free_result(pSql); taos_free_result(pSql);
...@@ -981,7 +1069,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { ...@@ -981,7 +1069,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
tscDoQuery(pSql); tscDoQuery(pSql);
tscDebug("%p load multi table meta result:%d %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); tscDebug("%p load multi table meta result:%d %s pObj:%p", pSql, pRes->code, taos_errstr(pSql), pObj);
if ((code = pRes->code) != TSDB_CODE_SUCCESS) { if ((code = pRes->code) != TSDB_CODE_SUCCESS) {
tscFreeSqlObj(pSql); tscFreeSqlObj(pSql);
} }
......
...@@ -168,7 +168,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf ...@@ -168,7 +168,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0);
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), true); taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), true);
taosTFree(pTableMetaInfo->vgroupList); pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList);
tscSetRetryTimer(pStream, pStream->pSql, retryDelay); tscSetRetryTimer(pStream, pStream->pSql, retryDelay);
return; return;
...@@ -274,8 +274,8 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf ...@@ -274,8 +274,8 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false);
tscFreeSqlResult(pSql); tscFreeSqlResult(pSql);
taosTFree(pSql->pSubs); taosTFree(pSql->pSubs);
pSql->numOfSubs = 0; pSql->subState.numOfSub = 0;
taosTFree(pTableMetaInfo->vgroupList); pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList);
tscSetNextLaunchTimer(pStream, pSql); tscSetNextLaunchTimer(pStream, pSql);
} }
} }
......
此差异已折叠。
...@@ -77,6 +77,7 @@ int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeCon ...@@ -77,6 +77,7 @@ int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeCon
return 0; return 0;
} }
void taos_init_imp(void) { void taos_init_imp(void) {
char temp[128]; char temp[128];
...@@ -124,8 +125,9 @@ void taos_init_imp(void) { ...@@ -124,8 +125,9 @@ void taos_init_imp(void) {
double factor = (tscEmbedded == 0)? 2.0:4.0; double factor = (tscEmbedded == 0)? 2.0:4.0;
tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor); tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor);
if (tscNumOfThreads < 2) {
if (tscNumOfThreads < 2) tscNumOfThreads = 2; tscNumOfThreads = 2;
}
tscQhandle = taosInitScheduler(queueSize, tscNumOfThreads, "tsc"); tscQhandle = taosInitScheduler(queueSize, tscNumOfThreads, "tsc");
if (NULL == tscQhandle) { if (NULL == tscQhandle) {
...@@ -140,8 +142,8 @@ void taos_init_imp(void) { ...@@ -140,8 +142,8 @@ void taos_init_imp(void) {
int64_t refreshTime = 10; // 10 seconds by default int64_t refreshTime = 10; // 10 seconds by default
if (tscMetaCache == NULL) { if (tscMetaCache == NULL) {
tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta"); tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta");
tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeSqlObjInCache, "sqlObj"); tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj");
} }
tscDebug("client is initialized successfully"); tscDebug("client is initialized successfully");
......
...@@ -114,9 +114,9 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { ...@@ -114,9 +114,9 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) {
} }
// for select query super table, the super table vgroup list can not be null in any cases. // for select query super table, the super table vgroup list can not be null in any cases.
if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
assert(pTableMetaInfo->vgroupList != NULL); // assert(pTableMetaInfo->vgroupList != NULL);
} // }
if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) { if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) {
return false; return false;
...@@ -360,26 +360,26 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { ...@@ -360,26 +360,26 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
tscFreeSqlResult(pSql); tscFreeSqlResult(pSql);
taosTFree(pSql->pSubs); taosTFree(pSql->pSubs);
pSql->numOfSubs = 0; pSql->subState.numOfSub = 0;
pSql->self = 0; pSql->self = 0;
tscResetSqlCmdObj(pCmd, false); tscResetSqlCmdObj(pCmd, false);
} }
static UNUSED_FUNC void tscFreeSubobj(SSqlObj* pSql) { static void tscFreeSubobj(SSqlObj* pSql) {
if (pSql->numOfSubs == 0) { if (pSql->subState.numOfSub == 0) {
return; return;
} }
tscDebug("%p start to free sub SqlObj, numOfSub:%d", pSql, pSql->numOfSubs); tscDebug("%p start to free sub SqlObj, numOfSub:%d", pSql, pSql->subState.numOfSub);
for(int32_t i = 0; i < pSql->numOfSubs; ++i) { for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
tscDebug("%p free sub SqlObj:%p, index:%d", pSql, pSql->pSubs[i], i); tscDebug("%p free sub SqlObj:%p, index:%d", pSql, pSql->pSubs[i], i);
taos_free_result(pSql->pSubs[i]); taos_free_result(pSql->pSubs[i]);
pSql->pSubs[i] = NULL; pSql->pSubs[i] = NULL;
} }
pSql->numOfSubs = 0; pSql->subState.numOfSub = 0;
} }
/** /**
...@@ -389,7 +389,7 @@ static UNUSED_FUNC void tscFreeSubobj(SSqlObj* pSql) { ...@@ -389,7 +389,7 @@ static UNUSED_FUNC void tscFreeSubobj(SSqlObj* pSql) {
* *
* @param pSql * @param pSql
*/ */
void tscFreeSqlObjInCache(void *pSql) { void tscFreeRegisteredSqlObj(void *pSql) {
assert(pSql != NULL); assert(pSql != NULL);
SSqlObj** p = (SSqlObj**)pSql; SSqlObj** p = (SSqlObj**)pSql;
...@@ -408,6 +408,24 @@ void tscFreeSqlObjInCache(void *pSql) { ...@@ -408,6 +408,24 @@ void tscFreeSqlObjInCache(void *pSql) {
} }
} }
void tscFreeTableMetaHelper(void *pTableMeta) {
STableMeta* p = (STableMeta*) pTableMeta;
int32_t numOfEps = p->vgroupInfo.numOfEps;
assert(numOfEps >= 0 && numOfEps <= TSDB_MAX_REPLICA);
for(int32_t i = 0; i < numOfEps; ++i) {
taosTFree(p->vgroupInfo.epAddr[i].fqdn);
}
int32_t numOfEps1 = p->corVgroupInfo.numOfEps;
assert(numOfEps1 >= 0 && numOfEps1 <= TSDB_MAX_REPLICA);
for(int32_t i = 0; i < numOfEps1; ++i) {
taosTFree(p->corVgroupInfo.epAddr[i].fqdn);
}
}
void tscFreeSqlObj(SSqlObj* pSql) { void tscFreeSqlObj(SSqlObj* pSql) {
if (pSql == NULL || pSql->signature != pSql) { if (pSql == NULL || pSql->signature != pSql) {
return; return;
...@@ -415,7 +433,9 @@ void tscFreeSqlObj(SSqlObj* pSql) { ...@@ -415,7 +433,9 @@ void tscFreeSqlObj(SSqlObj* pSql) {
tscDebug("%p start to free sqlObj", pSql); tscDebug("%p start to free sqlObj", pSql);
pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
tscFreeSubobj(pSql); tscFreeSubobj(pSql);
tscPartiallyFreeSqlObj(pSql); tscPartiallyFreeSqlObj(pSql);
pSql->signature = NULL; pSql->signature = NULL;
...@@ -684,16 +704,14 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) { ...@@ -684,16 +704,14 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) {
int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
SSqlCmd* pCmd = &pSql->cmd; SSqlCmd* pCmd = &pSql->cmd;
// the maximum expanded size in byte when a row-wise data is converted to SDataRow format
STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, 0);
int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta);
void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES);
size_t total = taosArrayGetSize(pTableDataBlockList); size_t total = taosArrayGetSize(pTableDataBlockList);
for (int32_t i = 0; i < total; ++i) { for (int32_t i = 0; i < total; ++i) {
pOneTableBlock = taosArrayGetP(pTableDataBlockList, i); // the maximum expanded size in byte when a row-wise data is converted to SDataRow format
STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, i);
int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta);
STableDataBlocks* dataBuf = NULL; STableDataBlocks* dataBuf = NULL;
int32_t ret = int32_t ret =
...@@ -827,35 +845,30 @@ TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) { ...@@ -827,35 +845,30 @@ TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) {
return f; return f;
} }
SFieldSupInfo* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) { SInternalField* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) {
assert(pFieldInfo != NULL); assert(pFieldInfo != NULL);
taosArrayPush(pFieldInfo->pFields, pField);
pFieldInfo->numOfOutput++; pFieldInfo->numOfOutput++;
struct SFieldSupInfo info = { struct SInternalField info = {
.pSqlExpr = NULL, .pSqlExpr = NULL,
.pArithExprInfo = NULL, .pArithExprInfo = NULL,
.visible = true, .visible = true,
}; };
return taosArrayPush(pFieldInfo->pSupportInfo, &info); info.field = *pField;
} return taosArrayPush(pFieldInfo->internalField, &info);
SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index) {
return TARRAY_GET_ELEM(pFieldInfo->pSupportInfo, index);
} }
SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) { SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) {
taosArrayInsert(pFieldInfo->pFields, index, field);
pFieldInfo->numOfOutput++; pFieldInfo->numOfOutput++;
struct SInternalField info = {
struct SFieldSupInfo info = {
.pSqlExpr = NULL, .pSqlExpr = NULL,
.pArithExprInfo = NULL, .pArithExprInfo = NULL,
.visible = true, .visible = true,
}; };
return taosArrayInsert(pFieldInfo->pSupportInfo, index, &info); info.field = *field;
return taosArrayInsert(pFieldInfo->internalField, index, &info);
} }
void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) { void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) {
...@@ -889,29 +902,18 @@ void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) { ...@@ -889,29 +902,18 @@ void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) {
} }
} }
void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src) { SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index) {
dst->numOfOutput = src->numOfOutput; assert(index < pFieldInfo->numOfOutput);
return TARRAY_GET_ELEM(pFieldInfo->internalField, index);
if (dst->pFields == NULL) {
dst->pFields = taosArrayClone(src->pFields);
} else {
taosArrayCopy(dst->pFields, src->pFields);
}
if (dst->pSupportInfo == NULL) {
dst->pSupportInfo = taosArrayClone(src->pSupportInfo);
} else {
taosArrayCopy(dst->pSupportInfo, src->pSupportInfo);
}
} }
TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) { TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) {
assert(index < pFieldInfo->numOfOutput); assert(index < pFieldInfo->numOfOutput);
return TARRAY_GET_ELEM(pFieldInfo->pFields, index); return &((SInternalField*)TARRAY_GET_ELEM(pFieldInfo->internalField, index))->field;
} }
int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) { int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) {
SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, index); SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, index);
assert(pInfo != NULL && pInfo->pSqlExpr != NULL); assert(pInfo != NULL && pInfo->pSqlExpr != NULL);
return pInfo->pSqlExpr->offset; return pInfo->pSqlExpr->offset;
...@@ -958,10 +960,8 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { ...@@ -958,10 +960,8 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) {
return; return;
} }
taosArrayDestroy(pFieldInfo->pFields);
for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) { for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) {
SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, i); SInternalField* pInfo = taosArrayGet(pFieldInfo->internalField, i);
if (pInfo->pArithExprInfo != NULL) { if (pInfo->pArithExprInfo != NULL) {
tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL); tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL);
...@@ -969,7 +969,9 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { ...@@ -969,7 +969,9 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) {
} }
} }
taosArrayDestroy(pFieldInfo->pSupportInfo); taosArrayDestroy(pFieldInfo->internalField);
taosTFree(pFieldInfo->final);
memset(pFieldInfo, 0, sizeof(SFieldInfo)); memset(pFieldInfo, 0, sizeof(SFieldInfo));
} }
...@@ -1516,13 +1518,6 @@ void tscSetFreeHeatBeat(STscObj* pObj) { ...@@ -1516,13 +1518,6 @@ void tscSetFreeHeatBeat(STscObj* pObj) {
pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
} }
bool tscShouldFreeHeartBeat(SSqlObj* pHb) {
assert(pHb == pHb->signature);
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pHb->cmd, 0);
return pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE;
}
/* /*
* the following four kinds of SqlObj should not be freed * the following four kinds of SqlObj should not be freed
* 1. SqlObj for stream computing * 1. SqlObj for stream computing
...@@ -1620,11 +1615,8 @@ STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, i ...@@ -1620,11 +1615,8 @@ STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, i
} }
void tscInitQueryInfo(SQueryInfo* pQueryInfo) { void tscInitQueryInfo(SQueryInfo* pQueryInfo) {
assert(pQueryInfo->fieldsInfo.pFields == NULL); assert(pQueryInfo->fieldsInfo.internalField == NULL);
pQueryInfo->fieldsInfo.pFields = taosArrayInit(4, sizeof(TAOS_FIELD)); pQueryInfo->fieldsInfo.internalField = taosArrayInit(4, sizeof(SInternalField));
assert(pQueryInfo->fieldsInfo.pSupportInfo == NULL);
pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo));
assert(pQueryInfo->exprList == NULL); assert(pQueryInfo->exprList == NULL);
pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES);
...@@ -1687,8 +1679,14 @@ void tscClearSubqueryInfo(SSqlCmd* pCmd) { ...@@ -1687,8 +1679,14 @@ void tscClearSubqueryInfo(SSqlCmd* pCmd) {
void tscFreeVgroupTableInfo(SArray* pVgroupTables) { void tscFreeVgroupTableInfo(SArray* pVgroupTables) {
if (pVgroupTables != NULL) { if (pVgroupTables != NULL) {
for (size_t i = 0; i < taosArrayGetSize(pVgroupTables); i++) { size_t num = taosArrayGetSize(pVgroupTables);
for (size_t i = 0; i < num; i++) {
SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i);
for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
taosTFree(pInfo->vgInfo.epAddr[j].fqdn);
}
taosArrayDestroy(pInfo->itemList); taosArrayDestroy(pInfo->itemList);
} }
taosArrayDestroy(pVgroupTables); taosArrayDestroy(pVgroupTables);
...@@ -1700,6 +1698,7 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool rem ...@@ -1700,6 +1698,7 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool rem
for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables);
tscClearTableMetaInfo(pTableMetaInfo, removeFromCache); tscClearTableMetaInfo(pTableMetaInfo, removeFromCache);
free(pTableMetaInfo); free(pTableMetaInfo);
...@@ -1732,13 +1731,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST ...@@ -1732,13 +1731,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
pTableMetaInfo->pTableMeta = pTableMeta; pTableMetaInfo->pTableMeta = pTableMeta;
if (vgroupList != NULL) { if (vgroupList != NULL) {
size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups; pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList);
pTableMetaInfo->vgroupList = malloc(size);
if (pTableMetaInfo->vgroupList == NULL) {
return NULL;
}
memcpy(pTableMetaInfo->vgroupList, vgroupList, size);
} }
pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES);
...@@ -1767,8 +1760,7 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) ...@@ -1767,8 +1760,7 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache)
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache);
} }
taosTFree(pTableMetaInfo->vgroupList); pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList);
tscColumnListDestroy(pTableMetaInfo->tagColList); tscColumnListDestroy(pTableMetaInfo->tagColList);
pTableMetaInfo->tagColList = NULL; pTableMetaInfo->tagColList = NULL;
} }
...@@ -1836,54 +1828,23 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm ...@@ -1836,54 +1828,23 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
return pNew; return pNew;
} }
// current sql function is not direct output result, so create a dummy output field
static void doSetNewFieldInfo(SQueryInfo* pNewQueryInfo, SSqlExpr* pExpr) {
TAOS_FIELD f = {.type = (uint8_t)pExpr->resType, .bytes = pExpr->resBytes};
tstrncpy(f.name, pExpr->aliasName, sizeof(f.name));
SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f);
pInfo1->pSqlExpr = pExpr;
pInfo1->visible = false;
}
static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* pNewQueryInfo, int64_t uid) { static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* pNewQueryInfo, int64_t uid) {
int32_t numOfOutput = (int32_t)tscSqlExprNumOfExprs(pNewQueryInfo); int32_t numOfOutput = (int32_t)tscSqlExprNumOfExprs(pNewQueryInfo);
if (numOfOutput == 0) { if (numOfOutput == 0) {
return; return;
} }
size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); // set the field info in pNewQueryInfo object according to sqlExpr information
SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; size_t numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo);
// set the field info in pNewQueryInfo object
for (int32_t i = 0; i < numOfExprs; ++i) { for (int32_t i = 0; i < numOfExprs; ++i) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); SSqlExpr* pExpr = tscSqlExprGet(pNewQueryInfo, i);
if (pExpr->uid == uid) {
if (i < pFieldInfo->numOfOutput) {
SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i);
if (pInfo->pSqlExpr != NULL) { TAOS_FIELD f = tscCreateField((int8_t) pExpr->resType, pExpr->aliasName, pExpr->resBytes);
TAOS_FIELD* p = tscFieldInfoGetField(pFieldInfo, i); SInternalField* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f);
assert(strcmp(p->name, pExpr->aliasName) == 0); pInfo1->pSqlExpr = pExpr;
SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, p);
*pInfo1 = *pInfo;
} else {
assert(pInfo->pArithExprInfo != NULL);
doSetNewFieldInfo(pNewQueryInfo, pExpr);
}
} else { // it is a arithmetic column, does not have actual field for sqlExpr, so build it
doSetNewFieldInfo(pNewQueryInfo, pExpr);
}
}
} }
// make sure the the sqlExpr for each fields is correct // update the pSqlExpr pointer in SInternalField according the field name
numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo);
// update the pSqlExpr pointer in SFieldSupInfo according the field name
// make sure the pSqlExpr point to the correct SqlExpr in pNewQueryInfo, not SqlExpr in pQueryInfo // make sure the pSqlExpr point to the correct SqlExpr in pNewQueryInfo, not SqlExpr in pQueryInfo
for (int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) { for (int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) {
TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f); TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f);
...@@ -1893,7 +1854,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p ...@@ -1893,7 +1854,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p
SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1); SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1);
if (strcmp(field->name, pExpr1->aliasName) == 0) { // establish link according to the result field name if (strcmp(field->name, pExpr1->aliasName) == 0) { // establish link according to the result field name
SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pNewQueryInfo->fieldsInfo, f); SInternalField* pInfo = tscFieldInfoGetInternalField(&pNewQueryInfo->fieldsInfo, f);
pInfo->pSqlExpr = pExpr1; pInfo->pSqlExpr = pExpr1;
matched = true; matched = true;
...@@ -2291,7 +2252,7 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { ...@@ -2291,7 +2252,7 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
* *
* For super table join with projection query, if anyone of the subquery is exhausted, the query completed. * For super table join with projection query, if anyone of the subquery is exhausted, the query completed.
*/ */
pSql->numOfSubs = 0; pSql->subState.numOfSub = 0;
pCmd->command = TSDB_SQL_SELECT; pCmd->command = TSDB_SQL_SELECT;
tscResetForNextRetrieve(pRes); tscResetForNextRetrieve(pRes);
...@@ -2323,7 +2284,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) { ...@@ -2323,7 +2284,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) {
pRes->numOfTotal = num; pRes->numOfTotal = num;
taosTFree(pSql->pSubs); taosTFree(pSql->pSubs);
pSql->numOfSubs = 0; pSql->subState.numOfSub = 0;
pSql->fp = fp; pSql->fp = fp;
tscDebug("%p try data in the next subclause:%d, total subclause:%d", pSql, pCmd->clauseIndex, pCmd->numOfClause); tscDebug("%p try data in the next subclause:%d, total subclause:%d", pSql, pCmd->clauseIndex, pCmd->numOfClause);
...@@ -2408,3 +2369,58 @@ void tscClearSqlOwner(SSqlObj* pSql) { ...@@ -2408,3 +2369,58 @@ void tscClearSqlOwner(SSqlObj* pSql) {
assert(taosCheckPthreadValid(pSql->owner)); assert(taosCheckPthreadValid(pSql->owner));
atomic_store_64(&pSql->owner, 0); atomic_store_64(&pSql->owner, 0);
} }
SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) {
if (vgroupList == NULL) {
return NULL;
}
size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups;
SVgroupsInfo* pNew = calloc(1, size);
if (pNew == NULL) {
return NULL;
}
pNew->numOfVgroups = vgroupList->numOfVgroups;
for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) {
SCMVgroupInfo* pNewVInfo = &pNew->vgroups[i];
SCMVgroupInfo* pvInfo = &vgroupList->vgroups[i];
pNewVInfo->vgId = pvInfo->vgId;
pNewVInfo->numOfEps = pvInfo->numOfEps;
for(int32_t j = 0; j < pvInfo->numOfEps; ++j) {
pNewVInfo->epAddr[j].fqdn = strdup(pvInfo->epAddr[j].fqdn);
pNewVInfo->epAddr[j].port = pvInfo->epAddr[j].port;
}
}
return pNew;
}
void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) {
if (vgroupList == NULL) {
return NULL;
}
for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) {
SCMVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i];
for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) {
taosTFree(pVgroupInfo->epAddr[j].fqdn);
}
}
taosTFree(vgroupList);
return NULL;
}
void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src) {
dst->vgId = src->vgId;
dst->numOfEps = src->numOfEps;
for(int32_t i = 0; i < dst->numOfEps; ++i) {
dst->epAddr[i].port = src->epAddr[i].port;
dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn);
}
}
...@@ -34,6 +34,7 @@ extern int32_t tsStatusInterval; ...@@ -34,6 +34,7 @@ extern int32_t tsStatusInterval;
extern int32_t tsNumOfMnodes; extern int32_t tsNumOfMnodes;
extern int32_t tsEnableVnodeBak; extern int32_t tsEnableVnodeBak;
extern int32_t tsEnableTelemetryReporting; extern int32_t tsEnableTelemetryReporting;
extern char tsEmail[];
// common // common
extern int tsRpcTimer; extern int tsRpcTimer;
......
...@@ -566,7 +566,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { ...@@ -566,7 +566,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) {
SKVRow nrow = NULL; SKVRow nrow = NULL;
void * ptr = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_GE); void * ptr = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_GE);
if (ptr == NULL || ((SColIdx *)ptr)->colId < colId) { // need to add a column value to the row if (ptr == NULL || ((SColIdx *)ptr)->colId > colId) { // need to add a column value to the row
int diff = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; int diff = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type];
nrow = malloc(kvRowLen(row) + sizeof(SColIdx) + diff); nrow = malloc(kvRowLen(row) + sizeof(SColIdx) + diff);
if (nrow == NULL) return -1; if (nrow == NULL) return -1;
......
...@@ -42,6 +42,7 @@ int32_t tsStatusInterval = 1; // second ...@@ -42,6 +42,7 @@ int32_t tsStatusInterval = 1; // second
int32_t tsNumOfMnodes = 3; int32_t tsNumOfMnodes = 3;
int32_t tsEnableVnodeBak = 1; int32_t tsEnableVnodeBak = 1;
int32_t tsEnableTelemetryReporting = 1; int32_t tsEnableTelemetryReporting = 1;
char tsEmail[TSDB_FQDN_LEN] = {0};
// common // common
int32_t tsRpcTimer = 1000; int32_t tsRpcTimer = 1000;
...@@ -307,6 +308,8 @@ bool taosCfgDynamicOptions(char *msg) { ...@@ -307,6 +308,8 @@ bool taosCfgDynamicOptions(char *msg) {
static void doInitGlobalConfig(void) { static void doInitGlobalConfig(void) {
osInit(); osInit();
srand(taosSafeRand());
SGlobalCfg cfg = {0}; SGlobalCfg cfg = {0};
// ip address // ip address
......
...@@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ...@@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED)
ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME}
POST_BUILD POST_BUILD
COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.0-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.8-dist.jar ${LIBRARY_OUTPUT_PATH}
COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
COMMENT "build jdbc driver") COMMENT "build jdbc driver")
ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME})
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.taosdata.jdbc</groupId> <groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId> <artifactId>taos-jdbcdriver</artifactId>
<version>2.0.0</version> <version>2.0.8</version>
<packaging>jar</packaging> <packaging>jar</packaging>
<name>JDBCDriver</name> <name>JDBCDriver</name>
<url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url> <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>
......
...@@ -587,7 +587,6 @@ public class TSDBDatabaseMetaData implements java.sql.DatabaseMetaData { ...@@ -587,7 +587,6 @@ public class TSDBDatabaseMetaData implements java.sql.DatabaseMetaData {
public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern)
throws SQLException { throws SQLException {
/** add by zyyang **********/ /** add by zyyang **********/
Statement stmt = null; Statement stmt = null;
if (null != conn && !conn.isClosed()) { if (null != conn && !conn.isClosed()) {
......
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF (TD_LINUX_64)
find_program(HAVE_ODBCINST NAMES odbcinst)
IF (HAVE_ODBCINST)
include(CheckSymbolExists)
# shall we revert CMAKE_REQUIRED_LIBRARIES and how?
set(CMAKE_REQUIRED_LIBRARIES odbc)
check_symbol_exists(SQLExecute "sql.h" HAVE_ODBC_DEV)
if(NOT (HAVE_ODBC_DEV))
unset(HAVE_ODBC_DEV CACHE)
message(WARNING "unixodbc-dev is not installed yet, you may install it under ubuntu by typing: sudo apt install unixodbc-dev")
else ()
message(STATUS "unixodbc/unixodbc-dev are installed, and odbc connector will be built")
AUX_SOURCE_DIRECTORY(src SRC)
# generate dynamic library (*.so)
ADD_LIBRARY(todbc SHARED ${SRC})
SET_TARGET_PROPERTIES(todbc PROPERTIES CLEAN_DIRECT_OUTPUT 1)
SET_TARGET_PROPERTIES(todbc PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1)
TARGET_LINK_LIBRARIES(todbc taos)
install(CODE "execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/install.sh ${CMAKE_BINARY_DIR})")
ADD_SUBDIRECTORY(tests)
endif()
ELSE ()
message(WARNING "unixodbc is not installed yet, you may install it under ubuntu by typing: sudo apt install unixodbc")
ENDIF ()
ENDIF ()
#!/bin/bash
set -u
BLD_DIR="$1"
rm -f "${BLD_DIR}/template.ini"
rm -f "${BLD_DIR}/template.dsn"
cat > "${BLD_DIR}/template.ini" <<EOF
[TAOS]
Description = taos odbc driver
Driver = ${BLD_DIR}/build/lib/libtodbc.so
EOF
cat > "${BLD_DIR}/template.dsn" <<EOF
[TAOS_DSN]
Description=Connection to TAOS
Driver=TAOS
EOF
sudo odbcinst -i -d -f "${BLD_DIR}/template.ini" &&
odbcinst -i -s -f "${BLD_DIR}/template.dsn" &&
echo "odbc install done"
此差异已折叠。
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "todbc_util.h"
#include "iconv.h"
#include <sql.h>
#include <sqltypes.h>
#include <sqlext.h>
#include <stdlib.h>
#include <string.h>
const char* sql_sql_type(int type) {
switch (type) {
case SQL_BIT: return "SQL_BIT";
case SQL_TINYINT: return "SQL_TINYINT";
case SQL_SMALLINT: return "SQL_SMALLINT";
case SQL_INTEGER: return "SQL_INTEGER";
case SQL_BIGINT: return "SQL_BIGINT";
case SQL_FLOAT: return "SQL_FLOAT";
case SQL_DOUBLE: return "SQL_DOUBLE";
case SQL_DECIMAL: return "SQL_DECIMAL";
case SQL_NUMERIC: return "SQL_NUMERIC";
case SQL_REAL: return "SQL_REAL";
case SQL_CHAR: return "SQL_CHAR";
case SQL_VARCHAR: return "SQL_VARCHAR";
case SQL_LONGVARCHAR: return "SQL_LONGVARCHAR";
case SQL_WCHAR: return "SQL_WCHAR";
case SQL_WVARCHAR: return "SQL_WVARCHAR";
case SQL_WLONGVARCHAR: return "SQL_WLONGVARCHAR";
case SQL_BINARY: return "SQL_BINARY";
case SQL_VARBINARY: return "SQL_VARBINARY";
case SQL_LONGVARBINARY: return "SQL_LONGVARBINARY";
case SQL_DATE: return "SQL_DATE";
case SQL_TIME: return "SQL_TIME";
case SQL_TIMESTAMP: return "SQL_TIMESTAMP";
case SQL_TYPE_DATE: return "SQL_TYPE_DATE";
case SQL_TYPE_TIME: return "SQL_TYPE_TIME";
case SQL_TYPE_TIMESTAMP: return "SQL_TYPE_TIMESTAMP";
case SQL_INTERVAL_MONTH: return "SQL_INTERVAL_MONTH";
case SQL_INTERVAL_YEAR: return "SQL_INTERVAL_YEAR";
case SQL_INTERVAL_YEAR_TO_MONTH: return "SQL_INTERVAL_YEAR_TO_MONTH";
case SQL_INTERVAL_DAY: return "SQL_INTERVAL_DAY";
case SQL_INTERVAL_HOUR: return "SQL_INTERVAL_HOUR";
case SQL_INTERVAL_MINUTE: return "SQL_INTERVAL_MINUTE";
case SQL_INTERVAL_SECOND: return "SQL_INTERVAL_SECOND";
case SQL_INTERVAL_DAY_TO_HOUR: return "SQL_INTERVAL_DAY_TO_HOUR";
case SQL_INTERVAL_DAY_TO_MINUTE: return "SQL_INTERVAL_DAY_TO_MINUTE";
case SQL_INTERVAL_DAY_TO_SECOND: return "SQL_INTERVAL_DAY_TO_SECOND";
case SQL_INTERVAL_HOUR_TO_MINUTE: return "SQL_INTERVAL_HOUR_TO_MINUTE";
case SQL_INTERVAL_HOUR_TO_SECOND: return "SQL_INTERVAL_HOUR_TO_SECOND";
case SQL_INTERVAL_MINUTE_TO_SECOND: return "SQL_INTERVAL_MINUTE_TO_SECOND";
case SQL_GUID: return "SQL_GUID";
default: return "UNKNOWN";
}
}
const char* sql_c_type(int type) {
switch (type) {
case SQL_C_CHAR: return "SQL_C_CHAR";
case SQL_C_WCHAR: return "SQL_C_WCHAR";
case SQL_C_SHORT: return "SQL_C_SHORT";
case SQL_C_SSHORT: return "SQL_C_SSHORT";
case SQL_C_USHORT: return "SQL_C_USHORT";
case SQL_C_LONG: return "SQL_C_LONG";
case SQL_C_SLONG: return "SQL_C_SLONG";
case SQL_C_ULONG: return "SQL_C_ULONG";
case SQL_C_FLOAT: return "SQL_C_FLOAT";
case SQL_C_DOUBLE: return "SQL_C_DOUBLE";
case SQL_C_BIT: return "SQL_C_BIT";
case SQL_C_TINYINT: return "SQL_C_TINYINT";
case SQL_C_STINYINT: return "SQL_C_STINYINT";
case SQL_C_UTINYINT: return "SQL_C_UTINYINT";
case SQL_C_SBIGINT: return "SQL_C_SBIGINT";
case SQL_C_UBIGINT: return "SQL_C_UBIGINT";
case SQL_C_BINARY: return "SQL_C_BINARY";
case SQL_C_DATE: return "SQL_C_DATE";
case SQL_C_TIME: return "SQL_C_TIME";
case SQL_C_TIMESTAMP: return "SQL_C_TIMESTAMP";
case SQL_C_TYPE_DATE: return "SQL_C_TYPE_DATE";
case SQL_C_TYPE_TIME: return "SQL_C_TYPE_TIME";
case SQL_C_TYPE_TIMESTAMP: return "SQL_C_TYPE_TIMESTAMP";
case SQL_C_NUMERIC: return "SQL_C_NUMERIC";
case SQL_C_GUID: return "SQL_C_GUID";
default: return "UNKNOWN";
}
}
int is_valid_sql_c_type(int type) {
const char *ctype = sql_c_type(type);
if (strcmp(ctype, "UNKNOWN")==0) return 0;
return 1;
}
int is_valid_sql_sql_type(int type) {
const char *sqltype = sql_sql_type(type);
if (strcmp(sqltype, "UNKNOWN")==0) return 0;
return 1;
}
int string_conv(const char *fromcode, const char *tocode,
const unsigned char *src, size_t sbytes,
unsigned char *dst, size_t dbytes,
size_t *consumed, size_t *generated)
{
if (consumed) *consumed = 0;
if (generated) *generated = 0;
if (dbytes <= 0) return -1;
dst[0] = '\0';
iconv_t conv = iconv_open(tocode, fromcode);
if (!conv) return -1;
int r = 0;
do {
char *s = (char*)src;
char *d = (char*)dst;
size_t sl = sbytes;
size_t dl = dbytes;
r = iconv(conv, &s, &sl, &d, &dl);
*d = '\0';
if (consumed) *consumed = sbytes - sl;
if (generated) *generated = dbytes - dl;
} while (0);
iconv_close(conv);
return r;
}
int utf8_chars(const char *src)
{
const char *fromcode = "UTF-8";
const char *tocode = "UCS-2LE";
iconv_t conv = iconv_open(tocode, fromcode);
if (!conv) return -1;
size_t slen = strlen(src);
char buf[4096];
size_t dlen = sizeof(buf);
char *ps = (char*)src;
char *pd = buf;
iconv(conv, &ps, &slen, &pd, &dlen);
DASSERT(slen==0);
size_t chars = (sizeof(buf) - dlen) / 2;
iconv_close(conv);
return chars;
}
unsigned char* utf8_to_ucs4le(const char *utf8, size_t *chars)
{
const char *tocode = "UCS-4LE";
const char *fromcode = "UTF-8";
iconv_t conv = iconv_open(tocode, fromcode);
if (!conv) return NULL;
unsigned char *ucs4le = NULL;
do {
size_t slen = strlen(utf8);
size_t dlen = slen * 4;
ucs4le = (unsigned char*)malloc(dlen+1);
if (!ucs4le) break;
char *src = (char*)utf8;
char *dst = (char*)ucs4le;
size_t s = slen;
size_t d = dlen;
iconv(conv, &src, &s, &dst, &d);
dst[0] = '\0';
if (chars) *chars = (dlen - d) / 4;
} while (0);
iconv_close(conv);
return ucs4le;
}
char* ucs4le_to_utf8(const unsigned char *ucs4le, size_t slen, size_t *chars)
{
const char *fromcode = "UCS-4LE";
const char *tocode = "UTF-8";
iconv_t conv = iconv_open(tocode, fromcode);
if (!conv) return NULL;
char *utf8 = NULL;
do {
size_t dlen = slen;
utf8 = (char*)malloc(dlen+1);
if (!utf8) break;
char *dst = utf8;
char *src = (char*)ucs4le;
size_t s = slen;
size_t d = dlen;
iconv(conv, &src, &s, &dst, &d);
dst[0] = '\0';
if (chars) *chars = (slen - s) / 4;
} while (0);
iconv_close(conv);
return utf8;
}
SQLCHAR* wchars_to_chars(const SQLWCHAR *wchars, size_t chs, size_t *bytes)
{
size_t dlen = chs * 4;
SQLCHAR *dst = (SQLCHAR*)malloc(dlen + 1);
if (!dst) return NULL;
string_conv("UCS-2LE", "UTF-8", (const unsigned char*)wchars, chs * sizeof(*wchars), dst, dlen + 1, NULL, bytes);
return dst;
}
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TODBC_UTIL_H_
#define _TODBC_UTIL_H_
#include <libgen.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <sql.h>
#define D(fmt, ...) \
fprintf(stderr, \
"%s[%d]:%s() " fmt "\n", \
basename((char*)__FILE__), __LINE__, __func__, \
##__VA_ARGS__)
#define DASSERT(statement) \
do { \
if (statement) break; \
D("Assertion failure: %s", #statement); \
abort(); \
} while (0)
#define DASSERTX(statement, fmt, ...) \
do { \
if (statement) break; \
D("Assertion failure: %s, " fmt "", #statement, ##__VA_ARGS__); \
abort(); \
} while (0)
const char* sql_sql_type(int type);
const char* sql_c_type(int type);
int is_valid_sql_c_type(int type);
int is_valid_sql_sql_type(int type);
int string_conv(const char *fromcode, const char *tocode,
const unsigned char *src, size_t sbytes,
unsigned char *dst, size_t dbytes,
size_t *consumed, size_t *generated);
int utf8_chars(const char *src);
unsigned char* utf8_to_ucs4le(const char *utf8, size_t *chars);
char* ucs4le_to_utf8(const unsigned char *ucs4le, size_t slen, size_t *chars);
SQLCHAR* wchars_to_chars(const SQLWCHAR *wchars, size_t chs, size_t *bytes);
#endif // _TODBC_UTIL_H_
PROJECT(TDengine)
IF (TD_LINUX)
AUX_SOURCE_DIRECTORY(. SRC)
ADD_EXECUTABLE(tcodbc main.c)
TARGET_LINK_LIBRARIES(tcodbc odbc)
ENDIF ()
#include <sql.h>
#include <sqlext.h>
#include <stdio.h>
#include <string.h>
#include "os.h"
// static const char *dsn = "TAOS_DSN";
// static const char *uid = "root";
// static const char *pwd = "taosdata";
typedef struct data_s data_t;
struct data_s {
int64_t ts;
int8_t b;
int8_t v1;
int16_t v2;
int32_t v4;
int64_t v8;
float f4;
double f8;
char bin[40+1];
char blob[40+1]; // why 80? ref: tests/examples/c/apitest.c
};
static const char *pre_stmts[] = {
"create database db",
"use db",
"create table t (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), blob nchar(10))"
};
static const char *pro_stmts[] = {
// "insert into t values ('2019-07-15 00:00:00', 1)",
// "insert into t values ('2019-07-15 01:00:00', 2)",
"select * from t"
// "drop database db"
};
#define CHK_RESULT(r, ht, h) \
do { \
if (r==0) break; \
SQLCHAR ss[10]; \
SQLINTEGER ne = 0; \
SQLCHAR es[4096]; \
SQLSMALLINT n = 0; \
ss[0] = '\0'; \
es[0] = '\0'; \
SQLRETURN ret = SQLGetDiagRec(ht, h, 1, ss, &ne, es, sizeof(es), &n); \
if (ret) break; \
fprintf(stderr, "%s%s\n", ss, es); \
} while (0)
static int do_statement(SQLHSTMT stmt, const char *statement) {
SQLRETURN r = 0;
do {
fprintf(stderr, "prepare [%s]\n", statement);
r = SQLPrepare(stmt, (SQLCHAR*)statement, strlen(statement));
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
fprintf(stderr, "execute [%s]\n", statement);
r = SQLExecute(stmt);
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
fprintf(stderr, "done\n");
} while (0);
fprintf(stderr, "r: [%x][%d]\n", r, r);
return r;
}
static int do_insert(SQLHSTMT stmt, data_t data) {
SQLRETURN r = 0;
SQLLEN lbin;
SQLLEN lblob;
const char *statement = "insert into t values (?, ?, ?, ?, ?, ?, ?, ?, ?,?)";
int ignored = 0;
do {
fprintf(stderr, "prepare [%s]\n", statement);
r = SQLPrepare(stmt, (SQLCHAR*)statement, strlen(statement));
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
fprintf(stderr, "bind 1 [%s]\n", statement);
r = SQLBindParameter(stmt, 1, SQL_PARAM_INPUT, SQL_C_SBIGINT, SQL_TIMESTAMP, ignored, ignored, &data.ts, ignored, NULL);
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
fprintf(stderr, "bind 2 [%s]\n", statement);
r = SQLBindParameter(stmt, 2, SQL_PARAM_INPUT, SQL_C_BIT, SQL_BIT, ignored, ignored, &data.b, ignored, NULL);
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
fprintf(stderr, "bind 3 [%s]\n", statement);
r = SQLBindParameter(stmt, 3, SQL_PARAM_INPUT, SQL_C_TINYINT, SQL_TINYINT, ignored, ignored, &data.v1, ignored, NULL);
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
fprintf(stderr, "bind 4 [%s]\n", statement);
r = SQLBindParameter(stmt, 4, SQL_PARAM_INPUT, SQL_C_SHORT, SQL_SMALLINT, ignored, ignored, &data.v2, ignored, NULL);
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
fprintf(stderr, "bind 5 [%s]\n", statement);
r = SQLBindParameter(stmt, 5, SQL_PARAM_INPUT, SQL_C_LONG, SQL_INTEGER, ignored, ignored, &data.v4, ignored, NULL);
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
fprintf(stderr, "bind 6 [%s]\n", statement);
r = SQLBindParameter(stmt, 6, SQL_PARAM_INPUT, SQL_C_SBIGINT, SQL_BIGINT, ignored, ignored, &data.v8, ignored, NULL);
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
fprintf(stderr, "bind 7 [%s]\n", statement);
r = SQLBindParameter(stmt, 7, SQL_PARAM_INPUT, SQL_C_FLOAT, SQL_FLOAT, ignored, ignored, &data.f4, ignored, NULL);
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
fprintf(stderr, "bind 8 [%s]\n", statement);
r = SQLBindParameter(stmt, 8, SQL_PARAM_INPUT, SQL_C_DOUBLE, SQL_DOUBLE, ignored, ignored, &data.f8, ignored, NULL);
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
fprintf(stderr, "bind 9 [%s]\n", statement);
lbin = SQL_NTS;
r = SQLBindParameter(stmt, 9, SQL_PARAM_INPUT, SQL_C_BINARY, SQL_VARBINARY, sizeof(data.bin)-1, ignored, &data.bin, ignored, &lbin);
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
fprintf(stderr, "bind 10 [%s]\n", statement);
lblob = SQL_NTS;
r = SQLBindParameter(stmt, 10, SQL_PARAM_INPUT, SQL_C_CHAR, SQL_VARCHAR, sizeof(data.blob)-1, ignored, &data.blob, ignored, &lblob);
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
fprintf(stderr, "execute [%s]\n", statement);
r = SQLExecute(stmt);
CHK_RESULT(r, SQL_HANDLE_STMT, stmt);
if (r) break;
// ts += 1;
// v = 2;
// fprintf(stderr, "execute [%s]\n", statement);
// r = SQLExecute(stmt);
// if (r) break;
fprintf(stderr, "done\n");
} while (0);
fprintf(stderr, "r: [%x][%d]\n", r, r);
return r;
}
int main(int argc, char *argv[]) {
if (argc < 4) return 1;
const char *dsn = argv[1];
const char *uid = argv[2];
const char *pwd = argv[3];
SQLRETURN r;
SQLHENV env = {0};
SQLHDBC conn = {0};
r = SQLAllocEnv(&env);
if (r!=SQL_SUCCESS) return 1;
do {
r = SQLAllocConnect(env, &conn);
CHK_RESULT(r, SQL_HANDLE_ENV, env);
if (r!=SQL_SUCCESS) break;
do {
r = SQLConnect(conn, (SQLCHAR*)dsn, strlen(dsn),
(SQLCHAR*)uid, strlen(uid),
(SQLCHAR*)pwd, strlen(pwd));
CHK_RESULT(r, SQL_HANDLE_DBC, conn);
if (r!=SQL_SUCCESS) break;
do {
SQLHSTMT stmt = {0};
r = SQLAllocHandle(SQL_HANDLE_STMT, conn, &stmt);
if (r!=SQL_SUCCESS) break;
do {
do_statement(stmt, "drop database db");
for (size_t i=0; i<sizeof(pre_stmts)/sizeof(pre_stmts[0]); ++i) {
r = do_statement(stmt, pre_stmts[i]);
if (r!=SQL_SUCCESS) break;
}
do {
data_t data = {0};
data.ts = 1591060628001;
data.b = 1;
data.v1 = 127;
data.v2 = 32767;
data.v4 = 2147483647;
data.v8 = 9223372036854775807;
data.f4 = 123.456;
data.f8 = 9999999.999999;
memset(data.bin, 0, sizeof(data.bin));
memset(data.blob, 0, sizeof(data.blob));
snprintf(data.bin, sizeof(data.bin), "hel我lo");
snprintf(data.blob, sizeof(data.blob), "world");
snprintf(data.blob, sizeof(data.blob), "wo人rld");
SQLHSTMT stmt = {0};
r = SQLAllocHandle(SQL_HANDLE_STMT, conn, &stmt);
if (r!=SQL_SUCCESS) break;
do {
r = do_insert(stmt, data);
if (r!=SQL_SUCCESS) break;
} while (0);
SQLFreeHandle(SQL_HANDLE_STMT, stmt);
// r = SQLAllocHandle(SQL_HANDLE_STMT, conn, &stmt);
// if (r!=SQL_SUCCESS) break;
// do {
// r = do_insert(stmt, ts++, v++);
// if (r!=SQL_SUCCESS) break;
// } while (0);
// SQLFreeHandle(SQL_HANDLE_STMT, stmt);
} while (0);
if (r!=SQL_SUCCESS) break;
for (size_t i=0; i<sizeof(pro_stmts)/sizeof(pro_stmts[0]); ++i) {
r = do_statement(stmt, pro_stmts[i]);
if (r!=SQL_SUCCESS) break;
}
} while (0);
SQLFreeHandle(SQL_HANDLE_STMT, stmt);
} while (0);
SQLDisconnect(conn);
} while (0);
SQLFreeConnect(conn);
} while (0);
SQLFreeEnv(env);
return r ? 1 : 0;
}
import pyodbc
cnxn = pyodbc.connect('DSN=TAOS_DSN;UID=root;PWD=taosdata', autocommit=True)
cnxn.setdecoding(pyodbc.SQL_CHAR, encoding='utf-8')
#cnxn.setdecoding(pyodbc.SQL_WCHAR, encoding='utf-8')
#cnxn.setencoding(encoding='utf-8')
cursor = cnxn.cursor()
cursor.execute("SELECT * from db.t")
row = cursor.fetchone()
while row:
print(row)
row = cursor.fetchone()
cursor.close()
#cursor = cnxn.cursor()
#cursor.execute("""
#INSERT INTO db.t values (?,?,?,?,?,?,?,?,?,?)
#""",
#"2020-12-12 00:00:00",
#1,
#27,
#32767,
#147483647,
#223372036854775807,
#23.456,
#899.999999,
#"foo",
#"bar")
cursor = cnxn.cursor()
cursor.execute("drop database if exists db");
cursor.close()
cursor = cnxn.cursor()
cursor.execute("create database db");
cursor.close()
cursor = cnxn.cursor()
cursor.execute("create table db.t (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), blob nchar(10))");
cursor.close()
cursor = cnxn.cursor()
cursor.execute("insert into db.t values('2020-10-13 06:44:00', 1, 127, 32767, 32768, 32769, 123.456, 789.987, 'hello', 'world')")
cursor.close()
cursor = cnxn.cursor()
cursor.execute("insert into db.t values(?,?,?,?,?,?,?,?,?,?)", "2020-10-13 07:06:00", 0, 127, 32767, 32768, 32769, 123.456, 789.987, "hel后lo", "wo哈rld");
cursor.close()
cursor = cnxn.cursor()
cursor.execute("SELECT * from db.t")
row = cursor.fetchone()
while row:
print(row)
row = cursor.fetchone()
cursor.close()
cursor = cnxn.cursor()
cursor.execute("drop database if exists db");
cursor.close()
cursor = cnxn.cursor()
cursor.execute("create database db");
cursor.close()
cursor = cnxn.cursor()
cursor.execute("create table db.t (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(4), blob nchar(4))");
cursor.close()
cursor = cnxn.cursor()
cursor.execute("insert into db.t values('2020-10-13 06:44:00', 1, 127, 32767, 32768, 32769, 123.456, 789.987, 'hell', 'worl')")
cursor.close()
cursor = cnxn.cursor()
cursor.execute("SELECT * from db.t")
row = cursor.fetchone()
while row:
print(row)
row = cursor.fetchone()
cursor.close()
cursor = cnxn.cursor()
cursor.execute("create table db.v (ts timestamp, v1 tinyint)")
cursor.close()
params = [ ('A', 1), ('B', 2), ('C', 3) ]
params = [ ('A', 1), ('B', 2), ('C', 3) ]
params = [ ('2020-10-16 00:00:00', 1),
('2020-10-16 00:00:01', 4),
('2020-10-16 00:00:02', 5),
('2020-10-16 00:00:03.009', 6) ]
cursor = cnxn.cursor()
cursor.fast_executemany = True
cursor.executemany("insert into db.v values (?, ?)", params)
cursor.close()
cursor = cnxn.cursor()
cursor.execute("SELECT * from db.v")
row = cursor.fetchone()
while row:
print(row)
row = cursor.fetchone()
cursor.close()
cursor = cnxn.cursor()
cursor.execute("SELECT * from db.v where v1 > ?", 4)
row = cursor.fetchone()
while row:
print(row)
row = cursor.fetchone()
cursor.close()
cursor = cnxn.cursor()
cursor.execute("SELECT * from db.v where v1 > ?", '5')
row = cursor.fetchone()
while row:
print(row)
row = cursor.fetchone()
cursor.close()
...@@ -225,6 +225,7 @@ class CTaosInterface(object): ...@@ -225,6 +225,7 @@ class CTaosInterface(object):
if connection.value == None: if connection.value == None:
print('connect to TDengine failed') print('connect to TDengine failed')
raise ConnectionError("connect to TDengine failed")
# sys.exit(1) # sys.exit(1)
else: else:
print('connect to TDengine success') print('connect to TDengine success')
......
...@@ -192,8 +192,10 @@ class TDengineCursor(object): ...@@ -192,8 +192,10 @@ class TDengineCursor(object):
buffer = [[] for i in range(len(self._fields))] buffer = [[] for i in range(len(self._fields))]
self._rowcount = 0 self._rowcount = 0
while True: while True:
block, num_of_fields = CTaosInterface.fetchBlock( block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result)
if errno != 0:
raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
if num_of_fields == 0: if num_of_fields == 0:
break break
self._rowcount += num_of_fields self._rowcount += num_of_fields
......
...@@ -225,6 +225,7 @@ class CTaosInterface(object): ...@@ -225,6 +225,7 @@ class CTaosInterface(object):
if connection.value == None: if connection.value == None:
print('connect to TDengine failed') print('connect to TDengine failed')
raise ConnectionError("connect to TDengine failed")
# sys.exit(1) # sys.exit(1)
#else: #else:
# print('connect to TDengine success') # print('connect to TDengine success')
......
...@@ -207,8 +207,10 @@ class TDengineCursor(object): ...@@ -207,8 +207,10 @@ class TDengineCursor(object):
buffer = [[] for i in range(len(self._fields))] buffer = [[] for i in range(len(self._fields))]
self._rowcount = 0 self._rowcount = 0
while True: while True:
block, num_of_fields = CTaosInterface.fetchBlock( block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result)
if errno != 0:
raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
if num_of_fields == 0: if num_of_fields == 0:
break break
self._rowcount += num_of_fields self._rowcount += num_of_fields
......
...@@ -142,6 +142,9 @@ class TDengineCursor(object): ...@@ -142,6 +142,9 @@ class TDengineCursor(object):
self._rowcount = 0 self._rowcount = 0
while True: while True:
block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
errno = CTaosInterface.libtaos.taos_errno(self._result)
if errno != 0:
raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
if num_of_fields == 0: break if num_of_fields == 0: break
self._rowcount += num_of_fields self._rowcount += num_of_fields
for i in range(len(self._fields)): for i in range(len(self._fields)):
......
...@@ -142,6 +142,9 @@ class TDengineCursor(object): ...@@ -142,6 +142,9 @@ class TDengineCursor(object):
self._rowcount = 0 self._rowcount = 0
while True: while True:
block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
errno = CTaosInterface.libtaos.taos_errno(self._result)
if errno != 0:
raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
if num_of_fields == 0: break if num_of_fields == 0: break
self._rowcount += num_of_fields self._rowcount += num_of_fields
for i in range(len(self._fields)): for i in range(len(self._fields)):
......
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册